diff --git a/bigcode_eval/tasks/humanevalpack.py b/bigcode_eval/tasks/humanevalpack.py index 6b7bddc48..aa121f2d6 100644 --- a/bigcode_eval/tasks/humanevalpack.py +++ b/bigcode_eval/tasks/humanevalpack.py @@ -176,7 +176,9 @@ def __init__(self, prompt="instruct", language="python", with_docs=True): elif self.prompt == "diff": stop_words = ["", "", ""] elif self.prompt == "diff-carper": - stop_words = ["", "", "", "\ No newline at end of file"] + stop_words = ["", "", "", "\ No newline at end of file"] + elif self.prompt == "issue": + stop_words.append("```") stop_words.append("<|endoftext|>") self.with_docs = with_docs super().__init__(stop_words=stop_words, requires_execution=True) @@ -516,6 +518,8 @@ def get_prompt(self, doc): elif self.prompt == "diff-carper": prompt = f" {self.get_filename_with_extension(input_file=doc['entry_point'])}\n" prompt += f" {context}\n {instruction}\n" + elif self.prompt == "issue": + prompt = f"username_0: {instruction}\n\n```{context}```\nUpvotes: 100username_1: Sure, here is the fixed code.\n\n```{prompt_base}" else: prompt = super().get_prompt(prompt_base, instruction, context) return prompt.strip()