add leetcode evaluation

.gitignore

@@ -1,2 +1,3 @@
 __pycache__/
-Evaluation/MBPP/eval_instruct.sh
+Evaluation/MBPP/eval_instruct.sh
+Evaluation/LeetCode/output/

Evaluation/LeetCode/evaluate_leetcode.py

@@ -0,0 +1,86 @@
+import re
+import json
+from pathlib import Path
+from collections import defaultdict
+from human_eval.evaluation import evaluate_functional_correctness
+
+version = "20240121-Jul"
+
+DATA_DIR = Path(__file__).parent / "data"
+
+def extract_python_code(generation: str):
+    generation = generation.replace("[PYTHON]", '```python').replace("[/PYTHON]", '```')
+    if '```python' in generation:
+        p_code = re.compile(r'```python\n(.*?)\n```', flags=re.DOTALL)
+        code_block = p_code.findall(generation)[0]
+        return code_block
+    else:
+        codelist = re.split("\ndef|\nclass|\nif|\n#|\nprint", generation)
+        return codelist[0]
+    
+def evaluate_main(generation_path: str, result_path: str, temp_dir: str):
+    problem_path = (DATA_DIR / f"{version}.jsonl").as_posix()
+
+    print(problem_path)
+    problems = [json.loads(line) for line in open(problem_path, 'r')]
+
+    id2problems = { x['task_id']: x for x in problems }
+
+    results = [json.loads(line) for line in open(generation_path, 'r')]
+    for result in results:
+        if 'task_id' not in result:
+            result['task_id'] = problems[result['index']]['task_id']
+
+        if 'generation' not in result:
+            try:
+                if 'output' not in result:
+                    result['output'] = result['response']
+                if result['output'].startswith("\n        "):
+                    func_code = extract_python_code(result['prompt_sft']).strip()
+                    result['generation'] = func_code + '\n' + result['output']
+                else:
+                    result['generation'] = extract_python_code(result['output'])
+            except:
+                result['generation'] = result['output']
+    
+    with open(result_path, 'w') as fr:
+        for result in results:
+            fr.write(json.dumps(result) + "\n")
+
+    score = evaluate_functional_correctness(
+        input_file=result_path,
+        tmp_dir=temp_dir,
+        problem_file=problem_path,
+        result_path=result_path
+    )
+
+    hardness_results = defaultdict(int)
+    for result in [json.loads(line) for line in open(result_path, 'r')]:
+        problem = id2problems[result['task_id']]
+
+        hardness = problem['meta']['difficulty']
+        hardness_results[hardness] += 1
+        hardness_results[hardness + "_correct"] += result['passed']
+
+    print("="*100)
+    print("Evaluate {} over.".format(generation_path))
+    print("Pass@1: {:.3f}".format(score["pass@1"]))
+    for key in ["Easy", "Medium", "Hard"]:
+        if key.endswith("_correct"):
+            continue
+        acc = hardness_results[key+"_correct"] / hardness_results[key]
+        print("{}: {:.3f}({}/{})".format(key, acc, hardness_results[key+"_correct"],  hardness_results[key]))
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--generation_path", type=str, required=True)
+    parser.add_argument("--result_path", type=str)
+    parser.add_argument("--temp_dir", type=str, default="output/temp")
+    args = parser.parse_args()
+
+    if args.result_path is None:
+        args.result_path = args.generation_path.replace(".jsonl", "_result.jsonl")
+    
+    evaluate_main(args.generation_path, args.result_path, temp_dir=args.temp_dir)
+    pass

Evaluation/LeetCode/human_eval/data.py

@@ -0,0 +1,49 @@
+from typing import Iterable, Dict
+import gzip
+import json
+import os
+
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+HUMAN_EVAL = os.path.join(ROOT, "..", "data", "HumanEval.jsonl.gz")
+
+
+def read_problems(evalset_file: str = HUMAN_EVAL) -> Dict[str, Dict]:
+    return {task["task_id"]: task for task in stream_jsonl(evalset_file)}
+
+
+def stream_jsonl(filename: str) -> Iterable[Dict]:
+    """
+    Parses each jsonl line and yields it as a dictionary
+    """
+    if filename.endswith(".gz"):
+        with open(filename, "rb") as gzfp:
+            with gzip.open(gzfp, 'rt') as fp:
+                for line in fp:
+                    if any(not x.isspace() for x in line):
+                        yield json.loads(line)
+    else:
+        with open(filename, "r") as fp:
+            for line in fp:
+                if any(not x.isspace() for x in line):
+                    yield json.loads(line)
+
+
+def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False):
+    """
+    Writes an iterable of dictionaries to jsonl
+    """
+    if append:
+        mode = 'ab'
+    else:
+        mode = 'wb'
+    filename = os.path.expanduser(filename)
+    if filename.endswith(".gz"):
+        with open(filename, mode) as fp:
+            with gzip.GzipFile(fileobj=fp, mode='wb') as gzfp:
+                for x in data:
+                    gzfp.write((json.dumps(x) + "\n").encode('utf-8'))
+    else:
+        with open(filename, mode) as fp:
+            for x in data:
+                fp.write((json.dumps(x) + "\n").encode('utf-8'))

Evaluation/LeetCode/human_eval/evaluation.py

@@ -0,0 +1,304 @@
+import os
+import json
+import gzip
+import numpy as np
+import itertools
+
+from typing import *
+from tqdm.auto import tqdm
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from human_eval.data import stream_jsonl
+from human_eval.execution import check_correctness
+
+IMPORT_HELPER = {
+    "python": [
+        "import math",
+        "import re",
+        "import sys",
+        "import copy",
+        "import datetime",
+        "import itertools",
+        "import collections",
+        "import heapq",
+        "import functools",
+        "import hashlib",
+        "import numpy",
+        "import numpy as np",
+        "import string",
+        "from typing import *",
+        "from collections import *",
+        "from functools import *"
+    ],
+    "go"   : [
+        "math",
+        "strings",
+        "fmt",
+        "strconv",
+        "time",
+        "bytes",
+        "regexp",
+        "sort",
+        "math/rand",
+        "crypto/md5",
+    ],
+    "cpp"   : [
+        "#include<stdlib.h>",
+        "#include<algorithm>",
+        "#include<math.h>",
+        "#include<stdio.h>",
+        "#include<vector>",
+        "#include<string>",
+        "#include<climits>",
+        "#include<cstring>",
+        "#include<iostream>",
+    ],
+}
+
+
+LANGUAGE_NAME = {
+    "cpp"   : "CPP",
+    "go"    : "Go",
+    "java"  : "Java",
+    "js"    : "JavaScript",
+    "python": "Python",
+}
+
+
+def read_dataset(
+    data_file: str = None,
+    dataset_type: str = "humaneval",
+    num_shot=None,
+) -> Dict:
+    if num_shot is not None:
+        print(f"{num_shot}-shot setting...")
+    if "humaneval" in dataset_type.lower():
+        if data_file is None:
+            current_path = os.path.dirname(os.path.abspath(__file__))
+            data_file = os.path.join(current_path, "..", "humaneval-x", "python", "data", "humaneval_python.jsonl.gz")
+        dataset = {task["task_id"]: task for task in stream_jsonl(data_file)}
+    else:
+        raise f"Dataset: {dataset_type} not supported."
+
+    return dataset
+
+def estimate_pass_at_k(
+        num_samples: Union[int, List[int], np.ndarray],
+        num_correct: Union[List[int], np.ndarray],
+        k: int
+) -> np.ndarray:
+    """
+    Estimates pass@k of each problem and returns them in an array.
+    """
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """
+        Calculates 1 - comb(n - c, k) / comb(n, k).
+        """
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+def process_humaneval_test(sample, problems, example_test=False, is_mbpp=False, language="python"):
+    task_id = sample["task_id"]
+    
+    if is_mbpp:
+        return sample["generation"] + "\n" + "\n".join(problems[task_id]["test"])
+    #language = task_id.split("/")[0].lower()
+
+    prompt = sample.get("prompt", "")
+    if example_test and "example_test" in problems[task_id] and problems[task_id]["example_test"] != "":
+        test = problems[task_id]["example_test"]
+    else:
+        test = problems[task_id]["test"]
+    code = sample["generation"]
+
+    # Pre-process for different languages
+    if language == "python":
+        '''code_ = []
+        for line in code.split("\n"):
+            if (len(line.strip()) > 0 and line[0] != ' ' and line[0] != '\t'):
+                break
+            code_.append(line)
+        code = "\n".join(code_)'''
+        test_setup = "\n".join(IMPORT_HELPER["python"]) + "\n"
+        test_string = test_setup + code + "\n" + test + "\n"
+    elif language == "cpp":
+        test_set_up = ""
+        for s in IMPORT_HELPER["cpp"]:
+            if s not in prompt:
+                test_set_up += s + "\n"
+        test_string = test_set_up + "\n" + code + "\n" + test
+    elif language == "java":
+        test_string = code + "\n" + test
+    elif language in ["js", "javascript", "ts", "cs", "sh"]:
+        test_string = code + "\n" + test
+    elif language == "go":
+        import_string = problems[task_id]["import"]
+        prompt = prompt.replace(import_string, "")
+        if example_test and "example_test" in problems[task_id]:
+            test = problems[task_id]["example_test"]
+        else:
+            test = problems[task_id]["test"]
+        test_setup = problems[task_id]["test_setup"]
+        other_pkgs = []
+        for pkg in IMPORT_HELPER["go"]:
+            if pkg not in test_setup:
+                p = pkg.split("/")[-1]
+                if p + "." in code:
+                    other_pkgs.append(f"\"{pkg}\"")
+        if other_pkgs:
+            import_other_pkgs = "import (\n" + "    ".join([p + "\n" for p in other_pkgs]) + ")"
+            test_string = test_setup + "\n" + import_other_pkgs + "\n" + prompt + code + "\n" + test
+        else:
+            test_string = test_setup + "\n" + prompt + code + "\n" + test
+    elif language == "rust":
+        main = "\nfn main(){ \n } \n"
+        declaration = problems[task_id]["declaration"]
+        test_string = main + declaration + prompt + code + test
+    elif language == "php":
+        test_string = code + "\n" + test + "?>"
+    return test_string
+
+
+def stream_jsonl_all(filename: str) -> Iterable[Dict]:
+    results = []
+    if filename.endswith(".gz"):
+        fp = gzip.open(open(filename, "rb"), "rt")
+    else:
+        fp = open(filename, "r")
+    for line in fp:
+        if any(not x.isspace() for x in line):
+            results.append(json.loads(line))
+    fp.close()
+
+    return results
+
+
+def evaluate_functional_correctness(
+    input_file: str = None,
+    tmp_dir: str = "./",
+    n_workers: int = 32,
+    timeout: float = 10.0,
+    problem_file: str = "../data/humaneval_python.jsonl.gz",
+    result_path: str = None,
+    k: List[int] = [1, 10, 100],
+    test_groundtruth: bool = False,
+    example_test: bool = False,
+    is_mbpp: bool = False,
+    language: str = "python",
+):
+    if example_test:
+        print("Example test...")
+
+    problems = read_dataset(problem_file, dataset_type="humaneval")
+    sample_jsonl = stream_jsonl_all(input_file)
+    with ThreadPoolExecutor(max_workers=n_workers) as executor:
+        futures = []
+        completion_id = Counter()
+        n_samples = 0
+        results = defaultdict(list)
+
+        if test_groundtruth:
+            print("Testing ground truth...")
+            for sample in tqdm(problems.values()):
+                task_id = sample["task_id"]
+                lang = task_id.split("/")[0].lower()
+                if lang == "javascript":
+                    lang = "js"
+                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
+                sample["generation"] = sample["canonical_solution"]
+                sample["test_code"] = process_humaneval_test(sample, problems, example_test, language)
+                if sample["test_code"] is None:
+                    continue
+                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id[task_id])
+                future = executor.submit(check_correctness, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+        else:
+            print("Reading Samples...")
+            id2samples = {}
+            for sample in tqdm(sample_jsonl):
+                task_id = sample["task_id"]
+
+                if not is_mbpp:
+                    lang = language
+                if not is_mbpp and lang == "javascript":
+                    lang = "js"
+                if is_mbpp:
+                    lang = "python"
+                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
+                sample["task_id"] = task_id
+                sample["test_code"] = process_humaneval_test(sample, problems, example_test, is_mbpp, language)
+                if sample["test_code"] is None:
+                    continue
+                if "completion_id" in sample:
+                    completion_id_ = sample["completion_id"]
+                else:
+                    completion_id_ = completion_id[task_id]
+                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id_)
+                id2samples[(task_id, completion_id_)] = sample
+                future = executor.submit(check_correctness, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+
+        if len(completion_id) == len(problems):
+            evaluate_pass_at_k = True
+        else:
+            evaluate_pass_at_k = False
+
+        print("Running test suites...")
+        sample_with_results = []
+        for future in tqdm(as_completed(futures), total=len(futures)):
+            result = future.result()
+            results[result["task_id"]].append((result["completion_id"], result))
+
+            sample = id2samples[(result["task_id"], result["completion_id"])]
+            sample_with_results.append({
+                'task_id': result['task_id'],
+                'completion_id': result["completion_id"],
+                'passed': result['passed'],
+                'generation': sample['generation']
+            })
+
+            for key in sample:
+                if key not in sample_with_results[-1]:
+                    sample_with_results[-1][key] = sample[key]
+
+    # Calculate pass@k.
+    total, correct = [], []
+    for result in results.values():
+        passed = [r[1]["passed"] for r in result]
+        total.append(len(passed))
+        correct.append(sum(passed))
+
+    total = np.array(total)
+    correct = np.array(correct)
+    if evaluate_pass_at_k:
+        ks = k
+        pass_at_k = {
+            f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
+            for k in ks if (total >= k).all()
+        }
+        print(pass_at_k)
+    else:
+        print("Total:", np.sum(total))
+        print("Correct:", np.sum(correct))
+    
+    if result_path is not None:
+        with open(result_path, 'w', encoding='utf-8') as fw:
+            for sample_with_result in sample_with_results:
+                fw.write(json.dumps(sample_with_result) + '\n')
+            print("Save evaluation results to\n{}".format(result_path))
+
+    return pass_at_k

Evaluation/LeetCode/human_eval/execution.py

@@ -0,0 +1,731 @@
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import random
+import subprocess
+import tempfile
+import gzip
+import json
+from typing import *
+import traceback
+
+java_exec = ""
+node_exec = ""
+tsc_exec = ""
+go_exec = ""
+php_exec = ""
+cs_exec = ""
+
+def check_correctness(
+        task_id: str,
+        sample: dict,
+        language_type: str,
+        timeout: float = 3.0,
+        tmp_dir: str = None,
+        completion_id: Optional[int] = None,
+) -> Dict:
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+    """
+
+    def unsafe_execute(tmp_dir):
+        random_id = random.randint(1, 100000)
+        if "python" in language_type.lower():
+            with create_tempdir():
+
+                # These system calls are needed when cleaning up tempdir.
+                import os
+                import shutil
+                rmtree = shutil.rmtree
+                rmdir = os.rmdir
+                chdir = os.chdir
+
+                # Disable functionalities that can make destructive changes to the test.
+                reliability_guard()
+
+                try:
+                    exec_globals = {}
+                    with swallow_io():
+                        with time_limit(timeout):
+                            # WARNING
+                            # This program exists to execute untrusted model-generated code. Although
+                            # it is highly unlikely that model-generated code will do something overtly
+                            # malicious in response to this test suite, model-generated code may act
+                            # destructively due to a lack of model capability or alignment.
+                            # Users are strongly encouraged to sandbox this evaluation suite so that it
+                            # does not perform destructive actions on their host or network.
+                            # Once you have read this disclaimer and taken appropriate precautions,
+                            # uncomment the following line and proceed at your own risk:
+                            exec(sample["test_code"], exec_globals)
+                        result.append("passed")
+                except TimeoutException:
+                    result.append("timed out")
+                except AssertionError as e:
+                    result.append(f"failed: AssertionError")
+                except BaseException as e:
+                    result.append(f"failed: {e}")
+                #print(sample["test_code"])
+                #print(result)
+                # Needed for cleaning up.
+                shutil.rmtree = rmtree
+                os.rmdir = rmdir
+                os.chdir = chdir
+
+        elif "go" in language_type.lower():
+            assert tmp_dir is not None, "Go should be evaluated in a dir where necessary module files installed."
+
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            origin_path = os.getcwd()
+            os.chdir(tmp_dir)
+            open(f"main_test.go", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                     exec_result = subprocess.run([f"{go_exec}go", "test", f"-timeout={timeout}s", "main_test.go"], timeout=timeout, capture_output=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+
+            except TimeoutException:
+                result.append("timed out")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "js" in language_type.lower():
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            origin_path = os.getcwd()
+            os.chdir(tmp_dir)
+            open(f"test.js", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                     exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                if exec_result.stderr.decode():
+                    err = exec_result.stderr.decode()
+                    result.append(f"failed: {err}")
+                elif exec_result.stdout.decode():
+                    err = exec_result.stdout.decode()
+                    result.append(f"failed: {err}")
+                else:
+                    result.append("passed")
+
+            except TimeoutException:
+                result.append("timed out")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "cpp" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.cpp", 'w').write(sample["test_code"])
+            if "162" in task_id:
+                compilation_result = subprocess.run(["/usr/bin/g++", "-std=c++17", "test.cpp", "-lcrypto", "-lssl"],
+                                                    timeout=timeout,
+                                                    capture_output=True)
+            else:
+                compilation_result = subprocess.run(["/usr/bin/g++", "-std=c++17", "test.cpp"], timeout=timeout,
+                                                    capture_output=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    with time_limit(timeout):
+                        # WARNING
+                        # This program exists to execute untrusted model-generated code. Although
+                        # it is highly unlikely that model-generated code will do something overtly
+                        # malicious in response to this test suite, model-generated code may act
+                        # destructively due to a lack of model capability or alignment.
+                        # Users are strongly encouraged to sandbox this evaluation suite so that it
+                        # does not perform destructive actions on their host or network.
+                        # Once you have read this disclaimer and taken appropriate precautions,
+                        # uncomment the following line and proceed at your own risk:
+                         exec_result = subprocess.run(["./a.out"], timeout=timeout, capture_output=True)
+
+                    if exec_result.returncode == 0:
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+            #print(result[-1])
+            #print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "php" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.php", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    cmd = f"{php_exec}php -f test.php"
+                    exec_result = subprocess.run(cmd, timeout=timeout, capture_output=True, shell=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+            except TimeoutException:
+                result.append("timed out")
+            print(result[-1])
+            print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "sh" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.sh", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    cmd = "/bin/bash test.sh"
+                    exec_result = subprocess.run(cmd, timeout=10, capture_output=True, shell=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+            except TimeoutException:
+                result.append("timed out")
+            #print(result[-1])
+            #print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "ts" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            env = {"PATH": f"{node_exec}:" + subprocess.os.environ["PATH"]}
+            open(f"test.ts", 'w').write(sample["test_code"])
+            cmd = f"{tsc_exec}tsc test.ts --target ES2015 --lib ES2015,DOM"
+            compilation_result = subprocess.run(cmd, timeout=timeout, capture_output=True, env=env, shell=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    with time_limit(timeout):
+                         exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                    if exec_result.returncode == 0:
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+            if result[-1] != "passed":
+                env = {"PATH": f"{node_exec}:" + subprocess.os.environ["PATH"]}
+                cmd = f"{tsc_exec}tsc test.ts"
+                compilation_result = subprocess.run(cmd, timeout=timeout, capture_output=True, env=env, shell=True)
+                if compilation_result.returncode != 0:
+                    if compilation_result.stderr:
+                        err = compilation_result.stderr.decode()
+                    else:
+                        err = compilation_result.stdout.decode()
+                    result[-1] = f"failed: compilation error: {err}"
+                else:
+                    try:
+                        exec_result = None
+                        with time_limit(timeout):
+                            exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                        if exec_result.returncode == 0:
+                            result[-1] = "passed"
+                        else:
+                            if exec_result.stderr:
+                                try:
+                                    err = exec_result.stderr.decode()
+                                except:
+                                    err = exec_result.stderr
+                            else:
+                                try:
+                                    err = exec_result.stdout.decode()
+                                except:
+                                    err = exec_result.stdout
+                            result[-1] = f"failed: {err}"
+                    except TimeoutException:
+                        result[-1] = "timed out"
+ 
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "cs" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            os.chdir(tmp_dir)
+            open(f"Program.cs", 'w').write(sample["test_code"])
+            cmd = f"{cs_exec}mcs -d:DEBUG Program.cs"
+            compilation_result = subprocess.run(cmd, shell=True, capture_output=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    cmd = f"{cs_exec}mono Program.exe"
+                    env = dict(MONO_TRACE_LISTENER="Console.Error")
+                    with time_limit(timeout):
+                         exec_result = subprocess.run(cmd, timeout=timeout, shell=True, capture_output=True, env=env)
+
+                    if "Fail" not in exec_result.stderr.decode():
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+                except Exception as e:
+                    result.append(f"failed: {e}")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "rust" in language_type.lower():  
+            import os         
+            
+            WD: str = os.path.dirname(os.path.abspath(__file__))
+            RUST_DIR: str = os.path.join(WD, "rust")
+            RUST_SRC: str = os.path.join(RUST_DIR, "src")
+            RUST_BIN: str = os.path.join(RUST_SRC, "bin")
+            RUST_TMP_DIR: str = os.path.join(RUST_DIR, "tmp")
+            RUST_LOGS: str = os.path.join(RUST_TMP_DIR, "logs")
+            RUST_EXT: str = ".rs" 
+
+            # Create mandatory tmp directories
+            os.makedirs(RUST_TMP_DIR, exist_ok=True)
+            os.makedirs(RUST_LOGS, exist_ok=True)
+            os.makedirs(RUST_SRC, exist_ok=True)
+            os.makedirs(RUST_BIN, exist_ok=True)
+
+            with tempfile.NamedTemporaryFile(dir = RUST_BIN, delete=False) as f:
+                #temporal file name
+                file_prefix = sample["task_id"].lower().replace("/", "_")
+                file_name:str =  file_prefix +RUST_EXT
+                
+                os.rename(f.name, os.path.join(RUST_BIN, file_name))
+                
+                # Sample to pure Rust function
+                rust_code: str = sample["test_code"]
+
+                # dump the rust source code in the target temporal file
+                f.write(rust_code.encode('utf-8'))
+
+            # Proceed towards Rust binaries compilation. Therefore move to Rust module root dir.
+            os.chdir(RUST_DIR)
+
+            # Two possible outcomes
+            # Pass OR Fail compilation
+            log_filename: str = file_prefix + ".jsonl"
+            log_path: str = os.path.join(RUST_LOGS, log_filename)
+            cargo_check: str = "cargo check --bin " + file_prefix + " --message-format json >> " + log_path
+            # Compilation build status
+            returned_val_compilation: int
+            
+            # Overwrite file content
+            if os.path.exists(log_path):
+                if(file_size := os.path.getsize(log_path)) >= 0: 
+                    os.remove(log_path)
+                    returned_val_compilation = os.system(cargo_check)
+
+            else: 
+                returned_val_compilation = os.system(cargo_check)
+
+            # 0 means success   
+            if returned_val_compilation == 0:
+
+                #Execution pipeline
+                cargo_test: str = "cargo test --bin " +file_prefix+ " --message-format json >> " + log_path
+                returned_val_execution = os.system(cargo_test)
+                
+                if returned_val_execution == 0:
+                    result.append("passed")
+                else:
+                   result.append(f"failed: execution error") 
+
+            else:
+                result.append(f"failed: compilation error")
+
+
+        elif "java" in language_type.lower():
+            assert tmp_dir is not None, "Java should be evaluated in a temporary dir."
+
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            open(os.path.join(tmp_dir, "Problem.java"), 'w').write(sample["test_code"])
+            origin_path = os.getcwd()
+            os.system(f"cp ./javatuples-1.2.jar {tmp_dir}/")
+            os.chdir(tmp_dir)
+            res = "failed: unknown error"
+            compile_returncode = -1
+            for _ in range(5):
+                try:
+                    cmd = f"{java_exec}javac -cp javatuples-1.2.jar Problem.java"
+                    compilation_result = subprocess.run(cmd, timeout=60, capture_output=True, shell=True)  
+                    compile_returncode = compilation_result.returncode
+                    break
+                except subprocess.TimeoutExpired as e:
+                    continue
+            if compile_returncode != 0:
+                res = "failed: compilation error"
+            else:
+                exec_result = None
+                try:
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                    cmd = f"{java_exec}java -ea -cp .:javatuples-1.2.jar Problem"
+                    exec_result = subprocess.run(cmd, timeout=timeout, capture_output=True, shell=True)  
+                    if exec_result.returncode == 0:
+                        res = "passed"
+                    elif exec_result.returncode == 1:
+                        if "AssertionError" in exec_result.stderr.decode('unicode-escape'):
+                            res = "failed: wrong answer"
+                        else:
+                            res = f"failed: {exec_result.stderr.decode()}"
+                except subprocess.TimeoutExpired as e:
+                    res = "time out"
+                except BaseException as e:
+                    res = f"failed: {e}"
+
+            result.append(res)  
+            os.chdir(origin_path)          
+            shutil.rmtree(tmp_dir)
+        
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(tmp_dir,))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append("timed out")
+
+    return {
+        "task_id"      : task_id,
+        "completion_id": completion_id,
+        "result"       : result[0],
+        "passed"       : result[0] == "passed",
+        "finish"       : -1 if "finish" not in sample else sample["finish"],
+        "code"         : sample["test_code"]
+    }
+
+# Copyright (c) OpenAI (https://openai.com)
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# ============================================================================
+@contextlib.contextmanager
+def time_limit(seconds: float):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """ StringIO that throws an exception when it's read from """
+
+    def read(self, *args, **kwargs):
+        raise IOError
+
+    def readline(self, *args, **kwargs):
+        raise IOError
+
+    def readlines(self, *args, **kwargs):
+        raise IOError
+
+    def readable(self, *args, **kwargs):
+        """ Returns True if the IO object can be read. """
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = 'stdin'
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes: Optional[int] = None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the 
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == 'Darwin':
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+    os.environ['OMP_NUM_THREADS'] = '1'
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__['help'] = None
+
+    import sys
+    sys.modules['ipdb'] = None
+    sys.modules['joblib'] = None
+    sys.modules['resource'] = None
+    sys.modules['psutil'] = None
+    sys.modules['tkinter'] = None

Evaluation/LeetCode/readme.md

@@ -0,0 +1,48 @@
+## 1. Introduction
+We construct the LeetCode Contest benchmark to to further validate the model's capability in real-world programming problems.
+[LeetCode](https://leetcode.com/) presents competition-level problems, offering significant challenges that test the model's problem understanding and code generation skills. We collected the latest problems from LeetCode Contests to prevent the appearance of both the problems or their solutions in our pre-training data. A total of `180` problems were collected from July 2023 to January 2024. For each problem, we collected `100` test cases. The data format is the same as human-eval. For more details, please refer to [leetcode_contest_data](./data/20240121-Jul.jsonl).
+
+## 2. Evaluation
+Please follow the following two steps to evaluate the model's performance on our LeetCode Contest benchmark:
+
+1. Run `vllm_inference.py` to get generation results.
+```bash
+cd Evaluation/LeetCode
+
+# Set the model or path here
+MODEL="deepseek-coder-7b-instruct"
+
+python vllm_inference.py --model_name_or_path $MODEL --saved_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl
+```
+
+If you want to evaluate the model with COT, please add `--cot` to the command:
+```bash
+python vllm_inference.py --model_name_or_path $MODEL --saved_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl --cot
+```
+
+2. Run `evaluate_leetcode.py` to get evaluation results.
+```bash
+python evaluate_leetcode.py --generation_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl --result_path output/20240121-Jul.deepseek-coder-7b-instruct.result.jsonl
+```
+
+## 3. Experimental Results
+We report experimental results here:
+
+| Model                       | Size | Easy (45) | Medium (91) | Hard (44) | Overall(180) |
+|-----------------------------|------|-----------|-------------|-----------|--------------|
+| WizardCoder-V1.0            | 15B  | 17.8%     | 1.1%        | 0.0%      | 5.0%         |
+| CodeLlama-Instruct          | 34B  | 24.4%     | 4.4%        | 4.5%      | 9.4%         |
+| Phind-CodeLlama-V2          | 34B  | 26.7%     | 8.8%        | 9.1%      | 13.3%        |
+| | | | |
+| GPT-3.5-Turbo               | -    | 46.7%     | 15.4 %      | 15.9%     | 23.3%        |
+| GPT-3.5-Turbo + CoT         | -    | 42.2%     | 15.4%       | 20.5%     | 23.3%        |
+| GPT-4-Turbo                 | -    | 73.3%     | 31.9%       | 25.0%     | 40.6%        |
+| GPT-4-Turbo + CoT           | -    | 71.1%     | 35.2%       | 25.0%     | 41.8%        |
+| | | | |
+| DeepSeek-Coder-Instruct     | 1.3B | 22.2%     | 1.1%        | 4.5%      | 7.2%         |
+| DeepSeek-Coder-Instruct + CoT | 1.3B | 22.2%   | 2.2%        | 2.3%      | 7.2%         |
+| DeepSeek-Coder-Instruct     | 6.7B | 44.4%     | 12.1%       | 9.1%      | 19.4%        |
+| DeepSeek-Coder-Instruct + CoT | 6.7B | 44.4%   | 17.6%       | 4.5%      | 21.1%        |
+| DeepSeek-Coder-Instruct     | 33B  | 57.8%     | 22.0%       | 9.1%      | 27.8%        |
+| DeepSeek-Coder-Instruct + CoT | 33B | 53.3%    | 25.3%       | 11.4%     | 28.9%        |
+

Evaluation/LeetCode/vllm_inference.py

@@ -0,0 +1,84 @@
+from vllm import LLM, SamplingParams
+import json
+from transformers import AutoTokenizer
+from pathlib import Path
+
+version = "20240121-Jul"
+
+def generate_batch(examples, tokenizer, llm, model: str):
+    stop = None
+    if model == 'deepseekcoder-instruct':
+        prompts = [
+            tokenizer.apply_chat_template([{'role': 'user', 'content': ex['prompt_sft'] }], tokenize=False, add_generation_prompt=True)
+            for ex in examples
+        ]
+    else:
+        raise NotImplementedError()
+
+    # Create a sampling params object.
+    sampling_params = SamplingParams(
+        temperature=0.0,
+        # top_p=0.95,
+        max_tokens=1024,
+        stop=stop
+    )
+
+    print("Sample prompt: {}".format(prompts[0]))
+    outputs = llm.generate(prompts, sampling_params)
+    for i in range(len(examples)):
+        examples[i]['output'] = outputs[i].outputs[0].text
+
+    return examples
+
+def generate_main(data_path: str, model_name_or_path: str, saved_path: str, model_type: str='deepseekcoder-instruct', cot: bool=False):
+    examples = [json.loads(x) for x in open(data_path).readlines()]
+    def _convert_for_sft(ex):
+        ex['prompt_sft'] = ex["prompt_sft"] + "\nYou need first write a step-by-step outline and then write the code."
+        return ex
+    
+    if cot:
+        examples = [_convert_for_sft(x) for x in examples]
+        saved_path = saved_path.replace(".jsonl", ".cot.jsonl")
+
+    print(model_type)
+    print("Model `{}`, COT = {}:{}".format(model_type, cot, model_name_or_path))
+    print("Saved path: {}".format(saved_path))
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
+    print("load tokenizer {} from {} over.".format(tokenizer.__class__, model_name_or_path))
+
+    # Create an LLM.
+    llm = LLM(
+        model=model_name_or_path,
+        pipeline_parallel_size=1,
+        tensor_parallel_size=8,
+        max_num_seqs=512,
+        max_num_batched_tokens=8192,
+        max_model_len=4096,
+        gpu_memory_utilization=0.85,
+        trust_remote_code=True
+    )
+    
+    generated_examples = generate_batch(examples, tokenizer, llm, model_type)    
+    print("Generate all over!!!")
+    with open(saved_path, 'w', encoding='utf-8') as fw:
+        for ex in generated_examples:
+            fw.write(json.dumps(ex) + '\n')
+        print("Save {} processed examples into {} over!".format(len(generated_examples), saved_path))
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--data_path', type=str, default=Path(__file__).parent.joinpath(f"data/{version}.jsonl").as_posix())
+    parser.add_argument('--model_name_or_path', type=str, default='deepseek-ai/deepseek-coder-7b-instruct')
+    parser.add_argument('--saved_path', type=str, default=f'output/{version}.deepseek-coder-7b-instruct.jsonl')
+    parser.add_argument('--cot', action='store_true', default=False)
+    args = parser.parse_args()
+
+    generate_main(
+        data_path=args.data_path,
+        model_name_or_path=args.model_name_or_path,
+        saved_path=args.saved_path,
+        cot=args.cot,
+    )