{
  "dataset": "https://github.com/openai/human-eval",
  "description": "Exploratory direct function generation using the same chat prompt, 192 tokens, greedy decoding and static screening; not official HumanEval pass@1; public dataset may be in pretraining. One filtered execution unresolved after one retry.",
  "groups": {
    "base": {
      "HumanEval/0": "pass",
      "HumanEval/1": "fail",
      "HumanEval/2": "pass",
      "HumanEval/3": "pass",
      "HumanEval/4": "syntax_invalid",
      "HumanEval/5": "pass",
      "HumanEval/6": "fail",
      "HumanEval/7": "syntax_invalid",
      "HumanEval/8": "pass",
      "HumanEval/9": "fail",
      "HumanEval/10": "syntax_invalid",
      "HumanEval/11": "fail",
      "HumanEval/12": "pass",
      "HumanEval/13": "pass",
      "HumanEval/14": "pass",
      "HumanEval/15": "fail",
      "HumanEval/16": "pass",
      "HumanEval/17": "fail",
      "HumanEval/18": "fail",
      "HumanEval/19": "fail",
      "HumanEval/20": "syntax_invalid",
      "HumanEval/21": "pass",
      "HumanEval/22": "pass",
      "HumanEval/23": "pass",
      "HumanEval/24": "fail",
      "HumanEval/25": "pass",
      "HumanEval/26": "fail",
      "HumanEval/27": "pass",
      "HumanEval/28": "pass",
      "HumanEval/29": "pass",
      "HumanEval/30": "syntax_invalid",
      "HumanEval/31": "pass",
      "HumanEval/32": "syntax_invalid",
      "HumanEval/33": "syntax_invalid",
      "HumanEval/34": "pass",
      "HumanEval/35": "pass",
      "HumanEval/36": "fail",
      "HumanEval/37": "syntax_invalid",
      "HumanEval/38": "fail",
      "HumanEval/39": "fail",
      "HumanEval/40": "fail",
      "HumanEval/41": "fail",
      "HumanEval/42": "pass",
      "HumanEval/43": "fail",
      "HumanEval/44": "fail",
      "HumanEval/45": "fail",
      "HumanEval/46": "fail",
      "HumanEval/47": "pass",
      "HumanEval/48": "pass",
      "HumanEval/49": "syntax_invalid",
      "HumanEval/50": "pass",
      "HumanEval/51": "pass",
      "HumanEval/52": "pass",
      "HumanEval/53": "pass",
      "HumanEval/54": "fail",
      "HumanEval/55": "pass",
      "HumanEval/56": "pass",
      "HumanEval/57": "fail",
      "HumanEval/58": "pass",
      "HumanEval/59": "pass",
      "HumanEval/60": "pass",
      "HumanEval/61": "pass",
      "HumanEval/62": "screened",
      "HumanEval/63": "syntax_invalid",
      "HumanEval/64": "fail",
      "HumanEval/65": "fail",
      "HumanEval/66": "fail",
      "HumanEval/67": "fail",
      "HumanEval/68": "fail",
      "HumanEval/69": "fail",
      "HumanEval/70": "fail",
      "HumanEval/71": "pass",
      "HumanEval/72": "syntax_invalid",
      "HumanEval/73": "fail",
      "HumanEval/74": "fail",
      "HumanEval/75": "fail",
      "HumanEval/76": "fail",
      "HumanEval/77": "fail",
      "HumanEval/78": "syntax_invalid",
      "HumanEval/79": "pass",
      "HumanEval/80": "fail",
      "HumanEval/81": "syntax_invalid",
      "HumanEval/82": "pass",
      "HumanEval/83": "fail",
      "HumanEval/84": "fail",
      "HumanEval/85": "fail",
      "HumanEval/86": "fail",
      "HumanEval/87": "fail",
      "HumanEval/88": "screened",
      "HumanEval/89": "fail",
      "HumanEval/90": "fail",
      "HumanEval/91": "fail",
      "HumanEval/92": "fail",
      "HumanEval/93": "fail",
      "HumanEval/94": "syntax_invalid",
      "HumanEval/95": "fail",
      "HumanEval/96": "fail",
      "HumanEval/97": "pass",
      "HumanEval/98": "fail",
      "HumanEval/99": "fail",
      "HumanEval/100": "fail",
      "HumanEval/101": "fail",
      "HumanEval/102": "syntax_invalid",
      "HumanEval/103": "fail",
      "HumanEval/104": "fail",
      "HumanEval/105": "syntax_invalid",
      "HumanEval/106": "pass",
      "HumanEval/107": "fail",
      "HumanEval/108": "syntax_invalid",
      "HumanEval/109": "fail",
      "HumanEval/110": "fail",
      "HumanEval/111": "fail",
      "HumanEval/112": "fail",
      "HumanEval/113": "fail",
      "HumanEval/114": "fail",
      "HumanEval/115": "screened",
      "HumanEval/116": "syntax_invalid",
      "HumanEval/117": "fail",
      "HumanEval/118": "syntax_invalid",
      "HumanEval/119": "fail",
      "HumanEval/120": "fail",
      "HumanEval/121": "fail",
      "HumanEval/122": "pass",
      "HumanEval/123": "syntax_invalid",
      "HumanEval/124": "syntax_invalid",
      "HumanEval/125": "fail",
      "HumanEval/126": "syntax_invalid",
      "HumanEval/127": "syntax_invalid",
      "HumanEval/128": "screened",
      "HumanEval/129": "fail",
      "HumanEval/130": "fail",
      "HumanEval/131": "fail",
      "HumanEval/132": "fail",
      "HumanEval/133": "pass",
      "HumanEval/134": "fail",
      "HumanEval/135": "fail",
      "HumanEval/136": "syntax_invalid",
      "HumanEval/137": "fail",
      "HumanEval/138": "fail",
      "HumanEval/139": "pass",
      "HumanEval/140": "fail",
      "HumanEval/141": "fail",
      "HumanEval/142": "syntax_invalid",
      "HumanEval/143": "fail",
      "HumanEval/144": "fail",
      "HumanEval/145": "fail",
      "HumanEval/146": "syntax_invalid",
      "HumanEval/147": "fail",
      "HumanEval/148": "fail",
      "HumanEval/149": "fail",
      "HumanEval/150": "pass",
      "HumanEval/151": "pass",
      "HumanEval/152": "pass",
      "HumanEval/153": "fail",
      "HumanEval/154": "fail",
      "HumanEval/155": "syntax_invalid",
      "HumanEval/156": "pass",
      "HumanEval/157": "syntax_invalid",
      "HumanEval/158": "syntax_invalid",
      "HumanEval/159": "pass",
      "HumanEval/160": "fail",
      "HumanEval/161": "fail",
      "HumanEval/162": "screened",
      "HumanEval/163": "fail"
    },
    "filtered": {
      "HumanEval/0": "pass",
      "HumanEval/1": "fail",
      "HumanEval/2": "pass",
      "HumanEval/3": "fail",
      "HumanEval/4": "fail",
      "HumanEval/5": "fail",
      "HumanEval/6": "fail",
      "HumanEval/7": "pass",
      "HumanEval/8": "pass",
      "HumanEval/9": "pass",
      "HumanEval/10": "syntax_invalid",
      "HumanEval/11": "pass",
      "HumanEval/12": "pass",
      "HumanEval/13": "pass",
      "HumanEval/14": "fail",
      "HumanEval/15": "syntax_invalid",
      "HumanEval/16": "fail",
      "HumanEval/17": "fail",
      "HumanEval/18": "pass",
      "HumanEval/19": "screened",
      "HumanEval/20": "fail",
      "HumanEval/21": "pass",
      "HumanEval/22": "pass",
      "HumanEval/23": "pass",
      "HumanEval/24": "fail",
      "HumanEval/25": "syntax_invalid",
      "HumanEval/26": "fail",
      "HumanEval/27": "pass",
      "HumanEval/28": "pass",
      "HumanEval/29": "pass",
      "HumanEval/30": "pass",
      "HumanEval/31": "pass",
      "HumanEval/32": "fail",
      "HumanEval/33": "syntax_invalid",
      "HumanEval/34": "pass",
      "HumanEval/35": "pass",
      "HumanEval/36": "fail",
      "HumanEval/37": "syntax_invalid",
      "HumanEval/38": "fail",
      "HumanEval/39": "fail",
      "HumanEval/40": "fail",
      "HumanEval/41": "fail",
      "HumanEval/42": "pass",
      "HumanEval/43": "syntax_invalid",
      "HumanEval/44": "pass",
      "HumanEval/45": "pass",
      "HumanEval/46": "fail",
      "HumanEval/47": "pass",
      "HumanEval/48": "syntax_invalid",
      "HumanEval/49": "pass",
      "HumanEval/50": "pass",
      "HumanEval/51": "fail",
      "HumanEval/52": "pass",
      "HumanEval/53": "pass",
      "HumanEval/54": "syntax_invalid",
      "HumanEval/55": "pass",
      "HumanEval/56": "pass",
      "HumanEval/57": "fail",
      "HumanEval/58": "fail",
      "HumanEval/59": "fail",
      "HumanEval/60": "pass",
      "HumanEval/61": "pass",
      "HumanEval/62": "fail",
      "HumanEval/63": "syntax_invalid",
      "HumanEval/64": "fail",
      "HumanEval/65": "syntax_invalid",
      "HumanEval/66": "syntax_invalid",
      "HumanEval/67": "syntax_invalid",
      "HumanEval/68": "fail",
      "HumanEval/69": "fail",
      "HumanEval/70": "fail",
      "HumanEval/71": "fail",
      "HumanEval/72": "fail",
      "HumanEval/73": "fail",
      "HumanEval/74": "fail",
      "HumanEval/75": "syntax_invalid",
      "HumanEval/76": "fail",
      "HumanEval/77": "fail",
      "HumanEval/78": "pass",
      "HumanEval/79": "fail",
      "HumanEval/80": "fail",
      "HumanEval/81": "syntax_invalid",
      "HumanEval/82": "fail",
      "HumanEval/83": "fail",
      "HumanEval/84": "fail",
      "HumanEval/85": "fail",
      "HumanEval/86": "fail",
      "HumanEval/87": "fail",
      "HumanEval/88": "fail",
      "HumanEval/89": "fail",
      "HumanEval/90": "fail",
      "HumanEval/91": "syntax_invalid",
      "HumanEval/92": "fail",
      "HumanEval/93": "fail",
      "HumanEval/94": "fail",
      "HumanEval/95": "fail",
      "HumanEval/96": "syntax_invalid",
      "HumanEval/97": "fail",
      "HumanEval/98": "fail",
      "HumanEval/99": "fail",
      "HumanEval/100": "unresolved",
      "HumanEval/101": "syntax_invalid",
      "HumanEval/102": "fail",
      "HumanEval/103": "fail",
      "HumanEval/104": "fail",
      "HumanEval/105": "fail",
      "HumanEval/106": "fail",
      "HumanEval/107": "syntax_invalid",
      "HumanEval/108": "fail",
      "HumanEval/109": "fail",
      "HumanEval/110": "fail",
      "HumanEval/111": "fail",
      "HumanEval/112": "fail",
      "HumanEval/113": "syntax_invalid",
      "HumanEval/114": "fail",
      "HumanEval/115": "fail",
      "HumanEval/116": "pass",
      "HumanEval/117": "fail",
      "HumanEval/118": "fail",
      "HumanEval/119": "fail",
      "HumanEval/120": "fail",
      "HumanEval/121": "pass",
      "HumanEval/122": "fail",
      "HumanEval/123": "syntax_invalid",
      "HumanEval/124": "screened",
      "HumanEval/125": "fail",
      "HumanEval/126": "fail",
      "HumanEval/127": "syntax_invalid",
      "HumanEval/128": "fail",
      "HumanEval/129": "syntax_invalid",
      "HumanEval/130": "fail",
      "HumanEval/131": "fail",
      "HumanEval/132": "fail",
      "HumanEval/133": "fail",
      "HumanEval/134": "fail",
      "HumanEval/135": "fail",
      "HumanEval/136": "fail",
      "HumanEval/137": "fail",
      "HumanEval/138": "fail",
      "HumanEval/139": "pass",
      "HumanEval/140": "fail",
      "HumanEval/141": "fail",
      "HumanEval/142": "fail",
      "HumanEval/143": "fail",
      "HumanEval/144": "fail",
      "HumanEval/145": "fail",
      "HumanEval/146": "fail",
      "HumanEval/147": "fail",
      "HumanEval/148": "fail",
      "HumanEval/149": "fail",
      "HumanEval/150": "fail",
      "HumanEval/151": "fail",
      "HumanEval/152": "fail",
      "HumanEval/153": "fail",
      "HumanEval/154": "pass",
      "HumanEval/155": "fail",
      "HumanEval/156": "syntax_invalid",
      "HumanEval/157": "fail",
      "HumanEval/158": "pass",
      "HumanEval/159": "pass",
      "HumanEval/160": "fail",
      "HumanEval/161": "fail",
      "HumanEval/162": "screened",
      "HumanEval/163": "fail"
    }
  }
}