{
  "description": "Qwen2.5-0.5B-Instruct, one seed; one visible example and two non-prompted dataset tests; one completion per model",
  "dataset": "https://huggingface.co/datasets/google-research-datasets/mbpp",
  "candidate_count": 2984,
  "training_tasks": 140,
  "excluded": {
    "no_pass": 127,
    "first_unusable": 100,
    "unresolved": 6
  },
  "dev_cases": 90,
  "test_cases": 100,
  "chosen_epochs": {
    "control": 3,
    "filtered": 3
  },
  "test_status_by_task": {
    "base": {
      "217": "fail",
      "451": "pass",
      "114": "fail",
      "64": "fail",
      "135": "pass",
      "241": "syntax_invalid",
      "403": "pass",
      "443": "fail",
      "413": "syntax_invalid",
      "329": "pass",
      "490": "fail",
      "454": "fail",
      "262": "pass",
      "463": "syntax_invalid",
      "253": "fail",
      "355": "fail",
      "292": "pass",
      "175": "pass",
      "307": "fail",
      "158": "fail",
      "369": "fail",
      "71": "syntax_invalid",
      "500": "fail",
      "142": "syntax_invalid",
      "218": "syntax_invalid",
      "28": "pass",
      "31": "fail",
      "78": "fail",
      "404": "pass",
      "173": "pass",
      "45": "fail",
      "251": "fail",
      "97": "syntax_invalid",
      "190": "fail",
      "113": "pass",
      "238": "pass",
      "450": "syntax_invalid",
      "234": "pass",
      "448": "fail",
      "249": "fail",
      "418": "pass",
      "66": "pass",
      "279": "syntax_invalid",
      "260": "syntax_invalid",
      "36": "syntax_invalid",
      "201": "pass",
      "434": "fail",
      "61": "fail",
      "273": "pass",
      "462": "fail",
      "191": "fail",
      "110": "fail",
      "325": "syntax_invalid",
      "76": "fail",
      "323": "syntax_invalid",
      "52": "pass",
      "499": "pass",
      "480": "pass",
      "374": "fail",
      "221": "fail",
      "446": "syntax_invalid",
      "364": "fail",
      "198": "syntax_invalid",
      "334": "syntax_invalid",
      "220": "fail",
      "345": "syntax_invalid",
      "237": "fail",
      "304": "syntax_invalid",
      "321": "fail",
      "226": "fail",
      "117": "fail",
      "427": "fail",
      "79": "pass",
      "318": "syntax_invalid",
      "258": "pass",
      "244": "syntax_invalid",
      "141": "syntax_invalid",
      "224": "pass",
      "254": "fail",
      "229": "fail",
      "209": "fail",
      "29": "pass",
      "473": "syntax_invalid",
      "231": "fail",
      "445": "fail",
      "54": "pass",
      "184": "fail",
      "389": "syntax_invalid",
      "395": "fail",
      "41": "pass",
      "189": "fail",
      "411": "pass",
      "104": "pass",
      "297": "syntax_invalid",
      "314": "syntax_invalid",
      "299": "syntax_invalid",
      "347": "fail",
      "140": "fail",
      "101": "fail",
      "193": "fail"
    },
    "control": {
      "217": "fail",
      "451": "pass",
      "114": "fail",
      "64": "fail",
      "135": "pass",
      "241": "fail",
      "403": "pass",
      "443": "fail",
      "413": "fail",
      "329": "pass",
      "490": "fail",
      "454": "fail",
      "262": "fail",
      "463": "fail",
      "253": "pass",
      "355": "fail",
      "292": "pass",
      "175": "pass",
      "307": "fail",
      "158": "fail",
      "369": "fail",
      "71": "fail",
      "500": "fail",
      "142": "fail",
      "218": "fail",
      "28": "fail",
      "31": "fail",
      "78": "pass",
      "404": "pass",
      "173": "pass",
      "45": "fail",
      "251": "fail",
      "97": "pass",
      "190": "fail",
      "113": "pass",
      "238": "fail",
      "450": "fail",
      "234": "pass",
      "448": "fail",
      "249": "fail",
      "418": "fail",
      "66": "pass",
      "279": "syntax_invalid",
      "260": "syntax_invalid",
      "36": "syntax_invalid",
      "201": "fail",
      "434": "fail",
      "61": "syntax_invalid",
      "273": "pass",
      "462": "fail",
      "191": "fail",
      "110": "fail",
      "325": "fail",
      "76": "fail",
      "323": "fail",
      "52": "pass",
      "499": "pass",
      "480": "pass",
      "374": "fail",
      "221": "pass",
      "446": "fail",
      "364": "fail",
      "198": "fail",
      "334": "pass",
      "220": "fail",
      "345": "fail",
      "237": "fail",
      "304": "fail",
      "321": "fail",
      "226": "fail",
      "117": "fail",
      "427": "fail",
      "79": "pass",
      "318": "fail",
      "258": "syntax_invalid",
      "244": "pass",
      "141": "pass",
      "224": "pass",
      "254": "fail",
      "229": "fail",
      "209": "fail",
      "29": "pass",
      "473": "fail",
      "231": "fail",
      "445": "fail",
      "54": "fail",
      "184": "fail",
      "389": "fail",
      "395": "pass",
      "41": "pass",
      "189": "fail",
      "411": "pass",
      "104": "pass",
      "297": "pass",
      "314": "fail",
      "299": "fail",
      "347": "fail",
      "140": "fail",
      "101": "fail",
      "193": "fail"
    },
    "filtered": {
      "217": "fail",
      "451": "pass",
      "114": "fail",
      "64": "pass",
      "135": "pass",
      "241": "screened",
      "403": "fail",
      "443": "fail",
      "413": "fail",
      "329": "pass",
      "490": "fail",
      "454": "fail",
      "262": "pass",
      "463": "fail",
      "253": "pass",
      "355": "fail",
      "292": "pass",
      "175": "pass",
      "307": "fail",
      "158": "fail",
      "369": "fail",
      "71": "fail",
      "500": "fail",
      "142": "fail",
      "218": "fail",
      "28": "fail",
      "31": "fail",
      "78": "pass",
      "404": "pass",
      "173": "pass",
      "45": "fail",
      "251": "fail",
      "97": "pass",
      "190": "fail",
      "113": "pass",
      "238": "pass",
      "450": "fail",
      "234": "pass",
      "448": "syntax_invalid",
      "249": "fail",
      "418": "pass",
      "66": "pass",
      "279": "fail",
      "260": "fail",
      "36": "fail",
      "201": "pass",
      "434": "fail",
      "61": "fail",
      "273": "pass",
      "462": "fail",
      "191": "fail",
      "110": "fail",
      "325": "fail",
      "76": "fail",
      "323": "fail",
      "52": "pass",
      "499": "pass",
      "480": "pass",
      "374": "fail",
      "221": "pass",
      "446": "fail",
      "364": "fail",
      "198": "fail",
      "334": "pass",
      "220": "fail",
      "345": "fail",
      "237": "fail",
      "304": "fail",
      "321": "fail",
      "226": "pass",
      "117": "fail",
      "427": "fail",
      "79": "pass",
      "318": "fail",
      "258": "fail",
      "244": "fail",
      "141": "fail",
      "224": "pass",
      "254": "fail",
      "229": "fail",
      "209": "fail",
      "29": "pass",
      "473": "fail",
      "231": "fail",
      "445": "fail",
      "54": "pass",
      "184": "fail",
      "389": "pass",
      "395": "pass",
      "41": "pass",
      "189": "pass",
      "411": "fail",
      "104": "pass",
      "297": "pass",
      "314": "syntax_invalid",
      "299": "fail",
      "347": "fail",
      "140": "fail",
      "101": "pass",
      "193": "fail"
    }
  },
  "seed43_replication": {
    "note": "initiated after seeing seed 42 results; same test task IDs; control dev-selected epoch 1, filtered epoch 3; fixed-step comparison uses epoch 3 for both",
    "chosen_epochs": {
      "control": 1,
      "filtered": 3
    },
    "test_status_by_task": {
      "base": {
        "217": "fail",
        "451": "pass",
        "114": "fail",
        "64": "fail",
        "135": "pass",
        "241": "syntax_invalid",
        "403": "pass",
        "443": "fail",
        "413": "syntax_invalid",
        "329": "pass",
        "490": "fail",
        "454": "fail",
        "262": "pass",
        "463": "syntax_invalid",
        "253": "fail",
        "355": "fail",
        "292": "pass",
        "175": "pass",
        "307": "fail",
        "158": "fail",
        "369": "fail",
        "71": "syntax_invalid",
        "500": "fail",
        "142": "syntax_invalid",
        "218": "syntax_invalid",
        "28": "pass",
        "31": "fail",
        "78": "fail",
        "404": "pass",
        "173": "pass",
        "45": "fail",
        "251": "fail",
        "97": "syntax_invalid",
        "190": "fail",
        "113": "pass",
        "238": "pass",
        "450": "syntax_invalid",
        "234": "pass",
        "448": "fail",
        "249": "fail",
        "418": "pass",
        "66": "pass",
        "279": "syntax_invalid",
        "260": "syntax_invalid",
        "36": "syntax_invalid",
        "201": "pass",
        "434": "fail",
        "61": "fail",
        "273": "pass",
        "462": "fail",
        "191": "fail",
        "110": "fail",
        "325": "syntax_invalid",
        "76": "fail",
        "323": "syntax_invalid",
        "52": "pass",
        "499": "pass",
        "480": "pass",
        "374": "fail",
        "221": "fail",
        "446": "syntax_invalid",
        "364": "fail",
        "198": "syntax_invalid",
        "334": "syntax_invalid",
        "220": "fail",
        "345": "syntax_invalid",
        "237": "fail",
        "304": "syntax_invalid",
        "321": "fail",
        "226": "fail",
        "117": "fail",
        "427": "fail",
        "79": "pass",
        "318": "syntax_invalid",
        "258": "pass",
        "244": "syntax_invalid",
        "141": "syntax_invalid",
        "224": "pass",
        "254": "fail",
        "229": "fail",
        "209": "fail",
        "29": "pass",
        "473": "syntax_invalid",
        "231": "fail",
        "445": "fail",
        "54": "pass",
        "184": "fail",
        "389": "syntax_invalid",
        "395": "fail",
        "41": "pass",
        "189": "fail",
        "411": "pass",
        "104": "pass",
        "297": "syntax_invalid",
        "314": "syntax_invalid",
        "299": "syntax_invalid",
        "347": "fail",
        "140": "fail",
        "101": "fail",
        "193": "fail"
      },
      "control": {
        "217": "fail",
        "451": "pass",
        "114": "fail",
        "64": "fail",
        "135": "fail",
        "241": "fail",
        "403": "fail",
        "443": "fail",
        "413": "fail",
        "329": "pass",
        "490": "fail",
        "454": "fail",
        "262": "fail",
        "463": "fail",
        "253": "pass",
        "355": "fail",
        "292": "pass",
        "175": "pass",
        "307": "fail",
        "158": "fail",
        "369": "pass",
        "71": "fail",
        "500": "fail",
        "142": "fail",
        "218": "fail",
        "28": "fail",
        "31": "fail",
        "78": "fail",
        "404": "pass",
        "173": "pass",
        "45": "pass",
        "251": "fail",
        "97": "fail",
        "190": "fail",
        "113": "pass",
        "238": "pass",
        "450": "pass",
        "234": "pass",
        "448": "fail",
        "249": "fail",
        "418": "fail",
        "66": "pass",
        "279": "fail",
        "260": "fail",
        "36": "fail",
        "201": "pass",
        "434": "fail",
        "61": "fail",
        "273": "pass",
        "462": "fail",
        "191": "fail",
        "110": "fail",
        "325": "fail",
        "76": "fail",
        "323": "fail",
        "52": "pass",
        "499": "pass",
        "480": "fail",
        "374": "fail",
        "221": "pass",
        "446": "fail",
        "364": "fail",
        "198": "fail",
        "334": "pass",
        "220": "fail",
        "345": "fail",
        "237": "fail",
        "304": "fail",
        "321": "fail",
        "226": "pass",
        "117": "fail",
        "427": "fail",
        "79": "pass",
        "318": "fail",
        "258": "pass",
        "244": "fail",
        "141": "fail",
        "224": "pass",
        "254": "fail",
        "229": "fail",
        "209": "fail",
        "29": "fail",
        "473": "fail",
        "231": "fail",
        "445": "fail",
        "54": "fail",
        "184": "fail",
        "389": "fail",
        "395": "pass",
        "41": "fail",
        "189": "fail",
        "411": "pass",
        "104": "pass",
        "297": "pass",
        "314": "fail",
        "299": "fail",
        "347": "fail",
        "140": "fail",
        "101": "pass",
        "193": "fail"
      },
      "filtered": {
        "217": "fail",
        "451": "pass",
        "114": "fail",
        "64": "pass",
        "135": "pass",
        "241": "screened",
        "403": "fail",
        "443": "fail",
        "413": "fail",
        "329": "pass",
        "490": "fail",
        "454": "fail",
        "262": "fail",
        "463": "fail",
        "253": "pass",
        "355": "fail",
        "292": "pass",
        "175": "pass",
        "307": "fail",
        "158": "fail",
        "369": "fail",
        "71": "syntax_invalid",
        "500": "fail",
        "142": "fail",
        "218": "fail",
        "28": "fail",
        "31": "fail",
        "78": "pass",
        "404": "pass",
        "173": "pass",
        "45": "fail",
        "251": "fail",
        "97": "pass",
        "190": "fail",
        "113": "pass",
        "238": "fail",
        "450": "fail",
        "234": "pass",
        "448": "fail",
        "249": "fail",
        "418": "pass",
        "66": "pass",
        "279": "fail",
        "260": "fail",
        "36": "fail",
        "201": "pass",
        "434": "fail",
        "61": "fail",
        "273": "pass",
        "462": "fail",
        "191": "fail",
        "110": "fail",
        "325": "fail",
        "76": "fail",
        "323": "fail",
        "52": "pass",
        "499": "pass",
        "480": "pass",
        "374": "fail",
        "221": "pass",
        "446": "fail",
        "364": "fail",
        "198": "fail",
        "334": "pass",
        "220": "fail",
        "345": "fail",
        "237": "fail",
        "304": "fail",
        "321": "fail",
        "226": "pass",
        "117": "fail",
        "427": "fail",
        "79": "pass",
        "318": "fail",
        "258": "pass",
        "244": "pass",
        "141": "fail",
        "224": "pass",
        "254": "fail",
        "229": "fail",
        "209": "syntax_invalid",
        "29": "pass",
        "473": "fail",
        "231": "fail",
        "445": "fail",
        "54": "pass",
        "184": "fail",
        "389": "pass",
        "395": "pass",
        "41": "pass",
        "189": "fail",
        "411": "pass",
        "104": "pass",
        "297": "pass",
        "314": "fail",
        "299": "fail",
        "347": "fail",
        "140": "fail",
        "101": "pass",
        "193": "fail"
      },
      "control-fixed-epoch3": {
        "217": "fail",
        "451": "pass",
        "114": "fail",
        "64": "fail",
        "135": "fail",
        "241": "fail",
        "403": "fail",
        "443": "fail",
        "413": "fail",
        "329": "fail",
        "490": "fail",
        "454": "fail",
        "262": "fail",
        "463": "fail",
        "253": "pass",
        "355": "fail",
        "292": "pass",
        "175": "pass",
        "307": "fail",
        "158": "fail",
        "369": "pass",
        "71": "fail",
        "500": "fail",
        "142": "fail",
        "218": "fail",
        "28": "fail",
        "31": "fail",
        "78": "syntax_invalid",
        "404": "pass",
        "173": "pass",
        "45": "fail",
        "251": "fail",
        "97": "fail",
        "190": "fail",
        "113": "pass",
        "238": "fail",
        "450": "fail",
        "234": "pass",
        "448": "fail",
        "249": "fail",
        "418": "pass",
        "66": "pass",
        "279": "fail",
        "260": "fail",
        "36": "fail",
        "201": "pass",
        "434": "fail",
        "61": "fail",
        "273": "pass",
        "462": "fail",
        "191": "fail",
        "110": "fail",
        "325": "fail",
        "76": "fail",
        "323": "fail",
        "52": "pass",
        "499": "pass",
        "480": "pass",
        "374": "fail",
        "221": "pass",
        "446": "fail",
        "364": "fail",
        "198": "syntax_invalid",
        "334": "fail",
        "220": "fail",
        "345": "fail",
        "237": "fail",
        "304": "fail",
        "321": "fail",
        "226": "pass",
        "117": "fail",
        "427": "fail",
        "79": "pass",
        "318": "fail",
        "258": "fail",
        "244": "fail",
        "141": "fail",
        "224": "pass",
        "254": "fail",
        "229": "fail",
        "209": "fail",
        "29": "fail",
        "473": "fail",
        "231": "fail",
        "445": "fail",
        "54": "pass",
        "184": "fail",
        "389": "fail",
        "395": "pass",
        "41": "pass",
        "189": "fail",
        "411": "fail",
        "104": "pass",
        "297": "pass",
        "314": "syntax_invalid",
        "299": "fail",
        "347": "fail",
        "140": "fail",
        "101": "fail",
        "193": "fail"
      }
    }
  }
}