{
  "humaneval": {
    "label": "HumanEval",
    "total_tasks": 164,
    "passed_count": 135,
    "success_pct": 82.32,
    "retry_flags": {},
    "report_file": "logs/gemma4_mac_overnight_20261017/humaneval_report.json",
    "report_sha256": "5225646dbec61b86e5b9eaba5ac9129f290fdebb3652061492151bb2b2985695",
    "timestamp": 1792313613
  },
  "tooluse": {
    "label": "Telos tool-use",
    "total_tasks": 30,
    "passed_count": 24,
    "success_pct": 80.0,
    "retry_flags": {},
    "report_file": "logs/gemma4_mac_overnight_20261017/tooluse_report.json",
    "report_sha256": "3fae1a9aed18cf93ea2a66defed542ef7b5dfcd299e46249cbae38ade5ad7dd2",
    "timestamp": 1792315849
  },
  "cyber": {
    "label": "Telos cybersecurity",
    "total_tasks": 50,
    "passed_count": 28,
    "success_pct": 56.0,
    "retry_flags": {},
    "report_file": "logs/gemma4_mac_overnight_20261017/cyber_report.json",
    "report_sha256": "feeca98b2c039330ae3740c65e857d331a24763eeb474f785af861b98a9d5877",
    "timestamp": 1792315869
  },
  "gpqa_diamond": {
    "label": "GPQA Diamond",
    "total_tasks": 198,
    "passed_count": 87,
    "success_pct": 43.94,
    "retry_flags": {},
    "report_file": "logs/gemma4_mac_overnight_20261017/gpqa_diamond_report.json",
    "report_sha256": "8de80f62a6505ac10319bcc03b77977f888666eb0b2d255f2204f88a187a453c",
    "timestamp": 1792284977
  },
  "mmlu_science": {
    "label": "MMLU Science selection",
    "total_tasks": 833,
    "passed_count": 488,
    "success_pct": 58.58,
    "retry_flags": {},
    "report_file": "logs/gemma4_mac_overnight_20261017/mmlu_science_report.json",
    "report_sha256": "9662874a435096c4ba3913926376b083de9f4d498090f592848d82c647a48c50",
    "timestamp": 1792274821
  },
  "competition_math": {
    "label": "Competition MATH selection",
    "total_tasks": 700,
    "passed_count": 470,
    "success_pct": 67.14,
    "retry_flags": {},
    "report_file": "logs/gemma4_mac_overnight_20261017/competition_math_report.json",
    "report_sha256": "0b36defb4f601af805758bdf7fd6e151a5b4c82a873c5854a7c995b98d92bce8",
    "timestamp": 1792292435
  },
  "arc": {
    "label": "ARC-Challenge",
    "total_tasks": 1172,
    "passed_count": 782,
    "success_pct": 66.72,
    "retry_flags": {},
    "report_file": "logs/gemma4_mac_overnight_20261017/arc_report.json",
    "report_sha256": "0979fefe275dc7ddee35f57d3530c01877ad5022e6ac084bc2bfb50003cfd69c",
    "timestamp": 1792265092
  },
  "_provenance": {
    "audited": "2026-10-28",
    "metric": "saved success rate; not controlled pass@1",
    "model_name": "Gemma 4 E4B \u00b7 4-bit MLX",
    "summary_discrepancies": [
      {
        "model": "gemma4_e4b",
        "suite": "humaneval",
        "field": "ast_validity_pct",
        "summary": 100.0,
        "detailed_report": 96.34
      },
      {
        "model": "gemma4_e4b",
        "suite": "humaneval",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 135,
          "FAILED_ASSERTION": 29
        },
        "detailed_report": {
          "PASSED": 135,
          "FAILED_ASSERTION": 20,
          "SYNTAX_ERROR": 6,
          "TIMEOUT": 1,
          "RUNTIME_EXCEPTION": 2,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "gemma4_e4b",
        "suite": "cyber",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 28,
          "FAILED_ASSERTION": 22
        },
        "detailed_report": {
          "PASSED": 28,
          "FAILED_ASSERTION": 22,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "gemma4_e4b",
        "suite": "gpqa_diamond",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 87,
          "FAILED_ASSERTION": 111
        },
        "detailed_report": {
          "PASSED": 87,
          "FAILED_ASSERTION": 111,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "gemma4_e4b",
        "suite": "mmlu_science",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 488,
          "FAILED_ASSERTION": 345
        },
        "detailed_report": {
          "PASSED": 488,
          "FAILED_ASSERTION": 345,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "gemma4_e4b",
        "suite": "competition_math",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 470,
          "FAILED_ASSERTION": 230
        },
        "detailed_report": {
          "PASSED": 470,
          "FAILED_ASSERTION": 230,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "gemma4_e4b",
        "suite": "arc",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 782,
          "FAILED_ASSERTION": 390
        },
        "detailed_report": {
          "PASSED": 782,
          "FAILED_ASSERTION": 390,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      }
    ]
  }
}
