{
  "humaneval": {
    "label": "HumanEval",
    "total_tasks": 164,
    "passed_count": 118,
    "success_pct": 71.95,
    "retry_flags": {},
    "report_file": "logs/ternary_bonsai_4b_mac_20261018/humaneval_report.json",
    "report_sha256": "6ec5593f0f4e93f7efa3e157654c055dc2e9a97cd36152a405139ae408c76ad2",
    "timestamp": 1792376533
  },
  "tooluse": {
    "label": "Telos tool-use",
    "total_tasks": 30,
    "passed_count": 19,
    "success_pct": 63.33,
    "retry_flags": {},
    "report_file": "logs/ternary_bonsai_4b_mac_20261018/tooluse_report.json",
    "report_sha256": "c543b29aa788275aee82146b9bd31e72778df70133a875c4352acf3bb7dad2e7",
    "timestamp": 1792377956
  },
  "cyber": {
    "label": "Telos cybersecurity",
    "total_tasks": 50,
    "passed_count": 8,
    "success_pct": 16.0,
    "retry_flags": {},
    "report_file": "logs/ternary_bonsai_4b_mac_20261018/cyber_report.json",
    "report_sha256": "508d3e7d4661862623d7213031b7b1365e49d4f39234072cc5c2c9d0506bfb14",
    "timestamp": 1792377964
  },
  "gpqa_diamond": {
    "label": "GPQA Diamond",
    "total_tasks": 198,
    "passed_count": 53,
    "success_pct": 26.77,
    "retry_flags": {},
    "report_file": "logs/ternary_bonsai_4b_mac_20261018/gpqa_diamond_report.json",
    "report_sha256": "7f0ebe1ab13f897b76313937c579b179c37198039d1a916fcca3922893726443",
    "timestamp": 1792340516
  },
  "mmlu_science": {
    "label": "MMLU Science selection",
    "total_tasks": 833,
    "passed_count": 521,
    "success_pct": 62.55,
    "retry_flags": {},
    "report_file": "logs/ternary_bonsai_4b_mac_20261018/mmlu_science_report.json",
    "report_sha256": "4893ede8e29f1b1ea8dcb662857f6c93f81c0f63f2c90b2ff0f97f7644c32159",
    "timestamp": 1792323731
  },
  "competition_math": {
    "label": "Competition MATH selection",
    "total_tasks": 700,
    "passed_count": 397,
    "success_pct": 56.71,
    "retry_flags": {},
    "report_file": "logs/ternary_bonsai_4b_mac_20261018/competition_math_report.json",
    "report_sha256": "7a17297bfed181b65d3dd798f0c4724f8e8d9cc853d40da9fbf3e39ff23978b3",
    "timestamp": 1792353619
  },
  "arc": {
    "label": "ARC-Challenge",
    "total_tasks": 1172,
    "passed_count": 640,
    "success_pct": 54.61,
    "retry_flags": {},
    "report_file": "logs/ternary_bonsai_4b_mac_20261018/arc_report.json",
    "report_sha256": "cbf1df4a9d64a60814cda6309509440df155dd97bf535a2153fb0c0cbd0c5346",
    "timestamp": 1792319010
  },
  "_provenance": {
    "audited": "2026-10-28",
    "metric": "saved success rate; not controlled pass@1",
    "model_name": "Ternary Bonsai 4B \u00b7 2-bit MLX",
    "summary_discrepancies": [
      {
        "model": "ternary_bonsai_4b",
        "suite": "humaneval",
        "field": "pass_at_1_pct",
        "summary": 73.78,
        "detailed_report": 71.95
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "humaneval",
        "field": "ast_validity_pct",
        "summary": 100.0,
        "detailed_report": 89.02
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "humaneval",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 121,
          "FAILED_ASSERTION": 43
        },
        "detailed_report": {
          "PASSED": 118,
          "FAILED_ASSERTION": 22,
          "SYNTAX_ERROR": 18,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 6,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "cyber",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 8,
          "FAILED_ASSERTION": 42
        },
        "detailed_report": {
          "PASSED": 8,
          "FAILED_ASSERTION": 42,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "gpqa_diamond",
        "field": "pass_at_1_pct",
        "summary": 32.32,
        "detailed_report": 26.77
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "gpqa_diamond",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 64,
          "FAILED_ASSERTION": 134
        },
        "detailed_report": {
          "PASSED": 53,
          "FAILED_ASSERTION": 145,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "mmlu_science",
        "field": "pass_at_1_pct",
        "summary": 63.75,
        "detailed_report": 62.55
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "mmlu_science",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 531,
          "FAILED_ASSERTION": 302
        },
        "detailed_report": {
          "PASSED": 521,
          "FAILED_ASSERTION": 312,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "competition_math",
        "field": "pass_at_1_pct",
        "summary": 58.14,
        "detailed_report": 56.71
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "competition_math",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 407,
          "FAILED_ASSERTION": 293
        },
        "detailed_report": {
          "PASSED": 397,
          "FAILED_ASSERTION": 303,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "arc",
        "field": "pass_at_1_pct",
        "summary": 55.38,
        "detailed_report": 54.61
      },
      {
        "model": "ternary_bonsai_4b",
        "suite": "arc",
        "field": "execution_outcomes",
        "summary": {
          "PASSED": 649,
          "FAILED_ASSERTION": 523
        },
        "detailed_report": {
          "PASSED": 640,
          "FAILED_ASSERTION": 532,
          "SYNTAX_ERROR": 0,
          "TIMEOUT": 0,
          "RUNTIME_EXCEPTION": 0,
          "MEMORY_EXCEEDED": 0
        }
      }
    ]
  }
}
