mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 13:26:48 +00:00
Andrew ParkandClaude Opus 4.7
dff6c5539e
Tests, ablation registries, rescore script, and m2 distillation configs that were left untracked alongside the recent feature commits. Junk excluded (minion_logs/, oj-debug.oj-debug.json — runtime artifacts). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
88 lines
3.1 KiB
Python
88 lines
3.1 KiB
Python
"""Tests for the ablation orchestrator's first-5 fail-fast kill heuristic.
|
|
|
|
Background — the bug:
|
|
The old kill condition fired on 5/5 score=0 regardless of answer content.
|
|
GPT-5-mini on GAIA scored 0/5 with REAL wrong answers ("Herbert Simon" vs
|
|
ref "Claude Shannon"), got killed by the orchestrator, but the cell would
|
|
have ended at acc=0.18 over 100 tasks. The fix tightens the 5-zero kill
|
|
to require empty answers (= wiring broken), not merely wrong ones.
|
|
|
|
Run:
|
|
.venv/bin/python -m pytest scripts/ablation/test_run_sweep.py -v
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from scripts.ablation.run_sweep import _check_first5_kill
|
|
|
|
|
|
def _errored_row(msg: str = "boom") -> dict:
|
|
return {"task_id": "t", "error": msg, "score": None, "answer": ""}
|
|
|
|
|
|
def _scored_row(score: float, answer: str = "") -> dict:
|
|
return {
|
|
"task_id": "t",
|
|
"error": None,
|
|
"score": {"score": score},
|
|
"answer": answer,
|
|
}
|
|
|
|
|
|
def test_kill_5_errors() -> None:
|
|
"""5 errored rows -> killed-5error."""
|
|
rows = [_errored_row(f"err {i}") for i in range(5)]
|
|
assert _check_first5_kill(rows) == "killed-5error"
|
|
|
|
|
|
def test_kill_5_empty_answers() -> None:
|
|
"""5 rows with score=0.0 AND empty answer -> killed-5zero (wiring broken)."""
|
|
rows = [_scored_row(0.0, "") for _ in range(5)]
|
|
assert _check_first5_kill(rows) == "killed-5zero"
|
|
|
|
|
|
def test_no_kill_5_wrong_answers() -> None:
|
|
"""5 rows with score=0.0 AND non-empty wrong answer -> NO kill.
|
|
|
|
REGRESSION TEST for the original bug: GPT-5-mini answering "Herbert Simon"
|
|
where the reference was "Claude Shannon" is legit poor performance, not
|
|
broken wiring. The orchestrator must let it ride.
|
|
"""
|
|
rows = [
|
|
_scored_row(0.0, "Herbert Simon"),
|
|
_scored_row(0.0, "Alan Turing"),
|
|
_scored_row(0.0, "John von Neumann"),
|
|
_scored_row(0.0, "Marvin Minsky"),
|
|
_scored_row(0.0, "Donald Knuth"),
|
|
]
|
|
assert _check_first5_kill(rows) is None
|
|
|
|
|
|
def test_no_kill_mixed() -> None:
|
|
"""4 errors + 1 success -> no kill (not all errored, not all scored-zero)."""
|
|
rows = [_errored_row() for _ in range(4)] + [_scored_row(1.0, "right answer")]
|
|
assert _check_first5_kill(rows) is None
|
|
|
|
|
|
def test_no_kill_under_5_rows() -> None:
|
|
"""Fewer than 5 rows -> None (not yet evaluable)."""
|
|
rows = [_errored_row() for _ in range(4)]
|
|
assert _check_first5_kill(rows) is None
|
|
|
|
|
|
def test_no_kill_4_empty_plus_1_nonempty() -> None:
|
|
"""4 empty + 1 non-empty wrong answer -> no kill (the non-empty saves it)."""
|
|
rows = [_scored_row(0.0, "") for _ in range(4)] + [_scored_row(0.0, "wrong")]
|
|
assert _check_first5_kill(rows) is None
|
|
|
|
|
|
def test_no_kill_whitespace_only_answer_treated_as_empty_but_with_score() -> None:
|
|
"""Whitespace-only answer is treated as empty -> killed-5zero fires."""
|
|
rows = [_scored_row(0.0, " \n\t") for _ in range(5)]
|
|
assert _check_first5_kill(rows) == "killed-5zero"
|
|
|
|
|
|
def test_only_first_5_inspected() -> None:
|
|
"""If the first 5 trigger a kill, later rows don't rescue it."""
|
|
rows = [_errored_row() for _ in range(5)] + [_scored_row(1.0, "ok")]
|
|
assert _check_first5_kill(rows) == "killed-5error"
|