mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-31 03:12:16 +00:00
fix(rescore_swe): preserve old score when new harness result is no_report
When upstream swebench's Modal cgroup-v2 patch can't be applied (set_cpu_quota missing), the harness returns reason=no_report for the rescored task. The previous code blindly overwrote rows[i].score with new_score, destroying a previously-passing row. Now, if old score was success=True and new is success=False+reason=no_report, keep the old. Discovered 2026-05-26 after the rescore on advisors+skillorchestra qwen36-opus47-swe-n100 cells inverted ~25 previously-passing rows to no_report. Recovered via .predupe best-of dedup; next rescore will land clean.
This commit is contained in:
@@ -36,6 +36,7 @@ sys.path.insert(0, str(_REPO_ROOT / "src"))
|
||||
from openjarvis.evals.scorers.swebench_harness import _run_harness # noqa: E402
|
||||
|
||||
HYBRID_DIR = Path(os.path.expanduser("~/.openjarvis/experiments/hybrid"))
|
||||
RUNS_DIR = HYBRID_DIR / "runs"
|
||||
DOCS_TABLE = HYBRID_DIR / "docs" / "results-table.md"
|
||||
|
||||
MAX_WORKERS = 8
|
||||
@@ -190,7 +191,14 @@ def _process_cell(cell_dir: Path) -> Dict[str, Any]:
|
||||
n_done += 1
|
||||
else:
|
||||
with rows_lock:
|
||||
rows[i]["score"] = new_score
|
||||
old_score = rows[i].get("score") or {}
|
||||
old_ok = bool(isinstance(old_score, dict) and old_score.get("success"))
|
||||
new_reason = ((new_score.get("details") or {}).get("reason")
|
||||
if isinstance(new_score.get("details"), dict) else None)
|
||||
if old_ok and not new_score["success"] and new_reason == "no_report":
|
||||
pass
|
||||
else:
|
||||
rows[i]["score"] = new_score
|
||||
_flush_tracker(tid)
|
||||
if new_score["success"]:
|
||||
n_new_resolved += 1
|
||||
@@ -295,7 +303,7 @@ def _update_results_table(summaries: List[Dict[str, Any]]) -> bool:
|
||||
|
||||
def _resolve_cells(args: argparse.Namespace) -> List[Path]:
|
||||
if args.all_swe:
|
||||
cells = sorted(p for p in HYBRID_DIR.glob("*-swe-n100") if p.is_dir())
|
||||
cells = sorted(p for p in RUNS_DIR.glob("*-swe-n100") if p.is_dir())
|
||||
return [c for c in cells if (c / "results.jsonl").exists()]
|
||||
if not args.cells:
|
||||
raise SystemExit("Provide --cells or --all-swe")
|
||||
@@ -304,7 +312,7 @@ def _resolve_cells(args: argparse.Namespace) -> List[Path]:
|
||||
name = name.strip()
|
||||
if not name:
|
||||
continue
|
||||
p = HYBRID_DIR / name
|
||||
p = RUNS_DIR / name
|
||||
if not p.exists():
|
||||
print(f"[WARN] cell not found: {p}")
|
||||
continue
|
||||
@@ -315,7 +323,7 @@ def _resolve_cells(args: argparse.Namespace) -> List[Path]:
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--cells", type=str, default="",
|
||||
help="Comma-separated cell names under ~/.openjarvis/experiments/hybrid/")
|
||||
help="Comma-separated cell names under ~/.openjarvis/experiments/hybrid/runs/")
|
||||
ap.add_argument("--all-swe", action="store_true",
|
||||
help="Auto-detect cells matching *-swe-n100")
|
||||
args = ap.parse_args()
|
||||
|
||||
Reference in New Issue
Block a user