mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-31 03:12:16 +00:00
b21463aab6
Two failure classes hit by a downstream team: 1) tmux/task-env death (TerminalBenchTaskEnv.__enter__, terminalbench_env.py): create_session ran inside the spin_up_terminal generator-CM with no exception safety — a tmux failure leaked the docker compose project (down deferred to GC, never if the env was retained) and surfaced as an opaque mid-run death. Now: exception-safe __enter__ with an idempotent _teardown(), a tmux/asciinema preflight in the container BEFORE the agent loop (TaskEnvironmentError naming the task image + remedy), and fail-that-task-cleanly semantics — the failure is recorded as a harness error (QueryTrace.error_kind="harness_error"), the container is downed, and the run continues. 2) OpenHands in-container SETUP hang misattributed as a model result: harness.run() had no bound (terminal-bench runs installed-agent setup with max_timeout_sec=inf inside the per-trial agent budget), and the summary conversion read the nonexistent results.trial_results attr and hardcoded errors=0, folding zero-model-request trials into resolve-rate as model misses. Now: global_agent_timeout_sec / global_timeout_multiplier are threaded config -> backend -> Harness kwargs (default 1800 s bound on SETUP+RUN; configurable per [run]/[[benchmarks]] TOML), and summarize_benchmark_results() classifies harness/infra failures out of the accuracy denominator keyed on token usage (zero/missing tokens + unresolved = the agent never contacted the model), NOT failure_mode — terminal-bench 0.2.18 leaves failure_mode UNSET on success AND on genuine misses, so a failure_mode-based check would misflag every real model miss. Genuine misses (tokens>0, is_resolved=false) stay in the denominator. QueryTrace gains error/error_kind (wired through to_dict/from_dict so the fields actually reach traces.jsonl; backward-compatible loads), the AgenticRunner flags zero-model-contact traces and TaskEnvironmentError as harness errors, and export/summary/console output exclude harness errors from resolve-rate while reporting them loudly. terminal-bench stays an undeclared dep on purpose: it requires Python >=3.12 while this project supports >=3.10,<3.14, so an unmarked extra would break uv lock. pyproject/uv.lock untouched. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
73 lines
2.0 KiB
Python
73 lines
2.0 KiB
Python
"""Tests for terminal-bench harness timeout plumbing through TOML configs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import textwrap
|
|
|
|
from openjarvis.evals.core.config import expand_suite, load_eval_config
|
|
|
|
|
|
def _write(tmp_path, body: str):
|
|
path = tmp_path / "suite.toml"
|
|
path.write_text(textwrap.dedent(body))
|
|
return path
|
|
|
|
|
|
BASE = """
|
|
[meta]
|
|
name = "timeouts"
|
|
|
|
[run]
|
|
output_dir = "results/"
|
|
{run_extra}
|
|
|
|
[[models]]
|
|
name = "test-model"
|
|
|
|
[[benchmarks]]
|
|
name = "terminalbench-native"
|
|
backend = "terminalbench-native"
|
|
{bench_extra}
|
|
"""
|
|
|
|
|
|
class TestTimeoutConfigPlumbing:
|
|
def test_run_level_timeouts_parse_and_expand(self, tmp_path):
|
|
path = _write(
|
|
tmp_path,
|
|
BASE.format(
|
|
run_extra=(
|
|
"global_agent_timeout_sec = 1200\n"
|
|
" global_timeout_multiplier = 1.5"
|
|
),
|
|
bench_extra="",
|
|
),
|
|
)
|
|
suite = load_eval_config(path)
|
|
assert suite.run.global_agent_timeout_sec == 1200.0
|
|
assert suite.run.global_timeout_multiplier == 1.5
|
|
|
|
(rc,) = expand_suite(suite)
|
|
assert rc.global_agent_timeout_sec == 1200.0
|
|
assert rc.global_timeout_multiplier == 1.5
|
|
|
|
def test_benchmark_override_wins(self, tmp_path):
|
|
path = _write(
|
|
tmp_path,
|
|
BASE.format(
|
|
run_extra="global_agent_timeout_sec = 1200",
|
|
bench_extra="global_agent_timeout_sec = 300",
|
|
),
|
|
)
|
|
(rc,) = expand_suite(load_eval_config(path))
|
|
assert rc.global_agent_timeout_sec == 300.0
|
|
|
|
def test_defaults_are_none(self, tmp_path):
|
|
path = _write(tmp_path, BASE.format(run_extra="", bench_extra=""))
|
|
suite = load_eval_config(path)
|
|
assert suite.run.global_agent_timeout_sec is None
|
|
assert suite.run.global_timeout_multiplier is None
|
|
(rc,) = expand_suite(suite)
|
|
assert rc.global_agent_timeout_sec is None
|
|
assert rc.global_timeout_multiplier is None
|