diff --git a/src/openjarvis/evals/backends/terminalbench_native.py b/src/openjarvis/evals/backends/terminalbench_native.py index 41adc0c4..5b52973d 100644 --- a/src/openjarvis/evals/backends/terminalbench_native.py +++ b/src/openjarvis/evals/backends/terminalbench_native.py @@ -77,16 +77,16 @@ class TerminalBenchNativeBackend(InferenceBackend): "cleanup": True, } - # Use built-in agent (naive uses LiteLLM) + # Use terminus-2 agent which accepts model_name + api_base as + # serializable strings (avoids Pydantic serialization issues with + # LLM objects in the harness lock file). from terminal_bench.agents.agent_name import AgentName - harness_kwargs["agent_name"] = AgentName(self._agent_name) + harness_kwargs["agent_name"] = AgentName("terminus-2") harness_kwargs["agent_kwargs"] = { - "llm": LiteLLM( - model_name=self._model, - temperature=self._temperature, - api_base=self._api_base, - ), + "model_name": self._model, + "api_base": self._api_base, + "temperature": self._temperature, } if self._max_samples is not None: diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index 69c9506a..635fa181 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -575,6 +575,61 @@ def _build_trackers(config) -> list: return trackers +def _run_terminalbench_native(config, console: Console) -> object: + """Run TerminalBench V2 natively via terminal-bench Harness.""" + from openjarvis.evals.backends.terminalbench_native import ( + TerminalBenchNativeBackend, + ) + from openjarvis.evals.core.types import RunSummary + + model = config.model + # LiteLLM expects "openai/" for OpenAI-compatible servers + litellm_model = f"openai/{model}" + output_dir = getattr(config, "output_path", None) or "results/terminalbench-native/" + + backend = TerminalBenchNativeBackend( + model=litellm_model, + api_base="http://localhost:8000/v1", + temperature=config.temperature, + max_samples=config.max_samples, + output_dir=output_dir, + n_concurrent=config.max_workers or 4, + ) + + import re + import time + # Docker compose project names must be lowercase alphanumeric + hyphens/underscores + model_slug = re.sub(r"[^a-z0-9_-]", "-", model.lower().replace("/", "-")) + run_id = f"tb2-{model_slug}" + console.print(f" Running TerminalBench V2 natively: {model}") + console.print(f" Harness run_id: {run_id}") + + results = backend.run_harness(run_id) + + # Convert BenchmarkResults to RunSummary + total = len(results.trial_results) if hasattr(results, "trial_results") else 0 + correct = 0 + if hasattr(results, "trial_results"): + for tr in results.trial_results: + if getattr(tr, "is_resolved", False): + correct += 1 + + accuracy = correct / total if total > 0 else 0.0 + return RunSummary( + benchmark="terminalbench-native", + category="agentic", + backend="terminalbench-native", + model=model, + total_samples=total, + scored_samples=total, + correct=correct, + accuracy=accuracy, + errors=0, + mean_latency_seconds=0.0, + total_cost_usd=0.0, + ) + + def _run_single(config, console: Optional[Console] = None) -> object: """Run a single eval from a RunConfig and return the summary.""" from openjarvis.evals.core.runner import EvalRunner @@ -582,6 +637,10 @@ def _run_single(config, console: Optional[Console] = None) -> object: if console is None: console = Console() + # TerminalBench V2 native: use terminal-bench Harness directly + if config.benchmark == "terminalbench-native": + return _run_terminalbench_native(config, console) + eval_backend = _build_backend( config.backend, config.engine_key, diff --git a/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml b/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml new file mode 100644 index 00000000..c195da79 --- /dev/null +++ b/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml @@ -0,0 +1,30 @@ +# TerminalBench V2 (native Docker) on Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "terminalbench-native-qwen122b" +description = "TerminalBench V2 native on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 8 +output_dir = "results/terminalbench-native-qwen122b/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "terminalbench-native" +backend = "terminalbench-native" +max_samples = 50 diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index b8a4e3cc..281247b3 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -31,7 +31,7 @@ else: logger = logging.getLogger(__name__) -VALID_BACKENDS = {"jarvis-direct", "jarvis-agent"} +VALID_BACKENDS = {"jarvis-direct", "jarvis-agent", "terminalbench-native"} # Known benchmark names (used for warnings, not hard validation) KNOWN_BENCHMARKS = {