feat: wire TerminalBench V2 native harness into eval CLI

- Add _run_terminalbench_native() to CLI for direct Harness invocation
- Use terminus-2 agent (serializable model_name + api_base kwargs)
- Lowercase Docker compose project names to satisfy validation
- Add terminalbench-native to valid backends in config
- Add eval config for Qwen3.5-122B-A10B-FP8

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

src/openjarvis/evals/backends/terminalbench_native.py

@@ -77,16 +77,16 @@ class TerminalBenchNativeBackend(InferenceBackend):
             "cleanup": True,
         }
 
-        # Use built-in agent (naive uses LiteLLM)
+        # Use terminus-2 agent which accepts model_name + api_base as
+        # serializable strings (avoids Pydantic serialization issues with
+        # LLM objects in the harness lock file).
         from terminal_bench.agents.agent_name import AgentName
 
-        harness_kwargs["agent_name"] = AgentName(self._agent_name)
+        harness_kwargs["agent_name"] = AgentName("terminus-2")
         harness_kwargs["agent_kwargs"] = {
-            "llm": LiteLLM(
-                model_name=self._model,
-                temperature=self._temperature,
-                api_base=self._api_base,
-            ),
+            "model_name": self._model,
+            "api_base": self._api_base,
+            "temperature": self._temperature,
         }
 
         if self._max_samples is not None:

src/openjarvis/evals/cli.py

@@ -575,6 +575,61 @@ def _build_trackers(config) -> list:
     return trackers
 
 
+def _run_terminalbench_native(config, console: Console) -> object:
+    """Run TerminalBench V2 natively via terminal-bench Harness."""
+    from openjarvis.evals.backends.terminalbench_native import (
+        TerminalBenchNativeBackend,
+    )
+    from openjarvis.evals.core.types import RunSummary
+
+    model = config.model
+    # LiteLLM expects "openai/<model>" for OpenAI-compatible servers
+    litellm_model = f"openai/{model}"
+    output_dir = getattr(config, "output_path", None) or "results/terminalbench-native/"
+
+    backend = TerminalBenchNativeBackend(
+        model=litellm_model,
+        api_base="http://localhost:8000/v1",
+        temperature=config.temperature,
+        max_samples=config.max_samples,
+        output_dir=output_dir,
+        n_concurrent=config.max_workers or 4,
+    )
+
+    import re
+    import time
+    # Docker compose project names must be lowercase alphanumeric + hyphens/underscores
+    model_slug = re.sub(r"[^a-z0-9_-]", "-", model.lower().replace("/", "-"))
+    run_id = f"tb2-{model_slug}"
+    console.print(f"  Running TerminalBench V2 natively: {model}")
+    console.print(f"  Harness run_id: {run_id}")
+
+    results = backend.run_harness(run_id)
+
+    # Convert BenchmarkResults to RunSummary
+    total = len(results.trial_results) if hasattr(results, "trial_results") else 0
+    correct = 0
+    if hasattr(results, "trial_results"):
+        for tr in results.trial_results:
+            if getattr(tr, "is_resolved", False):
+                correct += 1
+
+    accuracy = correct / total if total > 0 else 0.0
+    return RunSummary(
+        benchmark="terminalbench-native",
+        category="agentic",
+        backend="terminalbench-native",
+        model=model,
+        total_samples=total,
+        scored_samples=total,
+        correct=correct,
+        accuracy=accuracy,
+        errors=0,
+        mean_latency_seconds=0.0,
+        total_cost_usd=0.0,
+    )
+
+
 def _run_single(config, console: Optional[Console] = None) -> object:
     """Run a single eval from a RunConfig and return the summary."""
     from openjarvis.evals.core.runner import EvalRunner
@@ -582,6 +637,10 @@ def _run_single(config, console: Optional[Console] = None) -> object:
     if console is None:
         console = Console()
 
+    # TerminalBench V2 native: use terminal-bench Harness directly
+    if config.benchmark == "terminalbench-native":
+        return _run_terminalbench_native(config, console)
+
     eval_backend = _build_backend(
         config.backend,
         config.engine_key,

src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml

@@ -0,0 +1,30 @@
+# TerminalBench V2 (native Docker) on Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "terminalbench-native-qwen122b"
+description = "TerminalBench V2 native on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/terminalbench-native-qwen122b/"
+seed = 42
+telemetry = true
+gpu_metrics = true
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "terminalbench-native"
+backend = "terminalbench-native"
+max_samples = 50

src/openjarvis/evals/core/config.py

@@ -31,7 +31,7 @@ else:
 
 logger = logging.getLogger(__name__)
 
-VALID_BACKENDS = {"jarvis-direct", "jarvis-agent"}
+VALID_BACKENDS = {"jarvis-direct", "jarvis-agent", "terminalbench-native"}
 
 # Known benchmark names (used for warnings, not hard validation)
 KNOWN_BENCHMARKS = {