feat(evals): per-config max_turns + Phase 2 config bumps (#215)

Per-config max_turns:
- Add `max_turns` field to ExecutionConfig and RunConfig dataclasses
- Parse `[run] max_turns` from eval TOML configs
- Plumb through `_build_backend` to JarvisAgentBackend, which sets
  `builder._config.agent.max_turns` before building the system
- Default `None` falls back to `JarvisConfig.agent.max_turns` (10) so
  existing configs are unaffected

Why: Trinity-Large is a "thinking" model that consumes agent-loop turns
producing intermediate reasoning, not just tool calls. With the default
max_turns=10 it hit the cap on 25/50 GAIA tasks (50%) and 39/50
LiveResearchBench tasks (78%) before producing a final answer. With
max_turns=50 (set per-config in this commit's gaia-trinity-large.toml
and liveresearch-trinity-large.toml updates), Trinity LR jumped from
12.0% to 72.0% — equal to Qwen-27B. Without this field the only
workaround was a runtime `OPENJARVIS_CONFIG` TOML hack.

Misc config bumps:
- `gaia-qwen-2b.toml`, `liveresearch-qwen-2b.toml`,
  `taubench-telecom-qwen2b.toml`: max_workers 1 -> 8 to use the
  parallel runner from #207
- `gaia-trinity-large.toml`, `liveresearch-trinity-large.toml`:
  add `max_turns = 50` (using the new field)

Co-authored-by: Jon Saad-Falcon <jonsaadfalcon@gpu-dp-vg94q-lccz2.t-f6a9b26f-56c0-4f98-88fc-705e5d9b1714.svc.cluster.local>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

src/openjarvis/evals/backends/jarvis_agent.py

@@ -25,6 +25,7 @@ class JarvisAgentBackend(InferenceBackend):
         telemetry: bool = False,
         gpu_metrics: bool = False,
         model: Optional[str] = None,
+        max_turns: Optional[int] = None,
     ) -> None:
         from openjarvis.system import SystemBuilder
 
@@ -45,6 +46,12 @@ class JarvisAgentBackend(InferenceBackend):
         # creates a GpuMonitor when building the InstrumentedEngine.
         if gpu_metrics:
             builder._config.telemetry.gpu_metrics = True
+        # Override the agent's per-run turn budget. JarvisConfig.agent.max_turns
+        # defaults to 10, which is too low for thinking/reasoning models on
+        # multi-step agentic benchmarks (Trinity-Large hit the cap on 25/50
+        # GAIA tasks before this was configurable per-eval).
+        if max_turns is not None:
+            builder._config.agent.max_turns = max_turns
         self._system = builder.telemetry(telemetry).traces(True).build()
 
     def generate(

src/openjarvis/evals/cli.py

@@ -165,6 +165,7 @@ def _build_backend(
     telemetry: bool = False,
     gpu_metrics: bool = False,
     model: Optional[str] = None,
+    max_turns: Optional[int] = None,
 ):
     """Construct the appropriate backend."""
     if backend_name == "jarvis-agent":
@@ -177,6 +178,7 @@ def _build_backend(
             telemetry=telemetry,
             gpu_metrics=gpu_metrics,
             model=model,
+            max_turns=max_turns,
         )
     else:
         from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend
@@ -651,6 +653,7 @@ def _run_single(config, console: Optional[Console] = None) -> object:
         telemetry=getattr(config, "telemetry", False),
         gpu_metrics=getattr(config, "gpu_metrics", False),
         model=config.model,
+        max_turns=getattr(config, "max_turns", None),
     )
     dataset = _build_dataset(config.benchmark)
     # Inject engine config for benchmarks that run their own simulation

src/openjarvis/evals/configs/gaia-qwen-2b.toml

@@ -14,7 +14,7 @@ max_tokens = 4096
 engine = "cloud"
 
 [run]
-max_workers = 1
+max_workers = 8
 output_dir = "results/neurips-2026/baselines/qwen-2b/gaia/"
 seed = 42
 

src/openjarvis/evals/configs/gaia-trinity-large.toml

@@ -14,6 +14,10 @@ max_tokens = 4096
 
 [run]
 max_workers = 16
+# Trinity-Large is a "thinking" model — needs more turns to reach an answer.
+# Without this override the agent hits the default max_turns=10 cap on
+# ~50% of GAIA tasks before producing a final answer.
+max_turns = 50
 output_dir = "results/neurips-2026/baselines/gaia-trinity-large/"
 seed = 42
 

src/openjarvis/evals/configs/liveresearch-qwen-2b.toml

@@ -14,7 +14,7 @@ max_tokens = 4096
 engine = "cloud"
 
 [run]
-max_workers = 1
+max_workers = 8
 output_dir = "results/neurips-2026/baselines/qwen-2b/liveresearch/"
 seed = 42
 

src/openjarvis/evals/configs/liveresearch-trinity-large.toml

@@ -14,6 +14,10 @@ max_tokens = 4096
 
 [run]
 max_workers = 16
+# Trinity-Large is a "thinking" model — needs more turns to reach an answer.
+# Without this override the agent hits the default max_turns=10 cap on
+# ~78% of LiveResearchBench tasks before producing a final answer.
+max_turns = 50
 output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/"
 seed = 42
 

src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml

@@ -13,7 +13,7 @@ temperature = 0.0
 engine = "cloud"
 
 [run]
-max_workers = 1
+max_workers = 8
 output_dir = "results/neurips-2026/baselines/qwen-2b/taubench-telecom/"
 seed = 42
 

src/openjarvis/evals/core/config.py

@@ -133,6 +133,9 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig:
         sheets_spreadsheet_id=run_raw.get("sheets_spreadsheet_id", ""),
         sheets_worksheet=run_raw.get("sheets_worksheet", "Results"),
         sheets_credentials_path=run_raw.get("sheets_credentials_path", ""),
+        max_turns=(
+            int(run_raw["max_turns"]) if "max_turns" in run_raw else None
+        ),
     )
 
     # Parse [[models]]
@@ -293,6 +296,7 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]:
                     sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id,
                     sheets_worksheet=suite.run.sheets_worksheet,
                     sheets_credentials_path=suite.run.sheets_credentials_path,
+                    max_turns=suite.run.max_turns,
                 )
             )
 

src/openjarvis/evals/core/types.py

@@ -84,6 +84,11 @@ class RunConfig:
     system_prompt: str = ""
     episode_mode: bool = False
     dataset_subset: Optional[str] = None
+    # Override the agent harness's max_turns budget. Default None means use
+    # the JarvisConfig.agent.max_turns value (typically 10). Set higher when
+    # running thinking/reasoning models that consume turns on intermediate
+    # reasoning before producing tool calls.
+    max_turns: Optional[int] = None
 
 
 @dataclass(slots=True)
@@ -202,6 +207,10 @@ class ExecutionConfig:
     sheets_spreadsheet_id: str = ""
     sheets_worksheet: str = "Results"
     sheets_credentials_path: str = ""
+    # Override the agent harness's per-run max_turns budget. None falls back
+    # to JarvisConfig.agent.max_turns (default 10). Bump to 30-50 for
+    # thinking/reasoning models on agentic benchmarks (GAIA, LiveResearch).
+    max_turns: Optional[int] = None
 
 
 @dataclass(slots=True)