From 5388b9b75ab5587d646e1ad4ec0b23ba8d0e234b Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Wed, 8 Apr 2026 17:38:23 -0700 Subject: [PATCH] feat(evals): per-config max_turns + Phase 2 config bumps (#215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-config max_turns: - Add `max_turns` field to ExecutionConfig and RunConfig dataclasses - Parse `[run] max_turns` from eval TOML configs - Plumb through `_build_backend` to JarvisAgentBackend, which sets `builder._config.agent.max_turns` before building the system - Default `None` falls back to `JarvisConfig.agent.max_turns` (10) so existing configs are unaffected Why: Trinity-Large is a "thinking" model that consumes agent-loop turns producing intermediate reasoning, not just tool calls. With the default max_turns=10 it hit the cap on 25/50 GAIA tasks (50%) and 39/50 LiveResearchBench tasks (78%) before producing a final answer. With max_turns=50 (set per-config in this commit's gaia-trinity-large.toml and liveresearch-trinity-large.toml updates), Trinity LR jumped from 12.0% to 72.0% — equal to Qwen-27B. Without this field the only workaround was a runtime `OPENJARVIS_CONFIG` TOML hack. Misc config bumps: - `gaia-qwen-2b.toml`, `liveresearch-qwen-2b.toml`, `taubench-telecom-qwen2b.toml`: max_workers 1 -> 8 to use the parallel runner from #207 - `gaia-trinity-large.toml`, `liveresearch-trinity-large.toml`: add `max_turns = 50` (using the new field) Co-authored-by: Jon Saad-Falcon Co-authored-by: Claude Opus 4.6 (1M context) --- src/openjarvis/evals/backends/jarvis_agent.py | 7 +++++++ src/openjarvis/evals/cli.py | 3 +++ src/openjarvis/evals/configs/gaia-qwen-2b.toml | 2 +- src/openjarvis/evals/configs/gaia-trinity-large.toml | 4 ++++ src/openjarvis/evals/configs/liveresearch-qwen-2b.toml | 2 +- .../evals/configs/liveresearch-trinity-large.toml | 4 ++++ .../evals/configs/taubench-telecom-qwen2b.toml | 2 +- src/openjarvis/evals/core/config.py | 4 ++++ src/openjarvis/evals/core/types.py | 9 +++++++++ 9 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/openjarvis/evals/backends/jarvis_agent.py b/src/openjarvis/evals/backends/jarvis_agent.py index 61c70654..e4981fc4 100644 --- a/src/openjarvis/evals/backends/jarvis_agent.py +++ b/src/openjarvis/evals/backends/jarvis_agent.py @@ -25,6 +25,7 @@ class JarvisAgentBackend(InferenceBackend): telemetry: bool = False, gpu_metrics: bool = False, model: Optional[str] = None, + max_turns: Optional[int] = None, ) -> None: from openjarvis.system import SystemBuilder @@ -45,6 +46,12 @@ class JarvisAgentBackend(InferenceBackend): # creates a GpuMonitor when building the InstrumentedEngine. if gpu_metrics: builder._config.telemetry.gpu_metrics = True + # Override the agent's per-run turn budget. JarvisConfig.agent.max_turns + # defaults to 10, which is too low for thinking/reasoning models on + # multi-step agentic benchmarks (Trinity-Large hit the cap on 25/50 + # GAIA tasks before this was configurable per-eval). + if max_turns is not None: + builder._config.agent.max_turns = max_turns self._system = builder.telemetry(telemetry).traces(True).build() def generate( diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index e291e73f..92203b82 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -165,6 +165,7 @@ def _build_backend( telemetry: bool = False, gpu_metrics: bool = False, model: Optional[str] = None, + max_turns: Optional[int] = None, ): """Construct the appropriate backend.""" if backend_name == "jarvis-agent": @@ -177,6 +178,7 @@ def _build_backend( telemetry=telemetry, gpu_metrics=gpu_metrics, model=model, + max_turns=max_turns, ) else: from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend @@ -651,6 +653,7 @@ def _run_single(config, console: Optional[Console] = None) -> object: telemetry=getattr(config, "telemetry", False), gpu_metrics=getattr(config, "gpu_metrics", False), model=config.model, + max_turns=getattr(config, "max_turns", None), ) dataset = _build_dataset(config.benchmark) # Inject engine config for benchmarks that run their own simulation diff --git a/src/openjarvis/evals/configs/gaia-qwen-2b.toml b/src/openjarvis/evals/configs/gaia-qwen-2b.toml index 1c68053d..409decfd 100644 --- a/src/openjarvis/evals/configs/gaia-qwen-2b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen-2b.toml @@ -14,7 +14,7 @@ max_tokens = 4096 engine = "cloud" [run] -max_workers = 1 +max_workers = 8 output_dir = "results/neurips-2026/baselines/qwen-2b/gaia/" seed = 42 diff --git a/src/openjarvis/evals/configs/gaia-trinity-large.toml b/src/openjarvis/evals/configs/gaia-trinity-large.toml index 5d3ebad7..aecc3ea4 100644 --- a/src/openjarvis/evals/configs/gaia-trinity-large.toml +++ b/src/openjarvis/evals/configs/gaia-trinity-large.toml @@ -14,6 +14,10 @@ max_tokens = 4096 [run] max_workers = 16 +# Trinity-Large is a "thinking" model — needs more turns to reach an answer. +# Without this override the agent hits the default max_turns=10 cap on +# ~50% of GAIA tasks before producing a final answer. +max_turns = 50 output_dir = "results/neurips-2026/baselines/gaia-trinity-large/" seed = 42 diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml index d0beff38..fa27c3e2 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml @@ -14,7 +14,7 @@ max_tokens = 4096 engine = "cloud" [run] -max_workers = 1 +max_workers = 8 output_dir = "results/neurips-2026/baselines/qwen-2b/liveresearch/" seed = 42 diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml index 27d883d8..22a5d066 100644 --- a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml +++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml @@ -14,6 +14,10 @@ max_tokens = 4096 [run] max_workers = 16 +# Trinity-Large is a "thinking" model — needs more turns to reach an answer. +# Without this override the agent hits the default max_turns=10 cap on +# ~78% of LiveResearchBench tasks before producing a final answer. +max_turns = 50 output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/" seed = 42 diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml index d1fd6023..0c549d81 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml @@ -13,7 +13,7 @@ temperature = 0.0 engine = "cloud" [run] -max_workers = 1 +max_workers = 8 output_dir = "results/neurips-2026/baselines/qwen-2b/taubench-telecom/" seed = 42 diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index 281247b3..5a46ad2f 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -133,6 +133,9 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig: sheets_spreadsheet_id=run_raw.get("sheets_spreadsheet_id", ""), sheets_worksheet=run_raw.get("sheets_worksheet", "Results"), sheets_credentials_path=run_raw.get("sheets_credentials_path", ""), + max_turns=( + int(run_raw["max_turns"]) if "max_turns" in run_raw else None + ), ) # Parse [[models]] @@ -293,6 +296,7 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]: sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id, sheets_worksheet=suite.run.sheets_worksheet, sheets_credentials_path=suite.run.sheets_credentials_path, + max_turns=suite.run.max_turns, ) ) diff --git a/src/openjarvis/evals/core/types.py b/src/openjarvis/evals/core/types.py index 58e4bcb8..ee501d8d 100644 --- a/src/openjarvis/evals/core/types.py +++ b/src/openjarvis/evals/core/types.py @@ -84,6 +84,11 @@ class RunConfig: system_prompt: str = "" episode_mode: bool = False dataset_subset: Optional[str] = None + # Override the agent harness's max_turns budget. Default None means use + # the JarvisConfig.agent.max_turns value (typically 10). Set higher when + # running thinking/reasoning models that consume turns on intermediate + # reasoning before producing tool calls. + max_turns: Optional[int] = None @dataclass(slots=True) @@ -202,6 +207,10 @@ class ExecutionConfig: sheets_spreadsheet_id: str = "" sheets_worksheet: str = "Results" sheets_credentials_path: str = "" + # Override the agent harness's per-run max_turns budget. None falls back + # to JarvisConfig.agent.max_turns (default 10). Bump to 30-50 for + # thinking/reasoning models on agentic benchmarks (GAIA, LiveResearch). + max_turns: Optional[int] = None @dataclass(slots=True)