diff --git a/src/openjarvis/evals/backends/jarvis_agent.py b/src/openjarvis/evals/backends/jarvis_agent.py index 61c70654..e4981fc4 100644 --- a/src/openjarvis/evals/backends/jarvis_agent.py +++ b/src/openjarvis/evals/backends/jarvis_agent.py @@ -25,6 +25,7 @@ class JarvisAgentBackend(InferenceBackend): telemetry: bool = False, gpu_metrics: bool = False, model: Optional[str] = None, + max_turns: Optional[int] = None, ) -> None: from openjarvis.system import SystemBuilder @@ -45,6 +46,12 @@ class JarvisAgentBackend(InferenceBackend): # creates a GpuMonitor when building the InstrumentedEngine. if gpu_metrics: builder._config.telemetry.gpu_metrics = True + # Override the agent's per-run turn budget. JarvisConfig.agent.max_turns + # defaults to 10, which is too low for thinking/reasoning models on + # multi-step agentic benchmarks (Trinity-Large hit the cap on 25/50 + # GAIA tasks before this was configurable per-eval). + if max_turns is not None: + builder._config.agent.max_turns = max_turns self._system = builder.telemetry(telemetry).traces(True).build() def generate( diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index e291e73f..92203b82 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -165,6 +165,7 @@ def _build_backend( telemetry: bool = False, gpu_metrics: bool = False, model: Optional[str] = None, + max_turns: Optional[int] = None, ): """Construct the appropriate backend.""" if backend_name == "jarvis-agent": @@ -177,6 +178,7 @@ def _build_backend( telemetry=telemetry, gpu_metrics=gpu_metrics, model=model, + max_turns=max_turns, ) else: from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend @@ -651,6 +653,7 @@ def _run_single(config, console: Optional[Console] = None) -> object: telemetry=getattr(config, "telemetry", False), gpu_metrics=getattr(config, "gpu_metrics", False), model=config.model, + max_turns=getattr(config, "max_turns", None), ) dataset = _build_dataset(config.benchmark) # Inject engine config for benchmarks that run their own simulation diff --git a/src/openjarvis/evals/configs/gaia-qwen-2b.toml b/src/openjarvis/evals/configs/gaia-qwen-2b.toml index 1c68053d..409decfd 100644 --- a/src/openjarvis/evals/configs/gaia-qwen-2b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen-2b.toml @@ -14,7 +14,7 @@ max_tokens = 4096 engine = "cloud" [run] -max_workers = 1 +max_workers = 8 output_dir = "results/neurips-2026/baselines/qwen-2b/gaia/" seed = 42 diff --git a/src/openjarvis/evals/configs/gaia-trinity-large.toml b/src/openjarvis/evals/configs/gaia-trinity-large.toml index 5d3ebad7..aecc3ea4 100644 --- a/src/openjarvis/evals/configs/gaia-trinity-large.toml +++ b/src/openjarvis/evals/configs/gaia-trinity-large.toml @@ -14,6 +14,10 @@ max_tokens = 4096 [run] max_workers = 16 +# Trinity-Large is a "thinking" model — needs more turns to reach an answer. +# Without this override the agent hits the default max_turns=10 cap on +# ~50% of GAIA tasks before producing a final answer. +max_turns = 50 output_dir = "results/neurips-2026/baselines/gaia-trinity-large/" seed = 42 diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml index d0beff38..fa27c3e2 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml @@ -14,7 +14,7 @@ max_tokens = 4096 engine = "cloud" [run] -max_workers = 1 +max_workers = 8 output_dir = "results/neurips-2026/baselines/qwen-2b/liveresearch/" seed = 42 diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml index 27d883d8..22a5d066 100644 --- a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml +++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml @@ -14,6 +14,10 @@ max_tokens = 4096 [run] max_workers = 16 +# Trinity-Large is a "thinking" model — needs more turns to reach an answer. +# Without this override the agent hits the default max_turns=10 cap on +# ~78% of LiveResearchBench tasks before producing a final answer. +max_turns = 50 output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/" seed = 42 diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml index d1fd6023..0c549d81 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml @@ -13,7 +13,7 @@ temperature = 0.0 engine = "cloud" [run] -max_workers = 1 +max_workers = 8 output_dir = "results/neurips-2026/baselines/qwen-2b/taubench-telecom/" seed = 42 diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index 281247b3..5a46ad2f 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -133,6 +133,9 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig: sheets_spreadsheet_id=run_raw.get("sheets_spreadsheet_id", ""), sheets_worksheet=run_raw.get("sheets_worksheet", "Results"), sheets_credentials_path=run_raw.get("sheets_credentials_path", ""), + max_turns=( + int(run_raw["max_turns"]) if "max_turns" in run_raw else None + ), ) # Parse [[models]] @@ -293,6 +296,7 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]: sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id, sheets_worksheet=suite.run.sheets_worksheet, sheets_credentials_path=suite.run.sheets_credentials_path, + max_turns=suite.run.max_turns, ) ) diff --git a/src/openjarvis/evals/core/types.py b/src/openjarvis/evals/core/types.py index 58e4bcb8..ee501d8d 100644 --- a/src/openjarvis/evals/core/types.py +++ b/src/openjarvis/evals/core/types.py @@ -84,6 +84,11 @@ class RunConfig: system_prompt: str = "" episode_mode: bool = False dataset_subset: Optional[str] = None + # Override the agent harness's max_turns budget. Default None means use + # the JarvisConfig.agent.max_turns value (typically 10). Set higher when + # running thinking/reasoning models that consume turns on intermediate + # reasoning before producing tool calls. + max_turns: Optional[int] = None @dataclass(slots=True) @@ -202,6 +207,10 @@ class ExecutionConfig: sheets_spreadsheet_id: str = "" sheets_worksheet: str = "Results" sheets_credentials_path: str = "" + # Override the agent harness's per-run max_turns budget. None falls back + # to JarvisConfig.agent.max_turns (default 10). Bump to 30-50 for + # thinking/reasoning models on agentic benchmarks (GAIA, LiveResearch). + max_turns: Optional[int] = None @dataclass(slots=True)