fix: bump cloud_max_tokens default to 16384 for reasoning models

GPT-5 family and Gemini 2.5 Pro consume the output-token budget on hidden
chain-of-thought before emitting visible answer text. At max_tokens=4096
they silently truncate with empty answers (finish_reason=length / MAX_TOKENS)
on GAIA — surveyed n=100 cell logs show:
- cloud-only-gpt5-gaia: 26/100 empty answers
- cloud-only-gpt5mini-gaia: 6 empty (+41 short — partial truncations)
- cloud-only-gemini25pro-gaia: 18/100 empty answers
- cloud-only-opus47-gaia: 2/100 empty (Opus isn't a reasoning model in
  this sense, kept at 4096)

Adds _prices.is_reasoning_model + default_max_output_tokens helpers and
threads them through baseline_cloud's three call sites. Cells that opt in
via method_cfg.cloud_max_tokens override the new default unchanged.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrew Park
2026-05-18 01:32:12 -07:00
co-authored by Claude Opus 4.7
parent 08e37e294a
commit 79777371b1
2 changed files with 24 additions and 5 deletions
+14
View File
@@ -49,3 +49,17 @@ def supports_temperature(model: str) -> bool:
def is_gpt5_family(model: str) -> bool:
"""GPT-5 series requires ``max_completion_tokens`` and forced temp=1."""
return model.startswith("gpt-5")
def is_reasoning_model(model: str) -> bool:
"""Models that consume the output-token budget on hidden chain-of-thought
before emitting visible answer text. At max_tokens=4096 these silently
truncate with empty answers on GAIA (26/100 GPT-5, 18/100 Gemini Pro)."""
m = (model or "").lower()
return is_gpt5_family(model) or "gemini-2.5-pro" in m
def default_max_output_tokens(model: str) -> int:
"""Sane default for ``max_tokens`` per cloud call. Reasoning models get
a larger budget so their hidden thinking doesn't crowd out the answer."""
return 16384 if is_reasoning_model(model) else 4096
+10 -5
View File
@@ -32,6 +32,7 @@ from openjarvis.agents.hybrid._base import (
web_search_cfg,
)
from openjarvis.agents.hybrid._prices import cost as estimate_cost
from openjarvis.agents.hybrid._prices import default_max_output_tokens
from openjarvis.agents.hybrid.mini_swe_agent import run_swe_agent_loop
from openjarvis.core.registry import AgentRegistry
@@ -42,8 +43,12 @@ class BaselineCloudAgent(LocalCloudAgent):
Configurable knobs via ``cfg``:
- ``cloud_max_tokens`` (int, default 4096): max_tokens per GAIA call
and per turn of the SWE agent loop.
- ``cloud_max_tokens`` (int, default 4096 / 16384 for reasoning models):
max_tokens per GAIA call and per turn of the SWE agent loop. Default
jumps to 16384 for GPT-5 family and Gemini 2.5 Pro because those models
burn the budget on hidden chain-of-thought before emitting visible
answer text; at 4096 they silently truncated 1826% of GAIA cells with
empty answers. Override per-cell via ``method_cfg`` to opt out.
- ``swe_max_turns`` (int, default 30): SWE-bench loop turn cap.
- ``swe_bash_timeout_s`` (int, default 120): SWE-bench bash timeout.
"""
@@ -77,7 +82,7 @@ class BaselineCloudAgent(LocalCloudAgent):
max_turns=int(cfg.get("swe_max_turns", 30)),
bash_timeout=int(cfg.get("swe_bash_timeout_s", 120)),
output_cap=int(cfg.get("swe_output_cap", 10_000)),
turn_max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
turn_max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
trace_prefix="baseline_cloud",
)
meta = {
@@ -105,7 +110,7 @@ class BaselineCloudAgent(LocalCloudAgent):
text, p_tok, c_tok, n_searches, turns = self._call_anthropic_agent(
self._cloud_model,
user=input,
max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
temperature=0.0,
tools=[build_web_search_tool(ws_max_uses)],
max_turns=gaia_max_turns,
@@ -146,7 +151,7 @@ class BaselineCloudAgent(LocalCloudAgent):
# mini-SWE-agent loop above (now supports anthropic/openai/gemini).
text, p_tok, c_tok = self._call_cloud(
user=input,
max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
temperature=0.0,
)
meta = {