From 79777371b14330caeb335a3b6b5f184d2e828e49 Mon Sep 17 00:00:00 2001
From: Andrew Park <andysampark@gmail.com>
Date: Mon, 18 May 2026 01:32:12 -0700
Subject: [PATCH] fix: bump cloud_max_tokens default to 16384 for reasoning
 models
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

GPT-5 family and Gemini 2.5 Pro consume the output-token budget on hidden
chain-of-thought before emitting visible answer text. At max_tokens=4096
they silently truncate with empty answers (finish_reason=length / MAX_TOKENS)
on GAIA — surveyed n=100 cell logs show:
- cloud-only-gpt5-gaia: 26/100 empty answers
- cloud-only-gpt5mini-gaia: 6 empty (+41 short — partial truncations)
- cloud-only-gemini25pro-gaia: 18/100 empty answers
- cloud-only-opus47-gaia: 2/100 empty (Opus isn't a reasoning model in
  this sense, kept at 4096)

Adds _prices.is_reasoning_model + default_max_output_tokens helpers and
threads them through baseline_cloud's three call sites. Cells that opt in
via method_cfg.cloud_max_tokens override the new default unchanged.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 src/openjarvis/agents/hybrid/_prices.py        | 14 ++++++++++++++
 src/openjarvis/agents/hybrid/baseline_cloud.py | 15 ++++++++++-----
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/openjarvis/agents/hybrid/_prices.py b/src/openjarvis/agents/hybrid/_prices.py
index aa22e4a5..e9bad1d7 100644
--- a/src/openjarvis/agents/hybrid/_prices.py
+++ b/src/openjarvis/agents/hybrid/_prices.py
@@ -49,3 +49,17 @@ def supports_temperature(model: str) -> bool:
 def is_gpt5_family(model: str) -> bool:
     """GPT-5 series requires ``max_completion_tokens`` and forced temp=1."""
     return model.startswith("gpt-5")
+
+
+def is_reasoning_model(model: str) -> bool:
+    """Models that consume the output-token budget on hidden chain-of-thought
+    before emitting visible answer text. At max_tokens=4096 these silently
+    truncate with empty answers on GAIA (26/100 GPT-5, 18/100 Gemini Pro)."""
+    m = (model or "").lower()
+    return is_gpt5_family(model) or "gemini-2.5-pro" in m
+
+
+def default_max_output_tokens(model: str) -> int:
+    """Sane default for ``max_tokens`` per cloud call. Reasoning models get
+    a larger budget so their hidden thinking doesn't crowd out the answer."""
+    return 16384 if is_reasoning_model(model) else 4096
diff --git a/src/openjarvis/agents/hybrid/baseline_cloud.py b/src/openjarvis/agents/hybrid/baseline_cloud.py
index 71fca7f4..45ff0bf2 100644
--- a/src/openjarvis/agents/hybrid/baseline_cloud.py
+++ b/src/openjarvis/agents/hybrid/baseline_cloud.py
@@ -32,6 +32,7 @@ from openjarvis.agents.hybrid._base import (
     web_search_cfg,
 )
 from openjarvis.agents.hybrid._prices import cost as estimate_cost
+from openjarvis.agents.hybrid._prices import default_max_output_tokens
 from openjarvis.agents.hybrid.mini_swe_agent import run_swe_agent_loop
 from openjarvis.core.registry import AgentRegistry
 
@@ -42,8 +43,12 @@ class BaselineCloudAgent(LocalCloudAgent):
 
     Configurable knobs via ``cfg``:
 
-    - ``cloud_max_tokens`` (int, default 4096): max_tokens per GAIA call
-      and per turn of the SWE agent loop.
+    - ``cloud_max_tokens`` (int, default 4096 / 16384 for reasoning models):
+      max_tokens per GAIA call and per turn of the SWE agent loop. Default
+      jumps to 16384 for GPT-5 family and Gemini 2.5 Pro because those models
+      burn the budget on hidden chain-of-thought before emitting visible
+      answer text; at 4096 they silently truncated 18–26% of GAIA cells with
+      empty answers. Override per-cell via ``method_cfg`` to opt out.
     - ``swe_max_turns`` (int, default 30): SWE-bench loop turn cap.
     - ``swe_bash_timeout_s`` (int, default 120): SWE-bench bash timeout.
     """
@@ -77,7 +82,7 @@ class BaselineCloudAgent(LocalCloudAgent):
                 max_turns=int(cfg.get("swe_max_turns", 30)),
                 bash_timeout=int(cfg.get("swe_bash_timeout_s", 120)),
                 output_cap=int(cfg.get("swe_output_cap", 10_000)),
-                turn_max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
+                turn_max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
                 trace_prefix="baseline_cloud",
             )
             meta = {
@@ -105,7 +110,7 @@ class BaselineCloudAgent(LocalCloudAgent):
             text, p_tok, c_tok, n_searches, turns = self._call_anthropic_agent(
                 self._cloud_model,
                 user=input,
-                max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
+                max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
                 temperature=0.0,
                 tools=[build_web_search_tool(ws_max_uses)],
                 max_turns=gaia_max_turns,
@@ -146,7 +151,7 @@ class BaselineCloudAgent(LocalCloudAgent):
         # mini-SWE-agent loop above (now supports anthropic/openai/gemini).
         text, p_tok, c_tok = self._call_cloud(
             user=input,
-            max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
+            max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
             temperature=0.0,
         )
         meta = {