mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
fix: bump cloud_max_tokens default to 16384 for reasoning models
GPT-5 family and Gemini 2.5 Pro consume the output-token budget on hidden chain-of-thought before emitting visible answer text. At max_tokens=4096 they silently truncate with empty answers (finish_reason=length / MAX_TOKENS) on GAIA — surveyed n=100 cell logs show: - cloud-only-gpt5-gaia: 26/100 empty answers - cloud-only-gpt5mini-gaia: 6 empty (+41 short — partial truncations) - cloud-only-gemini25pro-gaia: 18/100 empty answers - cloud-only-opus47-gaia: 2/100 empty (Opus isn't a reasoning model in this sense, kept at 4096) Adds _prices.is_reasoning_model + default_max_output_tokens helpers and threads them through baseline_cloud's three call sites. Cells that opt in via method_cfg.cloud_max_tokens override the new default unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
08e37e294a
commit
79777371b1
@@ -49,3 +49,17 @@ def supports_temperature(model: str) -> bool:
|
||||
def is_gpt5_family(model: str) -> bool:
|
||||
"""GPT-5 series requires ``max_completion_tokens`` and forced temp=1."""
|
||||
return model.startswith("gpt-5")
|
||||
|
||||
|
||||
def is_reasoning_model(model: str) -> bool:
|
||||
"""Models that consume the output-token budget on hidden chain-of-thought
|
||||
before emitting visible answer text. At max_tokens=4096 these silently
|
||||
truncate with empty answers on GAIA (26/100 GPT-5, 18/100 Gemini Pro)."""
|
||||
m = (model or "").lower()
|
||||
return is_gpt5_family(model) or "gemini-2.5-pro" in m
|
||||
|
||||
|
||||
def default_max_output_tokens(model: str) -> int:
|
||||
"""Sane default for ``max_tokens`` per cloud call. Reasoning models get
|
||||
a larger budget so their hidden thinking doesn't crowd out the answer."""
|
||||
return 16384 if is_reasoning_model(model) else 4096
|
||||
|
||||
@@ -32,6 +32,7 @@ from openjarvis.agents.hybrid._base import (
|
||||
web_search_cfg,
|
||||
)
|
||||
from openjarvis.agents.hybrid._prices import cost as estimate_cost
|
||||
from openjarvis.agents.hybrid._prices import default_max_output_tokens
|
||||
from openjarvis.agents.hybrid.mini_swe_agent import run_swe_agent_loop
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
|
||||
@@ -42,8 +43,12 @@ class BaselineCloudAgent(LocalCloudAgent):
|
||||
|
||||
Configurable knobs via ``cfg``:
|
||||
|
||||
- ``cloud_max_tokens`` (int, default 4096): max_tokens per GAIA call
|
||||
and per turn of the SWE agent loop.
|
||||
- ``cloud_max_tokens`` (int, default 4096 / 16384 for reasoning models):
|
||||
max_tokens per GAIA call and per turn of the SWE agent loop. Default
|
||||
jumps to 16384 for GPT-5 family and Gemini 2.5 Pro because those models
|
||||
burn the budget on hidden chain-of-thought before emitting visible
|
||||
answer text; at 4096 they silently truncated 18–26% of GAIA cells with
|
||||
empty answers. Override per-cell via ``method_cfg`` to opt out.
|
||||
- ``swe_max_turns`` (int, default 30): SWE-bench loop turn cap.
|
||||
- ``swe_bash_timeout_s`` (int, default 120): SWE-bench bash timeout.
|
||||
"""
|
||||
@@ -77,7 +82,7 @@ class BaselineCloudAgent(LocalCloudAgent):
|
||||
max_turns=int(cfg.get("swe_max_turns", 30)),
|
||||
bash_timeout=int(cfg.get("swe_bash_timeout_s", 120)),
|
||||
output_cap=int(cfg.get("swe_output_cap", 10_000)),
|
||||
turn_max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
|
||||
turn_max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
|
||||
trace_prefix="baseline_cloud",
|
||||
)
|
||||
meta = {
|
||||
@@ -105,7 +110,7 @@ class BaselineCloudAgent(LocalCloudAgent):
|
||||
text, p_tok, c_tok, n_searches, turns = self._call_anthropic_agent(
|
||||
self._cloud_model,
|
||||
user=input,
|
||||
max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
|
||||
max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
|
||||
temperature=0.0,
|
||||
tools=[build_web_search_tool(ws_max_uses)],
|
||||
max_turns=gaia_max_turns,
|
||||
@@ -146,7 +151,7 @@ class BaselineCloudAgent(LocalCloudAgent):
|
||||
# mini-SWE-agent loop above (now supports anthropic/openai/gemini).
|
||||
text, p_tok, c_tok = self._call_cloud(
|
||||
user=input,
|
||||
max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
|
||||
max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
|
||||
temperature=0.0,
|
||||
)
|
||||
meta = {
|
||||
|
||||
Reference in New Issue
Block a user