From 39a8082af3d1bfcd4d90aed6cb25c613fe21da21 Mon Sep 17 00:00:00 2001 From: Avanika Narayan Date: Tue, 31 Mar 2026 19:29:55 -0700 Subject: [PATCH] fix(evals): add tool_choice=auto + fix traces thread safety (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes for eval accuracy and stability: 1. TauBench agent: add tool_choice="auto" to match tau2's native LLMAgent behavior. Without this, Qwen and GPT-5.4 score 10-14pp below leaderboard because the models don't receive explicit tool-calling guidance. 2. SystemBuilder: apply self._traces flag to config.traces.enabled. Previously builder.traces(False) was a no-op — traces stayed enabled, creating SQLite connections in the main thread that crashed when accessed from ThreadPoolExecutor worker threads in GAIA evals ("SQLite objects created in a thread can only be used in that same thread"). Co-authored-by: Jon Saad-Falcon Co-authored-by: Claude Opus 4.6 (1M context) --- src/openjarvis/evals/execution/taubench_env.py | 1 + src/openjarvis/system.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.py index cd9b25a8..34f838a1 100644 --- a/src/openjarvis/evals/execution/taubench_env.py +++ b/src/openjarvis/evals/execution/taubench_env.py @@ -177,6 +177,7 @@ class JarvisHalfDuplexAgent: "temperature": self._temperature, "max_tokens": self._max_tokens, "tools": openai_tools, + "tool_choice": "auto", } # Disable thinking mode for local models (Qwen3.5 etc.) # to avoid tags interfering with tool call parsing diff --git a/src/openjarvis/system.py b/src/openjarvis/system.py index a9ade2f8..92dbf40c 100644 --- a/src/openjarvis/system.py +++ b/src/openjarvis/system.py @@ -503,10 +503,15 @@ class SystemBuilder: # Resolve model model = self._resolve_model(config, engine) - # Compute telemetry_enabled once + # Compute telemetry_enabled and traces_enabled once telemetry_enabled = ( self._telemetry if self._telemetry is not None else config.telemetry.enabled ) + traces_enabled = ( + self._traces if self._traces is not None else config.traces.enabled + ) + # Apply traces flag to config so downstream code respects it + config.traces.enabled = traces_enabled gpu_monitor = None energy_monitor = None if telemetry_enabled and config.telemetry.gpu_metrics: