fix(evals): ToolCall-15 JSON parsing + traces thread safety (#172)

* fix(evals): ToolCall-15 JSON parsing + traces(True) → traces(telemetry)

Two fixes from sanity check:

1. ToolCall-15: Embed system prompt + tool descriptions + JSON format
   instructions into the problem text so jarvis-direct backend works.
   Fix scorer's JSON extraction to handle nested braces (balanced
   brace parser instead of regex). Models now correctly output tool
   calls and get scored. Claude/GPT/Gemini all score 40% (6/15).

2. JarvisAgentBackend: Change traces(True) back to traces(telemetry).
   The hardcoded True created SQLite trace connections that crash in
   ThreadPoolExecutor worker threads for GAIA and LiveResearchBench
   with jarvis-agent backend.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: remove unused tool_names variable

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Jon Saad-Falcon <jonsaadfalcon@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

src/openjarvis/evals/backends/jarvis_agent.py

@@ -45,7 +45,7 @@ class JarvisAgentBackend(InferenceBackend):
         # creates a GpuMonitor when building the InstrumentedEngine.
         if gpu_metrics:
             builder._config.telemetry.gpu_metrics = True
-        self._system = builder.telemetry(telemetry).traces(True).build()
+        self._system = builder.telemetry(telemetry).traces(telemetry).build()
 
     def generate(
         self,

src/openjarvis/evals/datasets/toolcall15.py

@@ -639,24 +639,45 @@ class ToolCall15Dataset(DatasetProvider):
         if max_samples is not None:
             scenarios = scenarios[:max_samples]
 
-        self._records = [
-            EvalRecord(
-                record_id=s["id"],
-                problem=s["user_message"],
-                reference="",  # scoring uses metadata, not reference text
-                category=s["category"],
-                subject=s["name"],
-                metadata={
-                    "system_prompt": SYSTEM_PROMPT,
-                    "tools": TOOLS,
-                    "mock_tool_outputs": s["mock_tool_outputs"],
-                    "reference_date": s.get("reference_date"),
-                    "scenario_id": s["id"],
-                    "scenario_name": s["name"],
-                },
+        self._records = []
+        for s in scenarios:
+            # Build a self-contained prompt that includes the system
+            # instructions, available tools, and user message so the
+            # model can respond with tool calls via any backend.
+            tool_descriptions = "\n".join(
+                f"- {t['function']['name']}: "
+                f"{t['function']['description']}"
+                for t in TOOLS
+            )
+            prompt = (
+                f"{SYSTEM_PROMPT}\n\n"
+                f"## Available Tools\n"
+                f"{tool_descriptions}\n\n"
+                f"When you need to use a tool, respond with ONLY "
+                f"a JSON object in this format:\n"
+                f'{{"tool": "<tool_name>", "arguments": {{...}}}}\n\n'
+                f"If the task requires multiple tools, call them "
+                f"one at a time. If no tool is needed, respond "
+                f"directly with your answer.\n\n"
+                f"## User Request\n{s['user_message']}"
+            )
+            self._records.append(
+                EvalRecord(
+                    record_id=s["id"],
+                    problem=prompt,
+                    reference="",
+                    category=s["category"],
+                    subject=s["name"],
+                    metadata={
+                        "system_prompt": SYSTEM_PROMPT,
+                        "tools": TOOLS,
+                        "mock_tool_outputs": s["mock_tool_outputs"],
+                        "reference_date": s.get("reference_date"),
+                        "scenario_id": s["id"],
+                        "scenario_name": s["name"],
+                    },
+                )
             )
-            for s in scenarios
-        ]
 
         LOGGER.info("ToolCall-15: loaded %d scenarios", len(self._records))
 

src/openjarvis/evals/scorers/toolcall15.py

@@ -21,11 +21,17 @@ from openjarvis.evals.core.types import EvalRecord
 LOGGER = logging.getLogger(__name__)
 
 
-def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:
-    """Extract tool calls from the record's query trace or metadata.
+def _extract_tool_calls(
+    record: EvalRecord,
+    model_answer: str = "",
+) -> List[Dict[str, Any]]:
+    """Extract tool calls from traces, metadata, or model text output.
 
     Returns a list of dicts with keys: name, arguments.
     """
+    import json
+    import re
+
     tool_calls: List[Dict[str, Any]] = []
 
     # Try query trace first (from EvalRunner)
@@ -39,7 +45,8 @@ def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:
                     "name": tc.get("name", ""),
                     "arguments": tc.get("arguments") or {},
                 })
-        return tool_calls
+        if tool_calls:
+            return tool_calls
 
     # Try tool_results list (from JarvisAgentBackend)
     tool_results = record.metadata.get("tool_results", [])
@@ -48,6 +55,48 @@ def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:
             "name": tr.get("tool_name", ""),
             "arguments": tr.get("arguments") or {},
         })
+    if tool_calls:
+        return tool_calls
+
+    # Parse tool calls from model text output (JSON format)
+    if model_answer:
+        # Extract balanced JSON objects from text
+        json_blocks: list[str] = []
+        # Try ```json blocks first
+        for m in re.finditer(
+            r"```(?:json)?\s*(\{.+?\})\s*```", model_answer, re.DOTALL
+        ):
+            json_blocks.append(m.group(1))
+        # Also extract bare balanced-brace JSON objects
+        depth = 0
+        start = -1
+        for i, ch in enumerate(model_answer):
+            if ch == "{":
+                if depth == 0:
+                    start = i
+                depth += 1
+            elif ch == "}":
+                depth -= 1
+                if depth == 0 and start >= 0:
+                    candidate = model_answer[start : i + 1]
+                    if '"tool"' in candidate or '"name"' in candidate:
+                        json_blocks.append(candidate)
+                    start = -1
+
+        for block in json_blocks:
+            try:
+                parsed = json.loads(block)
+                name = parsed.get("tool") or parsed.get("name", "")
+                args = parsed.get("arguments") or parsed.get("params") or {}
+                if isinstance(args, str):
+                    try:
+                        args = json.loads(args)
+                    except (json.JSONDecodeError, TypeError):
+                        args = {}
+                if name:
+                    tool_calls.append({"name": name, "arguments": args})
+            except json.JSONDecodeError:
+                continue
 
     return tool_calls
 
@@ -500,7 +549,7 @@ class ToolCall15Scorer(LLMJudgeScorer):
                 "scenario_id": scenario_id,
             }
 
-        tool_calls = _extract_tool_calls(record)
+        tool_calls = _extract_tool_calls(record, model_answer)
 
         points, reason = scorer_fn(tool_calls, model_answer)