From 30a29d64f5326cb7c010f8274acdfd32954ea56c Mon Sep 17 00:00:00 2001
From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com>
Date: Tue, 7 Apr 2026 19:57:53 -0700
Subject: [PATCH] feat(evals): parallel TauBench execution + Phase 2
 local-model configs (#207)

---
 .../evals/configs/gaia-nemotron-nano.toml     | 30 +++++++++++++++++++
 .../evals/configs/gaia-qwen-27b.toml          | 30 +++++++++++++++++++
 .../evals/configs/gaia-trinity-large.toml     | 30 +++++++++++++++++++
 .../configs/liveresearch-nemotron-nano.toml   | 30 +++++++++++++++++++
 .../evals/configs/liveresearch-qwen-27b.toml  | 30 +++++++++++++++++++
 .../configs/liveresearch-trinity-large.toml   | 30 +++++++++++++++++++
 .../taubench-telecom-nemotron-nano.toml       | 28 +++++++++++++++++
 .../configs/taubench-telecom-qwen-27b.toml    | 28 +++++++++++++++++
 .../taubench-telecom-trinity-large.toml       | 28 +++++++++++++++++
 src/openjarvis/evals/core/runner.py           | 21 ++++++++++++-
 src/openjarvis/evals/datasets/taubench.py     | 24 +++++++++++----
 .../evals/execution/taubench_env.py           |  6 ++++
 12 files changed, 308 insertions(+), 7 deletions(-)
 create mode 100644 src/openjarvis/evals/configs/gaia-nemotron-nano.toml
 create mode 100644 src/openjarvis/evals/configs/gaia-qwen-27b.toml
 create mode 100644 src/openjarvis/evals/configs/gaia-trinity-large.toml
 create mode 100644 src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml
 create mode 100644 src/openjarvis/evals/configs/liveresearch-qwen-27b.toml
 create mode 100644 src/openjarvis/evals/configs/liveresearch-trinity-large.toml
 create mode 100644 src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml
 create mode 100644 src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml
 create mode 100644 src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml

diff --git a/src/openjarvis/evals/configs/gaia-nemotron-nano.toml b/src/openjarvis/evals/configs/gaia-nemotron-nano.toml
new file mode 100644
index 00000000..f5afdffe
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-nemotron-nano.toml
@@ -0,0 +1,30 @@
+# GAIA eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
+[meta]
+name = "gaia-nemotron-nano"
+description = "GAIA on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/gaia-nemotron-nano/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/gaia-qwen-27b.toml b/src/openjarvis/evals/configs/gaia-qwen-27b.toml
new file mode 100644
index 00000000..0a1c1a13
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-qwen-27b.toml
@@ -0,0 +1,30 @@
+# GAIA eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
+[meta]
+name = "gaia-qwen-27b"
+description = "GAIA on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/gaia-qwen-27b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/gaia-trinity-large.toml b/src/openjarvis/evals/configs/gaia-trinity-large.toml
new file mode 100644
index 00000000..5d3ebad7
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-trinity-large.toml
@@ -0,0 +1,30 @@
+# GAIA eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
+[meta]
+name = "gaia-trinity-large"
+description = "GAIA on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/gaia-trinity-large/"
+seed = 42
+
+[[models]]
+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml
new file mode 100644
index 00000000..63ecf526
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml
@@ -0,0 +1,30 @@
+# LiveResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-nemotron-nano"
+description = "LiveResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/liveresearch-nemotron-nano/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml
new file mode 100644
index 00000000..05857481
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml
@@ -0,0 +1,30 @@
+# LiveResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
+[meta]
+name = "liveresearch-qwen-27b"
+description = "LiveResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/liveresearch-qwen-27b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml
new file mode 100644
index 00000000..27d883d8
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml
@@ -0,0 +1,30 @@
+# LiveResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
+[meta]
+name = "liveresearch-trinity-large"
+description = "LiveResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/"
+seed = 42
+
+[[models]]
+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml
new file mode 100644
index 00000000..332c0061
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml
@@ -0,0 +1,28 @@
+# TauBench V2 Telecom eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
+[meta]
+name = "taubench-telecom-nemotron-nano"
+description = "TauBench V2 Telecom on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (temperature=0.7)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 24
+output_dir = "results/neurips-2026/baselines/taubench-telecom-nemotron-nano/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml
new file mode 100644
index 00000000..18b8b518
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml
@@ -0,0 +1,28 @@
+# TauBench V2 Telecom eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
+[meta]
+name = "taubench-telecom-qwen-27b"
+description = "TauBench V2 Telecom on Qwen/Qwen3.5-27B-FP8 (temperature=0.7)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/taubench-telecom-qwen-27b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml b/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml
new file mode 100644
index 00000000..55487f12
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml
@@ -0,0 +1,28 @@
+# TauBench V2 Telecom eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
+[meta]
+name = "taubench-telecom-trinity-large"
+description = "TauBench V2 Telecom on arcee-ai/Trinity-Large-Thinking-FP8-Block (temperature=0.7)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/taubench-telecom-trinity-large/"
+seed = 42
+
+[[models]]
+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py
index 1c8eeadf..499eb721 100644
--- a/src/openjarvis/evals/core/runner.py
+++ b/src/openjarvis/evals/core/runner.py
@@ -131,6 +131,23 @@ class EvalRunner:
         except AttributeError:
             self._has_task_env = False
 
+        # Probe whether this task env is thread-safe (no CWD changes, no
+        # shared mutable globals). Datasets opt in by setting THREAD_SAFE = True
+        # on the env class returned by create_task_env. Default: False (safe).
+        self._task_env_thread_safe = False
+        if self._has_task_env:
+            try:
+                # Cheap probe: instantiate-free attribute lookup via a sample record
+                sample_records = list(self._dataset.iter_records())
+                if sample_records:
+                    probe_env = self._dataset.create_task_env(sample_records[0])
+                    if probe_env is not None and getattr(
+                        type(probe_env), "THREAD_SAFE", False
+                    ):
+                        self._task_env_thread_safe = True
+            except Exception as exc:  # pragma: no cover - probe is best-effort
+                LOGGER.debug("Task env thread-safety probe failed: %s", exc)
+
         records = list(self._dataset.iter_records())
         LOGGER.info(
             "Running %s: %d samples, backend=%s, model=%s, workers=%d, episode_mode=%s",
@@ -171,9 +188,11 @@ class EvalRunner:
         try:
             if cfg.episode_mode:
                 self._run_episode_mode(records, progress_callback, total)
-            elif self._has_task_env:
+            elif self._has_task_env and not self._task_env_thread_safe:
                 # Task environments (PinchBench etc.) change CWD —
                 # must process sequentially for thread safety.
+                # Envs that opt in via THREAD_SAFE=True fall through to the
+                # parallel ThreadPoolExecutor branch below.
                 for record in records:
                     result = self._process_one(record)
                     self._results.append(result)
diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.py
index 896a17f8..ed257f2c 100644
--- a/src/openjarvis/evals/datasets/taubench.py
+++ b/src/openjarvis/evals/datasets/taubench.py
@@ -9,6 +9,7 @@ Reference: https://github.com/sierra-research/tau2-bench
 from __future__ import annotations
 
 import logging
+import os
 import subprocess
 import sys
 from pathlib import Path
@@ -40,11 +41,20 @@ def _ensure_tau2() -> None:
                 capture_output=True,
             )
         LOGGER.info("Installing tau2-bench ...")
-        subprocess.run(
-            [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
-            check=True,
-            capture_output=True,
-        )
+        # Try `python -m pip` first; fall back to `uv pip` for uv-managed venvs
+        # which don't ship pip by default.
+        try:
+            subprocess.run(
+                [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
+                check=True,
+                capture_output=True,
+            )
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            subprocess.run(
+                ["uv", "pip", "install", "--python", sys.executable, "-e", str(CACHE_DIR)],
+                check=True,
+                capture_output=True,
+            )
 
 
 class TauBenchDataset(DatasetProvider):
@@ -68,7 +78,9 @@ class TauBenchDataset(DatasetProvider):
         self._temperature: float = 0.7
         self._max_tokens: int = 4096
         self._user_model: Optional[str] = None
-        self._num_trials: int = 3  # pass^k: best of k trials per task
+        # pass^k: best of k trials per task. Default 3, override via env var
+        # OPENJARVIS_TAUBENCH_TRIALS for faster runs (e.g. =1 for 3x speedup).
+        self._num_trials: int = int(os.environ.get("OPENJARVIS_TAUBENCH_TRIALS", "3"))
         self._telemetry: bool = False
         self._gpu_metrics: bool = False
 
diff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.py
index 925f4494..6c4425c6 100644
--- a/src/openjarvis/evals/execution/taubench_env.py
+++ b/src/openjarvis/evals/execution/taubench_env.py
@@ -222,6 +222,12 @@ class TauBenchTaskEnv:
     runs the simulation, and stores results in record.metadata for the scorer.
     """
 
+    # Thread-safety marker for the eval runner. tau2 simulations are pure
+    # in-process (LLM calls + tau2 orchestrator state held inside this
+    # instance) — no CWD changes, no shared mutable globals — so multiple
+    # TauBenchTaskEnv instances can run in parallel threads safely.
+    THREAD_SAFE = True
+
     def __init__(
         self,
         record: EvalRecord,