mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
feat(evals): parallel TauBench execution + Phase 2 local-model configs (#207)
This commit is contained in:
Jon Saad-Falcon
@@ -0,0 +1,30 @@
|
||||
# GAIA eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "gaia-nemotron-nano"
|
||||
description = "GAIA on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 8
|
||||
output_dir = "results/neurips-2026/baselines/gaia-nemotron-nano/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# GAIA eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
|
||||
[meta]
|
||||
name = "gaia-qwen-27b"
|
||||
description = "GAIA on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 8
|
||||
output_dir = "results/neurips-2026/baselines/gaia-qwen-27b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-27B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# GAIA eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
|
||||
[meta]
|
||||
name = "gaia-trinity-large"
|
||||
description = "GAIA on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 16
|
||||
output_dir = "results/neurips-2026/baselines/gaia-trinity-large/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
|
||||
engine = "vllm"
|
||||
num_gpus = 8
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# LiveResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "liveresearch-nemotron-nano"
|
||||
description = "LiveResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 8
|
||||
output_dir = "results/neurips-2026/baselines/liveresearch-nemotron-nano/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# LiveResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
|
||||
[meta]
|
||||
name = "liveresearch-qwen-27b"
|
||||
description = "LiveResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 8
|
||||
output_dir = "results/neurips-2026/baselines/liveresearch-qwen-27b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-27B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# LiveResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
|
||||
[meta]
|
||||
name = "liveresearch-trinity-large"
|
||||
description = "LiveResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 16
|
||||
output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
|
||||
engine = "vllm"
|
||||
num_gpus = 8
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,28 @@
|
||||
# TauBench V2 Telecom eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "taubench-telecom-nemotron-nano"
|
||||
description = "TauBench V2 Telecom on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (temperature=0.7)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 24
|
||||
output_dir = "results/neurips-2026/baselines/taubench-telecom-nemotron-nano/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,28 @@
|
||||
# TauBench V2 Telecom eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
|
||||
[meta]
|
||||
name = "taubench-telecom-qwen-27b"
|
||||
description = "TauBench V2 Telecom on Qwen/Qwen3.5-27B-FP8 (temperature=0.7)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 8
|
||||
output_dir = "results/neurips-2026/baselines/taubench-telecom-qwen-27b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-27B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,28 @@
|
||||
# TauBench V2 Telecom eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
|
||||
[meta]
|
||||
name = "taubench-telecom-trinity-large"
|
||||
description = "TauBench V2 Telecom on arcee-ai/Trinity-Large-Thinking-FP8-Block (temperature=0.7)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 16
|
||||
output_dir = "results/neurips-2026/baselines/taubench-telecom-trinity-large/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
|
||||
engine = "vllm"
|
||||
num_gpus = 8
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
split = "telecom"
|
||||
@@ -131,6 +131,23 @@ class EvalRunner:
|
||||
except AttributeError:
|
||||
self._has_task_env = False
|
||||
|
||||
# Probe whether this task env is thread-safe (no CWD changes, no
|
||||
# shared mutable globals). Datasets opt in by setting THREAD_SAFE = True
|
||||
# on the env class returned by create_task_env. Default: False (safe).
|
||||
self._task_env_thread_safe = False
|
||||
if self._has_task_env:
|
||||
try:
|
||||
# Cheap probe: instantiate-free attribute lookup via a sample record
|
||||
sample_records = list(self._dataset.iter_records())
|
||||
if sample_records:
|
||||
probe_env = self._dataset.create_task_env(sample_records[0])
|
||||
if probe_env is not None and getattr(
|
||||
type(probe_env), "THREAD_SAFE", False
|
||||
):
|
||||
self._task_env_thread_safe = True
|
||||
except Exception as exc: # pragma: no cover - probe is best-effort
|
||||
LOGGER.debug("Task env thread-safety probe failed: %s", exc)
|
||||
|
||||
records = list(self._dataset.iter_records())
|
||||
LOGGER.info(
|
||||
"Running %s: %d samples, backend=%s, model=%s, workers=%d, episode_mode=%s",
|
||||
@@ -171,9 +188,11 @@ class EvalRunner:
|
||||
try:
|
||||
if cfg.episode_mode:
|
||||
self._run_episode_mode(records, progress_callback, total)
|
||||
elif self._has_task_env:
|
||||
elif self._has_task_env and not self._task_env_thread_safe:
|
||||
# Task environments (PinchBench etc.) change CWD —
|
||||
# must process sequentially for thread safety.
|
||||
# Envs that opt in via THREAD_SAFE=True fall through to the
|
||||
# parallel ThreadPoolExecutor branch below.
|
||||
for record in records:
|
||||
result = self._process_one(record)
|
||||
self._results.append(result)
|
||||
|
||||
@@ -9,6 +9,7 @@ Reference: https://github.com/sierra-research/tau2-bench
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -40,11 +41,20 @@ def _ensure_tau2() -> None:
|
||||
capture_output=True,
|
||||
)
|
||||
LOGGER.info("Installing tau2-bench ...")
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
# Try `python -m pip` first; fall back to `uv pip` for uv-managed venvs
|
||||
# which don't ship pip by default.
|
||||
try:
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
subprocess.run(
|
||||
["uv", "pip", "install", "--python", sys.executable, "-e", str(CACHE_DIR)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
|
||||
class TauBenchDataset(DatasetProvider):
|
||||
@@ -68,7 +78,9 @@ class TauBenchDataset(DatasetProvider):
|
||||
self._temperature: float = 0.7
|
||||
self._max_tokens: int = 4096
|
||||
self._user_model: Optional[str] = None
|
||||
self._num_trials: int = 3 # pass^k: best of k trials per task
|
||||
# pass^k: best of k trials per task. Default 3, override via env var
|
||||
# OPENJARVIS_TAUBENCH_TRIALS for faster runs (e.g. =1 for 3x speedup).
|
||||
self._num_trials: int = int(os.environ.get("OPENJARVIS_TAUBENCH_TRIALS", "3"))
|
||||
self._telemetry: bool = False
|
||||
self._gpu_metrics: bool = False
|
||||
|
||||
|
||||
@@ -222,6 +222,12 @@ class TauBenchTaskEnv:
|
||||
runs the simulation, and stores results in record.metadata for the scorer.
|
||||
"""
|
||||
|
||||
# Thread-safety marker for the eval runner. tau2 simulations are pure
|
||||
# in-process (LLM calls + tau2 orchestrator state held inside this
|
||||
# instance) — no CWD changes, no shared mutable globals — so multiple
|
||||
# TauBenchTaskEnv instances can run in parallel threads safely.
|
||||
THREAD_SAFE = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
record: EvalRecord,
|
||||
|
||||
Reference in New Issue
Block a user