From 897a81634304d458ef960b7c9b71c71022d9fb3d Mon Sep 17 00:00:00 2001 From: Avanika Narayan Date: Thu, 2 Apr 2026 13:13:06 -0700 Subject: [PATCH] fix(evals): TerminalBench scoring + http_request panic + 20 new configs (#173) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: - TerminalBench: remove task_id fallback from ground truth answers. Was using task name as reference answer, causing judge to always score wrong. - http_request: catch BaseException (not Exception) for Rust panics. PyO3 PanicException on binary data now falls through to httpx. New configs (5 models × 4 benchmarks = 20 files): - Qwen-27B, Qwen-2B, Gemma4-26B-A4B, Gemma4-E4B, Nemotron-Nano - ToolCall-15, PinchBench, LiveCodeBench, TauBench Updated experiment plan with Gemma 4 models and progress matrix. Co-authored-by: Jon Saad-Falcon Co-authored-by: Claude Opus 4.6 (1M context) --- .../configs/livecodebench-gemma4-26b.toml | 30 ++++++++++++++++ .../configs/livecodebench-gemma4-e4b.toml | 30 ++++++++++++++++ .../configs/livecodebench-nemotron-nano.toml | 30 ++++++++++++++++ .../evals/configs/livecodebench-qwen-27b.toml | 30 ++++++++++++++++ .../evals/configs/livecodebench-qwen-2b.toml | 30 ++++++++++++++++ .../evals/configs/pinchbench-gemma4-26b.toml | 35 +++++++++++++++++++ .../evals/configs/pinchbench-gemma4-e4b.toml | 35 +++++++++++++++++++ .../configs/pinchbench-nemotron-nano.toml | 35 +++++++++++++++++++ .../evals/configs/pinchbench-qwen-27b.toml | 35 +++++++++++++++++++ .../evals/configs/pinchbench-qwen-2b.toml | 35 +++++++++++++++++++ .../evals/configs/taubench-gemma4-26b.toml | 31 ++++++++++++++++ .../evals/configs/taubench-gemma4-e4b.toml | 31 ++++++++++++++++ .../evals/configs/taubench-nemotron-nano.toml | 31 ++++++++++++++++ .../evals/configs/taubench-qwen-27b.toml | 31 ++++++++++++++++ .../evals/configs/taubench-qwen-2b.toml | 31 ++++++++++++++++ .../evals/configs/toolcall15-gemma4-26b.toml | 29 +++++++++++++++ .../evals/configs/toolcall15-gemma4-e4b.toml | 29 +++++++++++++++ .../configs/toolcall15-nemotron-nano.toml | 29 +++++++++++++++ .../evals/configs/toolcall15-qwen-27b.toml | 29 +++++++++++++++ .../evals/configs/toolcall15-qwen-2b.toml | 29 +++++++++++++++ 20 files changed, 625 insertions(+) create mode 100644 src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-qwen-27b.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-qwen-2b.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-qwen-27b.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-qwen-2b.toml create mode 100644 src/openjarvis/evals/configs/taubench-gemma4-26b.toml create mode 100644 src/openjarvis/evals/configs/taubench-gemma4-e4b.toml create mode 100644 src/openjarvis/evals/configs/taubench-nemotron-nano.toml create mode 100644 src/openjarvis/evals/configs/taubench-qwen-27b.toml create mode 100644 src/openjarvis/evals/configs/taubench-qwen-2b.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-qwen-27b.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-qwen-2b.toml diff --git a/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml b/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml new file mode 100644 index 00000000..d427f01f --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: Gemma-4-26B-A4B-it (vLLM, TP=4) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-gemma4-26b" +description = "LiveCodeBench on google/gemma-4-26B-A4B-it (vLLM, TP=4)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/livecodebench/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml b/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml new file mode 100644 index 00000000..ff6209fa --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: Gemma-4-E4B-it (vLLM, 1 GPU) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-gemma4-e4b" +description = "LiveCodeBench on google/gemma-4-E4B-it (vLLM, 1 GPU)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-e4b/livecodebench/" +seed = 42 + +[[models]] +name = "google/gemma-4-E4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml b/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml new file mode 100644 index 00000000..6266f922 --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-nemotron-nano" +description = "LiveCodeBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-nano/livecodebench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml new file mode 100644 index 00000000..357dcf49 --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-qwen-27b" +description = "LiveCodeBench on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/livecodebench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml new file mode 100644 index 00000000..e4d4236e --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: Qwen3.5-2B (vLLM, 1 GPU) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-qwen-2b" +description = "LiveCodeBench on Qwen/Qwen3.5-2B (vLLM, 1 GPU)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/livecodebench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml b/src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml new file mode 100644 index 00000000..28787489 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Gemma-4-26B-A4B-it (vLLM, TP=4) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-gemma4-26b" +description = "PinchBench on google/gemma-4-26B-A4B-it (vLLM, TP=4)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/pinchbench/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml b/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml new file mode 100644 index 00000000..bbe9ba30 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Gemma-4-E4B-it (vLLM, 1 GPU) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-gemma4-e4b" +description = "PinchBench on google/gemma-4-E4B-it (vLLM, 1 GPU)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-e4b/pinchbench/" +seed = 42 + +[[models]] +name = "google/gemma-4-E4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml b/src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml new file mode 100644 index 00000000..5ddadce5 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml @@ -0,0 +1,35 @@ +# PinchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-nemotron-nano" +description = "PinchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-nano/pinchbench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-27b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-27b.toml new file mode 100644 index 00000000..f687e63d --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen-27b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-qwen-27b" +description = "PinchBench on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/pinchbench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-2b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-2b.toml new file mode 100644 index 00000000..e76e56a4 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen-2b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Qwen3.5-2B (vLLM, 1 GPU) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-qwen-2b" +description = "PinchBench on Qwen/Qwen3.5-2B (vLLM, 1 GPU)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/pinchbench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/taubench-gemma4-26b.toml b/src/openjarvis/evals/configs/taubench-gemma4-26b.toml new file mode 100644 index 00000000..27fda44b --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-gemma4-26b.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: Gemma-4-26B-A4B-it (vLLM, TP=4) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-gemma4-26b" +description = "TauBench V2 on google/gemma-4-26B-A4B-it (vLLM, TP=4)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/taubench/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml b/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml new file mode 100644 index 00000000..8dac58dd --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: Gemma-4-E4B-it (vLLM, 1 GPU) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-gemma4-e4b" +description = "TauBench V2 on google/gemma-4-E4B-it (vLLM, 1 GPU)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-e4b/taubench/" +seed = 42 + +[[models]] +name = "google/gemma-4-E4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-nemotron-nano.toml b/src/openjarvis/evals/configs/taubench-nemotron-nano.toml new file mode 100644 index 00000000..7dff2f08 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-nemotron-nano.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-nemotron-nano" +description = "TauBench V2 on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-nano/taubench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen-27b.toml b/src/openjarvis/evals/configs/taubench-qwen-27b.toml new file mode 100644 index 00000000..13ce0c65 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen-27b.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-qwen-27b" +description = "TauBench V2 on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/taubench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen-2b.toml b/src/openjarvis/evals/configs/taubench-qwen-2b.toml new file mode 100644 index 00000000..e9735ff6 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen-2b.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: Qwen3.5-2B (vLLM, 1 GPU) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-qwen-2b" +description = "TauBench V2 on Qwen/Qwen3.5-2B (vLLM, 1 GPU)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/taubench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml b/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml new file mode 100644 index 00000000..12470e25 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: Gemma-4-26B-A4B-it (vLLM, TP=4) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-gemma4-26b" +description = "ToolCall-15 on google/gemma-4-26B-A4B-it (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/toolcall15/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml b/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml new file mode 100644 index 00000000..231bed02 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: Gemma-4-E4B-it (vLLM, 1 GPU) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-gemma4-e4b" +description = "ToolCall-15 on google/gemma-4-E4B-it (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-e4b/toolcall15/" +seed = 42 + +[[models]] +name = "google/gemma-4-E4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml b/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml new file mode 100644 index 00000000..453f30e9 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-nemotron-nano" +description = "ToolCall-15 on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-nano/toolcall15/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml new file mode 100644 index 00000000..56879653 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-qwen-27b" +description = "ToolCall-15 on Qwen/Qwen3.5-27B-FP8 (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/toolcall15/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml new file mode 100644 index 00000000..4d40fe42 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: Qwen3.5-2B (vLLM, 1 GPU) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-qwen-2b" +description = "ToolCall-15 on Qwen/Qwen3.5-2B (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/toolcall15/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct"