mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
fix(evals): TerminalBench scoring + http_request panic + 20 new configs (#173)
Fixes: - TerminalBench: remove task_id fallback from ground truth answers. Was using task name as reference answer, causing judge to always score wrong. - http_request: catch BaseException (not Exception) for Rust panics. PyO3 PanicException on binary data now falls through to httpx. New configs (5 models × 4 benchmarks = 20 files): - Qwen-27B, Qwen-2B, Gemma4-26B-A4B, Gemma4-E4B, Nemotron-Nano - ToolCall-15, PinchBench, LiveCodeBench, TauBench Updated experiment plan with Gemma 4 models and progress matrix. Co-authored-by: Jon Saad-Falcon <jonsaadfalcon@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Avanika Narayan
co-authored by
Jon Saad-Falcon
Claude Opus 4.6
Jon Saad-Falcon
Claude Opus 4.6
parent
908e28d42e
commit
897a816343
@@ -0,0 +1,30 @@
|
||||
# LiveCodeBench eval: Gemma-4-26B-A4B-it (vLLM, TP=4)
|
||||
# Competitive programming from LeetCode, AtCoder, CodeForces
|
||||
|
||||
[meta]
|
||||
name = "livecodebench-gemma4-26b"
|
||||
description = "LiveCodeBench on google/gemma-4-26B-A4B-it (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-26b/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-26B-A4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,30 @@
|
||||
# LiveCodeBench eval: Gemma-4-E4B-it (vLLM, 1 GPU)
|
||||
# Competitive programming from LeetCode, AtCoder, CodeForces
|
||||
|
||||
[meta]
|
||||
name = "livecodebench-gemma4-e4b"
|
||||
description = "LiveCodeBench on google/gemma-4-E4B-it (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-e4b/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-E4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,30 @@
|
||||
# LiveCodeBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
|
||||
# Competitive programming from LeetCode, AtCoder, CodeForces
|
||||
|
||||
[meta]
|
||||
name = "livecodebench-nemotron-nano"
|
||||
description = "LiveCodeBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-nano/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,30 @@
|
||||
# LiveCodeBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
|
||||
# Competitive programming from LeetCode, AtCoder, CodeForces
|
||||
|
||||
[meta]
|
||||
name = "livecodebench-qwen-27b"
|
||||
description = "LiveCodeBench on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-27b/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-27B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,30 @@
|
||||
# LiveCodeBench eval: Qwen3.5-2B (vLLM, 1 GPU)
|
||||
# Competitive programming from LeetCode, AtCoder, CodeForces
|
||||
|
||||
[meta]
|
||||
name = "livecodebench-qwen-2b"
|
||||
description = "LiveCodeBench on Qwen/Qwen3.5-2B (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-2b/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-2B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: Gemma-4-26B-A4B-it (vLLM, TP=4)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-gemma4-26b"
|
||||
description = "PinchBench on google/gemma-4-26B-A4B-it (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-26b/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-26B-A4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: Gemma-4-E4B-it (vLLM, 1 GPU)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-gemma4-e4b"
|
||||
description = "PinchBench on google/gemma-4-E4B-it (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-e4b/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-E4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-nemotron-nano"
|
||||
description = "PinchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-nano/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-qwen-27b"
|
||||
description = "PinchBench on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-27b/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-27B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: Qwen3.5-2B (vLLM, 1 GPU)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-qwen-2b"
|
||||
description = "PinchBench on Qwen/Qwen3.5-2B (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-2b/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-2B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,31 @@
|
||||
# TauBench V2 eval: Gemma-4-26B-A4B-it (vLLM, TP=4)
|
||||
# Multi-turn customer service benchmark — airline + retail splits
|
||||
|
||||
[meta]
|
||||
name = "taubench-gemma4-26b"
|
||||
description = "TauBench V2 on google/gemma-4-26B-A4B-it (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-26b/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-26B-A4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,31 @@
|
||||
# TauBench V2 eval: Gemma-4-E4B-it (vLLM, 1 GPU)
|
||||
# Multi-turn customer service benchmark — airline + retail splits
|
||||
|
||||
[meta]
|
||||
name = "taubench-gemma4-e4b"
|
||||
description = "TauBench V2 on google/gemma-4-E4B-it (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-e4b/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-E4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,31 @@
|
||||
# TauBench V2 eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
|
||||
# Multi-turn customer service benchmark — airline + retail splits
|
||||
|
||||
[meta]
|
||||
name = "taubench-nemotron-nano"
|
||||
description = "TauBench V2 on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-nano/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,31 @@
|
||||
# TauBench V2 eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
|
||||
# Multi-turn customer service benchmark — airline + retail splits
|
||||
|
||||
[meta]
|
||||
name = "taubench-qwen-27b"
|
||||
description = "TauBench V2 on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-27b/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-27B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,31 @@
|
||||
# TauBench V2 eval: Qwen3.5-2B (vLLM, 1 GPU)
|
||||
# Multi-turn customer service benchmark — airline + retail splits
|
||||
|
||||
[meta]
|
||||
name = "taubench-qwen-2b"
|
||||
description = "TauBench V2 on Qwen/Qwen3.5-2B (vLLM, 1 GPU)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-2b/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-2B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,29 @@
|
||||
# ToolCall-15 eval: Gemma-4-26B-A4B-it (vLLM, TP=4)
|
||||
# Lightweight tool calling benchmark — 15 scenarios, 5 categories
|
||||
|
||||
[meta]
|
||||
name = "toolcall15-gemma4-26b"
|
||||
description = "ToolCall-15 on google/gemma-4-26B-A4B-it (temperature=0)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-26b/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-26B-A4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
@@ -0,0 +1,29 @@
|
||||
# ToolCall-15 eval: Gemma-4-E4B-it (vLLM, 1 GPU)
|
||||
# Lightweight tool calling benchmark — 15 scenarios, 5 categories
|
||||
|
||||
[meta]
|
||||
name = "toolcall15-gemma4-e4b"
|
||||
description = "ToolCall-15 on google/gemma-4-E4B-it (temperature=0)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-e4b/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-E4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
@@ -0,0 +1,29 @@
|
||||
# ToolCall-15 eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
|
||||
# Lightweight tool calling benchmark — 15 scenarios, 5 categories
|
||||
|
||||
[meta]
|
||||
name = "toolcall15-nemotron-nano"
|
||||
description = "ToolCall-15 on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (temperature=0)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-nano/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
@@ -0,0 +1,29 @@
|
||||
# ToolCall-15 eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
|
||||
# Lightweight tool calling benchmark — 15 scenarios, 5 categories
|
||||
|
||||
[meta]
|
||||
name = "toolcall15-qwen-27b"
|
||||
description = "ToolCall-15 on Qwen/Qwen3.5-27B-FP8 (temperature=0)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-27b/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-27B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
@@ -0,0 +1,29 @@
|
||||
# ToolCall-15 eval: Qwen3.5-2B (vLLM, 1 GPU)
|
||||
# Lightweight tool calling benchmark — 15 scenarios, 5 categories
|
||||
|
||||
[meta]
|
||||
name = "toolcall15-qwen-2b"
|
||||
description = "ToolCall-15 on Qwen/Qwen3.5-2B (temperature=0)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-2b/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-2B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
Reference in New Issue
Block a user