mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
add eval configs (#208)
This commit is contained in:
Avanika Narayan
committed by
Jon Saad-Falcon
Jon Saad-Falcon
parent
30a29d64f5
commit
41ddab9099
+1
-4
@@ -90,7 +90,4 @@ CLAUDE.md
|
||||
research_mining_*
|
||||
.python-version
|
||||
src/openjarvis/channels/whatsapp_baileys_bridge/node_modules/
|
||||
|
||||
# SQLite in-memory artifacts
|
||||
:memory:
|
||||
MagicMock/
|
||||
scratch/
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# GAIA eval: gemini-3.1-pro-preview (cloud)
|
||||
[meta]
|
||||
name = "gaia-gemini-31-pro"
|
||||
description = "GAIA on gemini-3.1-pro-preview"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-gemini-31-pro/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "gemini-3.1-pro-preview"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# GAIA eval: Gemma-4-26B-A4B-it (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "gaia-gemma4-26b"
|
||||
description = "GAIA on google/gemma-4-26B-A4B-it"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-26b/gaia/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-26B-A4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,29 @@
|
||||
# GAIA eval: gpt-5.4 (cloud)
|
||||
[meta]
|
||||
name = "gaia-gpt54"
|
||||
description = "GAIA on gpt-5.4-2026-03-05"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-gpt54/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "gpt-5.4-2026-03-05"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Gaia: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
|
||||
[meta]
|
||||
name = "gaia-nemotron-super"
|
||||
description = "gaia on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-super/gaia/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
agent = "monitor_operative"
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# GAIA eval: Nemotron-3-Super-120B-A12B-FP8 (SGLang on port 8001)
|
||||
[meta]
|
||||
name = "gaia-nemotron3-super"
|
||||
description = "GAIA on Nemotron-3-Super-120B-A12B-FP8 (SGLang)"
|
||||
|
||||
[defaults]
|
||||
temperature = 1.0
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-nemotron3-super/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nemotron"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Gaia: Qwen/Qwen3.5-122B-A10B-FP8
|
||||
[meta]
|
||||
name = "gaia-qwen-122b"
|
||||
description = "gaia on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-122b/gaia/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
agent = "monitor_operative"
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# GAIA eval: Qwen3.5-2B (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "gaia-qwen-2b"
|
||||
description = "GAIA on Qwen/Qwen3.5-2B"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-2b/gaia/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-2B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Gaia: Qwen/Qwen3.5-35B-A3B-FP8
|
||||
[meta]
|
||||
name = "gaia-qwen-35b"
|
||||
description = "gaia on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-35b/gaia/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
agent = "monitor_operative"
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# GAIA eval: Qwen3.5-9B (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "gaia-qwen-9b"
|
||||
description = "GAIA on Qwen/Qwen3.5-9B"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-9b/gaia/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-9B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# GAIA eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4)
|
||||
[meta]
|
||||
name = "gaia-qwen122b"
|
||||
description = "GAIA on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-qwen122b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# GAIA eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4)
|
||||
[meta]
|
||||
name = "gaia-qwen35b"
|
||||
description = "GAIA on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-qwen35b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,29 @@
|
||||
# GAIA rerun with max_turns=30: claude-opus-4-6
|
||||
[meta]
|
||||
name = "gaia-rerun-claude-opus"
|
||||
description = "GAIA rerun with 30 max turns on claude-opus-4-6"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-rerun-claude-opus/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "claude-opus-4-6"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,29 @@
|
||||
# GAIA rerun with max_turns=30: claude-sonnet-4-6
|
||||
[meta]
|
||||
name = "gaia-rerun-claude-sonnet"
|
||||
description = "GAIA rerun with 30 max turns on claude-sonnet-4-6"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-rerun-claude-sonnet/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "claude-sonnet-4-6"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# GAIA rerun with max_turns=30: zai-org/GLM-4.7-Flash
|
||||
[meta]
|
||||
name = "gaia-rerun-glm47flash"
|
||||
description = "GAIA rerun with 30 max turns on zai-org/GLM-4.7-Flash"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-rerun-glm47flash/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "zai-org/GLM-4.7-Flash"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,29 @@
|
||||
# GAIA rerun with max_turns=30: gpt-5-mini-2025-08-07
|
||||
[meta]
|
||||
name = "gaia-rerun-gpt5mini"
|
||||
description = "GAIA rerun with 30 max turns on gpt-5-mini-2025-08-07"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-rerun-gpt5mini/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "gpt-5-mini-2025-08-07"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# GAIA rerun with max_turns=30: MiniMaxAI/MiniMax-M2.5
|
||||
[meta]
|
||||
name = "gaia-rerun-minimax-m25"
|
||||
description = "GAIA rerun with 30 max turns on MiniMaxAI/MiniMax-M2.5"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-rerun-minimax-m25/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "MiniMaxAI/MiniMax-M2.5"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,32 @@
|
||||
# GAIA rerun with max_turns=30: Qwen/Qwen3.5-122B-A10B-FP8
|
||||
[meta]
|
||||
name = "gaia-rerun-qwen122b"
|
||||
description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-rerun-qwen122b/"
|
||||
seed = 42
|
||||
telemetry = true
|
||||
gpu_metrics = true
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# GAIA rerun with max_turns=30: Qwen/Qwen3.5-35B-A3B-FP8
|
||||
[meta]
|
||||
name = "gaia-rerun-qwen35b"
|
||||
description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-rerun-qwen35b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# GAIA rerun with max_turns=30: Qwen/Qwen3.5-397B-A17B-FP8
|
||||
[meta]
|
||||
name = "gaia-rerun-qwen397b"
|
||||
description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-397B-A17B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/gaia-rerun-qwen397b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-397B-A17B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 8
|
||||
|
||||
[[benchmarks]]
|
||||
name = "gaia"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
|
||||
@@ -0,0 +1,39 @@
|
||||
# LiveCodeBench eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)
|
||||
# Competitive programming from LeetCode, AtCoder, CodeForces
|
||||
# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS
|
||||
#
|
||||
# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are
|
||||
# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs
|
||||
# yet (see github.com/ollama/ollama/issues/5245). This config is a
|
||||
# placeholder -- use vLLM or wait for Ollama sharded GGUF support.
|
||||
# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.
|
||||
|
||||
[meta]
|
||||
name = "livecodebench-kimi-k25"
|
||||
description = "LiveCodeBench on Kimi-K2.5 IQ4_XS via Ollama"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"
|
||||
engine = "ollama"
|
||||
param_count_b = 1000.0
|
||||
active_params_b = 32.0
|
||||
num_gpus = 7
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,36 @@
|
||||
# LiveCodeBench eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)
|
||||
# Competitive programming from LeetCode, AtCoder, CodeForces
|
||||
# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest
|
||||
# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not
|
||||
# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag
|
||||
# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).
|
||||
|
||||
[meta]
|
||||
name = "livecodebench-minimax-m25"
|
||||
description = "LiveCodeBench on MiniMax-M2.5 UD-TQ1_0 via Ollama"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"
|
||||
engine = "ollama"
|
||||
param_count_b = 229.0
|
||||
active_params_b = 45.6
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,28 @@
|
||||
# Livecodebench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
|
||||
[meta]
|
||||
name = "livecodebench-nemotron-super"
|
||||
description = "livecodebench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-super/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,28 @@
|
||||
# Livecodebench: Qwen/Qwen3.5-122B-A10B-FP8
|
||||
[meta]
|
||||
name = "livecodebench-qwen-122b"
|
||||
description = "livecodebench on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-122b/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,28 @@
|
||||
# Livecodebench: Qwen/Qwen3.5-35B-A3B-FP8
|
||||
[meta]
|
||||
name = "livecodebench-qwen-35b"
|
||||
description = "livecodebench on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-35b/livecodebench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "livecodebench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
@@ -0,0 +1,31 @@
|
||||
# LiveResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "liveresearch-gemma4-26b"
|
||||
description = "LiveResearchBench on google/gemma-4-26B-A4B-it"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-26b/liveresearch/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-26B-A4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Liveresearch: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
|
||||
[meta]
|
||||
name = "liveresearch-nemotron-super"
|
||||
description = "liveresearch on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-super/liveresearch/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
agent = "monitor_operative"
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Liveresearch: Qwen/Qwen3.5-122B-A10B-FP8
|
||||
[meta]
|
||||
name = "liveresearch-qwen-122b"
|
||||
description = "liveresearch on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-122b/liveresearch/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
agent = "monitor_operative"
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# LiveResearchBench: Qwen3.5-2B (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "liveresearch-qwen-2b"
|
||||
description = "LiveResearchBench on Qwen/Qwen3.5-2B"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-2b/liveresearch/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-2B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Liveresearch: Qwen/Qwen3.5-35B-A3B-FP8
|
||||
[meta]
|
||||
name = "liveresearch-qwen-35b"
|
||||
description = "liveresearch on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-35b/liveresearch/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
agent = "monitor_operative"
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# LiveResearchBench: Qwen3.5-9B (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "liveresearch-qwen-9b"
|
||||
description = "LiveResearchBench on Qwen/Qwen3.5-9B"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 16384
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-9b/liveresearch/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-9B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "liveresearch"
|
||||
backend = "jarvis-agent"
|
||||
agent = "monitor_operative"
|
||||
max_samples = 50
|
||||
tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
|
||||
@@ -0,0 +1,44 @@
|
||||
# PinchBench eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)
|
||||
# Agent: native_openhands -- all PinchBench-required tools enabled
|
||||
# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS
|
||||
#
|
||||
# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are
|
||||
# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs
|
||||
# yet (see github.com/ollama/ollama/issues/5245). This config is a
|
||||
# placeholder -- use vLLM or wait for Ollama sharded GGUF support.
|
||||
# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-kimi-k25"
|
||||
description = "PinchBench on Kimi-K2.5 IQ4_XS via Ollama"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"
|
||||
engine = "ollama"
|
||||
param_count_b = 1000.0
|
||||
active_params_b = 32.0
|
||||
num_gpus = 7
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,41 @@
|
||||
# PinchBench eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)
|
||||
# Agent: native_openhands -- all PinchBench-required tools enabled
|
||||
# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest
|
||||
# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not
|
||||
# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag
|
||||
# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-minimax-m25"
|
||||
description = "PinchBench on MiniMax-M2.5 UD-TQ1_0 via Ollama"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"
|
||||
engine = "ollama"
|
||||
param_count_b = 229.0
|
||||
active_params_b = 45.6
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
# Pinchbench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
|
||||
[meta]
|
||||
name = "pinchbench-nemotron-super"
|
||||
description = "pinchbench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-super/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 23
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
# Pinchbench: Qwen/Qwen3.5-122B-A10B-FP8
|
||||
[meta]
|
||||
name = "pinchbench-qwen-122b"
|
||||
description = "pinchbench on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-122b/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 23
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
# Pinchbench: Qwen/Qwen3.5-35B-A3B-FP8
|
||||
[meta]
|
||||
name = "pinchbench-qwen-35b"
|
||||
description = "pinchbench on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-35b/pinchbench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 23
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,40 @@
|
||||
# TauBench V2 eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)
|
||||
# Multi-turn customer service benchmark -- airline + retail splits
|
||||
# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS
|
||||
#
|
||||
# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are
|
||||
# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs
|
||||
# yet (see github.com/ollama/ollama/issues/5245). This config is a
|
||||
# placeholder -- use vLLM or wait for Ollama sharded GGUF support.
|
||||
# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.
|
||||
|
||||
[meta]
|
||||
name = "taubench-kimi-k25"
|
||||
description = "TauBench V2 on Kimi-K2.5 IQ4_XS via Ollama"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"
|
||||
engine = "ollama"
|
||||
param_count_b = 1000.0
|
||||
active_params_b = 32.0
|
||||
num_gpus = 7
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,37 @@
|
||||
# TauBench V2 eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)
|
||||
# Multi-turn customer service benchmark -- airline + retail splits
|
||||
# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest
|
||||
# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not
|
||||
# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag
|
||||
# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).
|
||||
|
||||
[meta]
|
||||
name = "taubench-minimax-m25"
|
||||
description = "TauBench V2 on MiniMax-M2.5 UD-TQ1_0 via Ollama"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"
|
||||
engine = "ollama"
|
||||
param_count_b = 229.0
|
||||
active_params_b = 45.6
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,29 @@
|
||||
# Taubench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
|
||||
[meta]
|
||||
name = "taubench-nemotron-super"
|
||||
description = "taubench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-super/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,29 @@
|
||||
# Taubench: Qwen/Qwen3.5-122B-A10B-FP8
|
||||
[meta]
|
||||
name = "taubench-qwen-122b"
|
||||
description = "taubench on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-122b/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,29 @@
|
||||
# Taubench: Qwen/Qwen3.5-35B-A3B-FP8
|
||||
[meta]
|
||||
name = "taubench-qwen-35b"
|
||||
description = "taubench on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-35b/taubench/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "airline,retail"
|
||||
@@ -0,0 +1,27 @@
|
||||
# TauBench V2 Telecom: Claude Opus 4.6
|
||||
[meta]
|
||||
name = "taubench-telecom-claude"
|
||||
description = "TauBench V2 telecom on claude-opus-4-6 (test split, pass^3)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/taubench-telecom-claude/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "claude-opus-4-6"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,29 @@
|
||||
# TauBench V2 Telecom: Gemma-4-26B-A4B-it (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "taubench-telecom-gemma4-26b"
|
||||
description = "TauBench V2 Telecom domain on google/gemma-4-26B-A4B-it"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/gemma4-26b/taubench-telecom/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "google/gemma-4-26B-A4B-it"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,27 @@
|
||||
# TauBench V2 Telecom: GPT-5.4
|
||||
[meta]
|
||||
name = "taubench-telecom-gpt54"
|
||||
description = "TauBench V2 telecom on gpt-5.4 (test split, pass^3)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/taubench-telecom-gpt54/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "gpt-5.4-2026-03-05"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,29 @@
|
||||
# Taubench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
|
||||
[meta]
|
||||
name = "taubench-telecom-nemotron-super"
|
||||
description = "taubench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-super/taubench-telecom/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,28 @@
|
||||
# TauBench V2 Telecom: Nemotron-3-Super (SGLang port 8001)
|
||||
[meta]
|
||||
name = "taubench-telecom-nemotron"
|
||||
description = "TauBench V2 telecom on Nemotron-3-Super (test split, pass^3)"
|
||||
|
||||
[defaults]
|
||||
temperature = 1.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/taubench-telecom-nemotron/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nemotron"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,29 @@
|
||||
# Taubench: Qwen/Qwen3.5-122B-A10B-FP8
|
||||
[meta]
|
||||
name = "taubench-telecom-qwen-122b"
|
||||
description = "taubench on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-122b/taubench-telecom/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,29 @@
|
||||
# Taubench: Qwen/Qwen3.5-35B-A3B-FP8
|
||||
[meta]
|
||||
name = "taubench-telecom-qwen-35b"
|
||||
description = "taubench on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-35b/taubench-telecom/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,29 @@
|
||||
# TauBench V2 Telecom: Qwen3.5-27B-FP8 (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "taubench-telecom-qwen27b"
|
||||
description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-27B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-27b/taubench-telecom/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-27B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,29 @@
|
||||
# TauBench V2 Telecom: Qwen3.5-2B (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "taubench-telecom-qwen2b"
|
||||
description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-2B"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-2b/taubench-telecom/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-2B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,29 @@
|
||||
# TauBench V2 Telecom: Qwen3.5-9B (vLLM, 1 GPU)
|
||||
[meta]
|
||||
name = "taubench-telecom-qwen9b"
|
||||
description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-9B"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.7
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-9b/taubench-telecom/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-9B"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "taubench"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 20
|
||||
split = "telecom"
|
||||
@@ -0,0 +1,38 @@
|
||||
# ToolCall-15 eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)
|
||||
# Lightweight tool calling benchmark -- 15 scenarios, 5 categories
|
||||
# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS
|
||||
#
|
||||
# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are
|
||||
# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs
|
||||
# yet (see github.com/ollama/ollama/issues/5245). This config is a
|
||||
# placeholder -- use vLLM or wait for Ollama sharded GGUF support.
|
||||
# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.
|
||||
|
||||
[meta]
|
||||
name = "toolcall15-kimi-k25"
|
||||
description = "ToolCall-15 on Kimi-K2.5 IQ4_XS via Ollama"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"
|
||||
engine = "ollama"
|
||||
param_count_b = 1000.0
|
||||
active_params_b = 32.0
|
||||
num_gpus = 7
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
@@ -0,0 +1,35 @@
|
||||
# ToolCall-15 eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)
|
||||
# Lightweight tool calling benchmark -- 15 scenarios, 5 categories
|
||||
# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest
|
||||
# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not
|
||||
# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag
|
||||
# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).
|
||||
|
||||
[meta]
|
||||
name = "toolcall15-minimax-m25"
|
||||
description = "ToolCall-15 on MiniMax-M2.5 UD-TQ1_0 via Ollama"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"
|
||||
engine = "ollama"
|
||||
param_count_b = 229.0
|
||||
active_params_b = 45.6
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
@@ -0,0 +1,28 @@
|
||||
# Toolcall15: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
|
||||
[meta]
|
||||
name = "toolcall15-nemotron-super"
|
||||
description = "toolcall15 on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/nemotron-super/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 15
|
||||
@@ -0,0 +1,28 @@
|
||||
# Toolcall15: Qwen/Qwen3.5-122B-A10B-FP8
|
||||
[meta]
|
||||
name = "toolcall15-qwen-122b"
|
||||
description = "toolcall15 on Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-122b/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 2
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 15
|
||||
@@ -0,0 +1,28 @@
|
||||
# Toolcall15: Qwen/Qwen3.5-35B-A3B-FP8
|
||||
[meta]
|
||||
name = "toolcall15-qwen-35b"
|
||||
description = "toolcall15 on Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 4096
|
||||
|
||||
[judge]
|
||||
model = "gpt-5-mini-2025-08-07"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/neurips-2026/baselines/qwen-35b/toolcall15/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "toolcall15"
|
||||
backend = "jarvis-direct"
|
||||
max_samples = 15
|
||||
Reference in New Issue
Block a user