From 41ddab9099ec0891c61c549fd3e7562b331a6f1d Mon Sep 17 00:00:00 2001 From: Avanika Narayan Date: Tue, 7 Apr 2026 20:51:42 -0700 Subject: [PATCH] add eval configs (#208) --- .gitignore | 5 +-- .../evals/configs/gaia-gemini-31-pro.toml | 29 ++++++++++++ .../evals/configs/gaia-gemma4-26b.toml | 31 +++++++++++++ src/openjarvis/evals/configs/gaia-gpt54.toml | 29 ++++++++++++ .../evals/configs/gaia-nemotron-super.toml | 30 +++++++++++++ .../evals/configs/gaia-nemotron3-super.toml | 31 +++++++++++++ .../evals/configs/gaia-qwen-122b.toml | 30 +++++++++++++ .../evals/configs/gaia-qwen-2b.toml | 31 +++++++++++++ .../evals/configs/gaia-qwen-35b.toml | 30 +++++++++++++ .../evals/configs/gaia-qwen-9b.toml | 31 +++++++++++++ .../evals/configs/gaia-qwen122b.toml | 30 +++++++++++++ .../evals/configs/gaia-qwen35b.toml | 30 +++++++++++++ .../evals/configs/gaia-rerun-claude-opus.toml | 29 ++++++++++++ .../configs/gaia-rerun-claude-sonnet.toml | 29 ++++++++++++ .../evals/configs/gaia-rerun-glm47flash.toml | 30 +++++++++++++ .../evals/configs/gaia-rerun-gpt5mini.toml | 29 ++++++++++++ .../evals/configs/gaia-rerun-minimax-m25.toml | 30 +++++++++++++ .../evals/configs/gaia-rerun-qwen122b.toml | 32 ++++++++++++++ .../evals/configs/gaia-rerun-qwen35b.toml | 30 +++++++++++++ .../evals/configs/gaia-rerun-qwen397b.toml | 30 +++++++++++++ .../evals/configs/livecodebench-kimi-k25.toml | 39 ++++++++++++++++ .../configs/livecodebench-minimax-m25.toml | 36 +++++++++++++++ .../configs/livecodebench-nemotron-super.toml | 28 ++++++++++++ .../configs/livecodebench-qwen-122b.toml | 28 ++++++++++++ .../evals/configs/livecodebench-qwen-35b.toml | 28 ++++++++++++ .../configs/liveresearch-gemma4-26b.toml | 31 +++++++++++++ .../configs/liveresearch-nemotron-super.toml | 30 +++++++++++++ .../evals/configs/liveresearch-qwen-122b.toml | 30 +++++++++++++ .../evals/configs/liveresearch-qwen-2b.toml | 31 +++++++++++++ .../evals/configs/liveresearch-qwen-35b.toml | 30 +++++++++++++ .../evals/configs/liveresearch-qwen-9b.toml | 31 +++++++++++++ .../evals/configs/pinchbench-kimi-k25.toml | 44 +++++++++++++++++++ .../evals/configs/pinchbench-minimax-m25.toml | 41 +++++++++++++++++ .../configs/pinchbench-nemotron-super.toml | 34 ++++++++++++++ .../evals/configs/pinchbench-qwen-122b.toml | 34 ++++++++++++++ .../evals/configs/pinchbench-qwen-35b.toml | 34 ++++++++++++++ .../evals/configs/taubench-kimi-k25.toml | 40 +++++++++++++++++ .../evals/configs/taubench-minimax-m25.toml | 37 ++++++++++++++++ .../configs/taubench-nemotron-super.toml | 29 ++++++++++++ .../evals/configs/taubench-qwen-122b.toml | 29 ++++++++++++ .../evals/configs/taubench-qwen-35b.toml | 29 ++++++++++++ .../configs/taubench-telecom-claude.toml | 27 ++++++++++++ .../configs/taubench-telecom-gemma4-26b.toml | 29 ++++++++++++ .../evals/configs/taubench-telecom-gpt54.toml | 27 ++++++++++++ .../taubench-telecom-nemotron-super.toml | 29 ++++++++++++ .../configs/taubench-telecom-nemotron.toml | 28 ++++++++++++ .../configs/taubench-telecom-qwen-122b.toml | 29 ++++++++++++ .../configs/taubench-telecom-qwen-35b.toml | 29 ++++++++++++ .../configs/taubench-telecom-qwen27b.toml | 29 ++++++++++++ .../configs/taubench-telecom-qwen2b.toml | 29 ++++++++++++ .../configs/taubench-telecom-qwen9b.toml | 29 ++++++++++++ .../evals/configs/toolcall15-kimi-k25.toml | 38 ++++++++++++++++ .../evals/configs/toolcall15-minimax-m25.toml | 35 +++++++++++++++ .../configs/toolcall15-nemotron-super.toml | 28 ++++++++++++ .../evals/configs/toolcall15-qwen-122b.toml | 28 ++++++++++++ .../evals/configs/toolcall15-qwen-35b.toml | 28 ++++++++++++ 56 files changed, 1707 insertions(+), 4 deletions(-) create mode 100644 src/openjarvis/evals/configs/gaia-gemini-31-pro.toml create mode 100644 src/openjarvis/evals/configs/gaia-gemma4-26b.toml create mode 100644 src/openjarvis/evals/configs/gaia-gpt54.toml create mode 100644 src/openjarvis/evals/configs/gaia-nemotron-super.toml create mode 100644 src/openjarvis/evals/configs/gaia-nemotron3-super.toml create mode 100644 src/openjarvis/evals/configs/gaia-qwen-122b.toml create mode 100644 src/openjarvis/evals/configs/gaia-qwen-2b.toml create mode 100644 src/openjarvis/evals/configs/gaia-qwen-35b.toml create mode 100644 src/openjarvis/evals/configs/gaia-qwen-9b.toml create mode 100644 src/openjarvis/evals/configs/gaia-qwen122b.toml create mode 100644 src/openjarvis/evals/configs/gaia-qwen35b.toml create mode 100644 src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml create mode 100644 src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml create mode 100644 src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml create mode 100644 src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml create mode 100644 src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml create mode 100644 src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml create mode 100644 src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml create mode 100644 src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-kimi-k25.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-minimax-m25.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-nemotron-super.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-qwen-122b.toml create mode 100644 src/openjarvis/evals/configs/livecodebench-qwen-35b.toml create mode 100644 src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml create mode 100644 src/openjarvis/evals/configs/liveresearch-nemotron-super.toml create mode 100644 src/openjarvis/evals/configs/liveresearch-qwen-122b.toml create mode 100644 src/openjarvis/evals/configs/liveresearch-qwen-2b.toml create mode 100644 src/openjarvis/evals/configs/liveresearch-qwen-35b.toml create mode 100644 src/openjarvis/evals/configs/liveresearch-qwen-9b.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-kimi-k25.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-minimax-m25.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-nemotron-super.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-qwen-122b.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-qwen-35b.toml create mode 100644 src/openjarvis/evals/configs/taubench-kimi-k25.toml create mode 100644 src/openjarvis/evals/configs/taubench-minimax-m25.toml create mode 100644 src/openjarvis/evals/configs/taubench-nemotron-super.toml create mode 100644 src/openjarvis/evals/configs/taubench-qwen-122b.toml create mode 100644 src/openjarvis/evals/configs/taubench-qwen-35b.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-claude.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-gpt54.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-nemotron.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml create mode 100644 src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-kimi-k25.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-minimax-m25.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-nemotron-super.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-qwen-122b.toml create mode 100644 src/openjarvis/evals/configs/toolcall15-qwen-35b.toml diff --git a/.gitignore b/.gitignore index a86fd343..a993fcf3 100644 --- a/.gitignore +++ b/.gitignore @@ -90,7 +90,4 @@ CLAUDE.md research_mining_* .python-version src/openjarvis/channels/whatsapp_baileys_bridge/node_modules/ - -# SQLite in-memory artifacts -:memory: -MagicMock/ +scratch/ diff --git a/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml b/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml new file mode 100644 index 00000000..1410012e --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml @@ -0,0 +1,29 @@ +# GAIA eval: gemini-3.1-pro-preview (cloud) +[meta] +name = "gaia-gemini-31-pro" +description = "GAIA on gemini-3.1-pro-preview" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-gemini-31-pro/" +seed = 42 + +[[models]] +name = "gemini-3.1-pro-preview" +engine = "cloud" + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-gemma4-26b.toml b/src/openjarvis/evals/configs/gaia-gemma4-26b.toml new file mode 100644 index 00000000..adf231ae --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-gemma4-26b.toml @@ -0,0 +1,31 @@ +# GAIA eval: Gemma-4-26B-A4B-it (vLLM, 1 GPU) +[meta] +name = "gaia-gemma4-26b" +description = "GAIA on google/gemma-4-26B-A4B-it" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/gaia/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-gpt54.toml b/src/openjarvis/evals/configs/gaia-gpt54.toml new file mode 100644 index 00000000..4a735d95 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-gpt54.toml @@ -0,0 +1,29 @@ +# GAIA eval: gpt-5.4 (cloud) +[meta] +name = "gaia-gpt54" +description = "GAIA on gpt-5.4-2026-03-05" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-gpt54/" +seed = 42 + +[[models]] +name = "gpt-5.4-2026-03-05" +engine = "cloud" + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-nemotron-super.toml b/src/openjarvis/evals/configs/gaia-nemotron-super.toml new file mode 100644 index 00000000..875e003c --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-nemotron-super.toml @@ -0,0 +1,30 @@ +# Gaia: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 +[meta] +name = "gaia-nemotron-super" +description = "gaia on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-super/gaia/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-nemotron3-super.toml b/src/openjarvis/evals/configs/gaia-nemotron3-super.toml new file mode 100644 index 00000000..3d6d9d56 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-nemotron3-super.toml @@ -0,0 +1,31 @@ +# GAIA eval: Nemotron-3-Super-120B-A12B-FP8 (SGLang on port 8001) +[meta] +name = "gaia-nemotron3-super" +description = "GAIA on Nemotron-3-Super-120B-A12B-FP8 (SGLang)" + +[defaults] +temperature = 1.0 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/gaia-nemotron3-super/" +seed = 42 + +[[models]] +name = "nemotron" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-qwen-122b.toml b/src/openjarvis/evals/configs/gaia-qwen-122b.toml new file mode 100644 index 00000000..94d5d4b0 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-qwen-122b.toml @@ -0,0 +1,30 @@ +# Gaia: Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "gaia-qwen-122b" +description = "gaia on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-122b/gaia/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-qwen-2b.toml b/src/openjarvis/evals/configs/gaia-qwen-2b.toml new file mode 100644 index 00000000..1c68053d --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-qwen-2b.toml @@ -0,0 +1,31 @@ +# GAIA eval: Qwen3.5-2B (vLLM, 1 GPU) +[meta] +name = "gaia-qwen-2b" +description = "GAIA on Qwen/Qwen3.5-2B" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/gaia/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-qwen-35b.toml b/src/openjarvis/evals/configs/gaia-qwen-35b.toml new file mode 100644 index 00000000..9df1a672 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-qwen-35b.toml @@ -0,0 +1,30 @@ +# Gaia: Qwen/Qwen3.5-35B-A3B-FP8 +[meta] +name = "gaia-qwen-35b" +description = "gaia on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-35b/gaia/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-qwen-9b.toml b/src/openjarvis/evals/configs/gaia-qwen-9b.toml new file mode 100644 index 00000000..8a3a2c91 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-qwen-9b.toml @@ -0,0 +1,31 @@ +# GAIA eval: Qwen3.5-9B (vLLM, 1 GPU) +[meta] +name = "gaia-qwen-9b" +description = "GAIA on Qwen/Qwen3.5-9B" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-9b/gaia/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-9B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-qwen122b.toml b/src/openjarvis/evals/configs/gaia-qwen122b.toml new file mode 100644 index 00000000..bb864424 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-qwen122b.toml @@ -0,0 +1,30 @@ +# GAIA eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4) +[meta] +name = "gaia-qwen122b" +description = "GAIA on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-qwen122b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-qwen35b.toml b/src/openjarvis/evals/configs/gaia-qwen35b.toml new file mode 100644 index 00000000..be8faa4c --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-qwen35b.toml @@ -0,0 +1,30 @@ +# GAIA eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4) +[meta] +name = "gaia-qwen35b" +description = "GAIA on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-qwen35b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml b/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml new file mode 100644 index 00000000..bde5c0dc --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml @@ -0,0 +1,29 @@ +# GAIA rerun with max_turns=30: claude-opus-4-6 +[meta] +name = "gaia-rerun-claude-opus" +description = "GAIA rerun with 30 max turns on claude-opus-4-6" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-rerun-claude-opus/" +seed = 42 + +[[models]] +name = "claude-opus-4-6" +engine = "cloud" + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml b/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml new file mode 100644 index 00000000..c63575e7 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml @@ -0,0 +1,29 @@ +# GAIA rerun with max_turns=30: claude-sonnet-4-6 +[meta] +name = "gaia-rerun-claude-sonnet" +description = "GAIA rerun with 30 max turns on claude-sonnet-4-6" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-rerun-claude-sonnet/" +seed = 42 + +[[models]] +name = "claude-sonnet-4-6" +engine = "cloud" + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml b/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml new file mode 100644 index 00000000..fe1763e0 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml @@ -0,0 +1,30 @@ +# GAIA rerun with max_turns=30: zai-org/GLM-4.7-Flash +[meta] +name = "gaia-rerun-glm47flash" +description = "GAIA rerun with 30 max turns on zai-org/GLM-4.7-Flash" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-rerun-glm47flash/" +seed = 42 + +[[models]] +name = "zai-org/GLM-4.7-Flash" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml b/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml new file mode 100644 index 00000000..8cdced92 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml @@ -0,0 +1,29 @@ +# GAIA rerun with max_turns=30: gpt-5-mini-2025-08-07 +[meta] +name = "gaia-rerun-gpt5mini" +description = "GAIA rerun with 30 max turns on gpt-5-mini-2025-08-07" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-rerun-gpt5mini/" +seed = 42 + +[[models]] +name = "gpt-5-mini-2025-08-07" +engine = "cloud" + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml b/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml new file mode 100644 index 00000000..d1a3cf78 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml @@ -0,0 +1,30 @@ +# GAIA rerun with max_turns=30: MiniMaxAI/MiniMax-M2.5 +[meta] +name = "gaia-rerun-minimax-m25" +description = "GAIA rerun with 30 max turns on MiniMaxAI/MiniMax-M2.5" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-rerun-minimax-m25/" +seed = 42 + +[[models]] +name = "MiniMaxAI/MiniMax-M2.5" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml new file mode 100644 index 00000000..286dcef5 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml @@ -0,0 +1,32 @@ +# GAIA rerun with max_turns=30: Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "gaia-rerun-qwen122b" +description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-rerun-qwen122b/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml new file mode 100644 index 00000000..51c51859 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml @@ -0,0 +1,30 @@ +# GAIA rerun with max_turns=30: Qwen/Qwen3.5-35B-A3B-FP8 +[meta] +name = "gaia-rerun-qwen35b" +description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-rerun-qwen35b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml new file mode 100644 index 00000000..edcbded5 --- /dev/null +++ b/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml @@ -0,0 +1,30 @@ +# GAIA rerun with max_turns=30: Qwen/Qwen3.5-397B-A17B-FP8 +[meta] +name = "gaia-rerun-qwen397b" +description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-397B-A17B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/gaia-rerun-qwen397b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-397B-A17B-FP8" +engine = "vllm" +num_gpus = 8 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"] diff --git a/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml b/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml new file mode 100644 index 00000000..655949b5 --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml @@ -0,0 +1,39 @@ +# LiveCodeBench eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF) +# Competitive programming from LeetCode, AtCoder, CodeForces +# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS +# +# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are +# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs +# yet (see github.com/ollama/ollama/issues/5245). This config is a +# placeholder -- use vLLM or wait for Ollama sharded GGUF support. +# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs. + +[meta] +name = "livecodebench-kimi-k25" +description = "LiveCodeBench on Kimi-K2.5 IQ4_XS via Ollama" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/livecodebench/" +seed = 42 + +[[models]] +name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS" +engine = "ollama" +param_count_b = 1000.0 +active_params_b = 32.0 +num_gpus = 7 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml b/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml new file mode 100644 index 00000000..73cf2930 --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml @@ -0,0 +1,36 @@ +# LiveCodeBench eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3) +# Competitive programming from LeetCode, AtCoder, CodeForces +# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest +# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not +# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag +# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB). + +[meta] +name = "livecodebench-minimax-m25" +description = "LiveCodeBench on MiniMax-M2.5 UD-TQ1_0 via Ollama" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/livecodebench/" +seed = 42 + +[[models]] +name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest" +engine = "ollama" +param_count_b = 229.0 +active_params_b = 45.6 +num_gpus = 2 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml b/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml new file mode 100644 index 00000000..30b8697b --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml @@ -0,0 +1,28 @@ +# Livecodebench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 +[meta] +name = "livecodebench-nemotron-super" +description = "livecodebench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-super/livecodebench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml new file mode 100644 index 00000000..3ac0c749 --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml @@ -0,0 +1,28 @@ +# Livecodebench: Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "livecodebench-qwen-122b" +description = "livecodebench on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-122b/livecodebench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml new file mode 100644 index 00000000..281a5e84 --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml @@ -0,0 +1,28 @@ +# Livecodebench: Qwen/Qwen3.5-35B-A3B-FP8 +[meta] +name = "livecodebench-qwen-35b" +description = "livecodebench on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-35b/livecodebench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml new file mode 100644 index 00000000..a398c7d0 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml @@ -0,0 +1,31 @@ +# LiveResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU) +[meta] +name = "liveresearch-gemma4-26b" +description = "LiveResearchBench on google/gemma-4-26B-A4B-it" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/liveresearch/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml new file mode 100644 index 00000000..f14956fa --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml @@ -0,0 +1,30 @@ +# Liveresearch: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 +[meta] +name = "liveresearch-nemotron-super" +description = "liveresearch on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-super/liveresearch/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml new file mode 100644 index 00000000..0e316fa2 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml @@ -0,0 +1,30 @@ +# Liveresearch: Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "liveresearch-qwen-122b" +description = "liveresearch on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-122b/liveresearch/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml new file mode 100644 index 00000000..d0beff38 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml @@ -0,0 +1,31 @@ +# LiveResearchBench: Qwen3.5-2B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-2b" +description = "LiveResearchBench on Qwen/Qwen3.5-2B" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/liveresearch/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml new file mode 100644 index 00000000..483229ee --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml @@ -0,0 +1,30 @@ +# Liveresearch: Qwen/Qwen3.5-35B-A3B-FP8 +[meta] +name = "liveresearch-qwen-35b" +description = "liveresearch on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-35b/liveresearch/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml new file mode 100644 index 00000000..002e02ae --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml @@ -0,0 +1,31 @@ +# LiveResearchBench: Qwen3.5-9B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-9b" +description = "LiveResearchBench on Qwen/Qwen3.5-9B" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-9b/liveresearch/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-9B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/pinchbench-kimi-k25.toml b/src/openjarvis/evals/configs/pinchbench-kimi-k25.toml new file mode 100644 index 00000000..e5ef8667 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-kimi-k25.toml @@ -0,0 +1,44 @@ +# PinchBench eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF) +# Agent: native_openhands -- all PinchBench-required tools enabled +# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS +# +# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are +# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs +# yet (see github.com/ollama/ollama/issues/5245). This config is a +# placeholder -- use vLLM or wait for Ollama sharded GGUF support. +# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs. + +[meta] +name = "pinchbench-kimi-k25" +description = "PinchBench on Kimi-K2.5 IQ4_XS via Ollama" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/pinchbench/" +seed = 42 + +[[models]] +name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS" +engine = "ollama" +param_count_b = 1000.0 +active_params_b = 32.0 +num_gpus = 7 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-minimax-m25.toml b/src/openjarvis/evals/configs/pinchbench-minimax-m25.toml new file mode 100644 index 00000000..e9662b3a --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-minimax-m25.toml @@ -0,0 +1,41 @@ +# PinchBench eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3) +# Agent: native_openhands -- all PinchBench-required tools enabled +# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest +# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not +# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag +# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB). + +[meta] +name = "pinchbench-minimax-m25" +description = "PinchBench on MiniMax-M2.5 UD-TQ1_0 via Ollama" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/pinchbench/" +seed = 42 + +[[models]] +name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest" +engine = "ollama" +param_count_b = 229.0 +active_params_b = 45.6 +num_gpus = 2 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml b/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml new file mode 100644 index 00000000..c66090ea --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml @@ -0,0 +1,34 @@ +# Pinchbench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 +[meta] +name = "pinchbench-nemotron-super" +description = "pinchbench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-super/pinchbench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +max_samples = 23 +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml new file mode 100644 index 00000000..5a91cc93 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml @@ -0,0 +1,34 @@ +# Pinchbench: Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "pinchbench-qwen-122b" +description = "pinchbench on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-122b/pinchbench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +max_samples = 23 +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml new file mode 100644 index 00000000..1f2188ee --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml @@ -0,0 +1,34 @@ +# Pinchbench: Qwen/Qwen3.5-35B-A3B-FP8 +[meta] +name = "pinchbench-qwen-35b" +description = "pinchbench on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-35b/pinchbench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +max_samples = 23 +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/taubench-kimi-k25.toml b/src/openjarvis/evals/configs/taubench-kimi-k25.toml new file mode 100644 index 00000000..639f3f05 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-kimi-k25.toml @@ -0,0 +1,40 @@ +# TauBench V2 eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF) +# Multi-turn customer service benchmark -- airline + retail splits +# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS +# +# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are +# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs +# yet (see github.com/ollama/ollama/issues/5245). This config is a +# placeholder -- use vLLM or wait for Ollama sharded GGUF support. +# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs. + +[meta] +name = "taubench-kimi-k25" +description = "TauBench V2 on Kimi-K2.5 IQ4_XS via Ollama" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/taubench/" +seed = 42 + +[[models]] +name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS" +engine = "ollama" +param_count_b = 1000.0 +active_params_b = 32.0 +num_gpus = 7 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-minimax-m25.toml b/src/openjarvis/evals/configs/taubench-minimax-m25.toml new file mode 100644 index 00000000..c962efd8 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-minimax-m25.toml @@ -0,0 +1,37 @@ +# TauBench V2 eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3) +# Multi-turn customer service benchmark -- airline + retail splits +# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest +# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not +# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag +# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB). + +[meta] +name = "taubench-minimax-m25" +description = "TauBench V2 on MiniMax-M2.5 UD-TQ1_0 via Ollama" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/taubench/" +seed = 42 + +[[models]] +name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest" +engine = "ollama" +param_count_b = 229.0 +active_params_b = 45.6 +num_gpus = 2 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-nemotron-super.toml b/src/openjarvis/evals/configs/taubench-nemotron-super.toml new file mode 100644 index 00000000..f7f5304b --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-nemotron-super.toml @@ -0,0 +1,29 @@ +# Taubench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 +[meta] +name = "taubench-nemotron-super" +description = "taubench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-super/taubench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen-122b.toml b/src/openjarvis/evals/configs/taubench-qwen-122b.toml new file mode 100644 index 00000000..a0f4329a --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen-122b.toml @@ -0,0 +1,29 @@ +# Taubench: Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "taubench-qwen-122b" +description = "taubench on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-122b/taubench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen-35b.toml b/src/openjarvis/evals/configs/taubench-qwen-35b.toml new file mode 100644 index 00000000..d902364b --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen-35b.toml @@ -0,0 +1,29 @@ +# Taubench: Qwen/Qwen3.5-35B-A3B-FP8 +[meta] +name = "taubench-qwen-35b" +description = "taubench on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-35b/taubench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-telecom-claude.toml b/src/openjarvis/evals/configs/taubench-telecom-claude.toml new file mode 100644 index 00000000..2710b225 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-claude.toml @@ -0,0 +1,27 @@ +# TauBench V2 Telecom: Claude Opus 4.6 +[meta] +name = "taubench-telecom-claude" +description = "TauBench V2 telecom on claude-opus-4-6 (test split, pass^3)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-telecom-claude/" +seed = 42 + +[[models]] +name = "claude-opus-4-6" +engine = "cloud" + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml b/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml new file mode 100644 index 00000000..380ab271 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml @@ -0,0 +1,29 @@ +# TauBench V2 Telecom: Gemma-4-26B-A4B-it (vLLM, 1 GPU) +[meta] +name = "taubench-telecom-gemma4-26b" +description = "TauBench V2 Telecom domain on google/gemma-4-26B-A4B-it" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/taubench-telecom/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml b/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml new file mode 100644 index 00000000..1dbb3f3a --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml @@ -0,0 +1,27 @@ +# TauBench V2 Telecom: GPT-5.4 +[meta] +name = "taubench-telecom-gpt54" +description = "TauBench V2 telecom on gpt-5.4 (test split, pass^3)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-telecom-gpt54/" +seed = 42 + +[[models]] +name = "gpt-5.4-2026-03-05" +engine = "cloud" + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml new file mode 100644 index 00000000..26cffbd2 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml @@ -0,0 +1,29 @@ +# Taubench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 +[meta] +name = "taubench-telecom-nemotron-super" +description = "taubench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-super/taubench-telecom/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml new file mode 100644 index 00000000..d71b37a2 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml @@ -0,0 +1,28 @@ +# TauBench V2 Telecom: Nemotron-3-Super (SGLang port 8001) +[meta] +name = "taubench-telecom-nemotron" +description = "TauBench V2 telecom on Nemotron-3-Super (test split, pass^3)" + +[defaults] +temperature = 1.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-telecom-nemotron/" +seed = 42 + +[[models]] +name = "nemotron" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml new file mode 100644 index 00000000..849fb30e --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml @@ -0,0 +1,29 @@ +# Taubench: Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "taubench-telecom-qwen-122b" +description = "taubench on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-122b/taubench-telecom/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml new file mode 100644 index 00000000..ccf5c8f6 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml @@ -0,0 +1,29 @@ +# Taubench: Qwen/Qwen3.5-35B-A3B-FP8 +[meta] +name = "taubench-telecom-qwen-35b" +description = "taubench on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-35b/taubench-telecom/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml new file mode 100644 index 00000000..c4a2de6f --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml @@ -0,0 +1,29 @@ +# TauBench V2 Telecom: Qwen3.5-27B-FP8 (vLLM, 1 GPU) +[meta] +name = "taubench-telecom-qwen27b" +description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-27B-FP8" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/taubench-telecom/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml new file mode 100644 index 00000000..d1fd6023 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml @@ -0,0 +1,29 @@ +# TauBench V2 Telecom: Qwen3.5-2B (vLLM, 1 GPU) +[meta] +name = "taubench-telecom-qwen2b" +description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-2B" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/taubench-telecom/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "telecom" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml new file mode 100644 index 00000000..85d8691e --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml @@ -0,0 +1,29 @@ +# TauBench V2 Telecom: Qwen3.5-9B (vLLM, 1 GPU) +[meta] +name = "taubench-telecom-qwen9b" +description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-9B" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-9b/taubench-telecom/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-9B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "telecom" diff --git a/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml b/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml new file mode 100644 index 00000000..2f0596bc --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml @@ -0,0 +1,38 @@ +# ToolCall-15 eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF) +# Lightweight tool calling benchmark -- 15 scenarios, 5 categories +# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS +# +# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are +# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs +# yet (see github.com/ollama/ollama/issues/5245). This config is a +# placeholder -- use vLLM or wait for Ollama sharded GGUF support. +# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs. + +[meta] +name = "toolcall15-kimi-k25" +description = "ToolCall-15 on Kimi-K2.5 IQ4_XS via Ollama" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/toolcall15/" +seed = 42 + +[[models]] +name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS" +engine = "ollama" +param_count_b = 1000.0 +active_params_b = 32.0 +num_gpus = 7 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml b/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml new file mode 100644 index 00000000..6d087419 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml @@ -0,0 +1,35 @@ +# ToolCall-15 eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3) +# Lightweight tool calling benchmark -- 15 scenarios, 5 categories +# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest +# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not +# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag +# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB). + +[meta] +name = "toolcall15-minimax-m25" +description = "ToolCall-15 on MiniMax-M2.5 UD-TQ1_0 via Ollama" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/toolcall15/" +seed = 42 + +[[models]] +name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest" +engine = "ollama" +param_count_b = 229.0 +active_params_b = 45.6 +num_gpus = 2 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml b/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml new file mode 100644 index 00000000..9b3fba0f --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml @@ -0,0 +1,28 @@ +# Toolcall15: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 +[meta] +name = "toolcall15-nemotron-super" +description = "toolcall15 on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-super/toolcall15/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" +max_samples = 15 diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml new file mode 100644 index 00000000..b88527e4 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml @@ -0,0 +1,28 @@ +# Toolcall15: Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "toolcall15-qwen-122b" +description = "toolcall15 on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-122b/toolcall15/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 2 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" +max_samples = 15 diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml new file mode 100644 index 00000000..7dd7ccee --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml @@ -0,0 +1,28 @@ +# Toolcall15: Qwen/Qwen3.5-35B-A3B-FP8 +[meta] +name = "toolcall15-qwen-35b" +description = "toolcall15 on Qwen/Qwen3.5-35B-A3B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-35b/toolcall15/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" +max_samples = 15