diff --git a/docs/experiments/neurips-2026-plan.md b/docs/experiments/neurips-2026-plan.md index 4539b4bf..ccd614f4 100644 --- a/docs/experiments/neurips-2026-plan.md +++ b/docs/experiments/neurips-2026-plan.md @@ -117,7 +117,7 @@ Key comparisons: | terminalbench | TerminalBench | varies | 20 | Implemented | | toolcall15 | ToolCall-15 | 15 | 15 (all) | TODO | | livecodebench | LiveCodeBench | ~100 | 20 | TODO | -| liveresearch | LiveResearchBench | 100 | 10 | TODO | +| liveresearch | DeepResearchBench | 100 | 10 | TODO | ## Metrics Captured Per Run - accuracy (benchmark-specific) @@ -138,7 +138,7 @@ Key comparisons: ### Phase 1a: Implement missing benchmarks - [ ] ToolCall-15 integration - [ ] LiveCodeBench integration -- [ ] LiveResearchBench integration +- [ ] DeepResearchBench integration - [ ] Wire telemetry capture to all eval runs ### Phase 1b: Run cloud baselines (no GPU needed) @@ -148,7 +148,7 @@ Key comparisons: TauBench Telecom (75%), GAIA (34.29%) - [x] Gemini 3.1 Pro — PinchBench (78.26%), TauBench A+R (58.33%), TauBench Telecom (77.5%), GAIA (47.06%) -- [ ] All 3 cloud baselines — ToolCall-15, LiveCodeBench, LiveResearchBench +- [ ] All 3 cloud baselines — ToolCall-15, LiveCodeBench, DeepResearchBench - [ ] All 3 cloud baselines — TerminalBench ### Phase 1c: Run local models (GPU required) @@ -230,7 +230,7 @@ Training targets: - Qwen 35B: TauBench telecom + GAIA running - ToolCall-15 integration: TODO - LiveCodeBench integration: TODO -- LiveResearchBench integration: TODO +- DeepResearchBench integration: TODO - Telemetry wiring: TODO ### Blocked diff --git a/src/openjarvis/evals/configs/liveresearch-claude-opus.toml b/src/openjarvis/evals/configs/liveresearch-claude-opus.toml index 71d173e0..e5bc67d2 100644 --- a/src/openjarvis/evals/configs/liveresearch-claude-opus.toml +++ b/src/openjarvis/evals/configs/liveresearch-claude-opus.toml @@ -1,7 +1,7 @@ -# LiveResearchBench: claude-opus-4-6 +# DeepResearchBench: claude-opus-4-6 [meta] name = "liveresearch-claude-opus" -description = "LiveResearchBench deep research benchmark on claude-opus-4-6" +description = "DeepResearchBench deep research benchmark on claude-opus-4-6" [defaults] temperature = 0.6 diff --git a/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml b/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml new file mode 100644 index 00000000..533bffbe --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml @@ -0,0 +1,29 @@ +# liveresearch: gemini-3.1-pro-preview (cloud) +[meta] +name = "liveresearch-gemini-31-pro" +description = "liveresearch on gemini-3.1-pro-preview" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/gemini-31-pro/liveresearch/" +seed = 42 + +[[models]] +name = "gemini-3.1-pro-preview" +engine = "cloud" + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml index a398c7d0..feaba829 100644 --- a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml +++ b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml @@ -1,7 +1,7 @@ -# LiveResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU) +# DeepResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU) [meta] name = "liveresearch-gemma4-26b" -description = "LiveResearchBench on google/gemma-4-26B-A4B-it" +description = "DeepResearchBench on google/gemma-4-26B-A4B-it" [defaults] temperature = 0.6 diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml new file mode 100644 index 00000000..da337db1 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml @@ -0,0 +1,30 @@ +# liveresearch: google/gemma-4-31B-it +[meta] +name = "liveresearch-gemma4-31b" +description = "liveresearch on google/gemma-4-31B-it" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 16 +output_dir = "results/neurips-2026/baselines/gemma4-31b/liveresearch/" +seed = 42 + +[[models]] +name = "google/gemma-4-31B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-gpt54.toml b/src/openjarvis/evals/configs/liveresearch-gpt54.toml new file mode 100644 index 00000000..904f32e7 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-gpt54.toml @@ -0,0 +1,29 @@ +# liveresearch: gpt-5.4-2026-03-05 (cloud) +[meta] +name = "liveresearch-gpt54" +description = "liveresearch on gpt-5.4-2026-03-05" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/gpt54/liveresearch/" +seed = 42 + +[[models]] +name = "gpt-5.4-2026-03-05" +engine = "cloud" + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml new file mode 100644 index 00000000..ad3816ca --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml @@ -0,0 +1,30 @@ +# liveresearch: granite33-agent +[meta] +name = "liveresearch-granite-3.3-8b-lora" +description = "liveresearch on granite33-agent" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 16 +output_dir = "results/neurips-2026/optimized-eval/track-a/granite-3.3-8b-lora-lora/liveresearch/" +seed = 42 + +[[models]] +name = "granite33-agent" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml new file mode 100644 index 00000000..7a29c69c --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml @@ -0,0 +1,30 @@ +# liveresearch: ibm-granite/granite-3.3-8b-instruct +[meta] +name = "liveresearch-granite-3.3-8b" +description = "liveresearch on ibm-granite/granite-3.3-8b-instruct" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 16 +output_dir = "results/neurips-2026/baselines/granite-3.3-8b/liveresearch/" +seed = 42 + +[[models]] +name = "ibm-granite/granite-3.3-8b-instruct" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml b/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml new file mode 100644 index 00000000..17a1a281 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml @@ -0,0 +1,30 @@ +# liveresearch: ibm-granite/granite-4.0-h-small +[meta] +name = "liveresearch-granite-4.0-h-small" +description = "liveresearch on ibm-granite/granite-4.0-h-small" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 16 +output_dir = "results/neurips-2026/baselines/granite-4.0-h-small/liveresearch/" +seed = 42 + +[[models]] +name = "ibm-granite/granite-4.0-h-small" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml b/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml new file mode 100644 index 00000000..d1089b36 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml @@ -0,0 +1,30 @@ +# liveresearch: ibm-granite/granite-4.0-micro +[meta] +name = "liveresearch-granite-4.0-micro" +description = "liveresearch on ibm-granite/granite-4.0-micro" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 16 +output_dir = "results/neurips-2026/baselines/granite-4.0-micro/liveresearch/" +seed = 42 + +[[models]] +name = "ibm-granite/granite-4.0-micro" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +max_samples = 50 +agent = "monitor_operative" +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml index 63ecf526..483aa3c2 100644 --- a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml +++ b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml @@ -1,7 +1,7 @@ -# LiveResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# DeepResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) [meta] name = "liveresearch-nemotron-nano" -description = "LiveResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent" +description = "DeepResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent" [defaults] temperature = 0.6 diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml new file mode 100644 index 00000000..808d5156 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml @@ -0,0 +1,30 @@ +# DeepResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +[meta] +name = "liveresearch-qwen-27b-lora-r32" +description = "DeepResearchBench on qwen27b-lora-r32 with monitor_operative agent" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/liveresearch-qwen-27b/" +seed = 42 + +[[models]] +name = "qwen27b-lora-r32" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml index 05857481..ecf09b6d 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml @@ -1,7 +1,7 @@ -# LiveResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# DeepResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4) [meta] name = "liveresearch-qwen-27b" -description = "LiveResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent" +description = "DeepResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent" [defaults] temperature = 0.6 diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml new file mode 100644 index 00000000..1cdb806e --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml @@ -0,0 +1,31 @@ +# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-2b-lora-r16" +description = "DeepResearchBench on Qwen/Qwen3.5-2B" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-2b-lora-r16/liveresearch/" +seed = 42 + +[[models]] +name = "qwen2b-lora-r16" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml new file mode 100644 index 00000000..da96ba7a --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml @@ -0,0 +1,31 @@ +# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-2b-lora-r32" +description = "DeepResearchBench on Qwen/Qwen3.5-2B" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-2b-lora-r32/liveresearch/" +seed = 42 + +[[models]] +name = "qwen2b-lora-r32" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml new file mode 100644 index 00000000..e9f26e54 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml @@ -0,0 +1,31 @@ +# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-2b-mixed" +description = "DeepResearchBench on qwen2b-mixed" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/optimized-eval/mixed/qwen-2b-lora/liveresearch/" +seed = 42 + +[[models]] +name = "qwen2b-mixed" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml new file mode 100644 index 00000000..e1777cc8 --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml @@ -0,0 +1,31 @@ +# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-2b-trackb" +description = "DeepResearchBench on qwen2b-trackb" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/optimized-eval/track-b/qwen-2b-lora/liveresearch/" +seed = 42 + +[[models]] +name = "qwen2b-trackb" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml index fa27c3e2..dbcd7b45 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml @@ -1,7 +1,7 @@ -# LiveResearchBench: Qwen3.5-2B (vLLM, 1 GPU) +# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU) [meta] name = "liveresearch-qwen-2b" -description = "LiveResearchBench on Qwen/Qwen3.5-2B" +description = "DeepResearchBench on Qwen/Qwen3.5-2B" [defaults] temperature = 0.6 diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml new file mode 100644 index 00000000..45e3cf7f --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml @@ -0,0 +1,31 @@ +# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-9b-lora-r32" +description = "DeepResearchBench on Qwen/Qwen3.5-9B" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-9b-lora-r32/liveresearch/" +seed = 42 + +[[models]] +name = "qwen9b-lora-r32" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml new file mode 100644 index 00000000..7c1af65d --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml @@ -0,0 +1,31 @@ +# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-9b-mixed" +description = "DeepResearchBench on qwen9b-mixed" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/optimized-eval/mixed/qwen-9b-lora/liveresearch/" +seed = 42 + +[[models]] +name = "qwen9b-mixed" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml new file mode 100644 index 00000000..8a802c5c --- /dev/null +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml @@ -0,0 +1,31 @@ +# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU) +[meta] +name = "liveresearch-qwen-9b-trackb" +description = "DeepResearchBench on qwen9b-trackb" + +[defaults] +temperature = 0.6 +max_tokens = 16384 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 +engine = "cloud" + +[run] +max_workers = 8 +output_dir = "results/neurips-2026/baselines/optimized-eval/track-b/qwen-9b-lora/liveresearch/" +seed = 42 + +[[models]] +name = "qwen9b-trackb" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "liveresearch" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"] diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml index 002e02ae..8da91a75 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml @@ -1,7 +1,7 @@ -# LiveResearchBench: Qwen3.5-9B (vLLM, 1 GPU) +# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU) [meta] name = "liveresearch-qwen-9b" -description = "LiveResearchBench on Qwen/Qwen3.5-9B" +description = "DeepResearchBench on Qwen/Qwen3.5-9B" [defaults] temperature = 0.6 diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml index 22a5d066..fd2e2214 100644 --- a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml +++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml @@ -1,7 +1,7 @@ -# LiveResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8) +# DeepResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8) [meta] name = "liveresearch-trinity-large" -description = "LiveResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent" +description = "DeepResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent" [defaults] temperature = 0.6 @@ -16,7 +16,7 @@ max_tokens = 4096 max_workers = 16 # Trinity-Large is a "thinking" model — needs more turns to reach an answer. # Without this override the agent hits the default max_turns=10 cap on -# ~78% of LiveResearchBench tasks before producing a final answer. +# ~78% of DeepResearchBench tasks before producing a final answer. max_turns = 50 output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/" seed = 42 diff --git a/src/openjarvis/evals/datasets/liveresearch.py b/src/openjarvis/evals/datasets/liveresearch.py index 100d1427..9eb058f8 100644 --- a/src/openjarvis/evals/datasets/liveresearch.py +++ b/src/openjarvis/evals/datasets/liveresearch.py @@ -1,4 +1,4 @@ -"""LiveResearchBench dataset provider — deep research benchmark. +"""DeepResearchBench dataset provider — deep research benchmark. Clones the deep_research_bench repo at runtime and parses query + criteria JSONL files into EvalRecords for use with AgenticRunner. @@ -50,14 +50,14 @@ def _build_criteria_index( class LiveResearchBenchDataset(DatasetProvider): - """LiveResearchBench — deep research with 100 expert-curated tasks. + """DeepResearchBench — deep research with 100 expert-curated tasks. Clones Ayanami0730/deep_research_bench from GitHub (or uses a local path) and parses query + criteria JSONL files into EvalRecords. """ dataset_id = "liveresearch" - dataset_name = "LiveResearchBench" + dataset_name = "DeepResearchBench" def __init__(self, path: Optional[str] = None) -> None: self._local_path = Path(path) if path else None @@ -68,7 +68,7 @@ class LiveResearchBenchDataset(DatasetProvider): issues: List[str] = [] if self._local_path is None and shutil.which("git") is None: issues.append( - "git binary not found. Install git to clone LiveResearchBench." + "git binary not found. Install git to clone DeepResearchBench." ) return issues @@ -77,12 +77,12 @@ class LiveResearchBenchDataset(DatasetProvider): if self._local_path is not None: if not self._local_path.exists(): raise FileNotFoundError( - f"LiveResearchBench path not found: {self._local_path}" + f"DeepResearchBench path not found: {self._local_path}" ) return self._local_path if not self._repo_dir.exists(): - LOGGER.info("Cloning LiveResearchBench from %s ...", LIVERESEARCH_REPO) + LOGGER.info("Cloning DeepResearchBench from %s ...", LIVERESEARCH_REPO) self._repo_dir.parent.mkdir(parents=True, exist_ok=True) subprocess.run( [ @@ -96,7 +96,7 @@ class LiveResearchBenchDataset(DatasetProvider): check=True, capture_output=True, ) - LOGGER.info("LiveResearchBench cloned to %s", self._repo_dir) + LOGGER.info("DeepResearchBench cloned to %s", self._repo_dir) return self._repo_dir @@ -179,7 +179,7 @@ class LiveResearchBenchDataset(DatasetProvider): ) ) - LOGGER.info("LiveResearchBench: loaded %d tasks", len(self._records)) + LOGGER.info("DeepResearchBench: loaded %d tasks", len(self._records)) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/scorers/liveresearch.py b/src/openjarvis/evals/scorers/liveresearch.py index a744fff0..7895e2f1 100644 --- a/src/openjarvis/evals/scorers/liveresearch.py +++ b/src/openjarvis/evals/scorers/liveresearch.py @@ -1,7 +1,7 @@ -"""LiveResearchBench scorer — LLM-as-judge for deep research quality. +"""DeepResearchBench scorer — LLM-as-judge for deep research quality. Evaluates research output quality across four dimensions from the -LiveResearchBench rubric: comprehensiveness, insight, instruction_following, +DeepResearchBench rubric: comprehensiveness, insight, instruction_following, and readability. Uses LLM-as-judge with per-task criteria when available, falling back to a generic research quality rubric. @@ -21,7 +21,7 @@ from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) -# The four scoring dimensions from LiveResearchBench +# The four scoring dimensions from DeepResearchBench DIMENSIONS = ["comprehensiveness", "insight", "instruction_following", "readability"] # Default dimension weights when task-specific weights are unavailable @@ -240,7 +240,7 @@ def _normalize_response(parsed: Dict[str, Any]) -> Dict[str, Any]: class LiveResearchBenchScorer(LLMJudgeScorer): - """LLM-as-judge scorer for LiveResearchBench deep research tasks. + """LLM-as-judge scorer for DeepResearchBench deep research tasks. Evaluates research reports across four dimensions: comprehensiveness, insight, instruction_following, readability. diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py index be7b0aba..5d9f74f8 100644 --- a/tests/evals/test_benchmark_datasets.py +++ b/tests/evals/test_benchmark_datasets.py @@ -136,12 +136,12 @@ class TestDatasetInstantiation: assert ds.dataset_id == "livecodebench" assert ds.dataset_name == "LiveCodeBench" - def test_liveresearch(self) -> None: + def test_deepresearch(self) -> None: from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset ds = LiveResearchBenchDataset() assert ds.dataset_id == "liveresearch" - assert ds.dataset_name == "LiveResearchBench" + assert ds.dataset_name == "DeepResearchBench" def test_toolcall15(self) -> None: from openjarvis.evals.datasets.toolcall15 import ToolCall15Dataset @@ -260,7 +260,7 @@ class TestScorerInstantiation: s = LiveCodeBenchScorer(_mock_backend(), "test-model") assert s.scorer_id == "livecodebench" - def test_liveresearch_scorer(self) -> None: + def test_deepresearch_scorer(self) -> None: from openjarvis.evals.scorers.liveresearch import LiveResearchBenchScorer s = LiveResearchBenchScorer(_mock_backend(), "test-model")