From 08fe3a6c980b1f9daeebfba3cf6f9dc3e2c137dc Mon Sep 17 00:00:00 2001 From: Andrew Park Date: Sat, 11 Jul 2026 11:56:06 -0700 Subject: [PATCH] evals: switch the default LLM judge to claude-haiku-4-5 Replaces gpt-5-mini-2025-08-07 as the default judge everywhere: the 147 eval configs plus the hardcoded defaults in the evals CLI, recipe composer, comparison config generator, taubench user-simulator, trial runner and OptimizeConfig. Haiku is roughly 3x faster on the judging path and we are out of OpenAI quota, so gpt-5-mini was failing closed on long eval runs. --- src/openjarvis/core/config.py | 2 +- src/openjarvis/evals/cli.py | 4 ++-- src/openjarvis/evals/comparison/make_configs.py | 2 +- .../evals/configs/deepplanning-monitor-operative.toml | 2 +- src/openjarvis/evals/configs/gaia-gemini-31-pro.toml | 2 +- src/openjarvis/evals/configs/gaia-gemma4-26b.toml | 2 +- src/openjarvis/evals/configs/gaia-gpt54.toml | 2 +- src/openjarvis/evals/configs/gaia-nemotron-nano.toml | 2 +- src/openjarvis/evals/configs/gaia-nemotron-super.toml | 2 +- src/openjarvis/evals/configs/gaia-nemotron3-super.toml | 2 +- src/openjarvis/evals/configs/gaia-qwen-122b.toml | 2 +- src/openjarvis/evals/configs/gaia-qwen-27b.toml | 2 +- src/openjarvis/evals/configs/gaia-qwen-2b.toml | 2 +- src/openjarvis/evals/configs/gaia-qwen-35b.toml | 2 +- src/openjarvis/evals/configs/gaia-qwen-9b.toml | 2 +- src/openjarvis/evals/configs/gaia-qwen122b.toml | 2 +- src/openjarvis/evals/configs/gaia-qwen35b.toml | 2 +- src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml | 2 +- src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml | 2 +- src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml | 2 +- src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml | 2 +- src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml | 2 +- src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml | 2 +- src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml | 2 +- src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml | 2 +- src/openjarvis/evals/configs/gaia-trinity-large.toml | 2 +- .../evals/configs/glm-4.7-flash-openhands-remaining.toml | 2 +- src/openjarvis/evals/configs/glm-4.7-flash-openhands.toml | 2 +- .../evals/configs/glm-4.7-fp8-openhands-gaia.toml | 2 +- .../evals/configs/glm5-fp8-monitor-operative.toml | 2 +- .../evals/configs/kimi-k2.5-monitor-operative.toml | 2 +- src/openjarvis/evals/configs/livecodebench-claude-opus.toml | 2 +- src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml | 2 +- src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml | 2 +- src/openjarvis/evals/configs/livecodebench-kimi-k25.toml | 2 +- src/openjarvis/evals/configs/livecodebench-minimax-m25.toml | 2 +- .../evals/configs/livecodebench-nemotron-nano.toml | 2 +- .../evals/configs/livecodebench-nemotron-super.toml | 2 +- src/openjarvis/evals/configs/livecodebench-qwen-122b.toml | 2 +- src/openjarvis/evals/configs/livecodebench-qwen-27b.toml | 2 +- src/openjarvis/evals/configs/livecodebench-qwen-2b.toml | 2 +- src/openjarvis/evals/configs/livecodebench-qwen-35b.toml | 2 +- src/openjarvis/evals/configs/livecodebench-qwen-9b.toml | 2 +- src/openjarvis/evals/configs/liveresearch-claude-opus.toml | 2 +- .../evals/configs/liveresearch-gemini-31-pro.toml | 2 +- src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml | 2 +- src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml | 2 +- src/openjarvis/evals/configs/liveresearch-gpt54.toml | 2 +- .../evals/configs/liveresearch-granite-3.3-8b-lora.toml | 2 +- .../evals/configs/liveresearch-granite-3.3-8b.toml | 2 +- .../evals/configs/liveresearch-granite-4.0-h-small.toml | 2 +- .../evals/configs/liveresearch-granite-4.0-micro.toml | 2 +- .../evals/configs/liveresearch-nemotron-nano.toml | 2 +- .../evals/configs/liveresearch-nemotron-super.toml | 2 +- src/openjarvis/evals/configs/liveresearch-qwen-122b.toml | 2 +- .../evals/configs/liveresearch-qwen-27b-lora-r32.toml | 2 +- src/openjarvis/evals/configs/liveresearch-qwen-27b.toml | 2 +- .../evals/configs/liveresearch-qwen-2b-lora-r16.toml | 2 +- .../evals/configs/liveresearch-qwen-2b-lora-r32.toml | 2 +- .../evals/configs/liveresearch-qwen-2b-mixed.toml | 2 +- .../evals/configs/liveresearch-qwen-2b-trackb.toml | 2 +- src/openjarvis/evals/configs/liveresearch-qwen-2b.toml | 2 +- src/openjarvis/evals/configs/liveresearch-qwen-35b.toml | 2 +- .../evals/configs/liveresearch-qwen-9b-lora-r32.toml | 2 +- .../evals/configs/liveresearch-qwen-9b-mixed.toml | 2 +- .../evals/configs/liveresearch-qwen-9b-trackb.toml | 2 +- src/openjarvis/evals/configs/liveresearch-qwen-9b.toml | 2 +- .../evals/configs/liveresearch-trinity-large.toml | 2 +- .../evals/configs/liveresearchbench-qwen-27b.toml | 2 +- src/openjarvis/evals/configs/liveresearchbench-qwen-2b.toml | 2 +- src/openjarvis/evals/configs/liveresearchbench-qwen-9b.toml | 2 +- src/openjarvis/evals/configs/long-horizon-claude-opus.toml | 2 +- .../evals/configs/long-horizon-claude-sonnet.toml | 2 +- src/openjarvis/evals/configs/long-horizon-glm47flash.toml | 2 +- src/openjarvis/evals/configs/long-horizon-gpt5mini.toml | 2 +- src/openjarvis/evals/configs/long-horizon-kimik25.toml | 2 +- src/openjarvis/evals/configs/long-horizon-minimax-m25.toml | 2 +- src/openjarvis/evals/configs/long-horizon-qwen122b.toml | 2 +- src/openjarvis/evals/configs/long-horizon-qwen35b.toml | 2 +- src/openjarvis/evals/configs/long-horizon-qwen397b.toml | 2 +- .../evals/configs/paperarena-monitor-operative.toml | 2 +- src/openjarvis/evals/configs/pinchbench-nemotron-super.toml | 2 +- src/openjarvis/evals/configs/pinchbench-qwen-122b.toml | 2 +- src/openjarvis/evals/configs/pinchbench-qwen-35b.toml | 2 +- src/openjarvis/evals/configs/qwen3.5-122b-kg-agentic.toml | 2 +- src/openjarvis/evals/configs/qwen3.5-122b-kg-hightoken.toml | 2 +- .../evals/configs/qwen3.5-122b-reasoning-suite.toml | 2 +- src/openjarvis/evals/configs/qwen3.5-35b-kg.toml | 2 +- .../evals/configs/qwen3.5-35b-monitor-operative.toml | 2 +- .../evals/configs/qwen3.5-397b-monitor-operative.toml | 2 +- src/openjarvis/evals/configs/smoke_amd.toml | 2 +- src/openjarvis/evals/configs/smoke_apple.toml | 2 +- src/openjarvis/evals/configs/smoke_nvidia.toml | 2 +- src/openjarvis/evals/configs/taubench-claude-opus.toml | 2 +- src/openjarvis/evals/configs/taubench-gemini-31-pro.toml | 2 +- src/openjarvis/evals/configs/taubench-gemma4-26b.toml | 2 +- src/openjarvis/evals/configs/taubench-gemma4-e4b.toml | 2 +- src/openjarvis/evals/configs/taubench-gpt54.toml | 2 +- src/openjarvis/evals/configs/taubench-kimi-k25.toml | 2 +- src/openjarvis/evals/configs/taubench-minimax-m25.toml | 2 +- src/openjarvis/evals/configs/taubench-nemotron-nano.toml | 2 +- src/openjarvis/evals/configs/taubench-nemotron-super.toml | 2 +- src/openjarvis/evals/configs/taubench-nemotron3-super.toml | 2 +- src/openjarvis/evals/configs/taubench-qwen-122b.toml | 2 +- src/openjarvis/evals/configs/taubench-qwen-27b.toml | 2 +- src/openjarvis/evals/configs/taubench-qwen-2b.toml | 2 +- src/openjarvis/evals/configs/taubench-qwen-35b.toml | 2 +- src/openjarvis/evals/configs/taubench-qwen-9b.toml | 2 +- src/openjarvis/evals/configs/taubench-qwen122b.toml | 2 +- src/openjarvis/evals/configs/taubench-qwen35b.toml | 2 +- src/openjarvis/evals/configs/taubench-qwen397b.toml | 2 +- src/openjarvis/evals/configs/taubench-telecom-claude.toml | 2 +- .../evals/configs/taubench-telecom-gemma4-26b.toml | 2 +- src/openjarvis/evals/configs/taubench-telecom-gpt54.toml | 2 +- .../evals/configs/taubench-telecom-nemotron-nano.toml | 2 +- .../evals/configs/taubench-telecom-nemotron-super.toml | 2 +- src/openjarvis/evals/configs/taubench-telecom-nemotron.toml | 2 +- .../evals/configs/taubench-telecom-qwen-122b.toml | 2 +- src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml | 2 +- src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml | 2 +- src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml | 2 +- src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml | 2 +- src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml | 2 +- .../evals/configs/taubench-telecom-trinity-large.toml | 2 +- .../evals/configs/terminalbench-native-qwen122b.toml | 2 +- src/openjarvis/evals/configs/terminalbench-qwen122b.toml | 2 +- src/openjarvis/evals/configs/toolcall15-claude-opus.toml | 2 +- src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml | 2 +- src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml | 2 +- src/openjarvis/evals/configs/toolcall15-kimi-k25.toml | 2 +- src/openjarvis/evals/configs/toolcall15-minimax-m25.toml | 2 +- src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml | 2 +- src/openjarvis/evals/configs/toolcall15-nemotron-super.toml | 2 +- src/openjarvis/evals/configs/toolcall15-qwen-122b.toml | 2 +- src/openjarvis/evals/configs/toolcall15-qwen-27b.toml | 2 +- src/openjarvis/evals/configs/toolcall15-qwen-2b.toml | 2 +- src/openjarvis/evals/configs/toolcall15-qwen-35b.toml | 2 +- src/openjarvis/evals/configs/toolcall15-qwen-9b.toml | 2 +- src/openjarvis/evals/configs/use_case_cloud_comparison.toml | 2 +- src/openjarvis/evals/configs/use_case_cloud_openhands.toml | 2 +- .../evals/configs/use_case_cloud_orchestrator.toml | 2 +- src/openjarvis/evals/configs/use_case_direct.toml | 2 +- src/openjarvis/evals/configs/use_case_ibm_granite.toml | 2 +- src/openjarvis/evals/configs/use_case_openhands.toml | 2 +- .../evals/configs/use_case_opensource_openhands.toml | 2 +- .../evals/configs/use_case_opensource_orchestrator.toml | 2 +- src/openjarvis/evals/configs/use_case_orchestrator.toml | 2 +- src/openjarvis/evals/configs/use_case_react.toml | 2 +- src/openjarvis/evals/configs/use_case_suite.toml | 2 +- src/openjarvis/evals/configs/use_case_v2_suite.toml | 2 +- src/openjarvis/evals/core/config.py | 2 +- src/openjarvis/evals/core/types.py | 4 ++-- src/openjarvis/evals/execution/taubench_env.py | 2 +- src/openjarvis/evals/tests/test_config.py | 6 +++--- src/openjarvis/evals/tests/test_types.py | 4 ++-- src/openjarvis/learning/optimize/trial_runner.py | 4 ++-- src/openjarvis/recipes/composer.py | 2 +- tests/learning/test_optimize_cmd.py | 2 +- tests/learning/test_trial_runner.py | 2 +- 159 files changed, 165 insertions(+), 165 deletions(-) diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py index 40c1f016..c68a228d 100644 --- a/src/openjarvis/core/config.py +++ b/src/openjarvis/core/config.py @@ -1408,7 +1408,7 @@ class OptimizeConfig: optimizer_provider: str = "anthropic" benchmark: str = "" max_samples: int = 50 - judge_model: str = "gpt-5-mini-2025-08-07" + judge_model: str = "claude-haiku-4-5-20251001" db_path: str = str(DEFAULT_CONFIG_DIR / "optimize.db") diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index 84e804ce..3af31beb 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -1141,7 +1141,7 @@ def main(): "-n", "--max-samples", type=int, default=None, help="Maximum samples to evaluate" ) @click.option("-w", "--max-workers", type=int, default=4, help="Parallel workers") -@click.option("--judge-model", default="gpt-5-mini-2025-08-07", help="LLM judge model") +@click.option("--judge-model", default="claude-haiku-4-5-20251001", help="LLM judge model") @click.option("-o", "--output", "output_path", default=None, help="Output JSONL path") @click.option("--seed", type=int, default=42, help="Random seed") @click.option("--split", "dataset_split", default=None, help="Dataset split override") @@ -1387,7 +1387,7 @@ def run( "-n", "--max-samples", type=int, default=None, help="Max samples per benchmark" ) @click.option("-w", "--max-workers", type=int, default=4, help="Parallel workers") -@click.option("--judge-model", default="gpt-5-mini-2025-08-07", help="LLM judge model") +@click.option("--judge-model", default="claude-haiku-4-5-20251001", help="LLM judge model") @click.option("--output-dir", default="results/", help="Output directory for results") @click.option("--seed", type=int, default=42, help="Random seed") @click.option("-v", "--verbose", is_flag=True, help="Verbose logging") diff --git a/src/openjarvis/evals/comparison/make_configs.py b/src/openjarvis/evals/comparison/make_configs.py index 6e03c9b1..6103cb4c 100644 --- a/src/openjarvis/evals/comparison/make_configs.py +++ b/src/openjarvis/evals/comparison/make_configs.py @@ -98,7 +98,7 @@ BENCHMARKS: Dict[str, Dict[str, object]] = { }, } -DEFAULT_JUDGE_MODEL = "gpt-5-mini-2025-08-07" +DEFAULT_JUDGE_MODEL = "claude-haiku-4-5-20251001" def _template_path() -> Path: diff --git a/src/openjarvis/evals/configs/deepplanning-monitor-operative.toml b/src/openjarvis/evals/configs/deepplanning-monitor-operative.toml index ac62fc08..783480ed 100644 --- a/src/openjarvis/evals/configs/deepplanning-monitor-operative.toml +++ b/src/openjarvis/evals/configs/deepplanning-monitor-operative.toml @@ -11,7 +11,7 @@ temperature = 0.6 max_tokens = 32768 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 1024 diff --git a/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml b/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml index 1410012e..f94e853c 100644 --- a/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml +++ b/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-gemma4-26b.toml b/src/openjarvis/evals/configs/gaia-gemma4-26b.toml index adf231ae..dae4e52b 100644 --- a/src/openjarvis/evals/configs/gaia-gemma4-26b.toml +++ b/src/openjarvis/evals/configs/gaia-gemma4-26b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/gaia-gpt54.toml b/src/openjarvis/evals/configs/gaia-gpt54.toml index fc495b92..551c524c 100644 --- a/src/openjarvis/evals/configs/gaia-gpt54.toml +++ b/src/openjarvis/evals/configs/gaia-gpt54.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-nemotron-nano.toml b/src/openjarvis/evals/configs/gaia-nemotron-nano.toml index f5afdffe..affaa020 100644 --- a/src/openjarvis/evals/configs/gaia-nemotron-nano.toml +++ b/src/openjarvis/evals/configs/gaia-nemotron-nano.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-nemotron-super.toml b/src/openjarvis/evals/configs/gaia-nemotron-super.toml index 875e003c..0a13b21d 100644 --- a/src/openjarvis/evals/configs/gaia-nemotron-super.toml +++ b/src/openjarvis/evals/configs/gaia-nemotron-super.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/gaia-nemotron3-super.toml b/src/openjarvis/evals/configs/gaia-nemotron3-super.toml index 3d6d9d56..d6d7d020 100644 --- a/src/openjarvis/evals/configs/gaia-nemotron3-super.toml +++ b/src/openjarvis/evals/configs/gaia-nemotron3-super.toml @@ -8,7 +8,7 @@ temperature = 1.0 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/gaia-qwen-122b.toml b/src/openjarvis/evals/configs/gaia-qwen-122b.toml index 94d5d4b0..559b4116 100644 --- a/src/openjarvis/evals/configs/gaia-qwen-122b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen-122b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/gaia-qwen-27b.toml b/src/openjarvis/evals/configs/gaia-qwen-27b.toml index 0a1c1a13..179d7c9c 100644 --- a/src/openjarvis/evals/configs/gaia-qwen-27b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen-27b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-qwen-2b.toml b/src/openjarvis/evals/configs/gaia-qwen-2b.toml index 359885de..05cb81eb 100644 --- a/src/openjarvis/evals/configs/gaia-qwen-2b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen-2b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/gaia-qwen-35b.toml b/src/openjarvis/evals/configs/gaia-qwen-35b.toml index 9df1a672..9842fb74 100644 --- a/src/openjarvis/evals/configs/gaia-qwen-35b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen-35b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/gaia-qwen-9b.toml b/src/openjarvis/evals/configs/gaia-qwen-9b.toml index 8a3a2c91..ebb252b9 100644 --- a/src/openjarvis/evals/configs/gaia-qwen-9b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen-9b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/gaia-qwen122b.toml b/src/openjarvis/evals/configs/gaia-qwen122b.toml index bb864424..17eeed67 100644 --- a/src/openjarvis/evals/configs/gaia-qwen122b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen122b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-qwen35b.toml b/src/openjarvis/evals/configs/gaia-qwen35b.toml index be8faa4c..194d7d5d 100644 --- a/src/openjarvis/evals/configs/gaia-qwen35b.toml +++ b/src/openjarvis/evals/configs/gaia-qwen35b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml b/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml index bde5c0dc..c13a78e8 100644 --- a/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml +++ b/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml b/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml index c63575e7..91159305 100644 --- a/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml +++ b/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml b/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml index fe1763e0..77dfb8b9 100644 --- a/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml +++ b/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml b/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml index 8cdced92..040d4e75 100644 --- a/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml +++ b/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml b/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml index d1a3cf78..2fc95c2b 100644 --- a/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml +++ b/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml index 286dcef5..f05e2ad7 100644 --- a/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml +++ b/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml index 51c51859..751274fd 100644 --- a/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml +++ b/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml index edcbded5..df10b35f 100644 --- a/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml +++ b/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/gaia-trinity-large.toml b/src/openjarvis/evals/configs/gaia-trinity-large.toml index aecc3ea4..171b9096 100644 --- a/src/openjarvis/evals/configs/gaia-trinity-large.toml +++ b/src/openjarvis/evals/configs/gaia-trinity-large.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/glm-4.7-flash-openhands-remaining.toml b/src/openjarvis/evals/configs/glm-4.7-flash-openhands-remaining.toml index 03f99083..a3cbd0d5 100644 --- a/src/openjarvis/evals/configs/glm-4.7-flash-openhands-remaining.toml +++ b/src/openjarvis/evals/configs/glm-4.7-flash-openhands-remaining.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 2048 diff --git a/src/openjarvis/evals/configs/glm-4.7-flash-openhands.toml b/src/openjarvis/evals/configs/glm-4.7-flash-openhands.toml index 99da4c8c..1128cc7c 100644 --- a/src/openjarvis/evals/configs/glm-4.7-flash-openhands.toml +++ b/src/openjarvis/evals/configs/glm-4.7-flash-openhands.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 1024 diff --git a/src/openjarvis/evals/configs/glm-4.7-fp8-openhands-gaia.toml b/src/openjarvis/evals/configs/glm-4.7-fp8-openhands-gaia.toml index 7acf8c27..b89ee29e 100644 --- a/src/openjarvis/evals/configs/glm-4.7-fp8-openhands-gaia.toml +++ b/src/openjarvis/evals/configs/glm-4.7-fp8-openhands-gaia.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 1024 diff --git a/src/openjarvis/evals/configs/glm5-fp8-monitor-operative.toml b/src/openjarvis/evals/configs/glm5-fp8-monitor-operative.toml index 23443286..237ae524 100644 --- a/src/openjarvis/evals/configs/glm5-fp8-monitor-operative.toml +++ b/src/openjarvis/evals/configs/glm5-fp8-monitor-operative.toml @@ -12,7 +12,7 @@ temperature = 0.6 max_tokens = 32768 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/kimi-k2.5-monitor-operative.toml b/src/openjarvis/evals/configs/kimi-k2.5-monitor-operative.toml index 86395bd6..cec98493 100644 --- a/src/openjarvis/evals/configs/kimi-k2.5-monitor-operative.toml +++ b/src/openjarvis/evals/configs/kimi-k2.5-monitor-operative.toml @@ -12,7 +12,7 @@ temperature = 1.0 # Kimi-K2.5 thinking mode recommended max_tokens = 32768 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/livecodebench-claude-opus.toml b/src/openjarvis/evals/configs/livecodebench-claude-opus.toml index ea2d4032..d8a4134c 100644 --- a/src/openjarvis/evals/configs/livecodebench-claude-opus.toml +++ b/src/openjarvis/evals/configs/livecodebench-claude-opus.toml @@ -11,7 +11,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 1024 diff --git a/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml b/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml index d427f01f..fe6302a7 100644 --- a/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml +++ b/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml b/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml index ff6209fa..78a2f210 100644 --- a/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml +++ b/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml b/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml index 655949b5..dddb06e5 100644 --- a/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml +++ b/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml @@ -17,7 +17,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml b/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml index 73cf2930..1d5d2061 100644 --- a/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml +++ b/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml @@ -14,7 +14,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml b/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml index 6266f922..d2ee1a1e 100644 --- a/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml +++ b/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml b/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml index 30b8697b..293a038b 100644 --- a/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml +++ b/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml @@ -8,7 +8,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml index 3ac0c749..6734c7d3 100644 --- a/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml +++ b/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml @@ -8,7 +8,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml index 357dcf49..319e6c46 100644 --- a/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml +++ b/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml index e4d4236e..32fb78e2 100644 --- a/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml +++ b/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml index 281a5e84..8d7f4366 100644 --- a/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml +++ b/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml @@ -8,7 +8,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-9b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-9b.toml index 2cef93d6..b2e9eb44 100644 --- a/src/openjarvis/evals/configs/livecodebench-qwen-9b.toml +++ b/src/openjarvis/evals/configs/livecodebench-qwen-9b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-claude-opus.toml b/src/openjarvis/evals/configs/liveresearch-claude-opus.toml index e5bc67d2..a16c1e55 100644 --- a/src/openjarvis/evals/configs/liveresearch-claude-opus.toml +++ b/src/openjarvis/evals/configs/liveresearch-claude-opus.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml b/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml index 533bffbe..e002d4e0 100644 --- a/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml +++ b/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml index feaba829..7fc5ca01 100644 --- a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml +++ b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml index da337db1..d5adeecf 100644 --- a/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml +++ b/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-gpt54.toml b/src/openjarvis/evals/configs/liveresearch-gpt54.toml index 904f32e7..99b10a84 100644 --- a/src/openjarvis/evals/configs/liveresearch-gpt54.toml +++ b/src/openjarvis/evals/configs/liveresearch-gpt54.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml index ad3816ca..9323e161 100644 --- a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml +++ b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml index 7a29c69c..5da2cc67 100644 --- a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml +++ b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml b/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml index 17a1a281..49c3f9d9 100644 --- a/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml +++ b/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml b/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml index d1089b36..00e39362 100644 --- a/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml +++ b/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml index 483aa3c2..c1afc4fe 100644 --- a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml +++ b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml index f14956fa..238a6efa 100644 --- a/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml +++ b/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml index 0e316fa2..9928e77f 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml index 808d5156..c302db26 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml index ecf09b6d..51d2b969 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml index 1cdb806e..4ab2eb7e 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml index da96ba7a..3219c65e 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml index e9f26e54..e8106960 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml index e1777cc8..ed81888b 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml index dbcd7b45..8c8d45f9 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml index 483229ee..9bf12e23 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml index 45e3cf7f..3526e9c6 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml index 7c1af65d..f06902d1 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml index 8a802c5c..8aa95c0e 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml index 8da91a75..34972c7a 100644 --- a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml +++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml index fd2e2214..f1beb5e5 100644 --- a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml +++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/liveresearchbench-qwen-27b.toml b/src/openjarvis/evals/configs/liveresearchbench-qwen-27b.toml index be8c8e05..80b1e279 100644 --- a/src/openjarvis/evals/configs/liveresearchbench-qwen-27b.toml +++ b/src/openjarvis/evals/configs/liveresearchbench-qwen-27b.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearchbench-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearchbench-qwen-2b.toml index f4d1df84..5b81c29a 100644 --- a/src/openjarvis/evals/configs/liveresearchbench-qwen-2b.toml +++ b/src/openjarvis/evals/configs/liveresearchbench-qwen-2b.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/liveresearchbench-qwen-9b.toml b/src/openjarvis/evals/configs/liveresearchbench-qwen-9b.toml index 818d257b..3640cfe9 100644 --- a/src/openjarvis/evals/configs/liveresearchbench-qwen-9b.toml +++ b/src/openjarvis/evals/configs/liveresearchbench-qwen-9b.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 16384 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 engine = "cloud" diff --git a/src/openjarvis/evals/configs/long-horizon-claude-opus.toml b/src/openjarvis/evals/configs/long-horizon-claude-opus.toml index b1f6b3aa..f293f123 100644 --- a/src/openjarvis/evals/configs/long-horizon-claude-opus.toml +++ b/src/openjarvis/evals/configs/long-horizon-claude-opus.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/long-horizon-claude-sonnet.toml b/src/openjarvis/evals/configs/long-horizon-claude-sonnet.toml index 67de5a30..eb96d6e7 100644 --- a/src/openjarvis/evals/configs/long-horizon-claude-sonnet.toml +++ b/src/openjarvis/evals/configs/long-horizon-claude-sonnet.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/long-horizon-glm47flash.toml b/src/openjarvis/evals/configs/long-horizon-glm47flash.toml index 5b5c0637..29e5f3fb 100644 --- a/src/openjarvis/evals/configs/long-horizon-glm47flash.toml +++ b/src/openjarvis/evals/configs/long-horizon-glm47flash.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/long-horizon-gpt5mini.toml b/src/openjarvis/evals/configs/long-horizon-gpt5mini.toml index c473ea12..09d3b554 100644 --- a/src/openjarvis/evals/configs/long-horizon-gpt5mini.toml +++ b/src/openjarvis/evals/configs/long-horizon-gpt5mini.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/long-horizon-kimik25.toml b/src/openjarvis/evals/configs/long-horizon-kimik25.toml index ed09e698..559eb26e 100644 --- a/src/openjarvis/evals/configs/long-horizon-kimik25.toml +++ b/src/openjarvis/evals/configs/long-horizon-kimik25.toml @@ -12,7 +12,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/long-horizon-minimax-m25.toml b/src/openjarvis/evals/configs/long-horizon-minimax-m25.toml index 3eeff22b..206008e2 100644 --- a/src/openjarvis/evals/configs/long-horizon-minimax-m25.toml +++ b/src/openjarvis/evals/configs/long-horizon-minimax-m25.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/long-horizon-qwen122b.toml b/src/openjarvis/evals/configs/long-horizon-qwen122b.toml index b4f07407..4a671496 100644 --- a/src/openjarvis/evals/configs/long-horizon-qwen122b.toml +++ b/src/openjarvis/evals/configs/long-horizon-qwen122b.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/long-horizon-qwen35b.toml b/src/openjarvis/evals/configs/long-horizon-qwen35b.toml index 811b385c..e3334a99 100644 --- a/src/openjarvis/evals/configs/long-horizon-qwen35b.toml +++ b/src/openjarvis/evals/configs/long-horizon-qwen35b.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/long-horizon-qwen397b.toml b/src/openjarvis/evals/configs/long-horizon-qwen397b.toml index 236319b4..92b7ebbc 100644 --- a/src/openjarvis/evals/configs/long-horizon-qwen397b.toml +++ b/src/openjarvis/evals/configs/long-horizon-qwen397b.toml @@ -10,7 +10,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/paperarena-monitor-operative.toml b/src/openjarvis/evals/configs/paperarena-monitor-operative.toml index 45f5b781..039b7c19 100644 --- a/src/openjarvis/evals/configs/paperarena-monitor-operative.toml +++ b/src/openjarvis/evals/configs/paperarena-monitor-operative.toml @@ -11,7 +11,7 @@ temperature = 0.6 max_tokens = 32768 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 1024 diff --git a/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml b/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml index c66090ea..9950983e 100644 --- a/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml +++ b/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml index 5a91cc93..c1d14811 100644 --- a/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml +++ b/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml index 1f2188ee..a6d0861a 100644 --- a/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml +++ b/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/qwen3.5-122b-kg-agentic.toml b/src/openjarvis/evals/configs/qwen3.5-122b-kg-agentic.toml index 84942507..92a91fb3 100644 --- a/src/openjarvis/evals/configs/qwen3.5-122b-kg-agentic.toml +++ b/src/openjarvis/evals/configs/qwen3.5-122b-kg-agentic.toml @@ -11,7 +11,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/qwen3.5-122b-kg-hightoken.toml b/src/openjarvis/evals/configs/qwen3.5-122b-kg-hightoken.toml index 48a4d7f9..e5f74577 100644 --- a/src/openjarvis/evals/configs/qwen3.5-122b-kg-hightoken.toml +++ b/src/openjarvis/evals/configs/qwen3.5-122b-kg-hightoken.toml @@ -11,7 +11,7 @@ temperature = 0.6 max_tokens = 8192 # 8x increase from baseline 1024 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/qwen3.5-122b-reasoning-suite.toml b/src/openjarvis/evals/configs/qwen3.5-122b-reasoning-suite.toml index d4154599..defa9fa1 100644 --- a/src/openjarvis/evals/configs/qwen3.5-122b-reasoning-suite.toml +++ b/src/openjarvis/evals/configs/qwen3.5-122b-reasoning-suite.toml @@ -10,7 +10,7 @@ temperature = 0.6 # Qwen3.5 recommended max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/qwen3.5-35b-kg.toml b/src/openjarvis/evals/configs/qwen3.5-35b-kg.toml index fa0b325e..6ad872e1 100644 --- a/src/openjarvis/evals/configs/qwen3.5-35b-kg.toml +++ b/src/openjarvis/evals/configs/qwen3.5-35b-kg.toml @@ -11,7 +11,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/qwen3.5-35b-monitor-operative.toml b/src/openjarvis/evals/configs/qwen3.5-35b-monitor-operative.toml index bd7085bd..47ae5fa8 100644 --- a/src/openjarvis/evals/configs/qwen3.5-35b-monitor-operative.toml +++ b/src/openjarvis/evals/configs/qwen3.5-35b-monitor-operative.toml @@ -14,7 +14,7 @@ temperature = 0.6 # Qwen3.5 thinking mode (precise analysis) max_tokens = 32768 # Qwen3.5 recommended for general queries [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 1024 diff --git a/src/openjarvis/evals/configs/qwen3.5-397b-monitor-operative.toml b/src/openjarvis/evals/configs/qwen3.5-397b-monitor-operative.toml index 22781617..ad6ae21e 100644 --- a/src/openjarvis/evals/configs/qwen3.5-397b-monitor-operative.toml +++ b/src/openjarvis/evals/configs/qwen3.5-397b-monitor-operative.toml @@ -14,7 +14,7 @@ temperature = 0.6 # Qwen3.5 thinking mode recommended max_tokens = 1024 # Keep short to prevent hangs; answers are brief [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/smoke_amd.toml b/src/openjarvis/evals/configs/smoke_amd.toml index 5021f693..88d00ab0 100644 --- a/src/openjarvis/evals/configs/smoke_amd.toml +++ b/src/openjarvis/evals/configs/smoke_amd.toml @@ -14,7 +14,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 512 diff --git a/src/openjarvis/evals/configs/smoke_apple.toml b/src/openjarvis/evals/configs/smoke_apple.toml index a058577c..bb5c7117 100644 --- a/src/openjarvis/evals/configs/smoke_apple.toml +++ b/src/openjarvis/evals/configs/smoke_apple.toml @@ -13,7 +13,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 512 diff --git a/src/openjarvis/evals/configs/smoke_nvidia.toml b/src/openjarvis/evals/configs/smoke_nvidia.toml index ef6b83c8..d0c69d53 100644 --- a/src/openjarvis/evals/configs/smoke_nvidia.toml +++ b/src/openjarvis/evals/configs/smoke_nvidia.toml @@ -13,7 +13,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 512 diff --git a/src/openjarvis/evals/configs/taubench-claude-opus.toml b/src/openjarvis/evals/configs/taubench-claude-opus.toml index d27f8f30..0549bb6a 100644 --- a/src/openjarvis/evals/configs/taubench-claude-opus.toml +++ b/src/openjarvis/evals/configs/taubench-claude-opus.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml b/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml index 4a089e67..413a1eda 100644 --- a/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml +++ b/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-gemma4-26b.toml b/src/openjarvis/evals/configs/taubench-gemma4-26b.toml index 27fda44b..9ec742c9 100644 --- a/src/openjarvis/evals/configs/taubench-gemma4-26b.toml +++ b/src/openjarvis/evals/configs/taubench-gemma4-26b.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml b/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml index 8dac58dd..2bf93e05 100644 --- a/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml +++ b/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-gpt54.toml b/src/openjarvis/evals/configs/taubench-gpt54.toml index f2b3ef7d..798bb613 100644 --- a/src/openjarvis/evals/configs/taubench-gpt54.toml +++ b/src/openjarvis/evals/configs/taubench-gpt54.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-kimi-k25.toml b/src/openjarvis/evals/configs/taubench-kimi-k25.toml index 639f3f05..0a60cc6e 100644 --- a/src/openjarvis/evals/configs/taubench-kimi-k25.toml +++ b/src/openjarvis/evals/configs/taubench-kimi-k25.toml @@ -17,7 +17,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-minimax-m25.toml b/src/openjarvis/evals/configs/taubench-minimax-m25.toml index c962efd8..98e48b04 100644 --- a/src/openjarvis/evals/configs/taubench-minimax-m25.toml +++ b/src/openjarvis/evals/configs/taubench-minimax-m25.toml @@ -14,7 +14,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-nemotron-nano.toml b/src/openjarvis/evals/configs/taubench-nemotron-nano.toml index 7dff2f08..cec26c1b 100644 --- a/src/openjarvis/evals/configs/taubench-nemotron-nano.toml +++ b/src/openjarvis/evals/configs/taubench-nemotron-nano.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-nemotron-super.toml b/src/openjarvis/evals/configs/taubench-nemotron-super.toml index f7f5304b..0fb04706 100644 --- a/src/openjarvis/evals/configs/taubench-nemotron-super.toml +++ b/src/openjarvis/evals/configs/taubench-nemotron-super.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-nemotron3-super.toml b/src/openjarvis/evals/configs/taubench-nemotron3-super.toml index 2afff9a7..d1b80c2a 100644 --- a/src/openjarvis/evals/configs/taubench-nemotron3-super.toml +++ b/src/openjarvis/evals/configs/taubench-nemotron3-super.toml @@ -8,7 +8,7 @@ temperature = 1.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-qwen-122b.toml b/src/openjarvis/evals/configs/taubench-qwen-122b.toml index a0f4329a..c84fc3e4 100644 --- a/src/openjarvis/evals/configs/taubench-qwen-122b.toml +++ b/src/openjarvis/evals/configs/taubench-qwen-122b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-qwen-27b.toml b/src/openjarvis/evals/configs/taubench-qwen-27b.toml index 13ce0c65..dffa7258 100644 --- a/src/openjarvis/evals/configs/taubench-qwen-27b.toml +++ b/src/openjarvis/evals/configs/taubench-qwen-27b.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-qwen-2b.toml b/src/openjarvis/evals/configs/taubench-qwen-2b.toml index e9735ff6..a113a0be 100644 --- a/src/openjarvis/evals/configs/taubench-qwen-2b.toml +++ b/src/openjarvis/evals/configs/taubench-qwen-2b.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-qwen-35b.toml b/src/openjarvis/evals/configs/taubench-qwen-35b.toml index d902364b..a1071c7c 100644 --- a/src/openjarvis/evals/configs/taubench-qwen-35b.toml +++ b/src/openjarvis/evals/configs/taubench-qwen-35b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-qwen-9b.toml b/src/openjarvis/evals/configs/taubench-qwen-9b.toml index d076895f..227d1061 100644 --- a/src/openjarvis/evals/configs/taubench-qwen-9b.toml +++ b/src/openjarvis/evals/configs/taubench-qwen-9b.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-qwen122b.toml b/src/openjarvis/evals/configs/taubench-qwen122b.toml index e403a462..fd9c17d4 100644 --- a/src/openjarvis/evals/configs/taubench-qwen122b.toml +++ b/src/openjarvis/evals/configs/taubench-qwen122b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-qwen35b.toml b/src/openjarvis/evals/configs/taubench-qwen35b.toml index cb779851..65c5a4b9 100644 --- a/src/openjarvis/evals/configs/taubench-qwen35b.toml +++ b/src/openjarvis/evals/configs/taubench-qwen35b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-qwen397b.toml b/src/openjarvis/evals/configs/taubench-qwen397b.toml index 3989aab5..73224b11 100644 --- a/src/openjarvis/evals/configs/taubench-qwen397b.toml +++ b/src/openjarvis/evals/configs/taubench-qwen397b.toml @@ -10,7 +10,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-claude.toml b/src/openjarvis/evals/configs/taubench-telecom-claude.toml index 2710b225..81bc2aa6 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-claude.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-claude.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml b/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml index 380ab271..c9ce1667 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml b/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml index 1dbb3f3a..7cfa91dc 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml index 332c0061..d25e41c2 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml index 26cffbd2..47bd9cda 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml index d71b37a2..3f24153d 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml @@ -8,7 +8,7 @@ temperature = 1.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml index 849fb30e..dea5c6a3 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml index 18b8b518..473d2b5d 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml index ccf5c8f6..15f132e1 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml index c4a2de6f..bf51978f 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml index 0c549d81..ceac8e7e 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml index 85d8691e..217903b2 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml b/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml index 55487f12..5d32ba13 100644 --- a/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml +++ b/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml @@ -8,7 +8,7 @@ temperature = 0.7 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml b/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml index c8dd0cea..e4c978b3 100644 --- a/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml +++ b/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/terminalbench-qwen122b.toml b/src/openjarvis/evals/configs/terminalbench-qwen122b.toml index 7566c045..a3c770a6 100644 --- a/src/openjarvis/evals/configs/terminalbench-qwen122b.toml +++ b/src/openjarvis/evals/configs/terminalbench-qwen122b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 8192 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 max_tokens = 4096 diff --git a/src/openjarvis/evals/configs/toolcall15-claude-opus.toml b/src/openjarvis/evals/configs/toolcall15-claude-opus.toml index fea53342..58ccfb9b 100644 --- a/src/openjarvis/evals/configs/toolcall15-claude-opus.toml +++ b/src/openjarvis/evals/configs/toolcall15-claude-opus.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml b/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml index 12470e25..a1609ecb 100644 --- a/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml +++ b/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml b/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml index 231bed02..14fc9cd5 100644 --- a/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml +++ b/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml b/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml index 2f0596bc..b4c93f8d 100644 --- a/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml +++ b/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml @@ -17,7 +17,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml b/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml index 6d087419..a0bd3bd6 100644 --- a/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml +++ b/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml @@ -14,7 +14,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml b/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml index 453f30e9..ae1f5412 100644 --- a/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml +++ b/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml b/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml index 9b3fba0f..421a5c02 100644 --- a/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml +++ b/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml index b88527e4..5e5f4298 100644 --- a/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml +++ b/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml index 56879653..25b0ded4 100644 --- a/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml +++ b/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml index 4d40fe42..f3ff1d4d 100644 --- a/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml +++ b/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml index 7dd7ccee..5ad8c2f1 100644 --- a/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml +++ b/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml @@ -8,7 +8,7 @@ temperature = 0.6 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-9b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-9b.toml index f30fa2b5..9cf55154 100644 --- a/src/openjarvis/evals/configs/toolcall15-qwen-9b.toml +++ b/src/openjarvis/evals/configs/toolcall15-qwen-9b.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" temperature = 0.0 engine = "cloud" diff --git a/src/openjarvis/evals/configs/use_case_cloud_comparison.toml b/src/openjarvis/evals/configs/use_case_cloud_comparison.toml index 08ffbbd1..960d8c97 100644 --- a/src/openjarvis/evals/configs/use_case_cloud_comparison.toml +++ b/src/openjarvis/evals/configs/use_case_cloud_comparison.toml @@ -14,7 +14,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_cloud_openhands.toml b/src/openjarvis/evals/configs/use_case_cloud_openhands.toml index 34c275aa..234fcc62 100644 --- a/src/openjarvis/evals/configs/use_case_cloud_openhands.toml +++ b/src/openjarvis/evals/configs/use_case_cloud_openhands.toml @@ -15,7 +15,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml b/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml index 6f535bb7..686c798e 100644 --- a/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml +++ b/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml @@ -15,7 +15,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_direct.toml b/src/openjarvis/evals/configs/use_case_direct.toml index 81d1c248..655daebb 100644 --- a/src/openjarvis/evals/configs/use_case_direct.toml +++ b/src/openjarvis/evals/configs/use_case_direct.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_ibm_granite.toml b/src/openjarvis/evals/configs/use_case_ibm_granite.toml index 4d8fbee7..c7d31c73 100644 --- a/src/openjarvis/evals/configs/use_case_ibm_granite.toml +++ b/src/openjarvis/evals/configs/use_case_ibm_granite.toml @@ -11,7 +11,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_openhands.toml b/src/openjarvis/evals/configs/use_case_openhands.toml index b66c507f..1c305bda 100644 --- a/src/openjarvis/evals/configs/use_case_openhands.toml +++ b/src/openjarvis/evals/configs/use_case_openhands.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_opensource_openhands.toml b/src/openjarvis/evals/configs/use_case_opensource_openhands.toml index e4f193dd..5afb8ab2 100644 --- a/src/openjarvis/evals/configs/use_case_opensource_openhands.toml +++ b/src/openjarvis/evals/configs/use_case_opensource_openhands.toml @@ -19,7 +19,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml b/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml index ddee471e..feeed7e4 100644 --- a/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml +++ b/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml @@ -19,7 +19,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_orchestrator.toml b/src/openjarvis/evals/configs/use_case_orchestrator.toml index 40257d4d..0f59f82d 100644 --- a/src/openjarvis/evals/configs/use_case_orchestrator.toml +++ b/src/openjarvis/evals/configs/use_case_orchestrator.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_react.toml b/src/openjarvis/evals/configs/use_case_react.toml index 08741fbc..7edbd9d5 100644 --- a/src/openjarvis/evals/configs/use_case_react.toml +++ b/src/openjarvis/evals/configs/use_case_react.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_suite.toml b/src/openjarvis/evals/configs/use_case_suite.toml index a4d2cc7a..c59d5a46 100644 --- a/src/openjarvis/evals/configs/use_case_suite.toml +++ b/src/openjarvis/evals/configs/use_case_suite.toml @@ -11,7 +11,7 @@ temperature = 0.0 max_tokens = 2048 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 4 diff --git a/src/openjarvis/evals/configs/use_case_v2_suite.toml b/src/openjarvis/evals/configs/use_case_v2_suite.toml index d706342b..beaa31b9 100644 --- a/src/openjarvis/evals/configs/use_case_v2_suite.toml +++ b/src/openjarvis/evals/configs/use_case_v2_suite.toml @@ -10,7 +10,7 @@ temperature = 0.0 max_tokens = 4096 [judge] -model = "gpt-5-mini-2025-08-07" +model = "claude-haiku-4-5-20251001" [run] max_workers = 2 diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index aeb16578..4f96a008 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -118,7 +118,7 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig: # Parse [judge] judge_raw = raw.get("judge", {}) judge = JudgeConfig( - model=judge_raw.get("model", "gpt-5-mini-2025-08-07"), + model=judge_raw.get("model", "claude-haiku-4-5-20251001"), engine=judge_raw.get("engine"), provider=judge_raw.get("provider"), temperature=float(judge_raw.get("temperature", 0.0)), diff --git a/src/openjarvis/evals/core/types.py b/src/openjarvis/evals/core/types.py index 0050a6b7..13ad5274 100644 --- a/src/openjarvis/evals/core/types.py +++ b/src/openjarvis/evals/core/types.py @@ -67,7 +67,7 @@ class RunConfig: max_workers: int = 4 temperature: float = 0.0 max_tokens: int = 2048 - judge_model: str = "gpt-5-mini-2025-08-07" + judge_model: str = "claude-haiku-4-5-20251001" judge_engine: str = "cloud" engine_key: Optional[str] = None agent_name: Optional[str] = None @@ -203,7 +203,7 @@ class DefaultsConfig: class JudgeConfig: """Configuration for the LLM judge.""" - model: str = "gpt-5-mini-2025-08-07" + model: str = "claude-haiku-4-5-20251001" engine: Optional[str] = None provider: Optional[str] = None temperature: float = 0.0 diff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.py index 320e17b2..c5fd6512 100644 --- a/src/openjarvis/evals/execution/taubench_env.py +++ b/src/openjarvis/evals/execution/taubench_env.py @@ -264,7 +264,7 @@ class TauBenchTaskEnv: self._model = model or "claude-opus-4-6" self._temperature = temperature self._max_tokens = max_tokens - self._user_model = user_model or "gpt-5-mini-2025-08-07" + self._user_model = user_model or "claude-haiku-4-5-20251001" self._telemetry = telemetry self._gpu_metrics = gpu_metrics self._system = None diff --git a/src/openjarvis/evals/tests/test_config.py b/src/openjarvis/evals/tests/test_config.py index 324297a2..6dd2f609 100644 --- a/src/openjarvis/evals/tests/test_config.py +++ b/src/openjarvis/evals/tests/test_config.py @@ -49,7 +49,7 @@ class TestDataclassDefaults: def test_judge_config_defaults(self): j = JudgeConfig() - assert j.model == "gpt-5-mini-2025-08-07" + assert j.model == "claude-haiku-4-5-20251001" assert j.provider is None assert j.temperature == 0.0 assert j.max_tokens == 1024 @@ -87,7 +87,7 @@ class TestDataclassDefaults: s = EvalSuiteConfig() assert s.meta.name == "" assert s.defaults.temperature == 0.0 - assert s.judge.model == "gpt-5-mini-2025-08-07" + assert s.judge.model == "claude-haiku-4-5-20251001" assert s.run.max_workers == 4 assert s.models == [] assert s.benchmarks == [] @@ -117,7 +117,7 @@ class TestLoadEvalConfig: assert suite.benchmarks[0].name == "supergpqa" # Defaults should be applied assert suite.defaults.temperature == 0.0 - assert suite.judge.model == "gpt-5-mini-2025-08-07" + assert suite.judge.model == "claude-haiku-4-5-20251001" def test_full_config(self, tmp_path): p = _write_toml( diff --git a/src/openjarvis/evals/tests/test_types.py b/src/openjarvis/evals/tests/test_types.py index 7aa88b98..fc7d55bf 100644 --- a/src/openjarvis/evals/tests/test_types.py +++ b/src/openjarvis/evals/tests/test_types.py @@ -113,7 +113,7 @@ class TestRunConfig: assert c.max_workers == 4 assert c.temperature == 0.0 assert c.max_tokens == 2048 - assert c.judge_model == "gpt-5-mini-2025-08-07" + assert c.judge_model == "claude-haiku-4-5-20251001" assert c.seed == 42 assert c.tools == [] assert c.telemetry is False @@ -312,7 +312,7 @@ class TestDefaultsConfig: class TestJudgeConfig: def test_defaults(self): j = JudgeConfig() - assert j.model == "gpt-5-mini-2025-08-07" + assert j.model == "claude-haiku-4-5-20251001" assert j.provider is None assert j.temperature == 0.0 assert j.max_tokens == 1024 diff --git a/src/openjarvis/learning/optimize/trial_runner.py b/src/openjarvis/learning/optimize/trial_runner.py index 19588062..c1240cb3 100644 --- a/src/openjarvis/learning/optimize/trial_runner.py +++ b/src/openjarvis/learning/optimize/trial_runner.py @@ -39,7 +39,7 @@ class TrialRunner: self, benchmark: str, max_samples: int = 50, - judge_model: str = "gpt-5-mini-2025-08-07", + judge_model: str = "claude-haiku-4-5-20251001", output_dir: str = "results/optimize/", ) -> None: self.benchmark = benchmark @@ -203,7 +203,7 @@ class MultiBenchTrialRunner: def __init__( self, benchmark_specs: List[BenchmarkSpec], - judge_model: str = "gpt-5-mini-2025-08-07", + judge_model: str = "claude-haiku-4-5-20251001", output_dir: str = "results/optimize/", ) -> None: self.benchmark_specs = benchmark_specs diff --git a/src/openjarvis/recipes/composer.py b/src/openjarvis/recipes/composer.py index 2983afa4..46438166 100644 --- a/src/openjarvis/recipes/composer.py +++ b/src/openjarvis/recipes/composer.py @@ -96,7 +96,7 @@ def recipe_to_eval_suite( max_tokens=2048, ), judge=JudgeConfig( - model=judge_model or recipe.eval_judge_model or "gpt-5-mini-2025-08-07", + model=judge_model or recipe.eval_judge_model or "claude-haiku-4-5-20251001", ), run=ExecutionConfig(), models=[model_cfg], diff --git a/tests/learning/test_optimize_cmd.py b/tests/learning/test_optimize_cmd.py index 3b5025e8..eaa55a1e 100644 --- a/tests/learning/test_optimize_cmd.py +++ b/tests/learning/test_optimize_cmd.py @@ -113,7 +113,7 @@ class TestOptimizeConfig: assert cfg.max_trials == 20 assert cfg.early_stop_patience == 5 assert cfg.optimizer_model == "claude-sonnet-4-6" - assert cfg.judge_model == "gpt-5-mini-2025-08-07" + assert cfg.judge_model == "claude-haiku-4-5-20251001" class TestNewEventTypes: diff --git a/tests/learning/test_trial_runner.py b/tests/learning/test_trial_runner.py index 93d73596..02daee15 100644 --- a/tests/learning/test_trial_runner.py +++ b/tests/learning/test_trial_runner.py @@ -16,7 +16,7 @@ class TestTrialRunnerInit: runner = TrialRunner(benchmark="supergpqa") assert runner.benchmark == "supergpqa" assert runner.max_samples == 50 - assert runner.judge_model == "gpt-5-mini-2025-08-07" + assert runner.judge_model == "claude-haiku-4-5-20251001" assert runner.output_dir == "results/optimize/" def test_custom_params(self) -> None: