From 3677dbab6c7bf786df4f0048cc547a19c82d1849 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon Date: Fri, 6 Mar 2026 23:28:57 +0000 Subject: [PATCH] style: fix all Ruff lint errors in src/ Shorten docstrings in backward-compat shims to fix E501 (line too long). Sort imports in learning/__init__.py. Zero Ruff violations remaining. Co-Authored-By: Claude Opus 4.6 --- src/openjarvis/engine/_openai_compat.py | 1 + src/openjarvis/engine/cloud.py | 84 ++++++- .../deepplanning-monitor-operative.toml | 42 ++++ .../configs/glm5-fp8-monitor-operative.toml | 74 ++++++ .../configs/kimi-k2.5-monitor-operative.toml | 74 ++++++ .../configs/paperarena-monitor-operative.toml | 42 ++++ .../qwen3.5-35b-monitor-operative.toml | 57 +++++ .../qwen3.5-397b-monitor-operative.toml | 50 ++++ .../configs/use_case_cloud_openhands.toml | 6 + .../configs/use_case_cloud_orchestrator.toml | 6 + .../use_case_opensource_openhands.toml | 2 +- .../use_case_opensource_orchestrator.toml | 2 +- src/openjarvis/evals/datasets/deepplanning.py | 238 ++++++++++++++++++ .../evals/datasets/lifelong_agent.py | 238 +++++++++++++++++- src/openjarvis/evals/datasets/paperarena.py | 170 +++++++++++++ .../evals/scorers/deepplanning_scorer.py | 87 +++++++ .../evals/scorers/paperarena_judge.py | 153 +++++++++++ src/openjarvis/learning/__init__.py | 6 +- src/openjarvis/learning/optimize/__init__.py | 10 +- src/openjarvis/optimize/feedback/__init__.py | 2 +- src/openjarvis/optimize/feedback/collector.py | 2 +- src/openjarvis/optimize/feedback/judge.py | 8 +- src/openjarvis/optimize/llm_optimizer.py | 2 +- src/openjarvis/optimize/optimizer.py | 2 +- src/openjarvis/optimize/personal/__init__.py | 2 +- src/openjarvis/optimize/personal/dataset.py | 2 +- src/openjarvis/optimize/personal/scorer.py | 2 +- .../optimize/personal/synthesizer.py | 2 +- src/openjarvis/optimize/search_space.py | 2 +- src/openjarvis/optimize/trial_runner.py | 2 +- src/openjarvis/optimize/types.py | 6 +- 31 files changed, 1339 insertions(+), 37 deletions(-) create mode 100644 src/openjarvis/evals/configs/deepplanning-monitor-operative.toml create mode 100644 src/openjarvis/evals/configs/glm5-fp8-monitor-operative.toml create mode 100644 src/openjarvis/evals/configs/kimi-k2.5-monitor-operative.toml create mode 100644 src/openjarvis/evals/configs/paperarena-monitor-operative.toml create mode 100644 src/openjarvis/evals/configs/qwen3.5-35b-monitor-operative.toml create mode 100644 src/openjarvis/evals/configs/qwen3.5-397b-monitor-operative.toml create mode 100644 src/openjarvis/evals/datasets/deepplanning.py create mode 100644 src/openjarvis/evals/datasets/paperarena.py create mode 100644 src/openjarvis/evals/scorers/deepplanning_scorer.py create mode 100644 src/openjarvis/evals/scorers/paperarena_judge.py diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.py index 0c60d0ca..f20eabc8 100644 --- a/src/openjarvis/engine/_openai_compat.py +++ b/src/openjarvis/engine/_openai_compat.py @@ -43,6 +43,7 @@ class _OpenAICompatibleEngine(InferenceEngine): "temperature": temperature, "max_tokens": max_tokens, "stream": False, + "chat_template_kwargs": {"enable_thinking": False}, **kwargs, } try: diff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.py index 35b81711..a1bd82bb 100644 --- a/src/openjarvis/engine/cloud.py +++ b/src/openjarvis/engine/cloud.py @@ -230,12 +230,54 @@ class CloudEngine(InferenceEngine): "ANTHROPIC_API_KEY and install " "openjarvis[inference-cloud]" ) - # Separate system message from conversation messages + # Separate system message and convert to Anthropic message format system_text = "" chat_msgs: List[Dict[str, Any]] = [] for m in messages: if m.role.value == "system": system_text = m.content + elif m.role.value == "tool": + # Anthropic expects tool results as role="user" with + # tool_result content blocks + tool_result_block = { + "type": "tool_result", + "tool_use_id": m.tool_call_id or "", + "content": m.content, + } + # Merge consecutive tool results into a single user message + if ( + chat_msgs + and chat_msgs[-1]["role"] == "user" + and isinstance(chat_msgs[-1]["content"], list) + and chat_msgs[-1]["content"] + and chat_msgs[-1]["content"][-1].get("type") == "tool_result" + ): + chat_msgs[-1]["content"].append(tool_result_block) + else: + chat_msgs.append({ + "role": "user", + "content": [tool_result_block], + }) + elif m.role.value == "assistant" and m.tool_calls: + # Convert assistant messages with tool_calls to Anthropic + # content blocks (text + tool_use) + content_blocks: List[Dict[str, Any]] = [] + if m.content: + content_blocks.append({"type": "text", "text": m.content}) + for tc in m.tool_calls: + args = tc.arguments + if isinstance(args, str): + try: + args = json.loads(args) + except (json.JSONDecodeError, TypeError): + args = {"input": args} + content_blocks.append({ + "type": "tool_use", + "id": tc.id, + "name": tc.name, + "input": args if isinstance(args, dict) else {}, + }) + chat_msgs.append({"role": "assistant", "content": content_blocks}) else: chat_msgs.append({"role": m.role.value, "content": m.content}) create_kwargs: Dict[str, Any] = { @@ -308,12 +350,50 @@ class CloudEngine(InferenceEngine): "GEMINI_API_KEY or GOOGLE_API_KEY and install " "openjarvis[inference-google]" ) - # Build contents from messages + # Build contents from messages, converting tool roles for Gemini system_text = "" contents: List[Dict[str, Any]] = [] for m in messages: if m.role.value == "system": system_text = m.content + elif m.role.value == "tool": + # Gemini expects function responses as role="user" with + # function_response parts + fn_resp_part = { + "function_response": { + "name": m.name or "unknown", + "response": {"result": m.content}, + } + } + # Merge consecutive tool results into a single user message + if ( + contents + and contents[-1]["role"] == "user" + and contents[-1]["parts"] + and "function_response" in contents[-1]["parts"][-1] + ): + contents[-1]["parts"].append(fn_resp_part) + else: + contents.append({"role": "user", "parts": [fn_resp_part]}) + elif m.role.value == "assistant" and m.tool_calls: + # Convert assistant tool_calls to function_call parts + parts: List[Dict[str, Any]] = [] + if m.content: + parts.append({"text": m.content}) + for tc in m.tool_calls: + args = tc.arguments + if isinstance(args, str): + try: + args = json.loads(args) + except (json.JSONDecodeError, TypeError): + args = {"input": args} + parts.append({ + "function_call": { + "name": tc.name, + "args": args if isinstance(args, dict) else {}, + } + }) + contents.append({"role": "model", "parts": parts}) elif m.role.value == "assistant": contents.append({"role": "model", "parts": [{"text": m.content}]}) else: diff --git a/src/openjarvis/evals/configs/deepplanning-monitor-operative.toml b/src/openjarvis/evals/configs/deepplanning-monitor-operative.toml new file mode 100644 index 00000000..ac62fc08 --- /dev/null +++ b/src/openjarvis/evals/configs/deepplanning-monitor-operative.toml @@ -0,0 +1,42 @@ +# DeepPlanning evaluation with MonitorOperativeAgent +# Benchmark: long-horizon planning with hard constraints (travel + shopping) +# Paper: arXiv:2601.18137, best frontier: GPT-5.2-high at 44.6% + +[meta] +name = "deepplanning-monitor-operative" +description = "DeepPlanning constraint satisfaction with MonitorOperativeAgent" + +[defaults] +temperature = 0.6 +max_tokens = 32768 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 1024 + +[run] +max_workers = 1 +output_dir = "results/deepplanning-monitor-operative/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B" +engine = "vllm" +param_count_b = 35.0 +active_params_b = 3.0 +gpu_peak_tflops = 989.5 +gpu_peak_bandwidth_gb_s = 3350.0 +num_gpus = 2 + +[[benchmarks]] +name = "deepplanning" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = [ + "think", "calculator", "code_interpreter", + "shell_exec", "memory_store", "memory_search", +] diff --git a/src/openjarvis/evals/configs/glm5-fp8-monitor-operative.toml b/src/openjarvis/evals/configs/glm5-fp8-monitor-operative.toml new file mode 100644 index 00000000..23443286 --- /dev/null +++ b/src/openjarvis/evals/configs/glm5-fp8-monitor-operative.toml @@ -0,0 +1,74 @@ +# MonitorOperativeAgent evaluation on long-horizon benchmarks +# Machine: 8x H100-80GB, engine: vLLM, model: GLM-5 (Q4_K_M GGUF) +# Inference: GLM-5 reasoning mode +# Benchmarks: LogHub, LifelongAgentBench, DeepPlanning + +[meta] +name = "glm5-q4km-monitor-operative" +description = "Evaluate MonitorOperativeAgent with GLM-5 Q4_K_M on long-horizon monitoring benchmarks" + +[defaults] +temperature = 0.6 +max_tokens = 32768 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/glm5-q4km-monitor-operative/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "GLM-5-Q4_K_M" +engine = "vllm" +param_count_b = 744.0 +active_params_b = 40.0 +gpu_peak_tflops = 989.5 +gpu_peak_bandwidth_gb_s = 3350.0 +num_gpus = 8 + +# --- Benchmarks --- + +[[benchmarks]] +name = "loghub" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 250 +tools = [ + "think", "calculator", "code_interpreter", + "shell_exec", "file_read", + "memory_store", "memory_search", "memory_retrieve", + "kg_add_entity", "kg_add_relation", "kg_query", "kg_neighbors", + "db_query", "web_search", "http_request", +] + +[[benchmarks]] +name = "lifelong-agent" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 250 +tools = [ + "think", "calculator", "code_interpreter", + "shell_exec", "file_read", + "memory_store", "memory_search", "memory_retrieve", + "kg_add_entity", "kg_add_relation", "kg_query", "kg_neighbors", + "db_query", "web_search", "http_request", +] + +[[benchmarks]] +name = "deepplanning" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 250 +tools = [ + "think", "calculator", "code_interpreter", + "shell_exec", "file_read", + "memory_store", "memory_search", "memory_retrieve", + "kg_add_entity", "kg_add_relation", "kg_query", "kg_neighbors", + "db_query", "web_search", "http_request", +] diff --git a/src/openjarvis/evals/configs/kimi-k2.5-monitor-operative.toml b/src/openjarvis/evals/configs/kimi-k2.5-monitor-operative.toml new file mode 100644 index 00000000..86395bd6 --- /dev/null +++ b/src/openjarvis/evals/configs/kimi-k2.5-monitor-operative.toml @@ -0,0 +1,74 @@ +# MonitorOperativeAgent evaluation on long-horizon benchmarks +# Machine: 8x H100-80GB, engine: vLLM, model: Kimi-K2.5 (IQ4_XS GGUF) +# Inference: Kimi-K2.5 thinking mode (temperature=1.0, top_p=0.95) +# Benchmarks: LogHub, LifelongAgentBench, DeepPlanning + +[meta] +name = "kimi-k2.5-monitor-operative" +description = "Evaluate MonitorOperativeAgent with Kimi-K2.5 on long-horizon monitoring benchmarks" + +[defaults] +temperature = 1.0 # Kimi-K2.5 thinking mode recommended +max_tokens = 32768 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 +output_dir = "results/kimi-k2.5-monitor-operative/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "Kimi-K2.5-IQ4_XS" +engine = "vllm" +param_count_b = 1000.0 +active_params_b = 32.0 +gpu_peak_tflops = 989.5 +gpu_peak_bandwidth_gb_s = 3350.0 +num_gpus = 8 + +# --- Benchmarks --- + +[[benchmarks]] +name = "loghub" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 250 +tools = [ + "think", "calculator", "code_interpreter", + "shell_exec", "file_read", + "memory_store", "memory_search", "memory_retrieve", + "kg_add_entity", "kg_add_relation", "kg_query", "kg_neighbors", + "db_query", "web_search", "http_request", +] + +[[benchmarks]] +name = "lifelong-agent" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 250 +tools = [ + "think", "calculator", "code_interpreter", + "shell_exec", "file_read", + "memory_store", "memory_search", "memory_retrieve", + "kg_add_entity", "kg_add_relation", "kg_query", "kg_neighbors", + "db_query", "web_search", "http_request", +] + +[[benchmarks]] +name = "deepplanning" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 250 +tools = [ + "think", "calculator", "code_interpreter", + "shell_exec", "file_read", + "memory_store", "memory_search", "memory_retrieve", + "kg_add_entity", "kg_add_relation", "kg_query", "kg_neighbors", + "db_query", "web_search", "http_request", +] diff --git a/src/openjarvis/evals/configs/paperarena-monitor-operative.toml b/src/openjarvis/evals/configs/paperarena-monitor-operative.toml new file mode 100644 index 00000000..45f5b781 --- /dev/null +++ b/src/openjarvis/evals/configs/paperarena-monitor-operative.toml @@ -0,0 +1,42 @@ +# PaperArena evaluation with MonitorOperativeAgent +# Benchmark: scientific literature reasoning (MC/CA/OA) +# Paper: arXiv:2510.10909, best frontier: Gemini 2.5 Pro at 38.8% + +[meta] +name = "paperarena-monitor-operative" +description = "PaperArena scientific reasoning with MonitorOperativeAgent" + +[defaults] +temperature = 0.6 +max_tokens = 32768 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 1024 + +[run] +max_workers = 1 +output_dir = "results/paperarena-monitor-operative/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B" +engine = "vllm" +param_count_b = 35.0 +active_params_b = 3.0 +gpu_peak_tflops = 989.5 +gpu_peak_bandwidth_gb_s = 3350.0 +num_gpus = 2 + +[[benchmarks]] +name = "paperarena" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = [ + "think", "calculator", "code_interpreter", + "web_search", "file_read", "memory_store", "memory_search", +] diff --git a/src/openjarvis/evals/configs/qwen3.5-35b-monitor-operative.toml b/src/openjarvis/evals/configs/qwen3.5-35b-monitor-operative.toml new file mode 100644 index 00000000..bd7085bd --- /dev/null +++ b/src/openjarvis/evals/configs/qwen3.5-35b-monitor-operative.toml @@ -0,0 +1,57 @@ +# MonitorOperativeAgent evaluation on Phase 25 text-to-text benchmarks +# Machine: 2x H100-80GB (GPUs 6-7), engine: vLLM, model: Qwen3.5-35B-A3B +# Inference: Qwen3.5 thinking mode (temperature=0.6, top_p=0.95, top_k=20) +# Benchmarks: LogHub (log anomaly detection), LifelongAgentBench (sequential SQL) +# Note: AMA-Bench excluded (no public dataset yet, paper arxiv:2602.22769) +# DeepPlanning/PaperArena: see dedicated config files + +[meta] +name = "qwen3.5-35b-monitor-operative" +description = "Evaluate MonitorOperativeAgent on Phase 25 text-to-text benchmarks" + +[defaults] +temperature = 0.6 # Qwen3.5 thinking mode (precise analysis) +max_tokens = 32768 # Qwen3.5 recommended for general queries + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 1024 + +[run] +max_workers = 1 # Sequential for agent state safety +output_dir = "results/qwen3.5-35b-monitor-operative/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B" +engine = "vllm" +param_count_b = 35.0 +active_params_b = 3.0 +gpu_peak_tflops = 989.5 # H100 SXM FP16 peak TFLOPS +gpu_peak_bandwidth_gb_s = 3350.0 # H100 SXM memory bandwidth +num_gpus = 2 + +# --- Phase 25 Text-to-Text Benchmarks --- + +[[benchmarks]] +name = "loghub" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = [ + "think", "calculator", "code_interpreter", + "memory_store", "memory_search", "memory_retrieve", +] + +[[benchmarks]] +name = "lifelong-agent" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = [ + "think", "calculator", "code_interpreter", + "memory_store", "memory_search", "memory_retrieve", +] diff --git a/src/openjarvis/evals/configs/qwen3.5-397b-monitor-operative.toml b/src/openjarvis/evals/configs/qwen3.5-397b-monitor-operative.toml new file mode 100644 index 00000000..ec8aacf2 --- /dev/null +++ b/src/openjarvis/evals/configs/qwen3.5-397b-monitor-operative.toml @@ -0,0 +1,50 @@ +# MonitorOperativeAgent evaluation on long-horizon benchmarks +# Machine: 4x H100-80GB (GPUs 0-3), engine: vLLM, model: Qwen3.5-122B-A10B-FP8 +# Inference: Qwen3.5 non-thinking mode (temperature=0.6) +# Benchmarks: LifelongAgentBench only (LogHub done at 94%, DeepPlanning needs >32K ctx) +# Note: PaperArena excluded (HF repo Melmaphother/PaperArena-Data is empty) +# AMA-Bench excluded (no public dataset yet, paper arxiv:2602.22769) + +[meta] +name = "qwen3.5-122b-monitor-operative" +description = "Evaluate MonitorOperativeAgent on long-horizon monitoring benchmarks" + +[defaults] +temperature = 0.6 # Qwen3.5 thinking mode recommended +max_tokens = 1024 # Keep short to prevent hangs; answers are brief + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 1 # Sequential for episode mode; 1 worker for agent state safety +output_dir = "results/qwen3.5-122b-monitor-operative/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +param_count_b = 122.0 +active_params_b = 10.0 +gpu_peak_tflops = 989.5 # H100 SXM FP16 peak TFLOPS +gpu_peak_bandwidth_gb_s = 3350.0 # H100 SXM memory bandwidth +num_gpus = 4 + +# --- Benchmarks --- + +[[benchmarks]] +name = "lifelong-agent" +backend = "jarvis-agent" +agent = "monitor_operative" +max_samples = 50 +tools = [ + "think", "calculator", "code_interpreter", + "shell_exec", "file_read", + "memory_store", "memory_search", "memory_retrieve", + "kg_add_entity", "kg_add_relation", "kg_query", "kg_neighbors", + "db_query", "web_search", "http_request", +] diff --git a/src/openjarvis/evals/configs/use_case_cloud_openhands.toml b/src/openjarvis/evals/configs/use_case_cloud_openhands.toml index 43c4c9e4..34c275aa 100644 --- a/src/openjarvis/evals/configs/use_case_cloud_openhands.toml +++ b/src/openjarvis/evals/configs/use_case_cloud_openhands.toml @@ -27,26 +27,32 @@ telemetry = true [[models]] name = "claude-opus-4-6" +engine = "cloud" provider = "anthropic" [[models]] name = "claude-sonnet-4-6" +engine = "cloud" provider = "anthropic" [[models]] name = "gpt-5.4" +engine = "cloud" provider = "openai" [[models]] name = "gpt-5-mini" +engine = "cloud" provider = "openai" [[models]] name = "gemini-3.1-pro-preview" +engine = "cloud" provider = "google" [[models]] name = "gemini-3-flash-preview" +engine = "cloud" provider = "google" # --- Benchmarks --- diff --git a/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml b/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml index 81e97695..6f535bb7 100644 --- a/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml +++ b/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml @@ -27,26 +27,32 @@ telemetry = true [[models]] name = "claude-opus-4-6" +engine = "cloud" provider = "anthropic" [[models]] name = "claude-sonnet-4-6" +engine = "cloud" provider = "anthropic" [[models]] name = "gpt-5.4" +engine = "cloud" provider = "openai" [[models]] name = "gpt-5-mini" +engine = "cloud" provider = "openai" [[models]] name = "gemini-3.1-pro-preview" +engine = "cloud" provider = "google" [[models]] name = "gemini-3-flash-preview" +engine = "cloud" provider = "google" # --- Benchmarks --- diff --git a/src/openjarvis/evals/configs/use_case_opensource_openhands.toml b/src/openjarvis/evals/configs/use_case_opensource_openhands.toml index 1969ed31..e4f193dd 100644 --- a/src/openjarvis/evals/configs/use_case_opensource_openhands.toml +++ b/src/openjarvis/evals/configs/use_case_opensource_openhands.toml @@ -38,7 +38,7 @@ active_params_b = 17.0 num_gpus = 8 [[models]] -name = "unsloth/Qwen3.5-122B-A10B-GGUF" +name = "Qwen/Qwen3.5-122B-A10B-FP8" engine = "vllm" param_count_b = 122.0 active_params_b = 10.0 diff --git a/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml b/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml index c40d60e1..ddee471e 100644 --- a/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml +++ b/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml @@ -38,7 +38,7 @@ active_params_b = 17.0 num_gpus = 8 [[models]] -name = "unsloth/Qwen3.5-122B-A10B-GGUF" +name = "Qwen/Qwen3.5-122B-A10B-FP8" engine = "vllm" param_count_b = 122.0 active_params_b = 10.0 diff --git a/src/openjarvis/evals/datasets/deepplanning.py b/src/openjarvis/evals/datasets/deepplanning.py new file mode 100644 index 00000000..6ab38dab --- /dev/null +++ b/src/openjarvis/evals/datasets/deepplanning.py @@ -0,0 +1,238 @@ +"""DeepPlanning: long-horizon planning with constraints. + +Evaluates agents on complex shopping tasks with hard constraints +(product attributes, ratings, stock, shipping). +Source: https://huggingface.co/datasets/Qwen/DeepPlanning +Paper: arXiv:2601.18137 + +The dataset contains shopping planning tasks at 3 difficulty levels +(120 total cases). Each case has a natural-language query, product +catalog, and ground-truth product selections with constraint metadata. +""" + +from __future__ import annotations + +import json +import logging +import random +import tarfile +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +logger = logging.getLogger(__name__) + +_SYSTEM_PROMPT = ( + "You are a shopping assistant. Given the user's request and a product catalog, " + "select the correct products that satisfy ALL stated constraints.\n\n" + "IMPORTANT INSTRUCTIONS:\n" + "- Read through the product catalog data directly — do NOT write code to parse it\n" + "- For each constraint, find products matching ALL requirements\n" + "- If a constraint references data not in the catalog (e.g., transport time " + "when only shipping provider is listed), use reasonable inference from " + "available shipping info\n" + "- For each selected product, state: name, brand, price, and the specific " + "attribute values that match each constraint\n" + "- Present your final answer as a clear list of selected products" +) + + +class DeepPlanningDataset(DatasetProvider): + """DeepPlanning long-horizon planning benchmark. + + Extracts shopping planning tasks from Qwen/DeepPlanning tar.gz archives. + Each case has a query with constraints and ground-truth product selections. + """ + + dataset_id = "deepplanning" + dataset_name = "DeepPlanning" + + def __init__( + self, + cache_dir: Optional[str] = None, + ) -> None: + self._cache_dir = cache_dir + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + snapshot_dir = self._find_snapshot_dir() + if snapshot_dir is None: + self._download_dataset() + snapshot_dir = self._find_snapshot_dir() + if snapshot_dir is None: + logger.error("Failed to download DeepPlanning dataset") + return + + records: List[EvalRecord] = [] + for level in [1, 2, 3]: + tar_path = snapshot_dir / f"database_level{level}.tar.gz" + if not tar_path.exists(): + logger.warning("Missing %s", tar_path) + continue + level_records = self._extract_cases(tar_path, level) + records.extend(level_records) + + logger.info("Loaded %d DeepPlanning cases", len(records)) + + if seed is not None: + random.Random(seed).shuffle(records) + if max_samples is not None: + records = records[:max_samples] + + self._records = records + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + def _find_snapshot_dir(self) -> Optional[Path]: + """Find the HF cache snapshot directory for Qwen/DeepPlanning.""" + base = Path.home() / ".cache" / "huggingface" / "hub" + ds_dir = base / "datasets--Qwen--DeepPlanning" / "snapshots" + if not ds_dir.exists(): + return None + snapshots = list(ds_dir.iterdir()) + return snapshots[0] if snapshots else None + + def _download_dataset(self) -> None: + """Trigger HF datasets download to populate the cache.""" + try: + from datasets import load_dataset + except ImportError: + raise ImportError( + "datasets required for DeepPlanning. " + "Install with: pip install datasets" + ) + logger.info("Downloading Qwen/DeepPlanning from HuggingFace...") + # Loading triggers the download even though we don't use the result + load_dataset("Qwen/DeepPlanning", split="train") + + def _extract_cases( + self, tar_path: Path, level: int, + ) -> List[EvalRecord]: + """Extract shopping cases from a tar.gz archive.""" + records: List[EvalRecord] = [] + try: + with tarfile.open(tar_path, "r:gz") as tf: + # Build index of all members by directory + members_by_dir: Dict[str, Dict[str, Any]] = {} + for m in tf.getmembers(): + parts = Path(m.name).parts + if len(parts) >= 2: + case_dir = parts[1] # e.g. "case_12" + fname = parts[-1] + members_by_dir.setdefault(case_dir, {})[fname] = m + + for case_name, files in members_by_dir.items(): + val_member = files.get("validation_cases.json") + prod_member = files.get("products.jsonl") + if val_member is None: + continue + + f = tf.extractfile(val_member) + if f is None: + continue + data = json.loads(f.read().decode("utf-8")) + + # Load product catalog + products_text = "" + if prod_member is not None: + pf = tf.extractfile(prod_member) + if pf is not None: + products_text = pf.read().decode("utf-8").strip() + + rec = self._case_to_record( + data, level, case_name, products_text, + ) + if rec: + records.append(rec) + except Exception: + logger.warning("Failed to read %s", tar_path, exc_info=True) + return records + + def _case_to_record( + self, + data: Dict[str, Any], + level: int, + case_name: str, + products_text: str = "", + ) -> Optional[EvalRecord]: + """Convert a validation_cases.json to an EvalRecord.""" + query = data.get("query", "") + if not query: + return None + + ground_truth = data.get("ground_truth_products", []) + meta_info = data.get("meta_info", []) + + # Build reference as structured summary + ref_parts = [] + for prod in ground_truth: + name = prod.get("name", "") + brand = prod.get("brand", "") + price = prod.get("price", "") + ref_parts.append(f"{brand} {name} (${price})") + reference = "; ".join(ref_parts) if ref_parts else "" + + # Count constraints + n_constraints = sum( + len(m.get("features", [])) for m in meta_info + ) + + difficulty = ( + "easy" if level == 1 + else "medium" if level == 2 + else "hard" + ) + + # Build product catalog section + catalog_section = "" + if products_text: + catalog_section = ( + f"## Product Catalog\n" + f"Below is the product database in JSONL format " + f"(one JSON object per line):\n\n" + f"```jsonl\n{products_text}\n```\n\n" + ) + + problem = ( + f"{_SYSTEM_PROMPT}\n\n" + f"## Shopping Request\n{query}\n\n" + f"{catalog_section}" + f"## Required Output\n" + f"List each product that satisfies ALL constraints. For each product:\n" + f"1. Product name and brand\n" + f"2. Price\n" + f"3. For each constraint, the matching attribute value from the catalog" + ) + + return EvalRecord( + record_id=f"deepplanning-L{level}-{case_name}", + problem=problem, + reference=reference, + category="agentic", + subject=f"shopping_L{level}", + metadata={ + "task_type": "shopping", + "level": level, + "difficulty": difficulty, + "case_name": case_name, + "n_products": len(ground_truth), + "n_constraints": n_constraints, + "ground_truth_products": ground_truth, + "meta_info": meta_info, + }, + ) + + +__all__ = ["DeepPlanningDataset"] diff --git a/src/openjarvis/evals/datasets/lifelong_agent.py b/src/openjarvis/evals/datasets/lifelong_agent.py index 98132ffe..fa17bb3e 100644 --- a/src/openjarvis/evals/datasets/lifelong_agent.py +++ b/src/openjarvis/evals/datasets/lifelong_agent.py @@ -24,6 +24,12 @@ _SYSTEM_PROMPT = ( "Previous tasks in this session may have modified the environment." ) +_KG_SYSTEM = ( + "You are a knowledge graph reasoning agent. Answer the question by " + "analyzing the provided knowledge graph operations and entity information. " + "Reason through each step and provide the final answer directly." +) + class LifelongAgentDataset(DatasetProvider): """LifelongAgentBench sequential task learning benchmark.""" @@ -56,19 +62,27 @@ class LifelongAgentDataset(DatasetProvider): if not data_dir.exists(): self._download(data_dir) - task_sequences = self._load_task_sequences(data_dir) + # Try Parquet first (HuggingFace format), then JSON/JSONL + records = self._load_from_parquet(data_dir) + if not records: + task_sequences = self._load_task_sequences(data_dir) + if seed is not None: + random.Random(seed).shuffle(task_sequences) + if max_samples is not None: + task_sequences = task_sequences[:max_samples] + self._episodes = [] + self._records = [] + for seq in task_sequences: + episode = self._sequence_to_episode(seq) + self._episodes.append(episode) + self._records.extend(episode) + return if seed is not None: - random.Random(seed).shuffle(task_sequences) + random.Random(seed).shuffle(records) if max_samples is not None: - task_sequences = task_sequences[:max_samples] - - self._episodes = [] - self._records = [] - for seq in task_sequences: - episode = self._sequence_to_episode(seq) - self._episodes.append(episode) - self._records.extend(episode) + records = records[:max_samples] + self._records = records def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) @@ -89,15 +103,214 @@ class LifelongAgentDataset(DatasetProvider): ) from exc data_dir.mkdir(parents=True, exist_ok=True) snapshot_download( - repo_id="LifelongAgentBench/LifelongAgentBench", + repo_id="csyq/LifelongAgentBench", repo_type="dataset", local_dir=str(data_dir), ) + def _load_from_parquet(self, data_dir: Path) -> List[EvalRecord]: + """Load records from Parquet files (HuggingFace dataset format).""" + parquet_files = list(data_dir.rglob("*.parquet")) + if not parquet_files: + return [] + + try: + import pandas as pd + except ImportError: + logger.warning("pandas required for Parquet loading") + return [] + + records: List[EvalRecord] = [] + for pf in sorted(parquet_files): + subset_name = pf.parent.name + df = pd.read_parquet(pf) + logger.info("Loading %d rows from %s", len(df), pf) + + for _, row in df.iterrows(): + row_dict = row.to_dict() + if subset_name == "knowledge_graph": + rec = self._kg_row_to_record(row_dict) + elif subset_name == "db_bench": + rec = self._db_row_to_record(row_dict) + elif subset_name == "os_interaction": + rec = self._os_row_to_record(row_dict) + else: + rec = self._generic_row_to_record(row_dict, subset_name) + if rec is not None: + records.append(rec) + + return records + + def _kg_row_to_record(self, row: Dict[str, Any]) -> Optional[EvalRecord]: + question = row.get("question", "") + if not question: + return None + + qid = row.get("qid", row.get("sample_index", "unknown")) + entity_dict = row.get("entity_dict", {}) + if isinstance(entity_dict, str): + try: + entity_dict = json.loads(entity_dict) + except (json.JSONDecodeError, TypeError): + entity_dict = {} + + entity_desc = "" + if isinstance(entity_dict, dict) and entity_dict: + entities = "\n".join( + f" - {name}: {mid}" for name, mid in entity_dict.items() + ) + entity_desc = f"Entities:\n{entities}" + + action_list = row.get("action_list", []) + action_desc = "" + if action_list: + if isinstance(action_list, str): + try: + action_list = json.loads(action_list) + except (json.JSONDecodeError, TypeError): + action_list = [] + if isinstance(action_list, list) and action_list: + steps = "\n".join(f" {i+1}. {a}" for i, a in enumerate(action_list)) + action_desc = f"\n\nKG Operations:\n{steps}" + + problem = ( + f"{_KG_SYSTEM}\n\n" + f"{entity_desc}" + f"{action_desc}\n\n" + f"Question: {question}" + ) + + answer_list = row.get("answer_list", []) + if isinstance(answer_list, str): + try: + answer_list = json.loads(answer_list) + except (json.JSONDecodeError, TypeError): + answer_list = [answer_list] + reference = ", ".join(str(a) for a in answer_list) if answer_list else "" + + return EvalRecord( + record_id=f"lifelong-kg-{qid}", + problem=problem, + reference=reference, + category="agentic", + subject="knowledge_graph", + metadata={ + "subset": "knowledge_graph", + "qid": qid, + "action_list": action_list, + "skill_list": row.get("skill_list", []), + }, + ) + + def _db_row_to_record(self, row: Dict[str, Any]) -> Optional[EvalRecord]: + instruction = row.get("instruction", "") + if not instruction: + return None + + idx = row.get("sample_index", "unknown") + table_info = row.get("table_info", {}) + if isinstance(table_info, str): + try: + table_info = json.loads(table_info) + except (json.JSONDecodeError, TypeError): + table_info = {} + + table_ctx = "" + if isinstance(table_info, dict) and table_info: + tname = table_info.get("name", "unknown") + cols = table_info.get("column_info_list", []) + if cols: + col_desc = ", ".join( + f"{c.get('name', '?')} ({c.get('type', '?')})" for c in cols + ) + table_ctx = f"\n\nTable: {tname}\nColumns: {col_desc}" + + problem = f"{_SYSTEM_PROMPT}\n\n## Task\n{instruction}{table_ctx}" + + answer_info = row.get("answer_info", {}) + if isinstance(answer_info, str): + try: + answer_info = json.loads(answer_info) + except (json.JSONDecodeError, TypeError): + answer_info = {} + + reference = "" + if isinstance(answer_info, dict): + if answer_info.get("direct") is not None: + reference = str(answer_info["direct"]) + elif answer_info.get("sql"): + reference = answer_info["sql"] + elif answer_info.get("md5"): + reference = answer_info["md5"] + + return EvalRecord( + record_id=f"lifelong-db-{idx}", + problem=problem, + reference=reference, + category="agentic", + subject="database", + metadata={ + "subset": "db_bench", + "sample_index": idx, + "skill_list": row.get("skill_list", []), + }, + ) + + def _os_row_to_record(self, row: Dict[str, Any]) -> Optional[EvalRecord]: + instruction = row.get("instruction", "") + if not instruction: + return None + + idx = row.get("sample_index", "unknown") + problem = f"{_SYSTEM_PROMPT}\n\n## Task\n{instruction}" + + eval_info = row.get("evaluation_info", {}) + if isinstance(eval_info, str): + try: + eval_info = json.loads(eval_info) + except (json.JSONDecodeError, TypeError): + eval_info = {} + + reference = "" + if isinstance(eval_info, dict): + cmd_item = eval_info.get("evaluation_command_item", {}) + if isinstance(cmd_item, dict): + reference = cmd_item.get("script", "") + + return EvalRecord( + record_id=f"lifelong-os-{idx}", + problem=problem, + reference=reference, + category="agentic", + subject="os", + metadata={ + "subset": "os_interaction", + "sample_index": idx, + "skill_list": row.get("skill_list", []), + }, + ) + + def _generic_row_to_record( + self, row: Dict[str, Any], subset_name: str, + ) -> Optional[EvalRecord]: + instruction = row.get("instruction", row.get("question", row.get("task", ""))) + if not instruction: + return None + idx = row.get("sample_index", row.get("id", "unknown")) + expected = row.get("answer", row.get("expected_output", "")) + return EvalRecord( + record_id=f"lifelong-{subset_name}-{idx}", + problem=f"{_SYSTEM_PROMPT}\n\n## Task\n{instruction}", + reference=str(expected), + category="agentic", + subject=subset_name, + metadata={"subset": subset_name, "sample_index": idx}, + ) + def _load_task_sequences( self, data_dir: Path, ) -> List[List[Dict[str, Any]]]: - """Load task sequences from disk.""" + """Load task sequences from JSON/JSONL files (legacy format).""" sequences: List[List[Dict[str, Any]]] = [] for p in sorted(data_dir.rglob("*.json")): @@ -105,7 +318,6 @@ class LifelongAgentDataset(DatasetProvider): with open(p) as f: data = json.load(f) if isinstance(data, list): - # Could be a sequence of tasks or list of sequences if data and isinstance(data[0], list): sequences.extend(data) elif data and isinstance(data[0], dict): diff --git a/src/openjarvis/evals/datasets/paperarena.py b/src/openjarvis/evals/datasets/paperarena.py new file mode 100644 index 00000000..786bad8a --- /dev/null +++ b/src/openjarvis/evals/datasets/paperarena.py @@ -0,0 +1,170 @@ +"""PaperArena: scientific literature reasoning benchmark. + +Evaluates agents on research paper comprehension with three question types: +MC (multiple choice), CA (closed answer), OA (open answer) across +easy/medium/hard difficulty. +Source: https://github.com/Melmaphother/PaperArena +Paper: arXiv:2510.10909 +""" + +from __future__ import annotations + +import json +import logging +import random +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +logger = logging.getLogger(__name__) + +_SYSTEM_PROMPT = ( + "You are a scientific research assistant. Read the paper context carefully " + "and answer the question. For multiple-choice questions, respond with just " + "the letter (A, B, C, or D). For other questions, provide a precise answer." +) + + +class PaperArenaDataset(DatasetProvider): + """PaperArena scientific literature reasoning benchmark. + + Three question types (MC, CA, OA) across three difficulty levels. + """ + + dataset_id = "paperarena" + dataset_name = "PaperArena" + + def __init__( + self, + cache_dir: Optional[str] = None, + ) -> None: + self._cache_dir = ( + Path(cache_dir) if cache_dir + else Path.home() / ".cache" / "paperarena" + ) + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + data_dir = self._cache_dir / "qa" + + if not data_dir.exists(): + self._download() + + records = self._load_records(data_dir) + + if seed is not None: + random.Random(seed).shuffle(records) + if max_samples is not None: + records = records[:max_samples] + + self._records = records + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + def _download(self) -> None: + """Download PaperArena data from HuggingFace or GitHub.""" + try: + from huggingface_hub import snapshot_download + except ImportError as exc: + raise ImportError( + "huggingface_hub required for PaperArena. " + "Install with: pip install huggingface_hub" + ) from exc + + self._cache_dir.mkdir(parents=True, exist_ok=True) + logger.info("Downloading PaperArena dataset...") + snapshot_download( + repo_id="Melmaphother/PaperArena-Data", + repo_type="dataset", + local_dir=str(self._cache_dir), + ) + + def _load_records(self, data_dir: Path) -> List[EvalRecord]: + """Load QA records from JSON/JSONL files.""" + records: List[EvalRecord] = [] + + for p in sorted(data_dir.rglob("*.json")): + try: + with open(p) as f: + data = json.load(f) + items = data if isinstance(data, list) else [data] + for item in items: + rec = self._item_to_record(item) + if rec is not None: + records.append(rec) + except (json.JSONDecodeError, OSError): + logger.debug("Skipping: %s", p) + + for p in sorted(data_dir.rglob("*.jsonl")): + try: + with open(p) as f: + for line in f: + line = line.strip() + if line: + item = json.loads(line) + rec = self._item_to_record(item) + if rec is not None: + records.append(rec) + except (json.JSONDecodeError, OSError): + logger.debug("Skipping: %s", p) + + return records + + def _item_to_record(self, item: Dict[str, Any]) -> Optional[EvalRecord]: + """Convert a raw QA item to an EvalRecord.""" + question_id = item.get("question_id", item.get("id", "")) + question_type = item.get("question_type", item.get("type", "OA")).upper() + difficulty = item.get("difficulty", "medium").lower() + question = item.get("question", "") + context = item.get("context", item.get("paper_context", "")) + reference = item.get("answer", item.get("reference", "")) + options = item.get("options", None) + tool_chain = item.get("minimal_tool_chain", []) + + if not question: + return None + + # Build problem prompt + prompt = f"{_SYSTEM_PROMPT}\n\n" + if context: + prompt += f"## Paper Context\n{context}\n\n" + prompt += f"## Question\n{question}" + + if options and question_type == "MC": + options_text = "\n".join( + f" {k}) {v}" for k, v in options.items() + ) if isinstance(options, dict) else "\n".join( + f" {chr(65 + i)}) {o}" for i, o in enumerate(options) + ) + prompt += f"\n\nOptions:\n{options_text}" + + subject = f"{difficulty}_{question_type.lower()}" + + return EvalRecord( + record_id=f"paperarena-{question_id}", + problem=prompt, + reference=str(reference), + category="agentic", + subject=subject, + metadata={ + "question_id": question_id, + "question_type": question_type, + "difficulty": difficulty, + "minimal_tool_chain": tool_chain, + }, + ) + + +__all__ = ["PaperArenaDataset"] diff --git a/src/openjarvis/evals/scorers/deepplanning_scorer.py b/src/openjarvis/evals/scorers/deepplanning_scorer.py new file mode 100644 index 00000000..24759370 --- /dev/null +++ b/src/openjarvis/evals/scorers/deepplanning_scorer.py @@ -0,0 +1,87 @@ +"""LLM-judge scorer for DeepPlanning constraint satisfaction.""" + +from __future__ import annotations + +import re +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord + +_JUDGE_PROMPT = """You are evaluating a planning task response. + +## Task Type: {task_type} + +## Original Task +{problem} + +## Reference Plan +{reference} + +## Agent's Plan +{model_answer} + +Evaluate whether the agent's plan satisfies ALL constraints from the task. + +For travel planning, check: +- Route consistency and feasibility +- Time feasibility (business hours, travel times) +- Budget accuracy (costs within stated limits) +- Personalization requirements met + +For shopping tasks, check: +- Correct products selected +- Coupon/discount rules applied correctly +- Cart total accuracy + +Respond with exactly: CORRECT or INCORRECT +Then provide a brief explanation.""" + + +class DeepPlanningScorer(LLMJudgeScorer): + """Score DeepPlanning responses via LLM judge on constraint satisfaction.""" + + scorer_id = "deepplanning" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + if not record.reference or not record.reference.strip(): + return None, {"reason": "no_ground_truth"} + + task_type = record.metadata.get("task_type", "planning") + + # Extract problem without system prompt prefix + problem = record.problem + if "## Task" in problem: + problem = problem.split("## Task", 1)[-1] + + prompt = _JUDGE_PROMPT.format( + task_type=task_type, + problem=problem, + reference=record.reference, + model_answer=model_answer, + ) + + try: + raw = self._ask_judge(prompt, max_tokens=4096) + is_correct = bool(re.search(r"\bCORRECT\b", raw, re.IGNORECASE)) + if re.search(r"\bINCORRECT\b", raw, re.IGNORECASE): + is_correct = False + + return is_correct, { + "match_type": "llm_judge", + "raw_judge_output": raw, + "task_type": task_type, + } + except Exception as exc: + return False, { + "match_type": "llm_judge_error", + "error": str(exc), + } + + +__all__ = ["DeepPlanningScorer"] diff --git a/src/openjarvis/evals/scorers/paperarena_judge.py b/src/openjarvis/evals/scorers/paperarena_judge.py new file mode 100644 index 00000000..be06d548 --- /dev/null +++ b/src/openjarvis/evals/scorers/paperarena_judge.py @@ -0,0 +1,153 @@ +"""Scorer for PaperArena: MC exact match + LLM judge for CA/OA.""" + +from __future__ import annotations + +import re +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord + +_JUDGE_PROMPT = """You are evaluating a scientific question answer. + +## Question +{question} + +## Reference Answer +{reference} + +## Agent's Answer +{model_answer} + +Is the agent's answer correct? Consider semantic equivalence, not exact wording. +For numerical answers, accept reasonable rounding. +Respond with exactly: CORRECT or INCORRECT""" + + +class PaperArenaScorer(LLMJudgeScorer): + """Score PaperArena: MC via letter extraction, CA/OA via LLM judge.""" + + scorer_id = "paperarena" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + if not record.reference or not record.reference.strip(): + return None, {"reason": "no_ground_truth"} + + question_type = record.metadata.get("question_type", "OA").upper() + + if question_type == "MC": + return self._score_mc(record, model_answer) + else: + return self._score_open(record, model_answer) + + def _score_mc( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + """Score multiple-choice via letter extraction.""" + ref_letter = record.reference.strip().upper() + if len(ref_letter) != 1 or ref_letter not in "ABCD": + # Reference is not a single letter; fall back to judge + return self._score_open(record, model_answer) + + # Try regex extraction for answer letter + extracted = self._extract_letter(model_answer) + + if extracted is None: + # Fall back to LLM extraction + extracted = self._extract_letter_with_llm( + record.problem, model_answer, + ) + + if extracted is None: + return None, { + "match_type": "exact_letter", + "reason": "no_letter_extracted", + } + + is_correct = extracted == ref_letter + return is_correct, { + "match_type": "exact_letter", + "reference_letter": ref_letter, + "candidate_letter": extracted, + } + + def _extract_letter(self, text: str) -> Optional[str]: + """Try to extract a single answer letter from text via regex.""" + # Common patterns: "The answer is A", "A)", "(A)", just "A" + patterns = [ + r"(?:the\s+answer\s+is|answer:?)\s*([A-D])\b", + r"\b([A-D])\)", + r"\(([A-D])\)", + ] + for pat in patterns: + m = re.search(pat, text, re.IGNORECASE) + if m: + return m.group(1).upper() + + # Last single capital letter on its own line + lines = text.strip().splitlines() + for line in reversed(lines): + stripped = line.strip() + if len(stripped) == 1 and stripped.upper() in "ABCD": + return stripped.upper() + + return None + + def _extract_letter_with_llm( + self, problem: str, model_answer: str, + ) -> Optional[str]: + """Use LLM to extract answer letter.""" + prompt = ( + f"Extract the final answer letter (A, B, C, or D) from this response.\n" + f"Problem: {problem[:2000]}\n" + f"Response: {model_answer[:2000]}\n\n" + f"Return ONLY a single letter (A-D), or NONE if no answer found." + ) + try: + raw = self._ask_judge(prompt, max_tokens=4096) + letter = raw.strip().upper() + m = re.search(r"([A-D])", letter) + if m: + return m.group(1) + except Exception: + pass + return None + + def _score_open( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + """Score CA/OA via LLM judge.""" + question = record.problem + if "## Question" in question: + question = question.split("## Question")[-1].strip() + + prompt = _JUDGE_PROMPT.format( + question=question, + reference=record.reference, + model_answer=model_answer, + ) + + try: + raw = self._ask_judge(prompt, max_tokens=4096) + is_correct = bool(re.search(r"\bCORRECT\b", raw, re.IGNORECASE)) + if re.search(r"\bINCORRECT\b", raw, re.IGNORECASE): + is_correct = False + + return is_correct, { + "match_type": "llm_judge", + "raw_judge_output": raw, + "question_type": record.metadata.get("question_type", "OA"), + } + except Exception as exc: + return False, { + "match_type": "llm_judge_error", + "error": str(exc), + } + + +__all__ = ["PaperArenaScorer"] diff --git a/src/openjarvis/learning/__init__.py b/src/openjarvis/learning/__init__.py index 064b8569..d9c68a8e 100644 --- a/src/openjarvis/learning/__init__.py +++ b/src/openjarvis/learning/__init__.py @@ -11,13 +11,13 @@ from openjarvis.learning._stubs import ( from openjarvis.learning.agent_evolver import AgentConfigEvolver from openjarvis.learning.heuristic_reward import HeuristicRewardFunction from openjarvis.learning.learning_orchestrator import LearningOrchestrator +from openjarvis.learning.optimize.llm_optimizer import LLMOptimizer +from openjarvis.learning.optimize.optimizer import OptimizationEngine +from openjarvis.learning.optimize.store import OptimizationStore from openjarvis.learning.router import ( HeuristicRouter, build_routing_context, ) -from openjarvis.learning.optimize.llm_optimizer import LLMOptimizer -from openjarvis.learning.optimize.optimizer import OptimizationEngine -from openjarvis.learning.optimize.store import OptimizationStore from openjarvis.learning.training.data import TrainingDataMiner from openjarvis.learning.training.lora import HAS_TORCH, LoRATrainer, LoRATrainingConfig diff --git a/src/openjarvis/learning/optimize/__init__.py b/src/openjarvis/learning/optimize/__init__.py index 55c2f7d3..ddfa3e5c 100644 --- a/src/openjarvis/learning/optimize/__init__.py +++ b/src/openjarvis/learning/optimize/__init__.py @@ -6,8 +6,14 @@ from openjarvis.learning.optimize.config import ( load_optimize_config, ) from openjarvis.learning.optimize.llm_optimizer import LLMOptimizer -from openjarvis.learning.optimize.optimizer import OptimizationEngine, compute_pareto_frontier -from openjarvis.learning.optimize.search_space import DEFAULT_SEARCH_SPACE, build_search_space +from openjarvis.learning.optimize.optimizer import ( + OptimizationEngine, + compute_pareto_frontier, +) +from openjarvis.learning.optimize.search_space import ( + DEFAULT_SEARCH_SPACE, + build_search_space, +) from openjarvis.learning.optimize.store import OptimizationStore from openjarvis.learning.optimize.trial_runner import ( BenchmarkSpec, diff --git a/src/openjarvis/optimize/feedback/__init__.py b/src/openjarvis/optimize/feedback/__init__.py index b32e5f5a..589fc3cc 100644 --- a/src/openjarvis/optimize/feedback/__init__.py +++ b/src/openjarvis/optimize/feedback/__init__.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.feedback moved to learning.optimize.feedback.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.feedback import * # noqa: F401,F403 from openjarvis.learning.optimize.feedback import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/feedback/collector.py b/src/openjarvis/optimize/feedback/collector.py index d7ac0f9b..995c176a 100644 --- a/src/openjarvis/optimize/feedback/collector.py +++ b/src/openjarvis/optimize/feedback/collector.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.feedback.collector moved to learning.optimize.feedback.collector.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.feedback.collector import * # noqa: F401,F403 from openjarvis.learning.optimize.feedback.collector import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/feedback/judge.py b/src/openjarvis/optimize/feedback/judge.py index bafac645..9010fc3a 100644 --- a/src/openjarvis/optimize/feedback/judge.py +++ b/src/openjarvis/optimize/feedback/judge.py @@ -1,4 +1,6 @@ -"""Backward-compatibility shim -- optimize.feedback.judge moved to learning.optimize.feedback.judge.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.feedback.judge import * # noqa: F401,F403 -from openjarvis.learning.optimize.feedback.judge import __all__ # noqa: F401 -from openjarvis.learning.optimize.feedback.judge import _parse_score # noqa: F401 +from openjarvis.learning.optimize.feedback.judge import ( + __all__, # noqa: F401 + _parse_score, # noqa: F401 +) diff --git a/src/openjarvis/optimize/llm_optimizer.py b/src/openjarvis/optimize/llm_optimizer.py index eb8ce46c..1246b280 100644 --- a/src/openjarvis/optimize/llm_optimizer.py +++ b/src/openjarvis/optimize/llm_optimizer.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.llm_optimizer moved to learning.optimize.llm_optimizer.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.llm_optimizer import * # noqa: F401,F403 from openjarvis.learning.optimize.llm_optimizer import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/optimizer.py b/src/openjarvis/optimize/optimizer.py index 96577982..67716608 100644 --- a/src/openjarvis/optimize/optimizer.py +++ b/src/openjarvis/optimize/optimizer.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.optimizer moved to learning.optimize.optimizer.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.optimizer import * # noqa: F401,F403 from openjarvis.learning.optimize.optimizer import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/personal/__init__.py b/src/openjarvis/optimize/personal/__init__.py index 9e78de8d..5b4a0623 100644 --- a/src/openjarvis/optimize/personal/__init__.py +++ b/src/openjarvis/optimize/personal/__init__.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.personal moved to learning.optimize.personal.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.personal import * # noqa: F401,F403 from openjarvis.learning.optimize.personal import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/personal/dataset.py b/src/openjarvis/optimize/personal/dataset.py index d5b1d0ed..bde2f478 100644 --- a/src/openjarvis/optimize/personal/dataset.py +++ b/src/openjarvis/optimize/personal/dataset.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.personal.dataset moved to learning.optimize.personal.dataset.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.personal.dataset import * # noqa: F401,F403 from openjarvis.learning.optimize.personal.dataset import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/personal/scorer.py b/src/openjarvis/optimize/personal/scorer.py index 88dc4a71..58737002 100644 --- a/src/openjarvis/optimize/personal/scorer.py +++ b/src/openjarvis/optimize/personal/scorer.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.personal.scorer moved to learning.optimize.personal.scorer.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.personal.scorer import * # noqa: F401,F403 from openjarvis.learning.optimize.personal.scorer import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/personal/synthesizer.py b/src/openjarvis/optimize/personal/synthesizer.py index 1af198bd..0fcc5718 100644 --- a/src/openjarvis/optimize/personal/synthesizer.py +++ b/src/openjarvis/optimize/personal/synthesizer.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.personal.synthesizer moved to learning.optimize.personal.synthesizer.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.personal.synthesizer import * # noqa: F401,F403 from openjarvis.learning.optimize.personal.synthesizer import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/search_space.py b/src/openjarvis/optimize/search_space.py index 251f1e25..3f28b613 100644 --- a/src/openjarvis/optimize/search_space.py +++ b/src/openjarvis/optimize/search_space.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.search_space moved to learning.optimize.search_space.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.search_space import * # noqa: F401,F403 from openjarvis.learning.optimize.search_space import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/trial_runner.py b/src/openjarvis/optimize/trial_runner.py index dfcf7cce..f1bd5abc 100644 --- a/src/openjarvis/optimize/trial_runner.py +++ b/src/openjarvis/optimize/trial_runner.py @@ -1,3 +1,3 @@ -"""Backward-compatibility shim -- optimize.trial_runner moved to learning.optimize.trial_runner.""" +"""Backward-compat shim: moved to learning.optimize.""" from openjarvis.learning.optimize.trial_runner import * # noqa: F401,F403 from openjarvis.learning.optimize.trial_runner import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/types.py b/src/openjarvis/optimize/types.py index 4acf0340..6b96442a 100644 --- a/src/openjarvis/optimize/types.py +++ b/src/openjarvis/optimize/types.py @@ -1,4 +1,6 @@ """Backward-compatibility shim -- optimize.types moved to learning.optimize.types.""" from openjarvis.learning.optimize.types import * # noqa: F401,F403 -from openjarvis.learning.optimize.types import __all__ # noqa: F401 -from openjarvis.learning.optimize.types import _PARAM_TO_RECIPE # noqa: F401 +from openjarvis.learning.optimize.types import ( + _PARAM_TO_RECIPE, # noqa: F401 + __all__, # noqa: F401 +)