diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index 92203b82..ccdbcea9 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -134,7 +134,15 @@ BENCHMARKS = { }, "liveresearch": { "category": "agentic", - "description": "LiveResearchBench deep research tasks", + "description": "DeepResearchBench report generation (alias: deepresearch)", + }, + "deepresearch": { + "category": "agentic", + "description": "DeepResearchBench deep research report generation", + }, + "liveresearchbench": { + "category": "reasoning", + "description": "LiveResearchBench recent research comprehension (Salesforce)", }, "toolcall15": { "category": "agentic", @@ -335,10 +343,16 @@ def _build_dataset(benchmark: str, subset: str | None = None): from openjarvis.evals.datasets.livecodebench import LiveCodeBenchDataset return LiveCodeBenchDataset() - elif benchmark == "liveresearch": + elif benchmark in ("liveresearch", "deepresearch"): from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset return LiveResearchBenchDataset(path=subset) + elif benchmark == "liveresearchbench": + from openjarvis.evals.datasets.liveresearchbench import ( + LiveResearchBenchSFDataset, + ) + + return LiveResearchBenchSFDataset() elif benchmark == "toolcall15": from openjarvis.evals.datasets.toolcall15 import ToolCall15Dataset @@ -487,10 +501,16 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str): from openjarvis.evals.scorers.livecodebench import LiveCodeBenchScorer return LiveCodeBenchScorer(judge_backend, judge_model) - elif benchmark == "liveresearch": + elif benchmark in ("liveresearch", "deepresearch"): from openjarvis.evals.scorers.liveresearch import LiveResearchBenchScorer return LiveResearchBenchScorer(judge_backend, judge_model) + elif benchmark == "liveresearchbench": + from openjarvis.evals.scorers.liveresearchbench import ( + LiveResearchBenchSFScorer, + ) + + return LiveResearchBenchSFScorer(judge_backend, judge_model) elif benchmark == "toolcall15": from openjarvis.evals.scorers.toolcall15 import ToolCall15Scorer diff --git a/src/openjarvis/evals/datasets/liveresearchbench.py b/src/openjarvis/evals/datasets/liveresearchbench.py new file mode 100644 index 00000000..bb4ae6eb --- /dev/null +++ b/src/openjarvis/evals/datasets/liveresearchbench.py @@ -0,0 +1,179 @@ +"""LiveResearchBench (Salesforce) dataset provider. + +80 expert-curated deep research tasks with per-question evaluation +checklists across three domains: daily life, enterprise, and academia. +543 checklist items total (grouped by question). + +Reference: https://github.com/SalesforceAIResearch/LiveResearchBench +HuggingFace: Salesforce/LiveResearchBench (gated — accept terms first) +""" + +from __future__ import annotations + +import logging +import random +import re +from collections import defaultdict +from datetime import datetime +from typing import Any, Dict, Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + +_HF_DATASET = "Salesforce/LiveResearchBench" + + +def _replace_date_placeholders(text: str) -> str: + """Replace dynamic date placeholders in queries.""" + now = datetime.now() + text = text.replace("{{current_year}}", str(now.year)) + text = text.replace("{{last_year}}", str(now.year - 1)) + text = text.replace("{{current_date}}", now.strftime("%Y-%m-%d")) + text = text.replace("{{date}}", now.strftime("%Y-%m-%d")) + text = re.sub(r"\{current_year\}", str(now.year), text) + text = re.sub(r"\{last_year\}", str(now.year - 1), text) + return text + + +class LiveResearchBenchSFDataset(DatasetProvider): + """Salesforce LiveResearchBench — 80 expert-curated research tasks. + + The HuggingFace dataset has 543 rows (multiple checklist items per + question). We group by ``qid`` to produce one EvalRecord per unique + question, with all checklist items aggregated in metadata. + """ + + dataset_id = "liveresearchbench" + dataset_name = "LiveResearchBench (Salesforce)" + + def __init__(self) -> None: + self._records: Optional[List[EvalRecord]] = None + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + try: + from datasets import load_dataset + except ImportError: + raise ImportError( + "datasets package required. Install with: pip install datasets" + ) + + import os + + hf_token = os.environ.get("HF_TOKEN") or os.environ.get( + "HUGGING_FACE_HUB_TOKEN" + ) + + # Try question_with_checklist first (has evaluation criteria) + hf_config = split or "question_with_checklist" + LOGGER.info( + "Loading LiveResearchBench from HuggingFace (%s, config=%s)", + _HF_DATASET, + hf_config, + ) + + try: + ds = load_dataset( + _HF_DATASET, hf_config, split="test", token=hf_token + ) + except Exception as exc: + raise RuntimeError( + f"Failed to load {_HF_DATASET}. This is a gated dataset — " + "visit https://huggingface.co/datasets/Salesforce/LiveResearchBench " + "to accept the terms, then set HF_TOKEN in your environment. " + f"Error: {exc}" + ) from exc + + # Group rows by qid (multiple checklist items per question) + questions: Dict[str, Dict[str, Any]] = {} + checklists_by_qid: Dict[str, List[str]] = defaultdict(list) + + for row in ds: + qid = str(row.get("qid", "")) + if not qid: + continue + + if qid not in questions: + question = row.get("question", "") or row.get( + "question_no_placeholder", "" + ) + questions[qid] = { + "question": question, + "category": row.get("category", ""), + } + + checklist = row.get("checklist", "") or row.get( + "checklist_no_placeholder", "" + ) + if checklist: + checklists_by_qid[qid].append(checklist) + + # Build EvalRecords + records: List[EvalRecord] = [] + for qid, info in questions.items(): + question = info["question"] + if not question: + continue + + question = _replace_date_placeholders(question) + + problem = ( + "You are a research assistant. Please conduct thorough " + "research on the following question and write a " + "comprehensive report with citations.\n\n" + f"{question}" + ) + + metadata: Dict[str, Any] = { + "qid": qid, + "original_question": question, + "category": info.get("category", ""), + "checklists": checklists_by_qid.get(qid, []), + } + + records.append( + EvalRecord( + record_id=f"lrb-{qid}", + problem=problem, + reference="", + category="liveresearchbench", + metadata=metadata, + ) + ) + + if seed is not None: + rng = random.Random(seed) + rng.shuffle(records) + + if max_samples is not None: + records = records[:max_samples] + + self._records = records + total_checklists = sum( + len(r.metadata.get("checklists", [])) for r in records + ) + LOGGER.info( + "LiveResearchBench: loaded %d tasks (%d checklist items)", + len(self._records), + total_checklists, + ) + + def iter_records(self) -> Iterable[EvalRecord]: + if self._records is None: + raise RuntimeError("Call .load() before iterating") + return iter(self._records) + + def size(self) -> int: + if self._records is None: + raise RuntimeError("Call .load() before size()") + return len(self._records) + + +__all__ = ["LiveResearchBenchSFDataset"] diff --git a/src/openjarvis/evals/scorers/liveresearchbench.py b/src/openjarvis/evals/scorers/liveresearchbench.py new file mode 100644 index 00000000..ea89ab55 --- /dev/null +++ b/src/openjarvis/evals/scorers/liveresearchbench.py @@ -0,0 +1,166 @@ +"""LiveResearchBench (Salesforce) scorer — checklist-based evaluation. + +Evaluates research reports using per-question checklists for coverage, +plus LLM-as-judge for presentation quality and citation adequacy. + +Reference: https://github.com/SalesforceAIResearch/LiveResearchBench +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + +_COVERAGE_PROMPT = """\ +You are evaluating a research report against a checklist of required topics/points. + +**Research Question:** +{question} + +**Report:** +{report} + +**Checklist items to verify (each should be covered in the report):** +{checklist_items} + +For each checklist item, determine if the report adequately covers it. +Respond with a JSON array of objects, one per checklist item: +[ + {{"item": "", "covered": true/false, "evidence": ""}}, + ... +] + +Then on the last line, provide the overall coverage score: +coverage_score: /""" + +_QUALITY_PROMPT = """\ +You are evaluating the quality of a research report. + +**Research Question:** +{question} + +**Report:** +{report} + +Rate the report on these dimensions (each 1-5): + +1. **Presentation**: Is the report well-structured, readable, and professional? +2. **Depth**: Does the report go beyond surface-level information? +3. **Citation**: Does the report reference specific sources, data, or evidence? +4. **Consistency**: Is the report internally consistent and free of contradictions? + +Respond in this exact format: +presentation: <1-5> +depth: <1-5> +citation: <1-5> +consistency: <1-5> +reasoning: """ + + +class LiveResearchBenchSFScorer(LLMJudgeScorer): + """Checklist + quality scorer for Salesforce LiveResearchBench.""" + + scorer_id = "liveresearchbench" + + def score( + self, + record: EvalRecord, + model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response", "score": 0.0} + + question = record.metadata.get("original_question", record.problem) + checklists = record.metadata.get("checklists", []) + + meta: Dict[str, Any] = {} + + # Phase 1: Checklist coverage (if available) + coverage_score = 0.0 + if checklists: + try: + checklist_text = "\n".join( + f"- {item}" for item in checklists + ) + prompt = _COVERAGE_PROMPT.format( + question=question, + report=model_answer[:8000], # Truncate long reports + checklist_items=checklist_text, + ) + raw = self._ask_judge( + prompt, temperature=1.0, max_tokens=4096 + ) + + # Parse coverage_score from last line + match = re.search( + r"coverage_score:\s*(\d+)\s*/\s*(\d+)", raw + ) + if match: + covered = int(match.group(1)) + total = int(match.group(2)) + coverage_score = covered / total if total > 0 else 0.0 + meta["coverage_covered"] = covered + meta["coverage_total"] = total + else: + # Fallback: count "covered": true occurrences + covered = raw.lower().count('"covered": true') + raw.lower().count('"covered":true') + total = len(checklists) + coverage_score = covered / total if total > 0 else 0.0 + meta["coverage_covered"] = covered + meta["coverage_total"] = total + + meta["coverage_score"] = coverage_score + meta["coverage_raw"] = raw[:500] + except Exception as exc: + LOGGER.warning( + "Coverage scoring failed for %s: %s", + record.record_id, + exc, + ) + meta["coverage_error"] = str(exc) + + # Phase 2: Quality dimensions + quality_score = 0.0 + try: + prompt = _QUALITY_PROMPT.format( + question=question, + report=model_answer[:8000], + ) + raw = self._ask_judge(prompt, temperature=1.0, max_tokens=1024) + + dims = {} + for dim in ["presentation", "depth", "citation", "consistency"]: + match = re.search(rf"{dim}:\s*(\d)", raw) + if match: + dims[dim] = int(match.group(1)) + + if dims: + quality_score = sum(dims.values()) / (5 * len(dims)) + meta["quality_dims"] = dims + meta["quality_score"] = quality_score + meta["quality_raw"] = raw[:500] + except Exception as exc: + LOGGER.warning( + "Quality scoring failed for %s: %s", record.record_id, exc + ) + meta["quality_error"] = str(exc) + + # Final score: weighted average of coverage and quality + if checklists: + final_score = 0.6 * coverage_score + 0.4 * quality_score + else: + final_score = quality_score + + meta["final_score"] = final_score + is_correct = final_score >= 0.5 + + return is_correct, meta + + +__all__ = ["LiveResearchBenchSFScorer"]