mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 14:07:55 +00:00
Merge pull request #241 from open-jarvis/feat/liveresearchbench-and-deepresearch-alias
feat: add Salesforce LiveResearchBench + deepresearch alias
This commit is contained in:
Avanika Narayan
@@ -134,7 +134,15 @@ BENCHMARKS = {
|
||||
},
|
||||
"liveresearch": {
|
||||
"category": "agentic",
|
||||
"description": "LiveResearchBench deep research tasks",
|
||||
"description": "DeepResearchBench report generation (alias: deepresearch)",
|
||||
},
|
||||
"deepresearch": {
|
||||
"category": "agentic",
|
||||
"description": "DeepResearchBench deep research report generation",
|
||||
},
|
||||
"liveresearchbench": {
|
||||
"category": "reasoning",
|
||||
"description": "LiveResearchBench recent research comprehension (Salesforce)",
|
||||
},
|
||||
"toolcall15": {
|
||||
"category": "agentic",
|
||||
@@ -335,10 +343,16 @@ def _build_dataset(benchmark: str, subset: str | None = None):
|
||||
from openjarvis.evals.datasets.livecodebench import LiveCodeBenchDataset
|
||||
|
||||
return LiveCodeBenchDataset()
|
||||
elif benchmark == "liveresearch":
|
||||
elif benchmark in ("liveresearch", "deepresearch"):
|
||||
from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset
|
||||
|
||||
return LiveResearchBenchDataset(path=subset)
|
||||
elif benchmark == "liveresearchbench":
|
||||
from openjarvis.evals.datasets.liveresearchbench import (
|
||||
LiveResearchBenchSFDataset,
|
||||
)
|
||||
|
||||
return LiveResearchBenchSFDataset()
|
||||
elif benchmark == "toolcall15":
|
||||
from openjarvis.evals.datasets.toolcall15 import ToolCall15Dataset
|
||||
|
||||
@@ -487,10 +501,16 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str):
|
||||
from openjarvis.evals.scorers.livecodebench import LiveCodeBenchScorer
|
||||
|
||||
return LiveCodeBenchScorer(judge_backend, judge_model)
|
||||
elif benchmark == "liveresearch":
|
||||
elif benchmark in ("liveresearch", "deepresearch"):
|
||||
from openjarvis.evals.scorers.liveresearch import LiveResearchBenchScorer
|
||||
|
||||
return LiveResearchBenchScorer(judge_backend, judge_model)
|
||||
elif benchmark == "liveresearchbench":
|
||||
from openjarvis.evals.scorers.liveresearchbench import (
|
||||
LiveResearchBenchSFScorer,
|
||||
)
|
||||
|
||||
return LiveResearchBenchSFScorer(judge_backend, judge_model)
|
||||
elif benchmark == "toolcall15":
|
||||
from openjarvis.evals.scorers.toolcall15 import ToolCall15Scorer
|
||||
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
"""LiveResearchBench (Salesforce) dataset provider.
|
||||
|
||||
80 expert-curated deep research tasks with per-question evaluation
|
||||
checklists across three domains: daily life, enterprise, and academia.
|
||||
543 checklist items total (grouped by question).
|
||||
|
||||
Reference: https://github.com/SalesforceAIResearch/LiveResearchBench
|
||||
HuggingFace: Salesforce/LiveResearchBench (gated — accept terms first)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from openjarvis.evals.core.dataset import DatasetProvider
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
_HF_DATASET = "Salesforce/LiveResearchBench"
|
||||
|
||||
|
||||
def _replace_date_placeholders(text: str) -> str:
|
||||
"""Replace dynamic date placeholders in queries."""
|
||||
now = datetime.now()
|
||||
text = text.replace("{{current_year}}", str(now.year))
|
||||
text = text.replace("{{last_year}}", str(now.year - 1))
|
||||
text = text.replace("{{current_date}}", now.strftime("%Y-%m-%d"))
|
||||
text = text.replace("{{date}}", now.strftime("%Y-%m-%d"))
|
||||
text = re.sub(r"\{current_year\}", str(now.year), text)
|
||||
text = re.sub(r"\{last_year\}", str(now.year - 1), text)
|
||||
return text
|
||||
|
||||
|
||||
class LiveResearchBenchSFDataset(DatasetProvider):
|
||||
"""Salesforce LiveResearchBench — 80 expert-curated research tasks.
|
||||
|
||||
The HuggingFace dataset has 543 rows (multiple checklist items per
|
||||
question). We group by ``qid`` to produce one EvalRecord per unique
|
||||
question, with all checklist items aggregated in metadata.
|
||||
"""
|
||||
|
||||
dataset_id = "liveresearchbench"
|
||||
dataset_name = "LiveResearchBench (Salesforce)"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._records: Optional[List[EvalRecord]] = None
|
||||
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
try:
|
||||
from datasets import load_dataset
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"datasets package required. Install with: pip install datasets"
|
||||
)
|
||||
|
||||
import os
|
||||
|
||||
hf_token = os.environ.get("HF_TOKEN") or os.environ.get(
|
||||
"HUGGING_FACE_HUB_TOKEN"
|
||||
)
|
||||
|
||||
# Try question_with_checklist first (has evaluation criteria)
|
||||
hf_config = split or "question_with_checklist"
|
||||
LOGGER.info(
|
||||
"Loading LiveResearchBench from HuggingFace (%s, config=%s)",
|
||||
_HF_DATASET,
|
||||
hf_config,
|
||||
)
|
||||
|
||||
try:
|
||||
ds = load_dataset(
|
||||
_HF_DATASET, hf_config, split="test", token=hf_token
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"Failed to load {_HF_DATASET}. This is a gated dataset — "
|
||||
"visit https://huggingface.co/datasets/Salesforce/LiveResearchBench "
|
||||
"to accept the terms, then set HF_TOKEN in your environment. "
|
||||
f"Error: {exc}"
|
||||
) from exc
|
||||
|
||||
# Group rows by qid (multiple checklist items per question)
|
||||
questions: Dict[str, Dict[str, Any]] = {}
|
||||
checklists_by_qid: Dict[str, List[str]] = defaultdict(list)
|
||||
|
||||
for row in ds:
|
||||
qid = str(row.get("qid", ""))
|
||||
if not qid:
|
||||
continue
|
||||
|
||||
if qid not in questions:
|
||||
question = row.get("question", "") or row.get(
|
||||
"question_no_placeholder", ""
|
||||
)
|
||||
questions[qid] = {
|
||||
"question": question,
|
||||
"category": row.get("category", ""),
|
||||
}
|
||||
|
||||
checklist = row.get("checklist", "") or row.get(
|
||||
"checklist_no_placeholder", ""
|
||||
)
|
||||
if checklist:
|
||||
checklists_by_qid[qid].append(checklist)
|
||||
|
||||
# Build EvalRecords
|
||||
records: List[EvalRecord] = []
|
||||
for qid, info in questions.items():
|
||||
question = info["question"]
|
||||
if not question:
|
||||
continue
|
||||
|
||||
question = _replace_date_placeholders(question)
|
||||
|
||||
problem = (
|
||||
"You are a research assistant. Please conduct thorough "
|
||||
"research on the following question and write a "
|
||||
"comprehensive report with citations.\n\n"
|
||||
f"{question}"
|
||||
)
|
||||
|
||||
metadata: Dict[str, Any] = {
|
||||
"qid": qid,
|
||||
"original_question": question,
|
||||
"category": info.get("category", ""),
|
||||
"checklists": checklists_by_qid.get(qid, []),
|
||||
}
|
||||
|
||||
records.append(
|
||||
EvalRecord(
|
||||
record_id=f"lrb-{qid}",
|
||||
problem=problem,
|
||||
reference="",
|
||||
category="liveresearchbench",
|
||||
metadata=metadata,
|
||||
)
|
||||
)
|
||||
|
||||
if seed is not None:
|
||||
rng = random.Random(seed)
|
||||
rng.shuffle(records)
|
||||
|
||||
if max_samples is not None:
|
||||
records = records[:max_samples]
|
||||
|
||||
self._records = records
|
||||
total_checklists = sum(
|
||||
len(r.metadata.get("checklists", [])) for r in records
|
||||
)
|
||||
LOGGER.info(
|
||||
"LiveResearchBench: loaded %d tasks (%d checklist items)",
|
||||
len(self._records),
|
||||
total_checklists,
|
||||
)
|
||||
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
if self._records is None:
|
||||
raise RuntimeError("Call .load() before iterating")
|
||||
return iter(self._records)
|
||||
|
||||
def size(self) -> int:
|
||||
if self._records is None:
|
||||
raise RuntimeError("Call .load() before size()")
|
||||
return len(self._records)
|
||||
|
||||
|
||||
__all__ = ["LiveResearchBenchSFDataset"]
|
||||
@@ -0,0 +1,166 @@
|
||||
"""LiveResearchBench (Salesforce) scorer — checklist-based evaluation.
|
||||
|
||||
Evaluates research reports using per-question checklists for coverage,
|
||||
plus LLM-as-judge for presentation quality and citation adequacy.
|
||||
|
||||
Reference: https://github.com/SalesforceAIResearch/LiveResearchBench
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from openjarvis.evals.core.scorer import LLMJudgeScorer
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
_COVERAGE_PROMPT = """\
|
||||
You are evaluating a research report against a checklist of required topics/points.
|
||||
|
||||
**Research Question:**
|
||||
{question}
|
||||
|
||||
**Report:**
|
||||
{report}
|
||||
|
||||
**Checklist items to verify (each should be covered in the report):**
|
||||
{checklist_items}
|
||||
|
||||
For each checklist item, determine if the report adequately covers it.
|
||||
Respond with a JSON array of objects, one per checklist item:
|
||||
[
|
||||
{{"item": "<checklist item text>", "covered": true/false, "evidence": "<brief quote or reason>"}},
|
||||
...
|
||||
]
|
||||
|
||||
Then on the last line, provide the overall coverage score:
|
||||
coverage_score: <number of covered items>/<total items>"""
|
||||
|
||||
_QUALITY_PROMPT = """\
|
||||
You are evaluating the quality of a research report.
|
||||
|
||||
**Research Question:**
|
||||
{question}
|
||||
|
||||
**Report:**
|
||||
{report}
|
||||
|
||||
Rate the report on these dimensions (each 1-5):
|
||||
|
||||
1. **Presentation**: Is the report well-structured, readable, and professional?
|
||||
2. **Depth**: Does the report go beyond surface-level information?
|
||||
3. **Citation**: Does the report reference specific sources, data, or evidence?
|
||||
4. **Consistency**: Is the report internally consistent and free of contradictions?
|
||||
|
||||
Respond in this exact format:
|
||||
presentation: <1-5>
|
||||
depth: <1-5>
|
||||
citation: <1-5>
|
||||
consistency: <1-5>
|
||||
reasoning: <brief explanation>"""
|
||||
|
||||
|
||||
class LiveResearchBenchSFScorer(LLMJudgeScorer):
|
||||
"""Checklist + quality scorer for Salesforce LiveResearchBench."""
|
||||
|
||||
scorer_id = "liveresearchbench"
|
||||
|
||||
def score(
|
||||
self,
|
||||
record: EvalRecord,
|
||||
model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
if not model_answer or not model_answer.strip():
|
||||
return False, {"reason": "empty_response", "score": 0.0}
|
||||
|
||||
question = record.metadata.get("original_question", record.problem)
|
||||
checklists = record.metadata.get("checklists", [])
|
||||
|
||||
meta: Dict[str, Any] = {}
|
||||
|
||||
# Phase 1: Checklist coverage (if available)
|
||||
coverage_score = 0.0
|
||||
if checklists:
|
||||
try:
|
||||
checklist_text = "\n".join(
|
||||
f"- {item}" for item in checklists
|
||||
)
|
||||
prompt = _COVERAGE_PROMPT.format(
|
||||
question=question,
|
||||
report=model_answer[:8000], # Truncate long reports
|
||||
checklist_items=checklist_text,
|
||||
)
|
||||
raw = self._ask_judge(
|
||||
prompt, temperature=1.0, max_tokens=4096
|
||||
)
|
||||
|
||||
# Parse coverage_score from last line
|
||||
match = re.search(
|
||||
r"coverage_score:\s*(\d+)\s*/\s*(\d+)", raw
|
||||
)
|
||||
if match:
|
||||
covered = int(match.group(1))
|
||||
total = int(match.group(2))
|
||||
coverage_score = covered / total if total > 0 else 0.0
|
||||
meta["coverage_covered"] = covered
|
||||
meta["coverage_total"] = total
|
||||
else:
|
||||
# Fallback: count "covered": true occurrences
|
||||
covered = raw.lower().count('"covered": true') + raw.lower().count('"covered":true')
|
||||
total = len(checklists)
|
||||
coverage_score = covered / total if total > 0 else 0.0
|
||||
meta["coverage_covered"] = covered
|
||||
meta["coverage_total"] = total
|
||||
|
||||
meta["coverage_score"] = coverage_score
|
||||
meta["coverage_raw"] = raw[:500]
|
||||
except Exception as exc:
|
||||
LOGGER.warning(
|
||||
"Coverage scoring failed for %s: %s",
|
||||
record.record_id,
|
||||
exc,
|
||||
)
|
||||
meta["coverage_error"] = str(exc)
|
||||
|
||||
# Phase 2: Quality dimensions
|
||||
quality_score = 0.0
|
||||
try:
|
||||
prompt = _QUALITY_PROMPT.format(
|
||||
question=question,
|
||||
report=model_answer[:8000],
|
||||
)
|
||||
raw = self._ask_judge(prompt, temperature=1.0, max_tokens=1024)
|
||||
|
||||
dims = {}
|
||||
for dim in ["presentation", "depth", "citation", "consistency"]:
|
||||
match = re.search(rf"{dim}:\s*(\d)", raw)
|
||||
if match:
|
||||
dims[dim] = int(match.group(1))
|
||||
|
||||
if dims:
|
||||
quality_score = sum(dims.values()) / (5 * len(dims))
|
||||
meta["quality_dims"] = dims
|
||||
meta["quality_score"] = quality_score
|
||||
meta["quality_raw"] = raw[:500]
|
||||
except Exception as exc:
|
||||
LOGGER.warning(
|
||||
"Quality scoring failed for %s: %s", record.record_id, exc
|
||||
)
|
||||
meta["quality_error"] = str(exc)
|
||||
|
||||
# Final score: weighted average of coverage and quality
|
||||
if checklists:
|
||||
final_score = 0.6 * coverage_score + 0.4 * quality_score
|
||||
else:
|
||||
final_score = quality_score
|
||||
|
||||
meta["final_score"] = final_score
|
||||
is_correct = final_score >= 0.5
|
||||
|
||||
return is_correct, meta
|
||||
|
||||
|
||||
__all__ = ["LiveResearchBenchSFScorer"]
|
||||
Reference in New Issue
Block a user