From bd4938320159a4d3de94cc3f39d56fc76ecd5b68 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon Date: Sun, 22 Feb 2026 23:48:43 +0000 Subject: [PATCH] Add evaluation framework and center README logo SVGs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evaluation framework (evals/): benchmarking system for measuring accuracy across four categories — Chat (WildChat), Reasoning (SuperGPQA), RAG (FRAMES), and Agentic (GAIA). Two backends: jarvis-direct (engine-level) and jarvis-agent (agent-level with tool calling), both supporting local and cloud models. Datasets adapted from IPW, scorers include exact match, LLM letter extraction, and LLM-as-judge. Parallel execution via ThreadPoolExecutor with incremental JSONL output. CLI: python -m evals {run,run-all,summarize,list}. 57 tests pass. SVG fix: center logo content within viewBox by wrapping icon+text in a translate(90,0) group, eliminating the left-shift visible in the README. Co-Authored-By: Claude Opus 4.6 --- assets/openjarvis-logo-dark.svg | 44 ++--- assets/openjarvis-logo-light.svg | 44 ++--- evals/__init__.py | 1 + evals/__main__.py | 6 + evals/backends/__init__.py | 1 + evals/backends/jarvis_agent.py | 88 +++++++++ evals/backends/jarvis_direct.py | 79 ++++++++ evals/cli.py | 301 +++++++++++++++++++++++++++++++ evals/core/__init__.py | 1 + evals/core/backend.py | 45 +++++ evals/core/dataset.py | 36 ++++ evals/core/runner.py | 236 ++++++++++++++++++++++++ evals/core/scorer.py | 53 ++++++ evals/core/types.py | 77 ++++++++ evals/datasets/__init__.py | 1 + evals/datasets/frames.py | 136 ++++++++++++++ evals/datasets/gaia.py | 168 +++++++++++++++++ evals/datasets/supergpqa.py | 121 +++++++++++++ evals/datasets/wildchat.py | 111 ++++++++++++ evals/pyproject.toml | 23 +++ evals/scorers/__init__.py | 1 + evals/scorers/frames_judge.py | 99 ++++++++++ evals/scorers/gaia_exact.py | 165 +++++++++++++++++ evals/scorers/supergpqa_mcq.py | 98 ++++++++++ evals/scorers/wildchat_judge.py | 161 +++++++++++++++++ evals/tests/__init__.py | 1 + evals/tests/conftest.py | 149 +++++++++++++++ evals/tests/test_backends.py | 138 ++++++++++++++ evals/tests/test_frames.py | 111 ++++++++++++ evals/tests/test_gaia.py | 140 ++++++++++++++ evals/tests/test_runner.py | 195 ++++++++++++++++++++ evals/tests/test_supergpqa.py | 105 +++++++++++ evals/tests/test_types.py | 88 +++++++++ evals/tests/test_wildchat.py | 117 ++++++++++++ pyproject.toml | 4 + 35 files changed, 3102 insertions(+), 42 deletions(-) create mode 100644 evals/__init__.py create mode 100644 evals/__main__.py create mode 100644 evals/backends/__init__.py create mode 100644 evals/backends/jarvis_agent.py create mode 100644 evals/backends/jarvis_direct.py create mode 100644 evals/cli.py create mode 100644 evals/core/__init__.py create mode 100644 evals/core/backend.py create mode 100644 evals/core/dataset.py create mode 100644 evals/core/runner.py create mode 100644 evals/core/scorer.py create mode 100644 evals/core/types.py create mode 100644 evals/datasets/__init__.py create mode 100644 evals/datasets/frames.py create mode 100644 evals/datasets/gaia.py create mode 100644 evals/datasets/supergpqa.py create mode 100644 evals/datasets/wildchat.py create mode 100644 evals/pyproject.toml create mode 100644 evals/scorers/__init__.py create mode 100644 evals/scorers/frames_judge.py create mode 100644 evals/scorers/gaia_exact.py create mode 100644 evals/scorers/supergpqa_mcq.py create mode 100644 evals/scorers/wildchat_judge.py create mode 100644 evals/tests/__init__.py create mode 100644 evals/tests/conftest.py create mode 100644 evals/tests/test_backends.py create mode 100644 evals/tests/test_frames.py create mode 100644 evals/tests/test_gaia.py create mode 100644 evals/tests/test_runner.py create mode 100644 evals/tests/test_supergpqa.py create mode 100644 evals/tests/test_types.py create mode 100644 evals/tests/test_wildchat.py diff --git a/assets/openjarvis-logo-dark.svg b/assets/openjarvis-logo-dark.svg index 4d64dc43..221ef061 100644 --- a/assets/openjarvis-logo-dark.svg +++ b/assets/openjarvis-logo-dark.svg @@ -1,24 +1,26 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + openjarvis - - openjarvis diff --git a/assets/openjarvis-logo-light.svg b/assets/openjarvis-logo-light.svg index d7b028e0..c9886c88 100644 --- a/assets/openjarvis-logo-light.svg +++ b/assets/openjarvis-logo-light.svg @@ -1,24 +1,26 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + openjarvis - - openjarvis diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 00000000..9893e3e1 --- /dev/null +++ b/evals/__init__.py @@ -0,0 +1 @@ +"""OpenJarvis Evaluation Framework.""" diff --git a/evals/__main__.py b/evals/__main__.py new file mode 100644 index 00000000..9b783560 --- /dev/null +++ b/evals/__main__.py @@ -0,0 +1,6 @@ +"""Allow running as ``python -m evals``.""" + +from evals.cli import main + +if __name__ == "__main__": + main() diff --git a/evals/backends/__init__.py b/evals/backends/__init__.py new file mode 100644 index 00000000..e7fd7b48 --- /dev/null +++ b/evals/backends/__init__.py @@ -0,0 +1 @@ +"""Inference backends for evaluation.""" diff --git a/evals/backends/jarvis_agent.py b/evals/backends/jarvis_agent.py new file mode 100644 index 00000000..8cc86d16 --- /dev/null +++ b/evals/backends/jarvis_agent.py @@ -0,0 +1,88 @@ +"""Jarvis Agent backend — agent-level inference with tool calling.""" + +from __future__ import annotations + +import time +from typing import Any, Dict, List, Optional + +from evals.core.backend import InferenceBackend + + +class JarvisAgentBackend(InferenceBackend): + """Agent-level inference via SystemBuilder + JarvisSystem.ask(). + + Supports tool calling via the agent harness. Works for both local + and cloud models. + """ + + backend_id = "jarvis-agent" + + def __init__( + self, + engine_key: Optional[str] = None, + agent_name: str = "orchestrator", + tools: Optional[List[str]] = None, + ) -> None: + from openjarvis.system import SystemBuilder + + self._agent_name = agent_name + self._tools = tools or [] + + builder = SystemBuilder() + if engine_key: + builder.engine(engine_key) + builder.agent(agent_name) + if tools: + builder.tools(tools) + self._system = builder.telemetry(False).traces(False).build() + + def generate( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> str: + result = self.generate_full( + prompt, model=model, system=system, + temperature=temperature, max_tokens=max_tokens, + ) + return result["content"] + + def generate_full( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> Dict[str, Any]: + t0 = time.monotonic() + result = self._system.ask( + prompt, + agent=self._agent_name, + tools=self._tools if self._tools else None, + temperature=temperature, + max_tokens=max_tokens, + ) + elapsed = time.monotonic() - t0 + + usage = result.get("usage", {}) + return { + "content": result.get("content", ""), + "usage": usage, + "model": result.get("model", model), + "latency_seconds": elapsed, + "cost_usd": result.get("cost_usd", 0.0), + "turns": result.get("turns", 1), + "tool_results": result.get("tool_results", []), + } + + def close(self) -> None: + self._system.close() + + +__all__ = ["JarvisAgentBackend"] diff --git a/evals/backends/jarvis_direct.py b/evals/backends/jarvis_direct.py new file mode 100644 index 00000000..69f6b5c6 --- /dev/null +++ b/evals/backends/jarvis_direct.py @@ -0,0 +1,79 @@ +"""Jarvis Direct backend — engine-level inference for local and cloud models.""" + +from __future__ import annotations + +import time +from typing import Any, Dict, Optional + +from evals.core.backend import InferenceBackend + + +class JarvisDirectBackend(InferenceBackend): + """Direct engine inference via SystemBuilder. + + Works for both local models (Ollama, vLLM, etc.) and cloud models + (OpenAI, Anthropic, Google) via the CloudEngine. + """ + + backend_id = "jarvis-direct" + + def __init__(self, engine_key: Optional[str] = None) -> None: + from openjarvis.system import SystemBuilder + + builder = SystemBuilder() + if engine_key: + builder.engine(engine_key) + self._system = builder.telemetry(False).traces(False).build() + + def generate( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> str: + result = self.generate_full( + prompt, model=model, system=system, + temperature=temperature, max_tokens=max_tokens, + ) + return result["content"] + + def generate_full( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> Dict[str, Any]: + from openjarvis.core.types import Message, Role + + messages = [] + if system: + messages.append(Message(role=Role.SYSTEM, content=system)) + messages.append(Message(role=Role.USER, content=prompt)) + + t0 = time.monotonic() + result = self._system.engine.generate( + messages, model=model, + temperature=temperature, max_tokens=max_tokens, + ) + elapsed = time.monotonic() - t0 + + usage = result.get("usage", {}) + return { + "content": result.get("content", ""), + "usage": usage, + "model": result.get("model", model), + "latency_seconds": elapsed, + "cost_usd": result.get("cost_usd", 0.0), + } + + def close(self) -> None: + self._system.close() + + +__all__ = ["JarvisDirectBackend"] diff --git a/evals/cli.py b/evals/cli.py new file mode 100644 index 00000000..0d959f71 --- /dev/null +++ b/evals/cli.py @@ -0,0 +1,301 @@ +"""CLI for the OpenJarvis evaluation framework.""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Optional + +import click + +# Registry of available benchmarks and their metadata +BENCHMARKS = { + "supergpqa": {"category": "reasoning", "description": "SuperGPQA multiple-choice"}, + "gaia": {"category": "agentic", "description": "GAIA agentic benchmark"}, + "frames": {"category": "rag", "description": "FRAMES multi-hop RAG"}, + "wildchat": {"category": "chat", "description": "WildChat conversation quality"}, +} + +BACKENDS = { + "jarvis-direct": "Engine-level inference (local or cloud)", + "jarvis-agent": "Agent-level inference with tool calling", +} + + +def _setup_logging(verbose: bool) -> None: + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + datefmt="%H:%M:%S", + ) + + +def _build_backend(backend_name: str, engine_key: Optional[str], + agent_name: str, tools: list[str]): + """Construct the appropriate backend.""" + if backend_name == "jarvis-agent": + from evals.backends.jarvis_agent import JarvisAgentBackend + return JarvisAgentBackend( + engine_key=engine_key, + agent_name=agent_name, + tools=tools, + ) + else: + from evals.backends.jarvis_direct import JarvisDirectBackend + return JarvisDirectBackend(engine_key=engine_key) + + +def _build_dataset(benchmark: str): + """Construct the dataset provider for a benchmark.""" + if benchmark == "supergpqa": + from evals.datasets.supergpqa import SuperGPQADataset + return SuperGPQADataset() + elif benchmark == "gaia": + from evals.datasets.gaia import GAIADataset + return GAIADataset() + elif benchmark == "frames": + from evals.datasets.frames import FRAMESDataset + return FRAMESDataset() + elif benchmark == "wildchat": + from evals.datasets.wildchat import WildChatDataset + return WildChatDataset() + else: + raise click.ClickException(f"Unknown benchmark: {benchmark}") + + +def _build_scorer(benchmark: str, judge_backend, judge_model: str): + """Construct the scorer for a benchmark.""" + if benchmark == "supergpqa": + from evals.scorers.supergpqa_mcq import SuperGPQAScorer + return SuperGPQAScorer(judge_backend, judge_model) + elif benchmark == "gaia": + from evals.scorers.gaia_exact import GAIAScorer + return GAIAScorer(judge_backend, judge_model) + elif benchmark == "frames": + from evals.scorers.frames_judge import FRAMESScorer + return FRAMESScorer(judge_backend, judge_model) + elif benchmark == "wildchat": + from evals.scorers.wildchat_judge import WildChatScorer + return WildChatScorer(judge_backend, judge_model) + else: + raise click.ClickException(f"Unknown benchmark: {benchmark}") + + +def _build_judge_backend(judge_model: str): + """Build the judge backend (always cloud for LLM-as-judge).""" + from evals.backends.jarvis_direct import JarvisDirectBackend + return JarvisDirectBackend(engine_key="cloud") + + +@click.group() +def main(): + """OpenJarvis Evaluation Framework.""" + + +@main.command() +@click.option("-b", "--benchmark", required=True, + type=click.Choice(list(BENCHMARKS.keys())), + help="Benchmark to run") +@click.option("--backend", default="jarvis-direct", + type=click.Choice(list(BACKENDS.keys())), + help="Inference backend") +@click.option("-m", "--model", required=True, help="Model identifier") +@click.option("-e", "--engine", "engine_key", default=None, + help="Engine key (ollama, vllm, cloud, ...)") +@click.option("--agent", "agent_name", default="orchestrator", + help="Agent name for jarvis-agent backend") +@click.option("--tools", default="", help="Comma-separated tool names") +@click.option("-n", "--max-samples", type=int, default=None, + help="Maximum samples to evaluate") +@click.option("-w", "--max-workers", type=int, default=4, + help="Parallel workers") +@click.option("--judge-model", default="gpt-4o", + help="LLM judge model") +@click.option("-o", "--output", "output_path", default=None, + help="Output JSONL path") +@click.option("--seed", type=int, default=42, help="Random seed") +@click.option("--split", "dataset_split", default=None, + help="Dataset split override") +@click.option("--temperature", type=float, default=0.0, + help="Generation temperature") +@click.option("--max-tokens", type=int, default=2048, + help="Max output tokens") +@click.option("-v", "--verbose", is_flag=True, help="Verbose logging") +def run(benchmark, backend, model, engine_key, agent_name, tools, + max_samples, max_workers, judge_model, output_path, seed, + dataset_split, temperature, max_tokens, verbose): + """Run a single benchmark evaluation.""" + _setup_logging(verbose) + + from evals.core.runner import EvalRunner + from evals.core.types import RunConfig + + tool_list = [t.strip() for t in tools.split(",") if t.strip()] if tools else [] + + config = RunConfig( + benchmark=benchmark, + backend=backend, + model=model, + max_samples=max_samples, + max_workers=max_workers, + temperature=temperature, + max_tokens=max_tokens, + judge_model=judge_model, + engine_key=engine_key, + agent_name=agent_name, + tools=tool_list, + output_path=output_path, + seed=seed, + dataset_split=dataset_split, + ) + + eval_backend = _build_backend(backend, engine_key, agent_name, tool_list) + dataset = _build_dataset(benchmark) + judge_backend = _build_judge_backend(judge_model) + scorer = _build_scorer(benchmark, judge_backend, judge_model) + + runner = EvalRunner(config, dataset, eval_backend, scorer) + + try: + summary = runner.run() + finally: + eval_backend.close() + judge_backend.close() + + # Print summary + click.echo(f"\n{'=' * 60}") + click.echo(f"Benchmark: {summary.benchmark}") + click.echo(f"Model: {summary.model}") + click.echo(f"Backend: {summary.backend}") + click.echo(f"Samples: {summary.total_samples}") + click.echo(f"Scored: {summary.scored_samples}") + click.echo(f"Correct: {summary.correct}") + click.echo(f"Accuracy: {summary.accuracy:.4f}") + click.echo(f"Errors: {summary.errors}") + click.echo(f"Latency: {summary.mean_latency_seconds:.2f}s (mean)") + click.echo(f"Cost: ${summary.total_cost_usd:.4f}") + if summary.per_subject: + click.echo("\nPer-subject breakdown:") + for subj, stats in sorted(summary.per_subject.items()): + click.echo(f" {subj}: {stats['accuracy']:.4f} " + f"({int(stats['correct'])}/{int(stats['scored'])})") + click.echo(f"{'=' * 60}") + + +@main.command("run-all") +@click.option("-m", "--model", required=True, help="Model identifier") +@click.option("-e", "--engine", "engine_key", default=None, + help="Engine key") +@click.option("-n", "--max-samples", type=int, default=None, + help="Max samples per benchmark") +@click.option("-w", "--max-workers", type=int, default=4, + help="Parallel workers") +@click.option("--judge-model", default="gpt-4o", help="LLM judge model") +@click.option("--output-dir", default="results/", + help="Output directory for results") +@click.option("--seed", type=int, default=42, help="Random seed") +@click.option("-v", "--verbose", is_flag=True, help="Verbose logging") +def run_all(model, engine_key, max_samples, max_workers, judge_model, + output_dir, seed, verbose): + """Run all benchmarks.""" + _setup_logging(verbose) + + from evals.core.runner import EvalRunner + from evals.core.types import RunConfig + + output_dir_path = Path(output_dir) + output_dir_path.mkdir(parents=True, exist_ok=True) + + model_slug = model.replace("/", "-").replace(":", "-") + summaries = [] + + for bench_name in BENCHMARKS: + click.echo(f"\n--- Running {bench_name} ---") + output_path = output_dir_path / f"{bench_name}_{model_slug}.jsonl" + + config = RunConfig( + benchmark=bench_name, + backend="jarvis-direct", + model=model, + max_samples=max_samples, + max_workers=max_workers, + judge_model=judge_model, + engine_key=engine_key, + output_path=str(output_path), + seed=seed, + ) + + eval_backend = _build_backend("jarvis-direct", engine_key, "orchestrator", []) + dataset = _build_dataset(bench_name) + judge_backend = _build_judge_backend(judge_model) + scorer = _build_scorer(bench_name, judge_backend, judge_model) + + runner = EvalRunner(config, dataset, eval_backend, scorer) + try: + summary = runner.run() + summaries.append(summary) + click.echo(f" {bench_name}: {summary.accuracy:.4f} " + f"({summary.correct}/{summary.scored_samples})") + except Exception as exc: + click.echo(f" {bench_name}: FAILED — {exc}", err=True) + finally: + eval_backend.close() + judge_backend.close() + + # Print overall summary + if summaries: + click.echo(f"\n{'=' * 60}") + click.echo("Overall Results:") + for s in summaries: + click.echo(f" {s.benchmark:12s} {s.accuracy:.4f} " + f"({s.correct}/{s.scored_samples})") + click.echo(f"{'=' * 60}") + + +@main.command() +@click.argument("jsonl_path", type=click.Path(exists=True)) +def summarize(jsonl_path): + """Summarize results from a JSONL output file.""" + records = [] + with open(jsonl_path) as f: + for line in f: + line = line.strip() + if line: + records.append(json.loads(line)) + + if not records: + click.echo("No records found.") + return + + total = len(records) + scored = [r for r in records if r.get("is_correct") is not None] + correct = [r for r in scored if r["is_correct"]] + errors = [r for r in records if r.get("error")] + accuracy = len(correct) / len(scored) if scored else 0.0 + + click.echo(f"File: {jsonl_path}") + click.echo(f"Benchmark: {records[0].get('benchmark', '?')}") + click.echo(f"Model: {records[0].get('model', '?')}") + click.echo(f"Total: {total}") + click.echo(f"Scored: {len(scored)}") + click.echo(f"Correct: {len(correct)}") + click.echo(f"Accuracy: {accuracy:.4f}") + click.echo(f"Errors: {len(errors)}") + + +@main.command("list") +def list_cmd(): + """List available benchmarks and backends.""" + click.echo("Benchmarks:") + for name, info in BENCHMARKS.items(): + click.echo(f" {name:12s} [{info['category']:10s}] {info['description']}") + + click.echo("\nBackends:") + for name, desc in BACKENDS.items(): + click.echo(f" {name:16s} {desc}") + + +if __name__ == "__main__": + main() diff --git a/evals/core/__init__.py b/evals/core/__init__.py new file mode 100644 index 00000000..339913e0 --- /dev/null +++ b/evals/core/__init__.py @@ -0,0 +1 @@ +"""Core evaluation types and ABCs.""" diff --git a/evals/core/backend.py b/evals/core/backend.py new file mode 100644 index 00000000..38c2feb3 --- /dev/null +++ b/evals/core/backend.py @@ -0,0 +1,45 @@ +"""Abstract base class for inference backends.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Dict + + +class InferenceBackend(ABC): + """Base class for all inference backends used in evaluation.""" + + backend_id: str + + @abstractmethod + def generate( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> str: + """Generate a response and return just the text content.""" + + @abstractmethod + def generate_full( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> Dict[str, Any]: + """Generate a response and return full details. + + Returns dict with keys: content, usage, model, latency_seconds, cost_usd. + """ + + def close(self) -> None: + """Release resources.""" + + +__all__ = ["InferenceBackend"] diff --git a/evals/core/dataset.py b/evals/core/dataset.py new file mode 100644 index 00000000..18271713 --- /dev/null +++ b/evals/core/dataset.py @@ -0,0 +1,36 @@ +"""Abstract base class for dataset providers.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Iterable, Optional + +from evals.core.types import EvalRecord + + +class DatasetProvider(ABC): + """Base class for all evaluation dataset providers.""" + + dataset_id: str + dataset_name: str + + @abstractmethod + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + """Load the dataset (possibly downloading from HuggingFace).""" + + @abstractmethod + def iter_records(self) -> Iterable[EvalRecord]: + """Iterate over loaded records.""" + + @abstractmethod + def size(self) -> int: + """Return the number of loaded records.""" + + +__all__ = ["DatasetProvider"] diff --git a/evals/core/runner.py b/evals/core/runner.py new file mode 100644 index 00000000..96e54dcb --- /dev/null +++ b/evals/core/runner.py @@ -0,0 +1,236 @@ +"""EvalRunner — parallel execution of evaluation samples.""" + +from __future__ import annotations + +import json +import logging +import time +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any, Dict, List, Optional + +from evals.core.backend import InferenceBackend +from evals.core.dataset import DatasetProvider +from evals.core.scorer import Scorer +from evals.core.types import EvalRecord, EvalResult, RunConfig, RunSummary + +LOGGER = logging.getLogger(__name__) + + +class EvalRunner: + """Runs an evaluation benchmark with parallel sample execution.""" + + def __init__( + self, + config: RunConfig, + dataset: DatasetProvider, + backend: InferenceBackend, + scorer: Scorer, + ) -> None: + self._config = config + self._dataset = dataset + self._backend = backend + self._scorer = scorer + self._results: List[EvalResult] = [] + self._output_file: Optional[Any] = None + + def run(self) -> RunSummary: + """Execute the evaluation and return a summary.""" + cfg = self._config + started_at = time.time() + + self._dataset.load( + max_samples=cfg.max_samples, + split=cfg.dataset_split, + seed=cfg.seed, + ) + records = list(self._dataset.iter_records()) + LOGGER.info( + "Running %s: %d samples, backend=%s, model=%s, workers=%d", + cfg.benchmark, len(records), cfg.backend, cfg.model, cfg.max_workers, + ) + + # Open output file for incremental JSONL writing + output_path = self._resolve_output_path() + if output_path: + output_path.parent.mkdir(parents=True, exist_ok=True) + self._output_file = open(output_path, "w") + + try: + with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool: + futures = { + pool.submit(self._process_one, r): r for r in records + } + for future in as_completed(futures): + result = future.result() + self._results.append(result) + self._flush_result(result) + finally: + if self._output_file: + self._output_file.close() + self._output_file = None + + ended_at = time.time() + summary = self._compute_summary(records, started_at, ended_at) + + # Write summary JSON alongside JSONL + if output_path: + summary_path = output_path.with_suffix(".summary.json") + with open(summary_path, "w") as f: + json.dump(_summary_to_dict(summary), f, indent=2) + LOGGER.info("Results written to %s", output_path) + LOGGER.info("Summary written to %s", summary_path) + + return summary + + def _process_one(self, record: EvalRecord) -> EvalResult: + """Process a single evaluation sample.""" + cfg = self._config + try: + full = self._backend.generate_full( + record.problem, + model=cfg.model, + temperature=cfg.temperature, + max_tokens=cfg.max_tokens, + ) + content = full.get("content", "") + usage = full.get("usage", {}) + latency = full.get("latency_seconds", 0.0) + cost = full.get("cost_usd", 0.0) + + is_correct, scoring_meta = self._scorer.score(record, content) + + return EvalResult( + record_id=record.record_id, + model_answer=content, + is_correct=is_correct, + score=1.0 if is_correct else (0.0 if is_correct is not None else None), + latency_seconds=latency, + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + cost_usd=cost, + scoring_metadata=scoring_meta, + ) + except Exception as exc: + LOGGER.error("Error processing %s: %s", record.record_id, exc) + return EvalResult( + record_id=record.record_id, + model_answer="", + error=str(exc), + ) + + def _flush_result(self, result: EvalResult) -> None: + """Append a single result to the output JSONL file.""" + if not self._output_file: + return + record_dict = { + "record_id": result.record_id, + "benchmark": self._config.benchmark, + "model": self._config.model, + "backend": self._config.backend, + "model_answer": result.model_answer, + "is_correct": result.is_correct, + "score": result.score, + "latency_seconds": result.latency_seconds, + "prompt_tokens": result.prompt_tokens, + "completion_tokens": result.completion_tokens, + "cost_usd": result.cost_usd, + "error": result.error, + "scoring_metadata": result.scoring_metadata, + } + self._output_file.write(json.dumps(record_dict) + "\n") + self._output_file.flush() + + def _resolve_output_path(self) -> Optional[Path]: + """Determine the output file path.""" + if self._config.output_path: + return Path(self._config.output_path) + # Auto-generate based on benchmark + model + model_slug = self._config.model.replace("/", "-").replace(":", "-") + name = f"{self._config.benchmark}_{model_slug}.jsonl" + return Path(name) + + def _compute_summary( + self, + records: List[EvalRecord], + started_at: float, + ended_at: float, + ) -> RunSummary: + """Compute aggregate statistics from results.""" + cfg = self._config + results = self._results + + scored = [r for r in results if r.is_correct is not None] + correct = [r for r in scored if r.is_correct] + errors = [r for r in results if r.error] + + latencies = [r.latency_seconds for r in results if r.latency_seconds > 0] + mean_latency = sum(latencies) / len(latencies) if latencies else 0.0 + total_cost = sum(r.cost_usd for r in results) + + # Per-subject breakdown + record_map = {r.record_id: r for r in records} + subject_groups: Dict[str, List[EvalResult]] = defaultdict(list) + for r in results: + rec = record_map.get(r.record_id) + subj = rec.subject if rec and rec.subject else "general" + subject_groups[subj].append(r) + + per_subject: Dict[str, Dict[str, float]] = {} + for subj, subj_results in sorted(subject_groups.items()): + subj_scored = [r for r in subj_results if r.is_correct is not None] + subj_correct = [r for r in subj_scored if r.is_correct] + subj_acc = len(subj_correct) / len(subj_scored) if subj_scored else 0.0 + per_subject[subj] = { + "accuracy": round(subj_acc, 4), + "total": float(len(subj_results)), + "scored": float(len(subj_scored)), + "correct": float(len(subj_correct)), + } + + # Determine category from records + categories = {r.category for r in records} + category = categories.pop() if len(categories) == 1 else cfg.benchmark + + accuracy = len(correct) / len(scored) if scored else 0.0 + + return RunSummary( + benchmark=cfg.benchmark, + category=category, + backend=cfg.backend, + model=cfg.model, + total_samples=len(results), + scored_samples=len(scored), + correct=len(correct), + accuracy=round(accuracy, 4), + errors=len(errors), + mean_latency_seconds=round(mean_latency, 4), + total_cost_usd=round(total_cost, 6), + per_subject=per_subject, + started_at=started_at, + ended_at=ended_at, + ) + + +def _summary_to_dict(s: RunSummary) -> Dict[str, Any]: + """Convert a RunSummary to a JSON-serializable dict.""" + return { + "benchmark": s.benchmark, + "category": s.category, + "backend": s.backend, + "model": s.model, + "total_samples": s.total_samples, + "scored_samples": s.scored_samples, + "correct": s.correct, + "accuracy": s.accuracy, + "errors": s.errors, + "mean_latency_seconds": s.mean_latency_seconds, + "total_cost_usd": s.total_cost_usd, + "per_subject": s.per_subject, + "started_at": s.started_at, + "ended_at": s.ended_at, + } + + +__all__ = ["EvalRunner"] diff --git a/evals/core/scorer.py b/evals/core/scorer.py new file mode 100644 index 00000000..88f51ca5 --- /dev/null +++ b/evals/core/scorer.py @@ -0,0 +1,53 @@ +"""Abstract base classes for scoring.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional, Tuple + +from evals.core.backend import InferenceBackend +from evals.core.types import EvalRecord + + +class Scorer(ABC): + """Base class for all scorers.""" + + scorer_id: str + + @abstractmethod + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + """Score a model answer against the reference. + + Returns (is_correct, metadata) where is_correct may be None + if scoring could not be determined. + """ + + +class LLMJudgeScorer(Scorer): + """Base for scorers that need an LLM to judge answers.""" + + def __init__(self, judge_backend: InferenceBackend, judge_model: str) -> None: + self._judge_backend = judge_backend + self._judge_model = judge_model + + def _ask_judge( + self, + prompt: str, + *, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 1024, + ) -> str: + """Send a prompt to the judge LLM and return the response text.""" + return self._judge_backend.generate( + prompt, + model=self._judge_model, + system=system, + temperature=temperature, + max_tokens=max_tokens, + ) + + +__all__ = ["LLMJudgeScorer", "Scorer"] diff --git a/evals/core/types.py b/evals/core/types.py new file mode 100644 index 00000000..cd5dd2fb --- /dev/null +++ b/evals/core/types.py @@ -0,0 +1,77 @@ +"""Core data types for the evaluation framework.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass(slots=True) +class EvalRecord: + """A single evaluation sample.""" + + record_id: str + problem: str + reference: str + category: str # "chat" | "reasoning" | "rag" | "agentic" + subject: str = "" + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class EvalResult: + """Result of evaluating a single sample.""" + + record_id: str + model_answer: str + is_correct: Optional[bool] = None + score: Optional[float] = None + latency_seconds: float = 0.0 + prompt_tokens: int = 0 + completion_tokens: int = 0 + cost_usd: float = 0.0 + error: Optional[str] = None + scoring_metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class RunConfig: + """Configuration for an evaluation run.""" + + benchmark: str + backend: str + model: str + max_samples: Optional[int] = None + max_workers: int = 4 + temperature: float = 0.0 + max_tokens: int = 2048 + judge_model: str = "gpt-4o" + engine_key: Optional[str] = None + agent_name: Optional[str] = None + tools: List[str] = field(default_factory=list) + output_path: Optional[str] = None + seed: int = 42 + dataset_split: Optional[str] = None + + +@dataclass(slots=True) +class RunSummary: + """Summary statistics for a completed evaluation run.""" + + benchmark: str + category: str + backend: str + model: str + total_samples: int + scored_samples: int + correct: int + accuracy: float + errors: int + mean_latency_seconds: float + total_cost_usd: float + per_subject: Dict[str, Dict[str, float]] = field(default_factory=dict) + started_at: float = 0.0 + ended_at: float = 0.0 + + +__all__ = ["EvalRecord", "EvalResult", "RunConfig", "RunSummary"] diff --git a/evals/datasets/__init__.py b/evals/datasets/__init__.py new file mode 100644 index 00000000..ecd71d7d --- /dev/null +++ b/evals/datasets/__init__.py @@ -0,0 +1 @@ +"""Dataset providers for evaluation benchmarks.""" diff --git a/evals/datasets/frames.py b/evals/datasets/frames.py new file mode 100644 index 00000000..bd5a3a69 --- /dev/null +++ b/evals/datasets/frames.py @@ -0,0 +1,136 @@ +"""FRAMES benchmark dataset (google/frames-benchmark). + +Adapted from IPW's frames.py dataset loader. +""" + +from __future__ import annotations + +import random +from typing import Iterable, List, MutableMapping, Optional, Sequence + +from evals.core.dataset import DatasetProvider +from evals.core.types import EvalRecord + +_DEFAULT_INPUT_PROMPT = """Please answer the question below. You should: + +- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings. +- If the answer is a number, return only the number without any units unless specified otherwise. +- If the answer is a string, don't include articles, and don't use abbreviations (e.g. for states). +- If the answer is a comma separated list, apply the above rules to each element in the list. +- This question may require multi-hop reasoning across multiple Wikipedia articles. +{wiki_context} + +Here is the question: + +{question}""" + + +class FRAMESDataset(DatasetProvider): + """FRAMES multi-hop factual retrieval benchmark.""" + + dataset_id = "frames" + dataset_name = "FRAMES" + + _hf_path = "google/frames-benchmark" + _default_split = "test" + + def __init__(self) -> None: + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + from datasets import load_dataset + + use_split = split or self._default_split + dataset = load_dataset(self._hf_path, split=use_split) + + rows: Sequence[MutableMapping[str, object]] + if hasattr(dataset, "to_list"): + rows = dataset.to_list() + else: + rows = list(dataset) + + if seed is not None: + rng = random.Random(seed) + rows = list(rows) + rng.shuffle(rows) + + if max_samples is not None: + rows = rows[:max_samples] + + self._records = [] + for idx, raw in enumerate(rows): + record = self._convert_row(raw, idx) + if record is not None: + self._records.append(record) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + def _convert_row( + self, raw: MutableMapping[str, object], idx: int, + ) -> Optional[EvalRecord]: + question = str( + raw.get("Prompt") or raw.get("prompt") or raw.get("question") or "" + ).strip() + answer = str( + raw.get("Answer") or raw.get("answer") or raw.get("gold_answer") or "" + ).strip() + + if not question or not answer: + return None + + # Extract reasoning types + reasoning = raw.get("reasoning_types", raw.get("reasoning_type", "")) + if isinstance(reasoning, list): + reasoning = ", ".join(str(r) for r in reasoning) + reasoning = str(reasoning) + + # Extract wiki links + wiki_links_raw = raw.get("wiki_links", raw.get("wikipedia_links", [])) + if isinstance(wiki_links_raw, str): + wiki_links = [link.strip() for link in wiki_links_raw.split(",") if link.strip()] + elif isinstance(wiki_links_raw, list): + wiki_links = [str(link) for link in wiki_links_raw] + else: + wiki_links = [] + + # Build wiki context + wiki_context = "" + if wiki_links: + wiki_context = ( + "\n\nRelevant Wikipedia articles that may help answer this question:\n" + + "\n".join(f"- {link}" for link in wiki_links) + ) + + problem = _DEFAULT_INPUT_PROMPT.format( + question=question, wiki_context=wiki_context, + ) + + subject = reasoning if reasoning else "general" + + metadata = { + "index": idx, + "reasoning_types": reasoning, + "wiki_links": wiki_links, + } + + return EvalRecord( + record_id=f"frames-{idx}", + problem=problem, + reference=answer, + category="rag", + subject=subject, + metadata=metadata, + ) + + +__all__ = ["FRAMESDataset"] diff --git a/evals/datasets/gaia.py b/evals/datasets/gaia.py new file mode 100644 index 00000000..418b8746 --- /dev/null +++ b/evals/datasets/gaia.py @@ -0,0 +1,168 @@ +"""GAIA benchmark dataset (gaia-benchmark/GAIA). + +Adapted from IPW's gaia.py dataset loader. +""" + +from __future__ import annotations + +import os +import random +import shutil +from pathlib import Path +from typing import Iterable, List, MutableMapping, Optional, Sequence + +from evals.core.dataset import DatasetProvider +from evals.core.types import EvalRecord + +_DEFAULT_CACHE_DIR = Path.home() / ".cache" / "gaia_benchmark" + +_DEFAULT_INPUT_PROMPT = """Please answer the question below. You should: + +- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings. +- If the answer is a number, return only the number without any units unless specified otherwise. +- If the answer is a string, don't include articles, and don't use abbreviations (e.g. for states). +- If the answer is a comma separated list, apply the above rules to each element in the list. + +{file} + +Here is the question: + +{question}""" + + +class GAIADataset(DatasetProvider): + """GAIA agentic benchmark dataset.""" + + dataset_id = "gaia" + dataset_name = "GAIA" + + _hf_path = "gaia-benchmark/GAIA" + _default_subset = "2023_all" + _default_split = "validation" + + def __init__(self, cache_dir: Optional[str] = None) -> None: + self._cache_dir = Path(cache_dir) if cache_dir else _DEFAULT_CACHE_DIR + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + from datasets import load_dataset + from huggingface_hub import snapshot_download + + use_split = split or self._default_split + + # Ensure dataset is downloaded + dataset_location = self._cache_dir / "GAIA" + if not dataset_location.exists(): + dataset_location.mkdir(parents=True, exist_ok=True) + try: + snapshot_download( + repo_id=self._hf_path, + repo_type="dataset", + local_dir=str(dataset_location), + ) + except Exception: + shutil.rmtree(dataset_location, ignore_errors=True) + raise + + dataset = load_dataset( + str(dataset_location), + name=self._default_subset, + split=use_split, + ) + + rows: Sequence[MutableMapping[str, object]] + if hasattr(dataset, "to_list"): + rows = dataset.to_list() + else: + rows = list(dataset) + + if seed is not None: + rng = random.Random(seed) + rows = list(rows) + rng.shuffle(rows) + + if max_samples is not None: + rows = rows[:max_samples] + + files_location = dataset_location / "2023" / use_split + + self._records = [] + for idx, raw in enumerate(rows): + record = self._convert_row(raw, files_location, idx) + if record is not None: + self._records.append(record) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + def _convert_row( + self, + raw: MutableMapping[str, object], + files_location: Path, + idx: int, + ) -> Optional[EvalRecord]: + task_id = str(raw.get("task_id") or "") + question = str(raw.get("Question") or "").strip() + answer = str(raw.get("Final answer") or "").strip() + level = raw.get("Level") + + if not question or not answer: + return None + + # Discover associated files + file_name: Optional[str] = None + file_path: Optional[Path] = None + if files_location.exists(): + files = [f for f in os.listdir(files_location) if task_id in f] + if files: + file_name = files[0] + file_path = files_location / file_name + + # Format the prompt + if file_name and file_path: + file_info = ( + f"The following file is referenced in the question below and you will " + f"likely need to use it in order to find the correct answer.\n" + f"File name: {file_name}\n" + f"File path: {file_path}\n" + f"Use the file reading tools to access this file." + ) + elif file_name: + file_info = ( + f"The following file is referenced in the question: {file_name}\n" + f"(Note: File path not available)" + ) + else: + file_info = "" + + problem = _DEFAULT_INPUT_PROMPT.format(file=file_info, question=question) + + subject = f"level_{level}" if level else "general" + + metadata = { + "task_id": task_id, + "level": level, + "file_name": file_name, + "file_path": str(file_path) if file_path else None, + } + + return EvalRecord( + record_id=f"gaia-{task_id or idx}", + problem=problem, + reference=answer, + category="agentic", + subject=subject, + metadata=metadata, + ) + + +__all__ = ["GAIADataset"] diff --git a/evals/datasets/supergpqa.py b/evals/datasets/supergpqa.py new file mode 100644 index 00000000..7291d827 --- /dev/null +++ b/evals/datasets/supergpqa.py @@ -0,0 +1,121 @@ +"""SuperGPQA dataset provider (m-a-p/SuperGPQA). + +Adapted from IPW's supergpqa.py dataset loader. +""" + +from __future__ import annotations + +import random +from typing import Iterable, List, MutableMapping, Optional, Sequence + +from evals.core.dataset import DatasetProvider +from evals.core.types import EvalRecord + + +def _format_options(options: Iterable[str]) -> str: + rendered = [] + for idx, option in enumerate(options): + letter = chr(ord("A") + idx) + rendered.append(f"{letter}. {option}") + return "\n".join(rendered) + + +class SuperGPQADataset(DatasetProvider): + """SuperGPQA multiple-choice benchmark dataset.""" + + dataset_id = "supergpqa" + dataset_name = "SuperGPQA" + + _hf_path = "m-a-p/SuperGPQA" + _default_split = "train" + + def __init__(self) -> None: + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + from datasets import load_dataset + + use_split = split or self._default_split + dataset = load_dataset(self._hf_path, split=use_split) + + rows: Sequence[MutableMapping[str, object]] + if hasattr(dataset, "to_list"): + rows = dataset.to_list() + else: + rows = list(dataset) + + if seed is not None: + rng = random.Random(seed) + rows = list(rows) + rng.shuffle(rows) + + if max_samples is not None: + rows = rows[:max_samples] + + self._records = [] + for idx, raw in enumerate(rows): + record = self._convert_row(raw, idx) + if record is not None: + self._records.append(record) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + def _convert_row( + self, raw: MutableMapping[str, object], idx: int, + ) -> Optional[EvalRecord]: + question = str(raw.get("question") or "").strip() + options_raw = raw.get("options") or [] + options = [str(o).strip() for o in options_raw if str(o).strip()] + answer_letter = str(raw.get("answer_letter") or "").strip().upper() + answer_text = str(raw.get("answer") or "").strip() + + subject = str( + raw.get("subfield") + or raw.get("field") + or raw.get("discipline") + or "general" + ).strip() or "general" + + if not question or not options or not answer_letter: + return None + + prompt_parts = [ + question, "", + "Options:", + _format_options(options), "", + "Respond with the correct letter only.", + ] + problem = "\n".join(part for part in prompt_parts if part).strip() + + metadata = { + "uuid": raw.get("uuid"), + "discipline": raw.get("discipline"), + "field": raw.get("field"), + "subfield": raw.get("subfield"), + "difficulty": raw.get("difficulty"), + "is_calculation": raw.get("is_calculation"), + "answer_text": answer_text, + "options": options, + } + + return EvalRecord( + record_id=f"supergpqa-{idx}", + problem=problem, + reference=answer_letter, + category="reasoning", + subject=subject, + metadata=metadata, + ) + + +__all__ = ["SuperGPQADataset"] diff --git a/evals/datasets/wildchat.py b/evals/datasets/wildchat.py new file mode 100644 index 00000000..3d69d0a5 --- /dev/null +++ b/evals/datasets/wildchat.py @@ -0,0 +1,111 @@ +"""WildChat dataset provider (allenai/WildChat-1M). + +Filters to English single-turn conversations for chat quality evaluation. +""" + +from __future__ import annotations + +import random +from typing import Iterable, List, MutableMapping, Optional, Sequence + +from evals.core.dataset import DatasetProvider +from evals.core.types import EvalRecord + + +class WildChatDataset(DatasetProvider): + """WildChat conversation quality benchmark.""" + + dataset_id = "wildchat" + dataset_name = "WildChat" + + _hf_path = "allenai/WildChat-1M" + _default_split = "train" + + def __init__(self) -> None: + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + from datasets import load_dataset + + use_split = split or self._default_split + dataset = load_dataset(self._hf_path, split=use_split) + + rows: Sequence[MutableMapping[str, object]] + if hasattr(dataset, "to_list"): + rows = dataset.to_list() + else: + rows = list(dataset) + + # Filter to English single-turn conversations + filtered: List[MutableMapping[str, object]] = [] + for row in rows: + if not isinstance(row, MutableMapping): + row = dict(row) + + language = str(row.get("language") or "").lower() + if language != "english": + continue + + conversation = row.get("conversation") + if not isinstance(conversation, list): + continue + + # Single-turn: exactly one user message and one assistant message + if len(conversation) != 2: + continue + + user_msg = conversation[0] + asst_msg = conversation[1] + if ( + str(user_msg.get("role", "")) != "user" + or str(asst_msg.get("role", "")) != "assistant" + ): + continue + + user_content = str(user_msg.get("content", "")).strip() + asst_content = str(asst_msg.get("content", "")).strip() + if not user_content or not asst_content: + continue + + row["_user_content"] = user_content + row["_asst_content"] = asst_content + filtered.append(row) + + # Shuffle with seed + if seed is not None: + rng = random.Random(seed) + rng.shuffle(filtered) + + if max_samples is not None: + filtered = filtered[:max_samples] + + self._records = [] + for idx, raw in enumerate(filtered): + self._records.append( + EvalRecord( + record_id=f"wildchat-{idx}", + problem=raw["_user_content"], + reference=raw["_asst_content"], + category="chat", + subject="conversation", + metadata={ + "model": str(raw.get("model", "")), + "language": "english", + }, + ) + ) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + +__all__ = ["WildChatDataset"] diff --git a/evals/pyproject.toml b/evals/pyproject.toml new file mode 100644 index 00000000..5c1227b2 --- /dev/null +++ b/evals/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "openjarvis-evals" +version = "0.1.0" +description = "Evaluation framework for OpenJarvis" +requires-python = ">=3.10" +dependencies = [ + "openjarvis>=1.0.0", + "click>=8", + "datasets>=2.14", + "huggingface-hub>=0.20", + "tqdm>=4.65", + "rich>=13", +] + +[project.optional-dependencies] +dev = ["pytest>=8", "pytest-cov>=5"] + +[project.scripts] +openjarvis-eval = "evals.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/evals/scorers/__init__.py b/evals/scorers/__init__.py new file mode 100644 index 00000000..613569d9 --- /dev/null +++ b/evals/scorers/__init__.py @@ -0,0 +1 @@ +"""Scoring implementations for evaluation benchmarks.""" diff --git a/evals/scorers/frames_judge.py b/evals/scorers/frames_judge.py new file mode 100644 index 00000000..966a3b64 --- /dev/null +++ b/evals/scorers/frames_judge.py @@ -0,0 +1,99 @@ +"""FRAMES scorer — LLM-as-judge for multi-hop factual retrieval. + +Adapted from IPW's frames.py evaluation handler. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +from evals.core.scorer import LLMJudgeScorer +from evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + +_GRADER_TEMPLATE = """You are evaluating an AI system's answer to a multi-hop factual question. + +Compare the predicted answer against the Ground Truth Answer and determine if the prediction is correct. + +## Evaluation Guidelines + +1. **Focus on semantic meaning**: Look for equivalent information - exact wording is not required. +2. **Assess factual accuracy**: Determine whether the essential facts from the Ground Truth are present in the answer. +3. **Ignore minor differences**: Capitalization, punctuation, formatting, and word order don't matter. +4. **Partial credit**: If the Ground Truth has multiple parts, all essential parts must be present for a correct rating. +5. **Additional information**: Extra correct information in the prediction is acceptable, but extra incorrect information is not. + +## Question +{question} + +## Ground Truth Answer +{ground_truth} + +## Predicted Answer +{predicted_answer} + +Your response MUST use exactly this format: +extracted_final_answer: +reasoning: +correct: """ + + +class FRAMESScorer(LLMJudgeScorer): + """LLM-as-judge evaluation for FRAMES multi-hop factual retrieval.""" + + scorer_id = "frames" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + reference = record.reference + if not reference or not reference.strip(): + return None, {"reason": "no_ground_truth"} + + prompt = _GRADER_TEMPLATE.format( + question=record.problem, + ground_truth=reference, + predicted_answer=model_answer, + ) + + try: + raw = self._ask_judge(prompt, temperature=0.0, max_tokens=1024) + + structured_match = re.search( + r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + ) + if structured_match: + is_correct = structured_match.group(1).lower() == "yes" + else: + response_upper = raw.upper().strip() + if "TRUE" in response_upper: + is_correct = True + elif "FALSE" in response_upper: + is_correct = False + else: + LOGGER.warning("Could not parse grade from response: %s", raw[:50]) + is_correct = False + + meta: Dict[str, Any] = { + "raw_judge_output": raw, + } + extracted = re.search( + r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + ) + if extracted: + meta["extracted_answer"] = extracted.group(1).strip() + + return is_correct, meta + + except Exception as exc: + LOGGER.error("FRAMES scoring failed: %s", exc) + return None, {"error": str(exc)} + + +__all__ = ["FRAMESScorer"] diff --git a/evals/scorers/gaia_exact.py b/evals/scorers/gaia_exact.py new file mode 100644 index 00000000..e8095060 --- /dev/null +++ b/evals/scorers/gaia_exact.py @@ -0,0 +1,165 @@ +"""GAIA scorer — normalized exact match with LLM fallback. + +Adapted from IPW's gaia.py evaluation handler. +""" + +from __future__ import annotations + +import logging +import re +import string +from typing import Any, Dict, Optional, Tuple + +from evals.core.scorer import LLMJudgeScorer +from evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Normalization helpers (ported from IPW) +# --------------------------------------------------------------------------- + + +def _normalize_number_str(number_str: str) -> float: + for char in ["$", "%", ","]: + number_str = number_str.replace(char, "") + try: + return float(number_str) + except ValueError: + return float("inf") + + +def _normalize_str(input_str: str, remove_punct: bool = True) -> str: + no_spaces = re.sub(r"\s", "", input_str) + if remove_punct: + translator = str.maketrans("", "", string.punctuation) + return no_spaces.lower().translate(translator) + return no_spaces.lower() + + +def _split_string(s: str, char_list: list[str] | None = None) -> list[str]: + if char_list is None: + char_list = [",", ";"] + pattern = f"[{''.join(char_list)}]" + return re.split(pattern, s) + + +def _is_float(element: object) -> bool: + try: + float(element) # type: ignore[arg-type] + return True + except (ValueError, TypeError): + return False + + +def exact_match(model_answer: str, ground_truth: str) -> bool: + """GAIA exact-match scorer with normalization for numbers, lists, and strings.""" + if model_answer is None: + model_answer = "None" + + if _is_float(ground_truth): + normalized = _normalize_number_str(model_answer) + return normalized == float(ground_truth) + + if any(char in ground_truth for char in [",", ";"]): + gt_elems = _split_string(ground_truth) + ma_elems = _split_string(model_answer) + if len(gt_elems) != len(ma_elems): + return False + comparisons = [] + for ma_elem, gt_elem in zip(ma_elems, gt_elems): + if _is_float(gt_elem): + comparisons.append( + _normalize_number_str(ma_elem) == float(gt_elem) + ) + else: + comparisons.append( + _normalize_str(ma_elem, remove_punct=False) + == _normalize_str(gt_elem, remove_punct=False) + ) + return all(comparisons) + + return _normalize_str(model_answer) == _normalize_str(ground_truth) + + +# --------------------------------------------------------------------------- +# LLM fallback prompt +# --------------------------------------------------------------------------- + +_LLM_FALLBACK_PROMPT = """Your job is to determine if the predicted answer is semantically equivalent to the gold target. + +Question: {question} +Gold target: {ground_truth} +Predicted answer: {response} + +Consider the following: +- Numerical answers should match exactly (accounting for different formats like $1,000 vs 1000) +- List answers should contain all elements (order may vary) +- String answers should have the same meaning (case and punctuation don't matter) + +Your response MUST use exactly this format: +extracted_final_answer: +reasoning: +correct: """ + + +class GAIAScorer(LLMJudgeScorer): + """GAIA evaluation: exact match with normalization + LLM fallback.""" + + scorer_id = "gaia" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + reference = record.reference + if not reference or not reference.strip(): + return None, {"reason": "no_ground_truth"} + + # Try exact match first (fast, no API call) + if exact_match(model_answer, reference): + return True, {"match_type": "exact"} + + # LLM fallback for semantic comparison + try: + prompt = _LLM_FALLBACK_PROMPT.format( + question=record.problem or "(No question provided)", + response=model_answer, + ground_truth=reference, + ) + raw = self._ask_judge(prompt, temperature=0.0, max_tokens=1024) + + structured_match = re.search( + r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + ) + if structured_match: + is_correct = structured_match.group(1).lower() == "yes" + else: + is_correct = ( + "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() + ) + + meta: Dict[str, Any] = { + "match_type": "llm_fallback", + "raw_judge_output": raw, + } + extracted_match = re.search( + r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + ) + if extracted_match: + meta["extracted_answer"] = extracted_match.group(1).strip() + + return is_correct, meta + + except Exception as exc: + LOGGER.error("GAIA LLM fallback failed: %s", exc) + return False, { + "match_type": "llm_fallback_error", + "error": str(exc), + } + + +__all__ = ["GAIAScorer", "exact_match"] diff --git a/evals/scorers/supergpqa_mcq.py b/evals/scorers/supergpqa_mcq.py new file mode 100644 index 00000000..97675a31 --- /dev/null +++ b/evals/scorers/supergpqa_mcq.py @@ -0,0 +1,98 @@ +"""SuperGPQA MCQ scorer — LLM-based letter extraction + exact match. + +Adapted from IPW's mcq.py and gpqa.py evaluation handlers. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +from evals.core.scorer import LLMJudgeScorer +from evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + + +class SuperGPQAScorer(LLMJudgeScorer): + """Score SuperGPQA responses by extracting answer letter via LLM.""" + + scorer_id = "supergpqa" + + def _valid_letters_from_options(self, metadata: Dict[str, Any]) -> str: + options = metadata.get("options") + if isinstance(options, list) and options: + n = len(options) + return "".join(chr(ord("A") + i) for i in range(n)) + return "ABCD" + + def _extract_answer_with_llm( + self, + problem: str, + model_answer: str, + valid_letters: str, + ) -> Optional[str]: + """Use the judge LLM to extract the answer letter from the response.""" + last_letter = valid_letters[-1] if valid_letters else "D" + + system_prompt = ( + f"You are an answer extraction assistant. Extract the final multiple choice answer " + f"from the response. Return ONLY a single letter (A-{last_letter}). " + f"If no valid answer letter is found, return 'NONE'." + ) + + user_prompt = ( + f"Problem: {problem}\nResponse: {model_answer}\n\n" + f"Extract the final answer letter:" + ) + + try: + raw_response = self._ask_judge( + user_prompt, system=system_prompt, + temperature=0.0, max_tokens=5, + ) + + extracted = raw_response.strip().upper() + + # Handle "The answer is: A" etc. + answer_match = re.search( + r"(?:THE ANSWER IS:?\s*)?([A-Z])", extracted, re.IGNORECASE, + ) + if answer_match: + extracted = answer_match.group(1).upper() + + if extracted in valid_letters: + return extracted + + return None + + except Exception as exc: + LOGGER.error("Error in LLM-based answer extraction: %s", exc) + return None + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + ref = record.reference.strip().upper() + if not ref: + return None, {"reason": "missing_reference_letter"} + + valid_letters = self._valid_letters_from_options(record.metadata) + + candidate = self._extract_answer_with_llm( + record.problem, model_answer, valid_letters, + ) + if not candidate: + return None, {"reason": "no_choice_letter_extracted"} + + is_correct = candidate == ref + meta = { + "reference_letter": ref, + "candidate_letter": candidate, + "valid_letters": valid_letters, + } + return is_correct, meta + + +__all__ = ["SuperGPQAScorer"] diff --git a/evals/scorers/wildchat_judge.py b/evals/scorers/wildchat_judge.py new file mode 100644 index 00000000..be6e9285 --- /dev/null +++ b/evals/scorers/wildchat_judge.py @@ -0,0 +1,161 @@ +"""WildChat scorer — dual-comparison LLM-as-judge. + +Adapted from IPW's wildchat.py evaluation handler. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +from evals.core.scorer import LLMJudgeScorer +from evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + +SYSTEM_PROMPT = """You are an impartial judge evaluating the quality of two AI-assistant replies to the same user prompt. + +Step 1 – Generate your own answer +Write the response *you* would give to the user. Keep it separate from later analysis. + +Step 2 – Decide the query type +Classify the user prompt as either +• **Subjective / open-ended** (creative writing, opinion, advice, brainstorming) +• **Objective / technical** (code, math, logical derivations with a single correct outcome) +If uncertain, default to "Subjective". + +Step 3 – Score each assistant with the correct rubric + +| Query type | Criteria | +|------------|----------| +| Subjective / open-ended | 1. Correctness / factual soundness 2. Helpfulness 3. Relevance 4. Conciseness 5. Creativity & novelty | +| Objective / technical | 1. Correctness only | + +When using the multi-criteria rubric, note strengths and weaknesses for **each** dimension. +When using the single-criterion rubric, focus exclusively on factual / functional accuracy and ignore style or flair. + +Step 4 – Compare & justify +Explain which assistant is better and why, correcting any mistakes you find. Highlight missing but important details. **Be concise.** + +Step 5 – Verdict +1. Assistant A is significantly better: [[A>>B]] +2. Assistant A is slightly better: [[A>B]] +3. Tie, Assistant A is equal: [[A=B]] +4. Assistant B is slightly better: [[B>A]] +5. Assistant B is significantly better: [[B>>A]] + +Choose exactly one token from: `[[A>>B]]`, `[[A>B]]`, `[[A=B]]`, `[[B>A]]`, `[[B>>A]]`. + +--- + +### Output format (strict) +Return **only** a JSON object that matches the provided schema: + + + +```json +{ +"query_type": "", +"explanation": " | (if query_type is \\"Objective / technical\\")", +"verdict": ">B]], [[A>B]], [[A=B]], [[B>A]], [[B>>A]]>" +} +```""" + + +class WildChatScorer(LLMJudgeScorer): + """Dual-comparison LLM-as-judge for chat quality.""" + + scorer_id = "wildchat" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + reference = record.reference + if not reference or not reference.strip(): + return None, {"reason": "empty_reference"} + + # Two comparisons: (model vs reference) and (reference vs model) + verdict1, response1 = self._get_judge_verdict( + record.problem, model_answer, reference, + ) + verdict2, response2 = self._get_judge_verdict( + record.problem, reference, model_answer, + ) + + if verdict1 is None or verdict2 is None: + return None, { + "reason": "missing_verdicts", + "verdict1": verdict1, + "verdict2": verdict2, + } + + result1 = self._verdict_to_bool(verdict1, generated_is_a=True) + result2 = self._verdict_to_bool(verdict2, generated_is_a=False) + + meta: Dict[str, Any] = { + "generated_as_a": {"verdict": verdict1, "response": response1}, + "generated_as_b": {"verdict": verdict2, "response": response2}, + } + + if result1 is None or result2 is None: + return None, meta + + final_result = result1 or result2 + return final_result, meta + + def _get_judge_verdict( + self, problem: str, response_a: str, response_b: str, + ) -> Tuple[Optional[str], Optional[str]]: + prompt = ( + f"<|User Prompt|>\n{problem}\n\n" + f"<|The Start of Assistant A's Answer|>\n{response_a}\n" + f"<|The End of Assistant A's Answer|>\n\n" + f"<|The Start of Assistant B's Answer|>\n{response_b}\n" + f"<|The End of Assistant B's Answer|>" + ) + + try: + raw = self._ask_judge( + prompt, system=SYSTEM_PROMPT, + temperature=0.0, max_tokens=1024, + ) + except Exception as exc: + LOGGER.error("WildChat judge call failed: %s", exc) + return None, None + + content = raw.strip() + verdict_match = re.search(r"\[\[([AB][><=]{1,2}[AB])\]\]", content) + if verdict_match: + return verdict_match.group(1), content + + return None, content + + @staticmethod + def _verdict_to_bool( + verdict: Optional[str], generated_is_a: bool, + ) -> Optional[bool]: + if not verdict: + return None + + verdict_map_a = { + "A>>B": True, + "A>B": True, + "A=B": True, + "B>A": False, + "B>>A": False, + } + + verdict_map_b = { + "A>>B": False, + "A>B": False, + "A=B": True, + "B>A": True, + "B>>A": True, + } + + verdict_map = verdict_map_a if generated_is_a else verdict_map_b + return verdict_map.get(verdict) + + +__all__ = ["WildChatScorer"] diff --git a/evals/tests/__init__.py b/evals/tests/__init__.py new file mode 100644 index 00000000..e80eaeb1 --- /dev/null +++ b/evals/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the evaluation framework.""" diff --git a/evals/tests/conftest.py b/evals/tests/conftest.py new file mode 100644 index 00000000..95aa59b4 --- /dev/null +++ b/evals/tests/conftest.py @@ -0,0 +1,149 @@ +"""Shared test fixtures for the evaluation framework.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +import pytest + +# Ensure evals package is importable from the repo root +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from evals.core.backend import InferenceBackend +from evals.core.dataset import DatasetProvider +from evals.core.scorer import Scorer +from evals.core.types import EvalRecord + +# --------------------------------------------------------------------------- +# Mock backend +# --------------------------------------------------------------------------- + + +class MockBackend(InferenceBackend): + """Backend that returns canned responses for testing.""" + + backend_id = "mock" + + def __init__(self, responses: Optional[Dict[str, str]] = None) -> None: + self._responses = responses or {} + self._default_response = "Mock response" + self._call_count = 0 + + def generate( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> str: + self._call_count += 1 + return self._responses.get(prompt, self._default_response) + + def generate_full( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> Dict[str, Any]: + content = self.generate( + prompt, model=model, system=system, + temperature=temperature, max_tokens=max_tokens, + ) + return { + "content": content, + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + }, + "model": model, + "latency_seconds": 0.1, + "cost_usd": 0.001, + } + + +class MockScorer(Scorer): + """Scorer that always returns a fixed result.""" + + scorer_id = "mock" + + def __init__(self, result: bool = True) -> None: + self._result = result + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + return self._result, {"mock": True} + + +class MockDataset(DatasetProvider): + """Dataset that yields fixed records.""" + + dataset_id = "mock" + dataset_name = "Mock" + + def __init__(self, records: Optional[list[EvalRecord]] = None) -> None: + self._records = records or [] + + def load(self, *, max_samples=None, split=None, seed=None) -> None: + pass + + def iter_records(self): + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def mock_backend(): + return MockBackend() + + +@pytest.fixture() +def mock_scorer(): + return MockScorer() + + +@pytest.fixture() +def sample_records(): + return [ + EvalRecord( + record_id="test-001", + problem="What is 2+2?", + reference="4", + category="reasoning", + subject="math", + ), + EvalRecord( + record_id="test-002", + problem="What is the capital of France?", + reference="Paris", + category="reasoning", + subject="geography", + ), + EvalRecord( + record_id="test-003", + problem="Hello, how are you?", + reference="I'm fine, thank you!", + category="chat", + subject="greeting", + ), + ] + + +@pytest.fixture() +def mock_dataset(sample_records): + return MockDataset(sample_records) diff --git a/evals/tests/test_backends.py b/evals/tests/test_backends.py new file mode 100644 index 00000000..5a431cc8 --- /dev/null +++ b/evals/tests/test_backends.py @@ -0,0 +1,138 @@ +"""Tests for backend construction with mocks.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + + +class TestJarvisDirectBackend: + @patch("openjarvis.system.SystemBuilder") + def test_construction_default(self, mock_builder_cls): + mock_builder = MagicMock() + mock_builder.engine.return_value = mock_builder + mock_builder.telemetry.return_value = mock_builder + mock_builder.traces.return_value = mock_builder + mock_system = MagicMock() + mock_builder.build.return_value = mock_system + mock_builder_cls.return_value = mock_builder + + from evals.backends.jarvis_direct import JarvisDirectBackend + + backend = JarvisDirectBackend() + assert backend.backend_id == "jarvis-direct" + mock_builder.telemetry.assert_called_with(False) + mock_builder.traces.assert_called_with(False) + mock_builder.build.assert_called_once() + + @patch("openjarvis.system.SystemBuilder") + def test_construction_with_engine_key(self, mock_builder_cls): + mock_builder = MagicMock() + mock_builder.engine.return_value = mock_builder + mock_builder.telemetry.return_value = mock_builder + mock_builder.traces.return_value = mock_builder + mock_builder.build.return_value = MagicMock() + mock_builder_cls.return_value = mock_builder + + from evals.backends.jarvis_direct import JarvisDirectBackend + + JarvisDirectBackend(engine_key="cloud") + mock_builder.engine.assert_called_with("cloud") + + @patch("openjarvis.system.SystemBuilder") + def test_generate_full(self, mock_builder_cls): + mock_builder = MagicMock() + mock_builder.engine.return_value = mock_builder + mock_builder.telemetry.return_value = mock_builder + mock_builder.traces.return_value = mock_builder + mock_system = MagicMock() + mock_system.engine.generate.return_value = { + "content": "42", + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + "model": "test-model", + "cost_usd": 0.001, + } + mock_builder.build.return_value = mock_system + mock_builder_cls.return_value = mock_builder + + from evals.backends.jarvis_direct import JarvisDirectBackend + + backend = JarvisDirectBackend() + result = backend.generate_full("What is 2+2?", model="test-model") + + assert result["content"] == "42" + assert result["cost_usd"] == 0.001 + assert "latency_seconds" in result + + @patch("openjarvis.system.SystemBuilder") + def test_generate(self, mock_builder_cls): + mock_builder = MagicMock() + mock_builder.engine.return_value = mock_builder + mock_builder.telemetry.return_value = mock_builder + mock_builder.traces.return_value = mock_builder + mock_system = MagicMock() + mock_system.engine.generate.return_value = { + "content": "Paris", + "usage": {}, + } + mock_builder.build.return_value = mock_system + mock_builder_cls.return_value = mock_builder + + from evals.backends.jarvis_direct import JarvisDirectBackend + + backend = JarvisDirectBackend() + text = backend.generate("Capital of France?", model="m") + assert text == "Paris" + + +class TestJarvisAgentBackend: + @patch("openjarvis.system.SystemBuilder") + def test_construction(self, mock_builder_cls): + mock_builder = MagicMock() + mock_builder.engine.return_value = mock_builder + mock_builder.agent.return_value = mock_builder + mock_builder.tools.return_value = mock_builder + mock_builder.telemetry.return_value = mock_builder + mock_builder.traces.return_value = mock_builder + mock_builder.build.return_value = MagicMock() + mock_builder_cls.return_value = mock_builder + + from evals.backends.jarvis_agent import JarvisAgentBackend + + backend = JarvisAgentBackend( + engine_key="cloud", agent_name="orchestrator", + tools=["calculator", "think"], + ) + assert backend.backend_id == "jarvis-agent" + mock_builder.engine.assert_called_with("cloud") + mock_builder.agent.assert_called_with("orchestrator") + mock_builder.tools.assert_called_with(["calculator", "think"]) + + @patch("openjarvis.system.SystemBuilder") + def test_generate_full(self, mock_builder_cls): + mock_builder = MagicMock() + mock_builder.engine.return_value = mock_builder + mock_builder.agent.return_value = mock_builder + mock_builder.tools.return_value = mock_builder + mock_builder.telemetry.return_value = mock_builder + mock_builder.traces.return_value = mock_builder + mock_system = MagicMock() + mock_system.ask.return_value = { + "content": "The answer is 4.", + "usage": {"prompt_tokens": 50, "completion_tokens": 20}, + "model": "gpt-4o", + "turns": 2, + "tool_results": [ + {"tool_name": "calculator", "content": "4", "success": True}, + ], + } + mock_builder.build.return_value = mock_system + mock_builder_cls.return_value = mock_builder + + from evals.backends.jarvis_agent import JarvisAgentBackend + + backend = JarvisAgentBackend(agent_name="orchestrator") + result = backend.generate_full("What is 2+2?", model="gpt-4o") + + assert result["content"] == "The answer is 4." + assert result["turns"] == 2 + assert len(result["tool_results"]) == 1 diff --git a/evals/tests/test_frames.py b/evals/tests/test_frames.py new file mode 100644 index 00000000..c9047f37 --- /dev/null +++ b/evals/tests/test_frames.py @@ -0,0 +1,111 @@ +"""Tests for FRAMES scorer (judge prompt formatting and verdict parsing).""" + +from __future__ import annotations + +from evals.core.types import EvalRecord +from evals.scorers.frames_judge import _GRADER_TEMPLATE, FRAMESScorer +from evals.tests.conftest import MockBackend + + +class TestGraderTemplate: + def test_template_formatting(self): + result = _GRADER_TEMPLATE.format( + question="What is the capital?", + ground_truth="Paris", + predicted_answer="The capital is Paris", + ) + assert "What is the capital?" in result + assert "Paris" in result + assert "The capital is Paris" in result + assert "correct: " in result + + def test_template_has_required_sections(self): + assert "## Question" in _GRADER_TEMPLATE + assert "## Ground Truth Answer" in _GRADER_TEMPLATE + assert "## Predicted Answer" in _GRADER_TEMPLATE + assert "extracted_final_answer:" in _GRADER_TEMPLATE + + +class TestFRAMESScorer: + def _make_record(self, reference="Paris"): + return EvalRecord( + record_id="frames-1", + problem="What is the capital of France?", + reference=reference, + category="rag", + subject="general", + ) + + def test_correct_answer(self): + backend = MockBackend() + backend._default_response = ( + "extracted_final_answer: Paris\n" + "reasoning: The answer correctly identifies Paris as the capital.\n" + "correct: yes" + ) + scorer = FRAMESScorer(backend, "gpt-4o") + + record = self._make_record("Paris") + is_correct, meta = scorer.score(record, "The capital is Paris") + + assert is_correct is True + assert "raw_judge_output" in meta + assert meta["extracted_answer"] == "Paris" + + def test_incorrect_answer(self): + backend = MockBackend() + backend._default_response = ( + "extracted_final_answer: London\n" + "reasoning: London is not the capital of France.\n" + "correct: no" + ) + scorer = FRAMESScorer(backend, "gpt-4o") + + record = self._make_record("Paris") + is_correct, meta = scorer.score(record, "London") + + assert is_correct is False + + def test_empty_response(self): + backend = MockBackend() + scorer = FRAMESScorer(backend, "gpt-4o") + + record = self._make_record("Paris") + is_correct, meta = scorer.score(record, "") + + assert is_correct is False + assert meta["reason"] == "empty_response" + + def test_no_ground_truth(self): + backend = MockBackend() + scorer = FRAMESScorer(backend, "gpt-4o") + + record = self._make_record("") + is_correct, meta = scorer.score(record, "Paris") + + assert is_correct is None + assert meta["reason"] == "no_ground_truth" + + def test_fallback_true_false_parsing(self): + backend = MockBackend() + backend._default_response = "The prediction is TRUE" + scorer = FRAMESScorer(backend, "gpt-4o") + + record = self._make_record("Paris") + is_correct, _ = scorer.score(record, "Paris") + + assert is_correct is True + + def test_judge_error(self): + class ErrorBackend(MockBackend): + def generate(self, prompt, **kw): + raise RuntimeError("API error") + + backend = ErrorBackend() + scorer = FRAMESScorer(backend, "gpt-4o") + + record = self._make_record("Paris") + is_correct, meta = scorer.score(record, "Paris") + + assert is_correct is None + assert "error" in meta diff --git a/evals/tests/test_gaia.py b/evals/tests/test_gaia.py new file mode 100644 index 00000000..9cdff9e7 --- /dev/null +++ b/evals/tests/test_gaia.py @@ -0,0 +1,140 @@ +"""Tests for GAIA scorer logic (normalization and exact match).""" + +from __future__ import annotations + +from evals.core.types import EvalRecord +from evals.scorers.gaia_exact import ( + GAIAScorer, + _is_float, + _normalize_number_str, + _normalize_str, + _split_string, + exact_match, +) +from evals.tests.conftest import MockBackend + + +class TestNormalization: + def test_normalize_number_str(self): + assert _normalize_number_str("1000") == 1000.0 + assert _normalize_number_str("$1,000") == 1000.0 + assert _normalize_number_str("50%") == 50.0 + assert _normalize_number_str("abc") == float("inf") + + def test_normalize_str(self): + assert _normalize_str("Hello World") == "helloworld" + assert _normalize_str("Hello, World!", remove_punct=True) == "helloworld" + assert _normalize_str("Hello, World!", remove_punct=False) == "hello,world!" + + def test_split_string(self): + assert _split_string("a, b, c") == ["a", " b", " c"] + assert _split_string("a; b") == ["a", " b"] + + def test_is_float(self): + assert _is_float("3.14") is True + assert _is_float("42") is True + assert _is_float("abc") is False + assert _is_float(None) is False + + +class TestExactMatch: + def test_number_match(self): + assert exact_match("42", "42") is True + assert exact_match("$1,000", "1000") is True + assert exact_match("43", "42") is False + + def test_string_match(self): + assert exact_match("Paris", "paris") is True + assert exact_match(" Paris ", "paris") is True + assert exact_match("London", "Paris") is False + + def test_list_match(self): + assert exact_match("1, 2, 3", "1, 2, 3") is True + assert exact_match("1, 2", "1, 2, 3") is False + + def test_none_answer(self): + assert exact_match(None, "42") is False + + def test_punctuation_handling(self): + assert exact_match("Hello!", "Hello") is True + assert exact_match("test.", "test") is True + + +class TestGAIAScorer: + def _make_record(self, reference="42"): + return EvalRecord( + record_id="gaia-001", + problem="What is the answer?", + reference=reference, + category="agentic", + subject="level_1", + ) + + def test_exact_match_correct(self): + backend = MockBackend() + scorer = GAIAScorer(backend, "gpt-4o") + + record = self._make_record("42") + is_correct, meta = scorer.score(record, "42") + + assert is_correct is True + assert meta["match_type"] == "exact" + + def test_exact_match_with_formatting(self): + backend = MockBackend() + scorer = GAIAScorer(backend, "gpt-4o") + + record = self._make_record("1000") + is_correct, meta = scorer.score(record, "$1,000") + + assert is_correct is True + assert meta["match_type"] == "exact" + + def test_llm_fallback(self): + backend = MockBackend() + backend._default_response = ( + "extracted_final_answer: 42\n" + "reasoning: The answer is semantically equivalent.\n" + "correct: yes" + ) + scorer = GAIAScorer(backend, "gpt-4o") + + record = self._make_record("42") + is_correct, meta = scorer.score(record, "The answer is forty-two") + + assert is_correct is True + assert meta["match_type"] == "llm_fallback" + + def test_llm_fallback_incorrect(self): + backend = MockBackend() + backend._default_response = ( + "extracted_final_answer: 43\n" + "reasoning: Different number.\n" + "correct: no" + ) + scorer = GAIAScorer(backend, "gpt-4o") + + record = self._make_record("42") + is_correct, meta = scorer.score(record, "The answer is 43") + + assert is_correct is False + + def test_empty_response(self): + backend = MockBackend() + scorer = GAIAScorer(backend, "gpt-4o") + + record = self._make_record("42") + is_correct, meta = scorer.score(record, "") + + assert is_correct is False + assert meta["reason"] == "empty_response" + + def test_no_ground_truth(self): + backend = MockBackend() + scorer = GAIAScorer(backend, "gpt-4o") + + record = self._make_record("") + is_correct, meta = scorer.score(record, "42") + + assert is_correct is None + assert meta["reason"] == "no_ground_truth" diff --git a/evals/tests/test_runner.py b/evals/tests/test_runner.py new file mode 100644 index 00000000..a2f51171 --- /dev/null +++ b/evals/tests/test_runner.py @@ -0,0 +1,195 @@ +"""Tests for the EvalRunner.""" + +from __future__ import annotations + +import json + +from evals.core.runner import EvalRunner +from evals.core.types import EvalRecord, RunConfig +from evals.tests.conftest import MockBackend, MockDataset, MockScorer + + +class TestEvalRunner: + def _make_records(self, n=5): + return [ + EvalRecord( + record_id=f"r{i}", + problem=f"Question {i}", + reference=f"Answer {i}", + category="reasoning", + subject="math" if i % 2 == 0 else "science", + ) + for i in range(n) + ] + + def test_basic_run(self, tmp_path): + records = self._make_records(5) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="test-model", + max_workers=1, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + summary = runner.run() + + assert summary.total_samples == 5 + assert summary.scored_samples == 5 + assert summary.correct == 5 + assert summary.accuracy == 1.0 + assert summary.errors == 0 + assert summary.benchmark == "test" + assert summary.model == "test-model" + + def test_with_errors(self, tmp_path): + records = self._make_records(3) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + # Backend that raises on second call + class FailingBackend(MockBackend): + def __init__(self): + super().__init__() + self._fail_count = 0 + + def generate_full(self, prompt, **kw): + self._fail_count += 1 + if self._fail_count == 2: + raise RuntimeError("test error") + return super().generate_full(prompt, **kw) + + dataset = MockDataset(records) + backend = FailingBackend() + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + summary = runner.run() + + assert summary.total_samples == 3 + assert summary.errors == 1 + + def test_per_subject_breakdown(self, tmp_path): + records = self._make_records(4) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + summary = runner.run() + + assert "math" in summary.per_subject + assert "science" in summary.per_subject + assert summary.per_subject["math"]["accuracy"] == 1.0 + + def test_jsonl_output(self, tmp_path): + records = self._make_records(3) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + runner.run() + + # Verify JSONL + lines = output_path.read_text().strip().split("\n") + assert len(lines) == 3 + first = json.loads(lines[0]) + assert "record_id" in first + assert "model_answer" in first + assert "is_correct" in first + + # Verify summary JSON + summary_path = output_path.with_suffix(".summary.json") + assert summary_path.exists() + summary_data = json.loads(summary_path.read_text()) + assert summary_data["total_samples"] == 3 + + def test_parallel_workers(self, tmp_path): + records = self._make_records(10) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=4, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + summary = runner.run() + + assert summary.total_samples == 10 + assert summary.correct == 10 + + def test_mixed_scoring(self, tmp_path): + records = self._make_records(4) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + # Scorer that alternates correct/incorrect + class AlternatingScorer(MockScorer): + def __init__(self): + super().__init__() + self._count = 0 + + def score(self, record, model_answer): + self._count += 1 + return (self._count % 2 == 0), {"count": self._count} + + dataset = MockDataset(records) + backend = MockBackend() + scorer = AlternatingScorer() + + runner = EvalRunner(config, dataset, backend, scorer) + summary = runner.run() + + assert summary.scored_samples == 4 + assert summary.correct == 2 + assert summary.accuracy == 0.5 diff --git a/evals/tests/test_supergpqa.py b/evals/tests/test_supergpqa.py new file mode 100644 index 00000000..5dedb361 --- /dev/null +++ b/evals/tests/test_supergpqa.py @@ -0,0 +1,105 @@ +"""Tests for SuperGPQA scorer logic.""" + +from __future__ import annotations + +from evals.core.types import EvalRecord +from evals.tests.conftest import MockBackend + + +class TestSuperGPQAScorer: + def _make_record(self, answer="B", options=None): + if options is None: + options = ["Option A", "Option B", "Option C", "Option D"] + return EvalRecord( + record_id="sgpqa-1", + problem=( + "What is X?\n\nOptions:\n" + "A. Option A\nB. Option B\n" + "C. Option C\nD. Option D\n\n" + "Respond with the correct letter only." + ), + reference=answer, + category="reasoning", + subject="math", + metadata={"options": options}, + ) + + def test_correct_extraction(self): + from evals.scorers.supergpqa_mcq import SuperGPQAScorer + + backend = MockBackend(responses={}) + backend._default_response = "B" + scorer = SuperGPQAScorer(backend, "gpt-4o") + + record = self._make_record(answer="B") + is_correct, meta = scorer.score(record, "The answer is B") + + assert is_correct is True + assert meta["reference_letter"] == "B" + assert meta["candidate_letter"] == "B" + + def test_incorrect_extraction(self): + from evals.scorers.supergpqa_mcq import SuperGPQAScorer + + backend = MockBackend() + backend._default_response = "A" + scorer = SuperGPQAScorer(backend, "gpt-4o") + + record = self._make_record(answer="B") + is_correct, meta = scorer.score(record, "I think A") + + assert is_correct is False + assert meta["candidate_letter"] == "A" + + def test_missing_reference(self): + from evals.scorers.supergpqa_mcq import SuperGPQAScorer + + backend = MockBackend() + scorer = SuperGPQAScorer(backend, "gpt-4o") + + record = self._make_record(answer="") + is_correct, meta = scorer.score(record, "B") + + assert is_correct is None + assert meta["reason"] == "missing_reference_letter" + + def test_no_extraction(self): + from evals.scorers.supergpqa_mcq import SuperGPQAScorer + + backend = MockBackend() + backend._default_response = "NONE" + scorer = SuperGPQAScorer(backend, "gpt-4o") + + record = self._make_record(answer="B") + is_correct, meta = scorer.score(record, "I don't know") + + assert is_correct is None + assert meta["reason"] == "no_choice_letter_extracted" + + def test_valid_letters_from_options(self): + from evals.scorers.supergpqa_mcq import SuperGPQAScorer + + backend = MockBackend() + scorer = SuperGPQAScorer(backend, "gpt-4o") + + # 5 options + metadata = {"options": ["A", "B", "C", "D", "E"]} + letters = scorer._valid_letters_from_options(metadata) + assert letters == "ABCDE" + + # No options + letters = scorer._valid_letters_from_options({}) + assert letters == "ABCD" + + def test_extraction_with_verbose_response(self): + from evals.scorers.supergpqa_mcq import SuperGPQAScorer + + backend = MockBackend() + backend._default_response = "THE ANSWER IS: C" + scorer = SuperGPQAScorer(backend, "gpt-4o") + + record = self._make_record(answer="C") + is_correct, meta = scorer.score(record, "After analysis, C is correct") + + assert is_correct is True + assert meta["candidate_letter"] == "C" diff --git a/evals/tests/test_types.py b/evals/tests/test_types.py new file mode 100644 index 00000000..64cfcfd6 --- /dev/null +++ b/evals/tests/test_types.py @@ -0,0 +1,88 @@ +"""Tests for core data types.""" + +from __future__ import annotations + +from evals.core.types import EvalRecord, EvalResult, RunConfig, RunSummary + + +class TestEvalRecord: + def test_creation(self): + r = EvalRecord( + record_id="r1", problem="What?", reference="42", + category="reasoning", + ) + assert r.record_id == "r1" + assert r.problem == "What?" + assert r.reference == "42" + assert r.category == "reasoning" + assert r.subject == "" + assert r.metadata == {} + + def test_with_subject_and_metadata(self): + r = EvalRecord( + record_id="r2", problem="Q", reference="A", + category="chat", subject="greet", + metadata={"key": "val"}, + ) + assert r.subject == "greet" + assert r.metadata == {"key": "val"} + + +class TestEvalResult: + def test_defaults(self): + r = EvalResult(record_id="r1", model_answer="42") + assert r.is_correct is None + assert r.score is None + assert r.latency_seconds == 0.0 + assert r.prompt_tokens == 0 + assert r.completion_tokens == 0 + assert r.cost_usd == 0.0 + assert r.error is None + assert r.scoring_metadata == {} + + def test_full(self): + r = EvalResult( + record_id="r1", model_answer="42", is_correct=True, + score=1.0, latency_seconds=1.5, prompt_tokens=100, + completion_tokens=50, cost_usd=0.01, + scoring_metadata={"match": "exact"}, + ) + assert r.is_correct is True + assert r.score == 1.0 + assert r.cost_usd == 0.01 + + +class TestRunConfig: + def test_defaults(self): + c = RunConfig(benchmark="supergpqa", backend="jarvis-direct", model="qwen3:8b") + assert c.max_samples is None + assert c.max_workers == 4 + assert c.temperature == 0.0 + assert c.max_tokens == 2048 + assert c.judge_model == "gpt-4o" + assert c.seed == 42 + assert c.tools == [] + + def test_with_agent(self): + c = RunConfig( + benchmark="gaia", backend="jarvis-agent", model="gpt-4o", + engine_key="cloud", agent_name="orchestrator", + tools=["calculator", "think"], + ) + assert c.agent_name == "orchestrator" + assert c.tools == ["calculator", "think"] + + +class TestRunSummary: + def test_creation(self): + s = RunSummary( + benchmark="supergpqa", category="reasoning", + backend="jarvis-direct", model="qwen3:8b", + total_samples=100, scored_samples=95, correct=47, + accuracy=0.495, errors=5, mean_latency_seconds=2.1, + total_cost_usd=0.0, + per_subject={"math": {"accuracy": 0.5}}, + ) + assert s.accuracy == 0.495 + assert s.per_subject["math"]["accuracy"] == 0.5 + assert s.started_at == 0.0 diff --git a/evals/tests/test_wildchat.py b/evals/tests/test_wildchat.py new file mode 100644 index 00000000..fae2c9d3 --- /dev/null +++ b/evals/tests/test_wildchat.py @@ -0,0 +1,117 @@ +"""Tests for WildChat scorer (verdict parsing and dual comparison).""" + +from __future__ import annotations + +from evals.core.types import EvalRecord +from evals.scorers.wildchat_judge import WildChatScorer +from evals.tests.conftest import MockBackend + + +class TestVerdictParsing: + def test_verdict_to_bool_generated_is_a(self): + assert WildChatScorer._verdict_to_bool("A>>B", generated_is_a=True) is True + assert WildChatScorer._verdict_to_bool("A>B", generated_is_a=True) is True + assert WildChatScorer._verdict_to_bool("A=B", generated_is_a=True) is True + assert WildChatScorer._verdict_to_bool("B>A", generated_is_a=True) is False + assert WildChatScorer._verdict_to_bool("B>>A", generated_is_a=True) is False + + def test_verdict_to_bool_generated_is_b(self): + assert WildChatScorer._verdict_to_bool("A>>B", generated_is_a=False) is False + assert WildChatScorer._verdict_to_bool("A>B", generated_is_a=False) is False + assert WildChatScorer._verdict_to_bool("A=B", generated_is_a=False) is True + assert WildChatScorer._verdict_to_bool("B>A", generated_is_a=False) is True + assert WildChatScorer._verdict_to_bool("B>>A", generated_is_a=False) is True + + def test_verdict_none(self): + assert WildChatScorer._verdict_to_bool(None, generated_is_a=True) is None + assert WildChatScorer._verdict_to_bool("", generated_is_a=True) is None + + def test_unknown_verdict(self): + assert WildChatScorer._verdict_to_bool("X>Y", generated_is_a=True) is None + + +class TestWildChatScorer: + def _make_record(self, reference="I'm fine, thanks!"): + return EvalRecord( + record_id="wc-1", + problem="How are you?", + reference=reference, + category="chat", + subject="conversation", + ) + + def test_model_wins(self): + backend = MockBackend() + # First call: model as A, verdict A>>B (model better) + # Second call: reference as A, verdict A>>B (reference better → model loses) + # But since it's OR logic, model wins if either comparison says it's good + call_count = 0 + + def mock_generate(prompt, **kw): + nonlocal call_count + call_count += 1 + if call_count == 1: + return '```json\n{"verdict": "[[A>>B]]"}\n```' + else: + return '```json\n{"verdict": "[[A>>B]]"}\n```' + + backend.generate = mock_generate + scorer = WildChatScorer(backend, "gpt-4o") + + record = self._make_record() + is_correct, meta = scorer.score(record, "I'm doing great!") + + assert is_correct is True + + def test_model_loses(self): + backend = MockBackend() + call_count = 0 + + def mock_generate(prompt, **kw): + nonlocal call_count + call_count += 1 + if call_count == 1: + # model as A, reference as B → B wins + return '```json\n{"verdict": "[[B>>A]]"}\n```' + else: + # reference as A, model as B → A wins (reference better) + return '```json\n{"verdict": "[[A>>B]]"}\n```' + + backend.generate = mock_generate + scorer = WildChatScorer(backend, "gpt-4o") + + record = self._make_record() + is_correct, meta = scorer.score(record, "Bad response") + + assert is_correct is False + + def test_tie(self): + backend = MockBackend() + backend._default_response = '{"verdict": "[[A=B]]"}' + scorer = WildChatScorer(backend, "gpt-4o") + + record = self._make_record() + is_correct, meta = scorer.score(record, "I'm fine, thanks!") + + assert is_correct is True # Tie counts as correct + + def test_empty_reference(self): + backend = MockBackend() + scorer = WildChatScorer(backend, "gpt-4o") + + record = self._make_record("") + is_correct, meta = scorer.score(record, "Hello") + + assert is_correct is None + assert meta["reason"] == "empty_reference" + + def test_missing_verdict(self): + backend = MockBackend() + backend._default_response = "I cannot decide between them." + scorer = WildChatScorer(backend, "gpt-4o") + + record = self._make_record() + is_correct, meta = scorer.score(record, "Hello") + + assert is_correct is None + assert "missing_verdicts" in str(meta.get("reason", "")) diff --git a/pyproject.toml b/pyproject.toml index 3b00aed4..81a68204 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,3 +85,7 @@ src = ["src", "tests"] [tool.ruff.lint] select = ["E", "F", "I", "W"] + +[tool.ruff.lint.per-file-ignores] +"evals/datasets/*.py" = ["E501"] +"evals/scorers/*.py" = ["E501"]