diff --git a/assets/openjarvis-logo-dark.svg b/assets/openjarvis-logo-dark.svg
index 4d64dc43..221ef061 100644
--- a/assets/openjarvis-logo-dark.svg
+++ b/assets/openjarvis-logo-dark.svg
@@ -1,24 +1,26 @@
diff --git a/assets/openjarvis-logo-light.svg b/assets/openjarvis-logo-light.svg
index d7b028e0..c9886c88 100644
--- a/assets/openjarvis-logo-light.svg
+++ b/assets/openjarvis-logo-light.svg
@@ -1,24 +1,26 @@
diff --git a/evals/__init__.py b/evals/__init__.py
new file mode 100644
index 00000000..9893e3e1
--- /dev/null
+++ b/evals/__init__.py
@@ -0,0 +1 @@
+"""OpenJarvis Evaluation Framework."""
diff --git a/evals/__main__.py b/evals/__main__.py
new file mode 100644
index 00000000..9b783560
--- /dev/null
+++ b/evals/__main__.py
@@ -0,0 +1,6 @@
+"""Allow running as ``python -m evals``."""
+
+from evals.cli import main
+
+if __name__ == "__main__":
+ main()
diff --git a/evals/backends/__init__.py b/evals/backends/__init__.py
new file mode 100644
index 00000000..e7fd7b48
--- /dev/null
+++ b/evals/backends/__init__.py
@@ -0,0 +1 @@
+"""Inference backends for evaluation."""
diff --git a/evals/backends/jarvis_agent.py b/evals/backends/jarvis_agent.py
new file mode 100644
index 00000000..8cc86d16
--- /dev/null
+++ b/evals/backends/jarvis_agent.py
@@ -0,0 +1,88 @@
+"""Jarvis Agent backend — agent-level inference with tool calling."""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, List, Optional
+
+from evals.core.backend import InferenceBackend
+
+
+class JarvisAgentBackend(InferenceBackend):
+ """Agent-level inference via SystemBuilder + JarvisSystem.ask().
+
+ Supports tool calling via the agent harness. Works for both local
+ and cloud models.
+ """
+
+ backend_id = "jarvis-agent"
+
+ def __init__(
+ self,
+ engine_key: Optional[str] = None,
+ agent_name: str = "orchestrator",
+ tools: Optional[List[str]] = None,
+ ) -> None:
+ from openjarvis.system import SystemBuilder
+
+ self._agent_name = agent_name
+ self._tools = tools or []
+
+ builder = SystemBuilder()
+ if engine_key:
+ builder.engine(engine_key)
+ builder.agent(agent_name)
+ if tools:
+ builder.tools(tools)
+ self._system = builder.telemetry(False).traces(False).build()
+
+ def generate(
+ self,
+ prompt: str,
+ *,
+ model: str,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 2048,
+ ) -> str:
+ result = self.generate_full(
+ prompt, model=model, system=system,
+ temperature=temperature, max_tokens=max_tokens,
+ )
+ return result["content"]
+
+ def generate_full(
+ self,
+ prompt: str,
+ *,
+ model: str,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 2048,
+ ) -> Dict[str, Any]:
+ t0 = time.monotonic()
+ result = self._system.ask(
+ prompt,
+ agent=self._agent_name,
+ tools=self._tools if self._tools else None,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ )
+ elapsed = time.monotonic() - t0
+
+ usage = result.get("usage", {})
+ return {
+ "content": result.get("content", ""),
+ "usage": usage,
+ "model": result.get("model", model),
+ "latency_seconds": elapsed,
+ "cost_usd": result.get("cost_usd", 0.0),
+ "turns": result.get("turns", 1),
+ "tool_results": result.get("tool_results", []),
+ }
+
+ def close(self) -> None:
+ self._system.close()
+
+
+__all__ = ["JarvisAgentBackend"]
diff --git a/evals/backends/jarvis_direct.py b/evals/backends/jarvis_direct.py
new file mode 100644
index 00000000..69f6b5c6
--- /dev/null
+++ b/evals/backends/jarvis_direct.py
@@ -0,0 +1,79 @@
+"""Jarvis Direct backend — engine-level inference for local and cloud models."""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, Optional
+
+from evals.core.backend import InferenceBackend
+
+
+class JarvisDirectBackend(InferenceBackend):
+ """Direct engine inference via SystemBuilder.
+
+ Works for both local models (Ollama, vLLM, etc.) and cloud models
+ (OpenAI, Anthropic, Google) via the CloudEngine.
+ """
+
+ backend_id = "jarvis-direct"
+
+ def __init__(self, engine_key: Optional[str] = None) -> None:
+ from openjarvis.system import SystemBuilder
+
+ builder = SystemBuilder()
+ if engine_key:
+ builder.engine(engine_key)
+ self._system = builder.telemetry(False).traces(False).build()
+
+ def generate(
+ self,
+ prompt: str,
+ *,
+ model: str,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 2048,
+ ) -> str:
+ result = self.generate_full(
+ prompt, model=model, system=system,
+ temperature=temperature, max_tokens=max_tokens,
+ )
+ return result["content"]
+
+ def generate_full(
+ self,
+ prompt: str,
+ *,
+ model: str,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 2048,
+ ) -> Dict[str, Any]:
+ from openjarvis.core.types import Message, Role
+
+ messages = []
+ if system:
+ messages.append(Message(role=Role.SYSTEM, content=system))
+ messages.append(Message(role=Role.USER, content=prompt))
+
+ t0 = time.monotonic()
+ result = self._system.engine.generate(
+ messages, model=model,
+ temperature=temperature, max_tokens=max_tokens,
+ )
+ elapsed = time.monotonic() - t0
+
+ usage = result.get("usage", {})
+ return {
+ "content": result.get("content", ""),
+ "usage": usage,
+ "model": result.get("model", model),
+ "latency_seconds": elapsed,
+ "cost_usd": result.get("cost_usd", 0.0),
+ }
+
+ def close(self) -> None:
+ self._system.close()
+
+
+__all__ = ["JarvisDirectBackend"]
diff --git a/evals/cli.py b/evals/cli.py
new file mode 100644
index 00000000..0d959f71
--- /dev/null
+++ b/evals/cli.py
@@ -0,0 +1,301 @@
+"""CLI for the OpenJarvis evaluation framework."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Optional
+
+import click
+
+# Registry of available benchmarks and their metadata
+BENCHMARKS = {
+ "supergpqa": {"category": "reasoning", "description": "SuperGPQA multiple-choice"},
+ "gaia": {"category": "agentic", "description": "GAIA agentic benchmark"},
+ "frames": {"category": "rag", "description": "FRAMES multi-hop RAG"},
+ "wildchat": {"category": "chat", "description": "WildChat conversation quality"},
+}
+
+BACKENDS = {
+ "jarvis-direct": "Engine-level inference (local or cloud)",
+ "jarvis-agent": "Agent-level inference with tool calling",
+}
+
+
+def _setup_logging(verbose: bool) -> None:
+ level = logging.DEBUG if verbose else logging.INFO
+ logging.basicConfig(
+ level=level,
+ format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+ datefmt="%H:%M:%S",
+ )
+
+
+def _build_backend(backend_name: str, engine_key: Optional[str],
+ agent_name: str, tools: list[str]):
+ """Construct the appropriate backend."""
+ if backend_name == "jarvis-agent":
+ from evals.backends.jarvis_agent import JarvisAgentBackend
+ return JarvisAgentBackend(
+ engine_key=engine_key,
+ agent_name=agent_name,
+ tools=tools,
+ )
+ else:
+ from evals.backends.jarvis_direct import JarvisDirectBackend
+ return JarvisDirectBackend(engine_key=engine_key)
+
+
+def _build_dataset(benchmark: str):
+ """Construct the dataset provider for a benchmark."""
+ if benchmark == "supergpqa":
+ from evals.datasets.supergpqa import SuperGPQADataset
+ return SuperGPQADataset()
+ elif benchmark == "gaia":
+ from evals.datasets.gaia import GAIADataset
+ return GAIADataset()
+ elif benchmark == "frames":
+ from evals.datasets.frames import FRAMESDataset
+ return FRAMESDataset()
+ elif benchmark == "wildchat":
+ from evals.datasets.wildchat import WildChatDataset
+ return WildChatDataset()
+ else:
+ raise click.ClickException(f"Unknown benchmark: {benchmark}")
+
+
+def _build_scorer(benchmark: str, judge_backend, judge_model: str):
+ """Construct the scorer for a benchmark."""
+ if benchmark == "supergpqa":
+ from evals.scorers.supergpqa_mcq import SuperGPQAScorer
+ return SuperGPQAScorer(judge_backend, judge_model)
+ elif benchmark == "gaia":
+ from evals.scorers.gaia_exact import GAIAScorer
+ return GAIAScorer(judge_backend, judge_model)
+ elif benchmark == "frames":
+ from evals.scorers.frames_judge import FRAMESScorer
+ return FRAMESScorer(judge_backend, judge_model)
+ elif benchmark == "wildchat":
+ from evals.scorers.wildchat_judge import WildChatScorer
+ return WildChatScorer(judge_backend, judge_model)
+ else:
+ raise click.ClickException(f"Unknown benchmark: {benchmark}")
+
+
+def _build_judge_backend(judge_model: str):
+ """Build the judge backend (always cloud for LLM-as-judge)."""
+ from evals.backends.jarvis_direct import JarvisDirectBackend
+ return JarvisDirectBackend(engine_key="cloud")
+
+
+@click.group()
+def main():
+ """OpenJarvis Evaluation Framework."""
+
+
+@main.command()
+@click.option("-b", "--benchmark", required=True,
+ type=click.Choice(list(BENCHMARKS.keys())),
+ help="Benchmark to run")
+@click.option("--backend", default="jarvis-direct",
+ type=click.Choice(list(BACKENDS.keys())),
+ help="Inference backend")
+@click.option("-m", "--model", required=True, help="Model identifier")
+@click.option("-e", "--engine", "engine_key", default=None,
+ help="Engine key (ollama, vllm, cloud, ...)")
+@click.option("--agent", "agent_name", default="orchestrator",
+ help="Agent name for jarvis-agent backend")
+@click.option("--tools", default="", help="Comma-separated tool names")
+@click.option("-n", "--max-samples", type=int, default=None,
+ help="Maximum samples to evaluate")
+@click.option("-w", "--max-workers", type=int, default=4,
+ help="Parallel workers")
+@click.option("--judge-model", default="gpt-4o",
+ help="LLM judge model")
+@click.option("-o", "--output", "output_path", default=None,
+ help="Output JSONL path")
+@click.option("--seed", type=int, default=42, help="Random seed")
+@click.option("--split", "dataset_split", default=None,
+ help="Dataset split override")
+@click.option("--temperature", type=float, default=0.0,
+ help="Generation temperature")
+@click.option("--max-tokens", type=int, default=2048,
+ help="Max output tokens")
+@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
+def run(benchmark, backend, model, engine_key, agent_name, tools,
+ max_samples, max_workers, judge_model, output_path, seed,
+ dataset_split, temperature, max_tokens, verbose):
+ """Run a single benchmark evaluation."""
+ _setup_logging(verbose)
+
+ from evals.core.runner import EvalRunner
+ from evals.core.types import RunConfig
+
+ tool_list = [t.strip() for t in tools.split(",") if t.strip()] if tools else []
+
+ config = RunConfig(
+ benchmark=benchmark,
+ backend=backend,
+ model=model,
+ max_samples=max_samples,
+ max_workers=max_workers,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ judge_model=judge_model,
+ engine_key=engine_key,
+ agent_name=agent_name,
+ tools=tool_list,
+ output_path=output_path,
+ seed=seed,
+ dataset_split=dataset_split,
+ )
+
+ eval_backend = _build_backend(backend, engine_key, agent_name, tool_list)
+ dataset = _build_dataset(benchmark)
+ judge_backend = _build_judge_backend(judge_model)
+ scorer = _build_scorer(benchmark, judge_backend, judge_model)
+
+ runner = EvalRunner(config, dataset, eval_backend, scorer)
+
+ try:
+ summary = runner.run()
+ finally:
+ eval_backend.close()
+ judge_backend.close()
+
+ # Print summary
+ click.echo(f"\n{'=' * 60}")
+ click.echo(f"Benchmark: {summary.benchmark}")
+ click.echo(f"Model: {summary.model}")
+ click.echo(f"Backend: {summary.backend}")
+ click.echo(f"Samples: {summary.total_samples}")
+ click.echo(f"Scored: {summary.scored_samples}")
+ click.echo(f"Correct: {summary.correct}")
+ click.echo(f"Accuracy: {summary.accuracy:.4f}")
+ click.echo(f"Errors: {summary.errors}")
+ click.echo(f"Latency: {summary.mean_latency_seconds:.2f}s (mean)")
+ click.echo(f"Cost: ${summary.total_cost_usd:.4f}")
+ if summary.per_subject:
+ click.echo("\nPer-subject breakdown:")
+ for subj, stats in sorted(summary.per_subject.items()):
+ click.echo(f" {subj}: {stats['accuracy']:.4f} "
+ f"({int(stats['correct'])}/{int(stats['scored'])})")
+ click.echo(f"{'=' * 60}")
+
+
+@main.command("run-all")
+@click.option("-m", "--model", required=True, help="Model identifier")
+@click.option("-e", "--engine", "engine_key", default=None,
+ help="Engine key")
+@click.option("-n", "--max-samples", type=int, default=None,
+ help="Max samples per benchmark")
+@click.option("-w", "--max-workers", type=int, default=4,
+ help="Parallel workers")
+@click.option("--judge-model", default="gpt-4o", help="LLM judge model")
+@click.option("--output-dir", default="results/",
+ help="Output directory for results")
+@click.option("--seed", type=int, default=42, help="Random seed")
+@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
+def run_all(model, engine_key, max_samples, max_workers, judge_model,
+ output_dir, seed, verbose):
+ """Run all benchmarks."""
+ _setup_logging(verbose)
+
+ from evals.core.runner import EvalRunner
+ from evals.core.types import RunConfig
+
+ output_dir_path = Path(output_dir)
+ output_dir_path.mkdir(parents=True, exist_ok=True)
+
+ model_slug = model.replace("/", "-").replace(":", "-")
+ summaries = []
+
+ for bench_name in BENCHMARKS:
+ click.echo(f"\n--- Running {bench_name} ---")
+ output_path = output_dir_path / f"{bench_name}_{model_slug}.jsonl"
+
+ config = RunConfig(
+ benchmark=bench_name,
+ backend="jarvis-direct",
+ model=model,
+ max_samples=max_samples,
+ max_workers=max_workers,
+ judge_model=judge_model,
+ engine_key=engine_key,
+ output_path=str(output_path),
+ seed=seed,
+ )
+
+ eval_backend = _build_backend("jarvis-direct", engine_key, "orchestrator", [])
+ dataset = _build_dataset(bench_name)
+ judge_backend = _build_judge_backend(judge_model)
+ scorer = _build_scorer(bench_name, judge_backend, judge_model)
+
+ runner = EvalRunner(config, dataset, eval_backend, scorer)
+ try:
+ summary = runner.run()
+ summaries.append(summary)
+ click.echo(f" {bench_name}: {summary.accuracy:.4f} "
+ f"({summary.correct}/{summary.scored_samples})")
+ except Exception as exc:
+ click.echo(f" {bench_name}: FAILED — {exc}", err=True)
+ finally:
+ eval_backend.close()
+ judge_backend.close()
+
+ # Print overall summary
+ if summaries:
+ click.echo(f"\n{'=' * 60}")
+ click.echo("Overall Results:")
+ for s in summaries:
+ click.echo(f" {s.benchmark:12s} {s.accuracy:.4f} "
+ f"({s.correct}/{s.scored_samples})")
+ click.echo(f"{'=' * 60}")
+
+
+@main.command()
+@click.argument("jsonl_path", type=click.Path(exists=True))
+def summarize(jsonl_path):
+ """Summarize results from a JSONL output file."""
+ records = []
+ with open(jsonl_path) as f:
+ for line in f:
+ line = line.strip()
+ if line:
+ records.append(json.loads(line))
+
+ if not records:
+ click.echo("No records found.")
+ return
+
+ total = len(records)
+ scored = [r for r in records if r.get("is_correct") is not None]
+ correct = [r for r in scored if r["is_correct"]]
+ errors = [r for r in records if r.get("error")]
+ accuracy = len(correct) / len(scored) if scored else 0.0
+
+ click.echo(f"File: {jsonl_path}")
+ click.echo(f"Benchmark: {records[0].get('benchmark', '?')}")
+ click.echo(f"Model: {records[0].get('model', '?')}")
+ click.echo(f"Total: {total}")
+ click.echo(f"Scored: {len(scored)}")
+ click.echo(f"Correct: {len(correct)}")
+ click.echo(f"Accuracy: {accuracy:.4f}")
+ click.echo(f"Errors: {len(errors)}")
+
+
+@main.command("list")
+def list_cmd():
+ """List available benchmarks and backends."""
+ click.echo("Benchmarks:")
+ for name, info in BENCHMARKS.items():
+ click.echo(f" {name:12s} [{info['category']:10s}] {info['description']}")
+
+ click.echo("\nBackends:")
+ for name, desc in BACKENDS.items():
+ click.echo(f" {name:16s} {desc}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/evals/core/__init__.py b/evals/core/__init__.py
new file mode 100644
index 00000000..339913e0
--- /dev/null
+++ b/evals/core/__init__.py
@@ -0,0 +1 @@
+"""Core evaluation types and ABCs."""
diff --git a/evals/core/backend.py b/evals/core/backend.py
new file mode 100644
index 00000000..38c2feb3
--- /dev/null
+++ b/evals/core/backend.py
@@ -0,0 +1,45 @@
+"""Abstract base class for inference backends."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict
+
+
+class InferenceBackend(ABC):
+ """Base class for all inference backends used in evaluation."""
+
+ backend_id: str
+
+ @abstractmethod
+ def generate(
+ self,
+ prompt: str,
+ *,
+ model: str,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 2048,
+ ) -> str:
+ """Generate a response and return just the text content."""
+
+ @abstractmethod
+ def generate_full(
+ self,
+ prompt: str,
+ *,
+ model: str,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 2048,
+ ) -> Dict[str, Any]:
+ """Generate a response and return full details.
+
+ Returns dict with keys: content, usage, model, latency_seconds, cost_usd.
+ """
+
+ def close(self) -> None:
+ """Release resources."""
+
+
+__all__ = ["InferenceBackend"]
diff --git a/evals/core/dataset.py b/evals/core/dataset.py
new file mode 100644
index 00000000..18271713
--- /dev/null
+++ b/evals/core/dataset.py
@@ -0,0 +1,36 @@
+"""Abstract base class for dataset providers."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Iterable, Optional
+
+from evals.core.types import EvalRecord
+
+
+class DatasetProvider(ABC):
+ """Base class for all evaluation dataset providers."""
+
+ dataset_id: str
+ dataset_name: str
+
+ @abstractmethod
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ """Load the dataset (possibly downloading from HuggingFace)."""
+
+ @abstractmethod
+ def iter_records(self) -> Iterable[EvalRecord]:
+ """Iterate over loaded records."""
+
+ @abstractmethod
+ def size(self) -> int:
+ """Return the number of loaded records."""
+
+
+__all__ = ["DatasetProvider"]
diff --git a/evals/core/runner.py b/evals/core/runner.py
new file mode 100644
index 00000000..96e54dcb
--- /dev/null
+++ b/evals/core/runner.py
@@ -0,0 +1,236 @@
+"""EvalRunner — parallel execution of evaluation samples."""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from evals.core.backend import InferenceBackend
+from evals.core.dataset import DatasetProvider
+from evals.core.scorer import Scorer
+from evals.core.types import EvalRecord, EvalResult, RunConfig, RunSummary
+
+LOGGER = logging.getLogger(__name__)
+
+
+class EvalRunner:
+ """Runs an evaluation benchmark with parallel sample execution."""
+
+ def __init__(
+ self,
+ config: RunConfig,
+ dataset: DatasetProvider,
+ backend: InferenceBackend,
+ scorer: Scorer,
+ ) -> None:
+ self._config = config
+ self._dataset = dataset
+ self._backend = backend
+ self._scorer = scorer
+ self._results: List[EvalResult] = []
+ self._output_file: Optional[Any] = None
+
+ def run(self) -> RunSummary:
+ """Execute the evaluation and return a summary."""
+ cfg = self._config
+ started_at = time.time()
+
+ self._dataset.load(
+ max_samples=cfg.max_samples,
+ split=cfg.dataset_split,
+ seed=cfg.seed,
+ )
+ records = list(self._dataset.iter_records())
+ LOGGER.info(
+ "Running %s: %d samples, backend=%s, model=%s, workers=%d",
+ cfg.benchmark, len(records), cfg.backend, cfg.model, cfg.max_workers,
+ )
+
+ # Open output file for incremental JSONL writing
+ output_path = self._resolve_output_path()
+ if output_path:
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ self._output_file = open(output_path, "w")
+
+ try:
+ with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool:
+ futures = {
+ pool.submit(self._process_one, r): r for r in records
+ }
+ for future in as_completed(futures):
+ result = future.result()
+ self._results.append(result)
+ self._flush_result(result)
+ finally:
+ if self._output_file:
+ self._output_file.close()
+ self._output_file = None
+
+ ended_at = time.time()
+ summary = self._compute_summary(records, started_at, ended_at)
+
+ # Write summary JSON alongside JSONL
+ if output_path:
+ summary_path = output_path.with_suffix(".summary.json")
+ with open(summary_path, "w") as f:
+ json.dump(_summary_to_dict(summary), f, indent=2)
+ LOGGER.info("Results written to %s", output_path)
+ LOGGER.info("Summary written to %s", summary_path)
+
+ return summary
+
+ def _process_one(self, record: EvalRecord) -> EvalResult:
+ """Process a single evaluation sample."""
+ cfg = self._config
+ try:
+ full = self._backend.generate_full(
+ record.problem,
+ model=cfg.model,
+ temperature=cfg.temperature,
+ max_tokens=cfg.max_tokens,
+ )
+ content = full.get("content", "")
+ usage = full.get("usage", {})
+ latency = full.get("latency_seconds", 0.0)
+ cost = full.get("cost_usd", 0.0)
+
+ is_correct, scoring_meta = self._scorer.score(record, content)
+
+ return EvalResult(
+ record_id=record.record_id,
+ model_answer=content,
+ is_correct=is_correct,
+ score=1.0 if is_correct else (0.0 if is_correct is not None else None),
+ latency_seconds=latency,
+ prompt_tokens=usage.get("prompt_tokens", 0),
+ completion_tokens=usage.get("completion_tokens", 0),
+ cost_usd=cost,
+ scoring_metadata=scoring_meta,
+ )
+ except Exception as exc:
+ LOGGER.error("Error processing %s: %s", record.record_id, exc)
+ return EvalResult(
+ record_id=record.record_id,
+ model_answer="",
+ error=str(exc),
+ )
+
+ def _flush_result(self, result: EvalResult) -> None:
+ """Append a single result to the output JSONL file."""
+ if not self._output_file:
+ return
+ record_dict = {
+ "record_id": result.record_id,
+ "benchmark": self._config.benchmark,
+ "model": self._config.model,
+ "backend": self._config.backend,
+ "model_answer": result.model_answer,
+ "is_correct": result.is_correct,
+ "score": result.score,
+ "latency_seconds": result.latency_seconds,
+ "prompt_tokens": result.prompt_tokens,
+ "completion_tokens": result.completion_tokens,
+ "cost_usd": result.cost_usd,
+ "error": result.error,
+ "scoring_metadata": result.scoring_metadata,
+ }
+ self._output_file.write(json.dumps(record_dict) + "\n")
+ self._output_file.flush()
+
+ def _resolve_output_path(self) -> Optional[Path]:
+ """Determine the output file path."""
+ if self._config.output_path:
+ return Path(self._config.output_path)
+ # Auto-generate based on benchmark + model
+ model_slug = self._config.model.replace("/", "-").replace(":", "-")
+ name = f"{self._config.benchmark}_{model_slug}.jsonl"
+ return Path(name)
+
+ def _compute_summary(
+ self,
+ records: List[EvalRecord],
+ started_at: float,
+ ended_at: float,
+ ) -> RunSummary:
+ """Compute aggregate statistics from results."""
+ cfg = self._config
+ results = self._results
+
+ scored = [r for r in results if r.is_correct is not None]
+ correct = [r for r in scored if r.is_correct]
+ errors = [r for r in results if r.error]
+
+ latencies = [r.latency_seconds for r in results if r.latency_seconds > 0]
+ mean_latency = sum(latencies) / len(latencies) if latencies else 0.0
+ total_cost = sum(r.cost_usd for r in results)
+
+ # Per-subject breakdown
+ record_map = {r.record_id: r for r in records}
+ subject_groups: Dict[str, List[EvalResult]] = defaultdict(list)
+ for r in results:
+ rec = record_map.get(r.record_id)
+ subj = rec.subject if rec and rec.subject else "general"
+ subject_groups[subj].append(r)
+
+ per_subject: Dict[str, Dict[str, float]] = {}
+ for subj, subj_results in sorted(subject_groups.items()):
+ subj_scored = [r for r in subj_results if r.is_correct is not None]
+ subj_correct = [r for r in subj_scored if r.is_correct]
+ subj_acc = len(subj_correct) / len(subj_scored) if subj_scored else 0.0
+ per_subject[subj] = {
+ "accuracy": round(subj_acc, 4),
+ "total": float(len(subj_results)),
+ "scored": float(len(subj_scored)),
+ "correct": float(len(subj_correct)),
+ }
+
+ # Determine category from records
+ categories = {r.category for r in records}
+ category = categories.pop() if len(categories) == 1 else cfg.benchmark
+
+ accuracy = len(correct) / len(scored) if scored else 0.0
+
+ return RunSummary(
+ benchmark=cfg.benchmark,
+ category=category,
+ backend=cfg.backend,
+ model=cfg.model,
+ total_samples=len(results),
+ scored_samples=len(scored),
+ correct=len(correct),
+ accuracy=round(accuracy, 4),
+ errors=len(errors),
+ mean_latency_seconds=round(mean_latency, 4),
+ total_cost_usd=round(total_cost, 6),
+ per_subject=per_subject,
+ started_at=started_at,
+ ended_at=ended_at,
+ )
+
+
+def _summary_to_dict(s: RunSummary) -> Dict[str, Any]:
+ """Convert a RunSummary to a JSON-serializable dict."""
+ return {
+ "benchmark": s.benchmark,
+ "category": s.category,
+ "backend": s.backend,
+ "model": s.model,
+ "total_samples": s.total_samples,
+ "scored_samples": s.scored_samples,
+ "correct": s.correct,
+ "accuracy": s.accuracy,
+ "errors": s.errors,
+ "mean_latency_seconds": s.mean_latency_seconds,
+ "total_cost_usd": s.total_cost_usd,
+ "per_subject": s.per_subject,
+ "started_at": s.started_at,
+ "ended_at": s.ended_at,
+ }
+
+
+__all__ = ["EvalRunner"]
diff --git a/evals/core/scorer.py b/evals/core/scorer.py
new file mode 100644
index 00000000..88f51ca5
--- /dev/null
+++ b/evals/core/scorer.py
@@ -0,0 +1,53 @@
+"""Abstract base classes for scoring."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.backend import InferenceBackend
+from evals.core.types import EvalRecord
+
+
+class Scorer(ABC):
+ """Base class for all scorers."""
+
+ scorer_id: str
+
+ @abstractmethod
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ """Score a model answer against the reference.
+
+ Returns (is_correct, metadata) where is_correct may be None
+ if scoring could not be determined.
+ """
+
+
+class LLMJudgeScorer(Scorer):
+ """Base for scorers that need an LLM to judge answers."""
+
+ def __init__(self, judge_backend: InferenceBackend, judge_model: str) -> None:
+ self._judge_backend = judge_backend
+ self._judge_model = judge_model
+
+ def _ask_judge(
+ self,
+ prompt: str,
+ *,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 1024,
+ ) -> str:
+ """Send a prompt to the judge LLM and return the response text."""
+ return self._judge_backend.generate(
+ prompt,
+ model=self._judge_model,
+ system=system,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ )
+
+
+__all__ = ["LLMJudgeScorer", "Scorer"]
diff --git a/evals/core/types.py b/evals/core/types.py
new file mode 100644
index 00000000..cd5dd2fb
--- /dev/null
+++ b/evals/core/types.py
@@ -0,0 +1,77 @@
+"""Core data types for the evaluation framework."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass(slots=True)
+class EvalRecord:
+ """A single evaluation sample."""
+
+ record_id: str
+ problem: str
+ reference: str
+ category: str # "chat" | "reasoning" | "rag" | "agentic"
+ subject: str = ""
+ metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class EvalResult:
+ """Result of evaluating a single sample."""
+
+ record_id: str
+ model_answer: str
+ is_correct: Optional[bool] = None
+ score: Optional[float] = None
+ latency_seconds: float = 0.0
+ prompt_tokens: int = 0
+ completion_tokens: int = 0
+ cost_usd: float = 0.0
+ error: Optional[str] = None
+ scoring_metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class RunConfig:
+ """Configuration for an evaluation run."""
+
+ benchmark: str
+ backend: str
+ model: str
+ max_samples: Optional[int] = None
+ max_workers: int = 4
+ temperature: float = 0.0
+ max_tokens: int = 2048
+ judge_model: str = "gpt-4o"
+ engine_key: Optional[str] = None
+ agent_name: Optional[str] = None
+ tools: List[str] = field(default_factory=list)
+ output_path: Optional[str] = None
+ seed: int = 42
+ dataset_split: Optional[str] = None
+
+
+@dataclass(slots=True)
+class RunSummary:
+ """Summary statistics for a completed evaluation run."""
+
+ benchmark: str
+ category: str
+ backend: str
+ model: str
+ total_samples: int
+ scored_samples: int
+ correct: int
+ accuracy: float
+ errors: int
+ mean_latency_seconds: float
+ total_cost_usd: float
+ per_subject: Dict[str, Dict[str, float]] = field(default_factory=dict)
+ started_at: float = 0.0
+ ended_at: float = 0.0
+
+
+__all__ = ["EvalRecord", "EvalResult", "RunConfig", "RunSummary"]
diff --git a/evals/datasets/__init__.py b/evals/datasets/__init__.py
new file mode 100644
index 00000000..ecd71d7d
--- /dev/null
+++ b/evals/datasets/__init__.py
@@ -0,0 +1 @@
+"""Dataset providers for evaluation benchmarks."""
diff --git a/evals/datasets/frames.py b/evals/datasets/frames.py
new file mode 100644
index 00000000..bd5a3a69
--- /dev/null
+++ b/evals/datasets/frames.py
@@ -0,0 +1,136 @@
+"""FRAMES benchmark dataset (google/frames-benchmark).
+
+Adapted from IPW's frames.py dataset loader.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+_DEFAULT_INPUT_PROMPT = """Please answer the question below. You should:
+
+- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings.
+- If the answer is a number, return only the number without any units unless specified otherwise.
+- If the answer is a string, don't include articles, and don't use abbreviations (e.g. for states).
+- If the answer is a comma separated list, apply the above rules to each element in the list.
+- This question may require multi-hop reasoning across multiple Wikipedia articles.
+{wiki_context}
+
+Here is the question:
+
+{question}"""
+
+
+class FRAMESDataset(DatasetProvider):
+ """FRAMES multi-hop factual retrieval benchmark."""
+
+ dataset_id = "frames"
+ dataset_name = "FRAMES"
+
+ _hf_path = "google/frames-benchmark"
+ _default_split = "test"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ question = str(
+ raw.get("Prompt") or raw.get("prompt") or raw.get("question") or ""
+ ).strip()
+ answer = str(
+ raw.get("Answer") or raw.get("answer") or raw.get("gold_answer") or ""
+ ).strip()
+
+ if not question or not answer:
+ return None
+
+ # Extract reasoning types
+ reasoning = raw.get("reasoning_types", raw.get("reasoning_type", ""))
+ if isinstance(reasoning, list):
+ reasoning = ", ".join(str(r) for r in reasoning)
+ reasoning = str(reasoning)
+
+ # Extract wiki links
+ wiki_links_raw = raw.get("wiki_links", raw.get("wikipedia_links", []))
+ if isinstance(wiki_links_raw, str):
+ wiki_links = [link.strip() for link in wiki_links_raw.split(",") if link.strip()]
+ elif isinstance(wiki_links_raw, list):
+ wiki_links = [str(link) for link in wiki_links_raw]
+ else:
+ wiki_links = []
+
+ # Build wiki context
+ wiki_context = ""
+ if wiki_links:
+ wiki_context = (
+ "\n\nRelevant Wikipedia articles that may help answer this question:\n"
+ + "\n".join(f"- {link}" for link in wiki_links)
+ )
+
+ problem = _DEFAULT_INPUT_PROMPT.format(
+ question=question, wiki_context=wiki_context,
+ )
+
+ subject = reasoning if reasoning else "general"
+
+ metadata = {
+ "index": idx,
+ "reasoning_types": reasoning,
+ "wiki_links": wiki_links,
+ }
+
+ return EvalRecord(
+ record_id=f"frames-{idx}",
+ problem=problem,
+ reference=answer,
+ category="rag",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+__all__ = ["FRAMESDataset"]
diff --git a/evals/datasets/gaia.py b/evals/datasets/gaia.py
new file mode 100644
index 00000000..418b8746
--- /dev/null
+++ b/evals/datasets/gaia.py
@@ -0,0 +1,168 @@
+"""GAIA benchmark dataset (gaia-benchmark/GAIA).
+
+Adapted from IPW's gaia.py dataset loader.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import shutil
+from pathlib import Path
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+_DEFAULT_CACHE_DIR = Path.home() / ".cache" / "gaia_benchmark"
+
+_DEFAULT_INPUT_PROMPT = """Please answer the question below. You should:
+
+- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings.
+- If the answer is a number, return only the number without any units unless specified otherwise.
+- If the answer is a string, don't include articles, and don't use abbreviations (e.g. for states).
+- If the answer is a comma separated list, apply the above rules to each element in the list.
+
+{file}
+
+Here is the question:
+
+{question}"""
+
+
+class GAIADataset(DatasetProvider):
+ """GAIA agentic benchmark dataset."""
+
+ dataset_id = "gaia"
+ dataset_name = "GAIA"
+
+ _hf_path = "gaia-benchmark/GAIA"
+ _default_subset = "2023_all"
+ _default_split = "validation"
+
+ def __init__(self, cache_dir: Optional[str] = None) -> None:
+ self._cache_dir = Path(cache_dir) if cache_dir else _DEFAULT_CACHE_DIR
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+ from huggingface_hub import snapshot_download
+
+ use_split = split or self._default_split
+
+ # Ensure dataset is downloaded
+ dataset_location = self._cache_dir / "GAIA"
+ if not dataset_location.exists():
+ dataset_location.mkdir(parents=True, exist_ok=True)
+ try:
+ snapshot_download(
+ repo_id=self._hf_path,
+ repo_type="dataset",
+ local_dir=str(dataset_location),
+ )
+ except Exception:
+ shutil.rmtree(dataset_location, ignore_errors=True)
+ raise
+
+ dataset = load_dataset(
+ str(dataset_location),
+ name=self._default_subset,
+ split=use_split,
+ )
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ files_location = dataset_location / "2023" / use_split
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, files_location, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self,
+ raw: MutableMapping[str, object],
+ files_location: Path,
+ idx: int,
+ ) -> Optional[EvalRecord]:
+ task_id = str(raw.get("task_id") or "")
+ question = str(raw.get("Question") or "").strip()
+ answer = str(raw.get("Final answer") or "").strip()
+ level = raw.get("Level")
+
+ if not question or not answer:
+ return None
+
+ # Discover associated files
+ file_name: Optional[str] = None
+ file_path: Optional[Path] = None
+ if files_location.exists():
+ files = [f for f in os.listdir(files_location) if task_id in f]
+ if files:
+ file_name = files[0]
+ file_path = files_location / file_name
+
+ # Format the prompt
+ if file_name and file_path:
+ file_info = (
+ f"The following file is referenced in the question below and you will "
+ f"likely need to use it in order to find the correct answer.\n"
+ f"File name: {file_name}\n"
+ f"File path: {file_path}\n"
+ f"Use the file reading tools to access this file."
+ )
+ elif file_name:
+ file_info = (
+ f"The following file is referenced in the question: {file_name}\n"
+ f"(Note: File path not available)"
+ )
+ else:
+ file_info = ""
+
+ problem = _DEFAULT_INPUT_PROMPT.format(file=file_info, question=question)
+
+ subject = f"level_{level}" if level else "general"
+
+ metadata = {
+ "task_id": task_id,
+ "level": level,
+ "file_name": file_name,
+ "file_path": str(file_path) if file_path else None,
+ }
+
+ return EvalRecord(
+ record_id=f"gaia-{task_id or idx}",
+ problem=problem,
+ reference=answer,
+ category="agentic",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+__all__ = ["GAIADataset"]
diff --git a/evals/datasets/supergpqa.py b/evals/datasets/supergpqa.py
new file mode 100644
index 00000000..7291d827
--- /dev/null
+++ b/evals/datasets/supergpqa.py
@@ -0,0 +1,121 @@
+"""SuperGPQA dataset provider (m-a-p/SuperGPQA).
+
+Adapted from IPW's supergpqa.py dataset loader.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+
+def _format_options(options: Iterable[str]) -> str:
+ rendered = []
+ for idx, option in enumerate(options):
+ letter = chr(ord("A") + idx)
+ rendered.append(f"{letter}. {option}")
+ return "\n".join(rendered)
+
+
+class SuperGPQADataset(DatasetProvider):
+ """SuperGPQA multiple-choice benchmark dataset."""
+
+ dataset_id = "supergpqa"
+ dataset_name = "SuperGPQA"
+
+ _hf_path = "m-a-p/SuperGPQA"
+ _default_split = "train"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ question = str(raw.get("question") or "").strip()
+ options_raw = raw.get("options") or []
+ options = [str(o).strip() for o in options_raw if str(o).strip()]
+ answer_letter = str(raw.get("answer_letter") or "").strip().upper()
+ answer_text = str(raw.get("answer") or "").strip()
+
+ subject = str(
+ raw.get("subfield")
+ or raw.get("field")
+ or raw.get("discipline")
+ or "general"
+ ).strip() or "general"
+
+ if not question or not options or not answer_letter:
+ return None
+
+ prompt_parts = [
+ question, "",
+ "Options:",
+ _format_options(options), "",
+ "Respond with the correct letter only.",
+ ]
+ problem = "\n".join(part for part in prompt_parts if part).strip()
+
+ metadata = {
+ "uuid": raw.get("uuid"),
+ "discipline": raw.get("discipline"),
+ "field": raw.get("field"),
+ "subfield": raw.get("subfield"),
+ "difficulty": raw.get("difficulty"),
+ "is_calculation": raw.get("is_calculation"),
+ "answer_text": answer_text,
+ "options": options,
+ }
+
+ return EvalRecord(
+ record_id=f"supergpqa-{idx}",
+ problem=problem,
+ reference=answer_letter,
+ category="reasoning",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+__all__ = ["SuperGPQADataset"]
diff --git a/evals/datasets/wildchat.py b/evals/datasets/wildchat.py
new file mode 100644
index 00000000..3d69d0a5
--- /dev/null
+++ b/evals/datasets/wildchat.py
@@ -0,0 +1,111 @@
+"""WildChat dataset provider (allenai/WildChat-1M).
+
+Filters to English single-turn conversations for chat quality evaluation.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+
+class WildChatDataset(DatasetProvider):
+ """WildChat conversation quality benchmark."""
+
+ dataset_id = "wildchat"
+ dataset_name = "WildChat"
+
+ _hf_path = "allenai/WildChat-1M"
+ _default_split = "train"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ # Filter to English single-turn conversations
+ filtered: List[MutableMapping[str, object]] = []
+ for row in rows:
+ if not isinstance(row, MutableMapping):
+ row = dict(row)
+
+ language = str(row.get("language") or "").lower()
+ if language != "english":
+ continue
+
+ conversation = row.get("conversation")
+ if not isinstance(conversation, list):
+ continue
+
+ # Single-turn: exactly one user message and one assistant message
+ if len(conversation) != 2:
+ continue
+
+ user_msg = conversation[0]
+ asst_msg = conversation[1]
+ if (
+ str(user_msg.get("role", "")) != "user"
+ or str(asst_msg.get("role", "")) != "assistant"
+ ):
+ continue
+
+ user_content = str(user_msg.get("content", "")).strip()
+ asst_content = str(asst_msg.get("content", "")).strip()
+ if not user_content or not asst_content:
+ continue
+
+ row["_user_content"] = user_content
+ row["_asst_content"] = asst_content
+ filtered.append(row)
+
+ # Shuffle with seed
+ if seed is not None:
+ rng = random.Random(seed)
+ rng.shuffle(filtered)
+
+ if max_samples is not None:
+ filtered = filtered[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(filtered):
+ self._records.append(
+ EvalRecord(
+ record_id=f"wildchat-{idx}",
+ problem=raw["_user_content"],
+ reference=raw["_asst_content"],
+ category="chat",
+ subject="conversation",
+ metadata={
+ "model": str(raw.get("model", "")),
+ "language": "english",
+ },
+ )
+ )
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+
+__all__ = ["WildChatDataset"]
diff --git a/evals/pyproject.toml b/evals/pyproject.toml
new file mode 100644
index 00000000..5c1227b2
--- /dev/null
+++ b/evals/pyproject.toml
@@ -0,0 +1,23 @@
+[project]
+name = "openjarvis-evals"
+version = "0.1.0"
+description = "Evaluation framework for OpenJarvis"
+requires-python = ">=3.10"
+dependencies = [
+ "openjarvis>=1.0.0",
+ "click>=8",
+ "datasets>=2.14",
+ "huggingface-hub>=0.20",
+ "tqdm>=4.65",
+ "rich>=13",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "pytest-cov>=5"]
+
+[project.scripts]
+openjarvis-eval = "evals.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
diff --git a/evals/scorers/__init__.py b/evals/scorers/__init__.py
new file mode 100644
index 00000000..613569d9
--- /dev/null
+++ b/evals/scorers/__init__.py
@@ -0,0 +1 @@
+"""Scoring implementations for evaluation benchmarks."""
diff --git a/evals/scorers/frames_judge.py b/evals/scorers/frames_judge.py
new file mode 100644
index 00000000..966a3b64
--- /dev/null
+++ b/evals/scorers/frames_judge.py
@@ -0,0 +1,99 @@
+"""FRAMES scorer — LLM-as-judge for multi-hop factual retrieval.
+
+Adapted from IPW's frames.py evaluation handler.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+_GRADER_TEMPLATE = """You are evaluating an AI system's answer to a multi-hop factual question.
+
+Compare the predicted answer against the Ground Truth Answer and determine if the prediction is correct.
+
+## Evaluation Guidelines
+
+1. **Focus on semantic meaning**: Look for equivalent information - exact wording is not required.
+2. **Assess factual accuracy**: Determine whether the essential facts from the Ground Truth are present in the answer.
+3. **Ignore minor differences**: Capitalization, punctuation, formatting, and word order don't matter.
+4. **Partial credit**: If the Ground Truth has multiple parts, all essential parts must be present for a correct rating.
+5. **Additional information**: Extra correct information in the prediction is acceptable, but extra incorrect information is not.
+
+## Question
+{question}
+
+## Ground Truth Answer
+{ground_truth}
+
+## Predicted Answer
+{predicted_answer}
+
+Your response MUST use exactly this format:
+extracted_final_answer:
+reasoning:
+correct: """
+
+
+class FRAMESScorer(LLMJudgeScorer):
+ """LLM-as-judge evaluation for FRAMES multi-hop factual retrieval."""
+
+ scorer_id = "frames"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ reference = record.reference
+ if not reference or not reference.strip():
+ return None, {"reason": "no_ground_truth"}
+
+ prompt = _GRADER_TEMPLATE.format(
+ question=record.problem,
+ ground_truth=reference,
+ predicted_answer=model_answer,
+ )
+
+ try:
+ raw = self._ask_judge(prompt, temperature=0.0, max_tokens=1024)
+
+ structured_match = re.search(
+ r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
+ )
+ if structured_match:
+ is_correct = structured_match.group(1).lower() == "yes"
+ else:
+ response_upper = raw.upper().strip()
+ if "TRUE" in response_upper:
+ is_correct = True
+ elif "FALSE" in response_upper:
+ is_correct = False
+ else:
+ LOGGER.warning("Could not parse grade from response: %s", raw[:50])
+ is_correct = False
+
+ meta: Dict[str, Any] = {
+ "raw_judge_output": raw,
+ }
+ extracted = re.search(
+ r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
+ )
+ if extracted:
+ meta["extracted_answer"] = extracted.group(1).strip()
+
+ return is_correct, meta
+
+ except Exception as exc:
+ LOGGER.error("FRAMES scoring failed: %s", exc)
+ return None, {"error": str(exc)}
+
+
+__all__ = ["FRAMESScorer"]
diff --git a/evals/scorers/gaia_exact.py b/evals/scorers/gaia_exact.py
new file mode 100644
index 00000000..e8095060
--- /dev/null
+++ b/evals/scorers/gaia_exact.py
@@ -0,0 +1,165 @@
+"""GAIA scorer — normalized exact match with LLM fallback.
+
+Adapted from IPW's gaia.py evaluation handler.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import string
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Normalization helpers (ported from IPW)
+# ---------------------------------------------------------------------------
+
+
+def _normalize_number_str(number_str: str) -> float:
+ for char in ["$", "%", ","]:
+ number_str = number_str.replace(char, "")
+ try:
+ return float(number_str)
+ except ValueError:
+ return float("inf")
+
+
+def _normalize_str(input_str: str, remove_punct: bool = True) -> str:
+ no_spaces = re.sub(r"\s", "", input_str)
+ if remove_punct:
+ translator = str.maketrans("", "", string.punctuation)
+ return no_spaces.lower().translate(translator)
+ return no_spaces.lower()
+
+
+def _split_string(s: str, char_list: list[str] | None = None) -> list[str]:
+ if char_list is None:
+ char_list = [",", ";"]
+ pattern = f"[{''.join(char_list)}]"
+ return re.split(pattern, s)
+
+
+def _is_float(element: object) -> bool:
+ try:
+ float(element) # type: ignore[arg-type]
+ return True
+ except (ValueError, TypeError):
+ return False
+
+
+def exact_match(model_answer: str, ground_truth: str) -> bool:
+ """GAIA exact-match scorer with normalization for numbers, lists, and strings."""
+ if model_answer is None:
+ model_answer = "None"
+
+ if _is_float(ground_truth):
+ normalized = _normalize_number_str(model_answer)
+ return normalized == float(ground_truth)
+
+ if any(char in ground_truth for char in [",", ";"]):
+ gt_elems = _split_string(ground_truth)
+ ma_elems = _split_string(model_answer)
+ if len(gt_elems) != len(ma_elems):
+ return False
+ comparisons = []
+ for ma_elem, gt_elem in zip(ma_elems, gt_elems):
+ if _is_float(gt_elem):
+ comparisons.append(
+ _normalize_number_str(ma_elem) == float(gt_elem)
+ )
+ else:
+ comparisons.append(
+ _normalize_str(ma_elem, remove_punct=False)
+ == _normalize_str(gt_elem, remove_punct=False)
+ )
+ return all(comparisons)
+
+ return _normalize_str(model_answer) == _normalize_str(ground_truth)
+
+
+# ---------------------------------------------------------------------------
+# LLM fallback prompt
+# ---------------------------------------------------------------------------
+
+_LLM_FALLBACK_PROMPT = """Your job is to determine if the predicted answer is semantically equivalent to the gold target.
+
+Question: {question}
+Gold target: {ground_truth}
+Predicted answer: {response}
+
+Consider the following:
+- Numerical answers should match exactly (accounting for different formats like $1,000 vs 1000)
+- List answers should contain all elements (order may vary)
+- String answers should have the same meaning (case and punctuation don't matter)
+
+Your response MUST use exactly this format:
+extracted_final_answer:
+reasoning:
+correct: """
+
+
+class GAIAScorer(LLMJudgeScorer):
+ """GAIA evaluation: exact match with normalization + LLM fallback."""
+
+ scorer_id = "gaia"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ reference = record.reference
+ if not reference or not reference.strip():
+ return None, {"reason": "no_ground_truth"}
+
+ # Try exact match first (fast, no API call)
+ if exact_match(model_answer, reference):
+ return True, {"match_type": "exact"}
+
+ # LLM fallback for semantic comparison
+ try:
+ prompt = _LLM_FALLBACK_PROMPT.format(
+ question=record.problem or "(No question provided)",
+ response=model_answer,
+ ground_truth=reference,
+ )
+ raw = self._ask_judge(prompt, temperature=0.0, max_tokens=1024)
+
+ structured_match = re.search(
+ r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
+ )
+ if structured_match:
+ is_correct = structured_match.group(1).lower() == "yes"
+ else:
+ is_correct = (
+ "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper()
+ )
+
+ meta: Dict[str, Any] = {
+ "match_type": "llm_fallback",
+ "raw_judge_output": raw,
+ }
+ extracted_match = re.search(
+ r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
+ )
+ if extracted_match:
+ meta["extracted_answer"] = extracted_match.group(1).strip()
+
+ return is_correct, meta
+
+ except Exception as exc:
+ LOGGER.error("GAIA LLM fallback failed: %s", exc)
+ return False, {
+ "match_type": "llm_fallback_error",
+ "error": str(exc),
+ }
+
+
+__all__ = ["GAIAScorer", "exact_match"]
diff --git a/evals/scorers/supergpqa_mcq.py b/evals/scorers/supergpqa_mcq.py
new file mode 100644
index 00000000..97675a31
--- /dev/null
+++ b/evals/scorers/supergpqa_mcq.py
@@ -0,0 +1,98 @@
+"""SuperGPQA MCQ scorer — LLM-based letter extraction + exact match.
+
+Adapted from IPW's mcq.py and gpqa.py evaluation handlers.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+
+class SuperGPQAScorer(LLMJudgeScorer):
+ """Score SuperGPQA responses by extracting answer letter via LLM."""
+
+ scorer_id = "supergpqa"
+
+ def _valid_letters_from_options(self, metadata: Dict[str, Any]) -> str:
+ options = metadata.get("options")
+ if isinstance(options, list) and options:
+ n = len(options)
+ return "".join(chr(ord("A") + i) for i in range(n))
+ return "ABCD"
+
+ def _extract_answer_with_llm(
+ self,
+ problem: str,
+ model_answer: str,
+ valid_letters: str,
+ ) -> Optional[str]:
+ """Use the judge LLM to extract the answer letter from the response."""
+ last_letter = valid_letters[-1] if valid_letters else "D"
+
+ system_prompt = (
+ f"You are an answer extraction assistant. Extract the final multiple choice answer "
+ f"from the response. Return ONLY a single letter (A-{last_letter}). "
+ f"If no valid answer letter is found, return 'NONE'."
+ )
+
+ user_prompt = (
+ f"Problem: {problem}\nResponse: {model_answer}\n\n"
+ f"Extract the final answer letter:"
+ )
+
+ try:
+ raw_response = self._ask_judge(
+ user_prompt, system=system_prompt,
+ temperature=0.0, max_tokens=5,
+ )
+
+ extracted = raw_response.strip().upper()
+
+ # Handle "The answer is: A" etc.
+ answer_match = re.search(
+ r"(?:THE ANSWER IS:?\s*)?([A-Z])", extracted, re.IGNORECASE,
+ )
+ if answer_match:
+ extracted = answer_match.group(1).upper()
+
+ if extracted in valid_letters:
+ return extracted
+
+ return None
+
+ except Exception as exc:
+ LOGGER.error("Error in LLM-based answer extraction: %s", exc)
+ return None
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ ref = record.reference.strip().upper()
+ if not ref:
+ return None, {"reason": "missing_reference_letter"}
+
+ valid_letters = self._valid_letters_from_options(record.metadata)
+
+ candidate = self._extract_answer_with_llm(
+ record.problem, model_answer, valid_letters,
+ )
+ if not candidate:
+ return None, {"reason": "no_choice_letter_extracted"}
+
+ is_correct = candidate == ref
+ meta = {
+ "reference_letter": ref,
+ "candidate_letter": candidate,
+ "valid_letters": valid_letters,
+ }
+ return is_correct, meta
+
+
+__all__ = ["SuperGPQAScorer"]
diff --git a/evals/scorers/wildchat_judge.py b/evals/scorers/wildchat_judge.py
new file mode 100644
index 00000000..be6e9285
--- /dev/null
+++ b/evals/scorers/wildchat_judge.py
@@ -0,0 +1,161 @@
+"""WildChat scorer — dual-comparison LLM-as-judge.
+
+Adapted from IPW's wildchat.py evaluation handler.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+SYSTEM_PROMPT = """You are an impartial judge evaluating the quality of two AI-assistant replies to the same user prompt.
+
+Step 1 – Generate your own answer
+Write the response *you* would give to the user. Keep it separate from later analysis.
+
+Step 2 – Decide the query type
+Classify the user prompt as either
+• **Subjective / open-ended** (creative writing, opinion, advice, brainstorming)
+• **Objective / technical** (code, math, logical derivations with a single correct outcome)
+If uncertain, default to "Subjective".
+
+Step 3 – Score each assistant with the correct rubric
+
+| Query type | Criteria |
+|------------|----------|
+| Subjective / open-ended | 1. Correctness / factual soundness 2. Helpfulness 3. Relevance 4. Conciseness 5. Creativity & novelty |
+| Objective / technical | 1. Correctness only |
+
+When using the multi-criteria rubric, note strengths and weaknesses for **each** dimension.
+When using the single-criterion rubric, focus exclusively on factual / functional accuracy and ignore style or flair.
+
+Step 4 – Compare & justify
+Explain which assistant is better and why, correcting any mistakes you find. Highlight missing but important details. **Be concise.**
+
+Step 5 – Verdict
+1. Assistant A is significantly better: [[A>>B]]
+2. Assistant A is slightly better: [[A>B]]
+3. Tie, Assistant A is equal: [[A=B]]
+4. Assistant B is slightly better: [[B>A]]
+5. Assistant B is significantly better: [[B>>A]]
+
+Choose exactly one token from: `[[A>>B]]`, `[[A>B]]`, `[[A=B]]`, `[[B>A]]`, `[[B>>A]]`.
+
+---
+
+### Output format (strict)
+Return **only** a JSON object that matches the provided schema:
+
+
+
+```json
+{
+"query_type": "",
+"explanation": " | (if query_type is \\"Objective / technical\\")",
+"verdict": ">B]], [[A>B]], [[A=B]], [[B>A]], [[B>>A]]>"
+}
+```"""
+
+
+class WildChatScorer(LLMJudgeScorer):
+ """Dual-comparison LLM-as-judge for chat quality."""
+
+ scorer_id = "wildchat"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ reference = record.reference
+ if not reference or not reference.strip():
+ return None, {"reason": "empty_reference"}
+
+ # Two comparisons: (model vs reference) and (reference vs model)
+ verdict1, response1 = self._get_judge_verdict(
+ record.problem, model_answer, reference,
+ )
+ verdict2, response2 = self._get_judge_verdict(
+ record.problem, reference, model_answer,
+ )
+
+ if verdict1 is None or verdict2 is None:
+ return None, {
+ "reason": "missing_verdicts",
+ "verdict1": verdict1,
+ "verdict2": verdict2,
+ }
+
+ result1 = self._verdict_to_bool(verdict1, generated_is_a=True)
+ result2 = self._verdict_to_bool(verdict2, generated_is_a=False)
+
+ meta: Dict[str, Any] = {
+ "generated_as_a": {"verdict": verdict1, "response": response1},
+ "generated_as_b": {"verdict": verdict2, "response": response2},
+ }
+
+ if result1 is None or result2 is None:
+ return None, meta
+
+ final_result = result1 or result2
+ return final_result, meta
+
+ def _get_judge_verdict(
+ self, problem: str, response_a: str, response_b: str,
+ ) -> Tuple[Optional[str], Optional[str]]:
+ prompt = (
+ f"<|User Prompt|>\n{problem}\n\n"
+ f"<|The Start of Assistant A's Answer|>\n{response_a}\n"
+ f"<|The End of Assistant A's Answer|>\n\n"
+ f"<|The Start of Assistant B's Answer|>\n{response_b}\n"
+ f"<|The End of Assistant B's Answer|>"
+ )
+
+ try:
+ raw = self._ask_judge(
+ prompt, system=SYSTEM_PROMPT,
+ temperature=0.0, max_tokens=1024,
+ )
+ except Exception as exc:
+ LOGGER.error("WildChat judge call failed: %s", exc)
+ return None, None
+
+ content = raw.strip()
+ verdict_match = re.search(r"\[\[([AB][><=]{1,2}[AB])\]\]", content)
+ if verdict_match:
+ return verdict_match.group(1), content
+
+ return None, content
+
+ @staticmethod
+ def _verdict_to_bool(
+ verdict: Optional[str], generated_is_a: bool,
+ ) -> Optional[bool]:
+ if not verdict:
+ return None
+
+ verdict_map_a = {
+ "A>>B": True,
+ "A>B": True,
+ "A=B": True,
+ "B>A": False,
+ "B>>A": False,
+ }
+
+ verdict_map_b = {
+ "A>>B": False,
+ "A>B": False,
+ "A=B": True,
+ "B>A": True,
+ "B>>A": True,
+ }
+
+ verdict_map = verdict_map_a if generated_is_a else verdict_map_b
+ return verdict_map.get(verdict)
+
+
+__all__ = ["WildChatScorer"]
diff --git a/evals/tests/__init__.py b/evals/tests/__init__.py
new file mode 100644
index 00000000..e80eaeb1
--- /dev/null
+++ b/evals/tests/__init__.py
@@ -0,0 +1 @@
+"""Tests for the evaluation framework."""
diff --git a/evals/tests/conftest.py b/evals/tests/conftest.py
new file mode 100644
index 00000000..95aa59b4
--- /dev/null
+++ b/evals/tests/conftest.py
@@ -0,0 +1,149 @@
+"""Shared test fixtures for the evaluation framework."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+import pytest
+
+# Ensure evals package is importable from the repo root
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+
+from evals.core.backend import InferenceBackend
+from evals.core.dataset import DatasetProvider
+from evals.core.scorer import Scorer
+from evals.core.types import EvalRecord
+
+# ---------------------------------------------------------------------------
+# Mock backend
+# ---------------------------------------------------------------------------
+
+
+class MockBackend(InferenceBackend):
+ """Backend that returns canned responses for testing."""
+
+ backend_id = "mock"
+
+ def __init__(self, responses: Optional[Dict[str, str]] = None) -> None:
+ self._responses = responses or {}
+ self._default_response = "Mock response"
+ self._call_count = 0
+
+ def generate(
+ self,
+ prompt: str,
+ *,
+ model: str,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 2048,
+ ) -> str:
+ self._call_count += 1
+ return self._responses.get(prompt, self._default_response)
+
+ def generate_full(
+ self,
+ prompt: str,
+ *,
+ model: str,
+ system: str = "",
+ temperature: float = 0.0,
+ max_tokens: int = 2048,
+ ) -> Dict[str, Any]:
+ content = self.generate(
+ prompt, model=model, system=system,
+ temperature=temperature, max_tokens=max_tokens,
+ )
+ return {
+ "content": content,
+ "usage": {
+ "prompt_tokens": 100,
+ "completion_tokens": 50,
+ "total_tokens": 150,
+ },
+ "model": model,
+ "latency_seconds": 0.1,
+ "cost_usd": 0.001,
+ }
+
+
+class MockScorer(Scorer):
+ """Scorer that always returns a fixed result."""
+
+ scorer_id = "mock"
+
+ def __init__(self, result: bool = True) -> None:
+ self._result = result
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ return self._result, {"mock": True}
+
+
+class MockDataset(DatasetProvider):
+ """Dataset that yields fixed records."""
+
+ dataset_id = "mock"
+ dataset_name = "Mock"
+
+ def __init__(self, records: Optional[list[EvalRecord]] = None) -> None:
+ self._records = records or []
+
+ def load(self, *, max_samples=None, split=None, seed=None) -> None:
+ pass
+
+ def iter_records(self):
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def mock_backend():
+ return MockBackend()
+
+
+@pytest.fixture()
+def mock_scorer():
+ return MockScorer()
+
+
+@pytest.fixture()
+def sample_records():
+ return [
+ EvalRecord(
+ record_id="test-001",
+ problem="What is 2+2?",
+ reference="4",
+ category="reasoning",
+ subject="math",
+ ),
+ EvalRecord(
+ record_id="test-002",
+ problem="What is the capital of France?",
+ reference="Paris",
+ category="reasoning",
+ subject="geography",
+ ),
+ EvalRecord(
+ record_id="test-003",
+ problem="Hello, how are you?",
+ reference="I'm fine, thank you!",
+ category="chat",
+ subject="greeting",
+ ),
+ ]
+
+
+@pytest.fixture()
+def mock_dataset(sample_records):
+ return MockDataset(sample_records)
diff --git a/evals/tests/test_backends.py b/evals/tests/test_backends.py
new file mode 100644
index 00000000..5a431cc8
--- /dev/null
+++ b/evals/tests/test_backends.py
@@ -0,0 +1,138 @@
+"""Tests for backend construction with mocks."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+
+class TestJarvisDirectBackend:
+ @patch("openjarvis.system.SystemBuilder")
+ def test_construction_default(self, mock_builder_cls):
+ mock_builder = MagicMock()
+ mock_builder.engine.return_value = mock_builder
+ mock_builder.telemetry.return_value = mock_builder
+ mock_builder.traces.return_value = mock_builder
+ mock_system = MagicMock()
+ mock_builder.build.return_value = mock_system
+ mock_builder_cls.return_value = mock_builder
+
+ from evals.backends.jarvis_direct import JarvisDirectBackend
+
+ backend = JarvisDirectBackend()
+ assert backend.backend_id == "jarvis-direct"
+ mock_builder.telemetry.assert_called_with(False)
+ mock_builder.traces.assert_called_with(False)
+ mock_builder.build.assert_called_once()
+
+ @patch("openjarvis.system.SystemBuilder")
+ def test_construction_with_engine_key(self, mock_builder_cls):
+ mock_builder = MagicMock()
+ mock_builder.engine.return_value = mock_builder
+ mock_builder.telemetry.return_value = mock_builder
+ mock_builder.traces.return_value = mock_builder
+ mock_builder.build.return_value = MagicMock()
+ mock_builder_cls.return_value = mock_builder
+
+ from evals.backends.jarvis_direct import JarvisDirectBackend
+
+ JarvisDirectBackend(engine_key="cloud")
+ mock_builder.engine.assert_called_with("cloud")
+
+ @patch("openjarvis.system.SystemBuilder")
+ def test_generate_full(self, mock_builder_cls):
+ mock_builder = MagicMock()
+ mock_builder.engine.return_value = mock_builder
+ mock_builder.telemetry.return_value = mock_builder
+ mock_builder.traces.return_value = mock_builder
+ mock_system = MagicMock()
+ mock_system.engine.generate.return_value = {
+ "content": "42",
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5},
+ "model": "test-model",
+ "cost_usd": 0.001,
+ }
+ mock_builder.build.return_value = mock_system
+ mock_builder_cls.return_value = mock_builder
+
+ from evals.backends.jarvis_direct import JarvisDirectBackend
+
+ backend = JarvisDirectBackend()
+ result = backend.generate_full("What is 2+2?", model="test-model")
+
+ assert result["content"] == "42"
+ assert result["cost_usd"] == 0.001
+ assert "latency_seconds" in result
+
+ @patch("openjarvis.system.SystemBuilder")
+ def test_generate(self, mock_builder_cls):
+ mock_builder = MagicMock()
+ mock_builder.engine.return_value = mock_builder
+ mock_builder.telemetry.return_value = mock_builder
+ mock_builder.traces.return_value = mock_builder
+ mock_system = MagicMock()
+ mock_system.engine.generate.return_value = {
+ "content": "Paris",
+ "usage": {},
+ }
+ mock_builder.build.return_value = mock_system
+ mock_builder_cls.return_value = mock_builder
+
+ from evals.backends.jarvis_direct import JarvisDirectBackend
+
+ backend = JarvisDirectBackend()
+ text = backend.generate("Capital of France?", model="m")
+ assert text == "Paris"
+
+
+class TestJarvisAgentBackend:
+ @patch("openjarvis.system.SystemBuilder")
+ def test_construction(self, mock_builder_cls):
+ mock_builder = MagicMock()
+ mock_builder.engine.return_value = mock_builder
+ mock_builder.agent.return_value = mock_builder
+ mock_builder.tools.return_value = mock_builder
+ mock_builder.telemetry.return_value = mock_builder
+ mock_builder.traces.return_value = mock_builder
+ mock_builder.build.return_value = MagicMock()
+ mock_builder_cls.return_value = mock_builder
+
+ from evals.backends.jarvis_agent import JarvisAgentBackend
+
+ backend = JarvisAgentBackend(
+ engine_key="cloud", agent_name="orchestrator",
+ tools=["calculator", "think"],
+ )
+ assert backend.backend_id == "jarvis-agent"
+ mock_builder.engine.assert_called_with("cloud")
+ mock_builder.agent.assert_called_with("orchestrator")
+ mock_builder.tools.assert_called_with(["calculator", "think"])
+
+ @patch("openjarvis.system.SystemBuilder")
+ def test_generate_full(self, mock_builder_cls):
+ mock_builder = MagicMock()
+ mock_builder.engine.return_value = mock_builder
+ mock_builder.agent.return_value = mock_builder
+ mock_builder.tools.return_value = mock_builder
+ mock_builder.telemetry.return_value = mock_builder
+ mock_builder.traces.return_value = mock_builder
+ mock_system = MagicMock()
+ mock_system.ask.return_value = {
+ "content": "The answer is 4.",
+ "usage": {"prompt_tokens": 50, "completion_tokens": 20},
+ "model": "gpt-4o",
+ "turns": 2,
+ "tool_results": [
+ {"tool_name": "calculator", "content": "4", "success": True},
+ ],
+ }
+ mock_builder.build.return_value = mock_system
+ mock_builder_cls.return_value = mock_builder
+
+ from evals.backends.jarvis_agent import JarvisAgentBackend
+
+ backend = JarvisAgentBackend(agent_name="orchestrator")
+ result = backend.generate_full("What is 2+2?", model="gpt-4o")
+
+ assert result["content"] == "The answer is 4."
+ assert result["turns"] == 2
+ assert len(result["tool_results"]) == 1
diff --git a/evals/tests/test_frames.py b/evals/tests/test_frames.py
new file mode 100644
index 00000000..c9047f37
--- /dev/null
+++ b/evals/tests/test_frames.py
@@ -0,0 +1,111 @@
+"""Tests for FRAMES scorer (judge prompt formatting and verdict parsing)."""
+
+from __future__ import annotations
+
+from evals.core.types import EvalRecord
+from evals.scorers.frames_judge import _GRADER_TEMPLATE, FRAMESScorer
+from evals.tests.conftest import MockBackend
+
+
+class TestGraderTemplate:
+ def test_template_formatting(self):
+ result = _GRADER_TEMPLATE.format(
+ question="What is the capital?",
+ ground_truth="Paris",
+ predicted_answer="The capital is Paris",
+ )
+ assert "What is the capital?" in result
+ assert "Paris" in result
+ assert "The capital is Paris" in result
+ assert "correct: " in result
+
+ def test_template_has_required_sections(self):
+ assert "## Question" in _GRADER_TEMPLATE
+ assert "## Ground Truth Answer" in _GRADER_TEMPLATE
+ assert "## Predicted Answer" in _GRADER_TEMPLATE
+ assert "extracted_final_answer:" in _GRADER_TEMPLATE
+
+
+class TestFRAMESScorer:
+ def _make_record(self, reference="Paris"):
+ return EvalRecord(
+ record_id="frames-1",
+ problem="What is the capital of France?",
+ reference=reference,
+ category="rag",
+ subject="general",
+ )
+
+ def test_correct_answer(self):
+ backend = MockBackend()
+ backend._default_response = (
+ "extracted_final_answer: Paris\n"
+ "reasoning: The answer correctly identifies Paris as the capital.\n"
+ "correct: yes"
+ )
+ scorer = FRAMESScorer(backend, "gpt-4o")
+
+ record = self._make_record("Paris")
+ is_correct, meta = scorer.score(record, "The capital is Paris")
+
+ assert is_correct is True
+ assert "raw_judge_output" in meta
+ assert meta["extracted_answer"] == "Paris"
+
+ def test_incorrect_answer(self):
+ backend = MockBackend()
+ backend._default_response = (
+ "extracted_final_answer: London\n"
+ "reasoning: London is not the capital of France.\n"
+ "correct: no"
+ )
+ scorer = FRAMESScorer(backend, "gpt-4o")
+
+ record = self._make_record("Paris")
+ is_correct, meta = scorer.score(record, "London")
+
+ assert is_correct is False
+
+ def test_empty_response(self):
+ backend = MockBackend()
+ scorer = FRAMESScorer(backend, "gpt-4o")
+
+ record = self._make_record("Paris")
+ is_correct, meta = scorer.score(record, "")
+
+ assert is_correct is False
+ assert meta["reason"] == "empty_response"
+
+ def test_no_ground_truth(self):
+ backend = MockBackend()
+ scorer = FRAMESScorer(backend, "gpt-4o")
+
+ record = self._make_record("")
+ is_correct, meta = scorer.score(record, "Paris")
+
+ assert is_correct is None
+ assert meta["reason"] == "no_ground_truth"
+
+ def test_fallback_true_false_parsing(self):
+ backend = MockBackend()
+ backend._default_response = "The prediction is TRUE"
+ scorer = FRAMESScorer(backend, "gpt-4o")
+
+ record = self._make_record("Paris")
+ is_correct, _ = scorer.score(record, "Paris")
+
+ assert is_correct is True
+
+ def test_judge_error(self):
+ class ErrorBackend(MockBackend):
+ def generate(self, prompt, **kw):
+ raise RuntimeError("API error")
+
+ backend = ErrorBackend()
+ scorer = FRAMESScorer(backend, "gpt-4o")
+
+ record = self._make_record("Paris")
+ is_correct, meta = scorer.score(record, "Paris")
+
+ assert is_correct is None
+ assert "error" in meta
diff --git a/evals/tests/test_gaia.py b/evals/tests/test_gaia.py
new file mode 100644
index 00000000..9cdff9e7
--- /dev/null
+++ b/evals/tests/test_gaia.py
@@ -0,0 +1,140 @@
+"""Tests for GAIA scorer logic (normalization and exact match)."""
+
+from __future__ import annotations
+
+from evals.core.types import EvalRecord
+from evals.scorers.gaia_exact import (
+ GAIAScorer,
+ _is_float,
+ _normalize_number_str,
+ _normalize_str,
+ _split_string,
+ exact_match,
+)
+from evals.tests.conftest import MockBackend
+
+
+class TestNormalization:
+ def test_normalize_number_str(self):
+ assert _normalize_number_str("1000") == 1000.0
+ assert _normalize_number_str("$1,000") == 1000.0
+ assert _normalize_number_str("50%") == 50.0
+ assert _normalize_number_str("abc") == float("inf")
+
+ def test_normalize_str(self):
+ assert _normalize_str("Hello World") == "helloworld"
+ assert _normalize_str("Hello, World!", remove_punct=True) == "helloworld"
+ assert _normalize_str("Hello, World!", remove_punct=False) == "hello,world!"
+
+ def test_split_string(self):
+ assert _split_string("a, b, c") == ["a", " b", " c"]
+ assert _split_string("a; b") == ["a", " b"]
+
+ def test_is_float(self):
+ assert _is_float("3.14") is True
+ assert _is_float("42") is True
+ assert _is_float("abc") is False
+ assert _is_float(None) is False
+
+
+class TestExactMatch:
+ def test_number_match(self):
+ assert exact_match("42", "42") is True
+ assert exact_match("$1,000", "1000") is True
+ assert exact_match("43", "42") is False
+
+ def test_string_match(self):
+ assert exact_match("Paris", "paris") is True
+ assert exact_match(" Paris ", "paris") is True
+ assert exact_match("London", "Paris") is False
+
+ def test_list_match(self):
+ assert exact_match("1, 2, 3", "1, 2, 3") is True
+ assert exact_match("1, 2", "1, 2, 3") is False
+
+ def test_none_answer(self):
+ assert exact_match(None, "42") is False
+
+ def test_punctuation_handling(self):
+ assert exact_match("Hello!", "Hello") is True
+ assert exact_match("test.", "test") is True
+
+
+class TestGAIAScorer:
+ def _make_record(self, reference="42"):
+ return EvalRecord(
+ record_id="gaia-001",
+ problem="What is the answer?",
+ reference=reference,
+ category="agentic",
+ subject="level_1",
+ )
+
+ def test_exact_match_correct(self):
+ backend = MockBackend()
+ scorer = GAIAScorer(backend, "gpt-4o")
+
+ record = self._make_record("42")
+ is_correct, meta = scorer.score(record, "42")
+
+ assert is_correct is True
+ assert meta["match_type"] == "exact"
+
+ def test_exact_match_with_formatting(self):
+ backend = MockBackend()
+ scorer = GAIAScorer(backend, "gpt-4o")
+
+ record = self._make_record("1000")
+ is_correct, meta = scorer.score(record, "$1,000")
+
+ assert is_correct is True
+ assert meta["match_type"] == "exact"
+
+ def test_llm_fallback(self):
+ backend = MockBackend()
+ backend._default_response = (
+ "extracted_final_answer: 42\n"
+ "reasoning: The answer is semantically equivalent.\n"
+ "correct: yes"
+ )
+ scorer = GAIAScorer(backend, "gpt-4o")
+
+ record = self._make_record("42")
+ is_correct, meta = scorer.score(record, "The answer is forty-two")
+
+ assert is_correct is True
+ assert meta["match_type"] == "llm_fallback"
+
+ def test_llm_fallback_incorrect(self):
+ backend = MockBackend()
+ backend._default_response = (
+ "extracted_final_answer: 43\n"
+ "reasoning: Different number.\n"
+ "correct: no"
+ )
+ scorer = GAIAScorer(backend, "gpt-4o")
+
+ record = self._make_record("42")
+ is_correct, meta = scorer.score(record, "The answer is 43")
+
+ assert is_correct is False
+
+ def test_empty_response(self):
+ backend = MockBackend()
+ scorer = GAIAScorer(backend, "gpt-4o")
+
+ record = self._make_record("42")
+ is_correct, meta = scorer.score(record, "")
+
+ assert is_correct is False
+ assert meta["reason"] == "empty_response"
+
+ def test_no_ground_truth(self):
+ backend = MockBackend()
+ scorer = GAIAScorer(backend, "gpt-4o")
+
+ record = self._make_record("")
+ is_correct, meta = scorer.score(record, "42")
+
+ assert is_correct is None
+ assert meta["reason"] == "no_ground_truth"
diff --git a/evals/tests/test_runner.py b/evals/tests/test_runner.py
new file mode 100644
index 00000000..a2f51171
--- /dev/null
+++ b/evals/tests/test_runner.py
@@ -0,0 +1,195 @@
+"""Tests for the EvalRunner."""
+
+from __future__ import annotations
+
+import json
+
+from evals.core.runner import EvalRunner
+from evals.core.types import EvalRecord, RunConfig
+from evals.tests.conftest import MockBackend, MockDataset, MockScorer
+
+
+class TestEvalRunner:
+ def _make_records(self, n=5):
+ return [
+ EvalRecord(
+ record_id=f"r{i}",
+ problem=f"Question {i}",
+ reference=f"Answer {i}",
+ category="reasoning",
+ subject="math" if i % 2 == 0 else "science",
+ )
+ for i in range(n)
+ ]
+
+ def test_basic_run(self, tmp_path):
+ records = self._make_records(5)
+ output_path = tmp_path / "results.jsonl"
+
+ config = RunConfig(
+ benchmark="test",
+ backend="mock",
+ model="test-model",
+ max_workers=1,
+ output_path=str(output_path),
+ )
+
+ dataset = MockDataset(records)
+ backend = MockBackend()
+ scorer = MockScorer(result=True)
+
+ runner = EvalRunner(config, dataset, backend, scorer)
+ summary = runner.run()
+
+ assert summary.total_samples == 5
+ assert summary.scored_samples == 5
+ assert summary.correct == 5
+ assert summary.accuracy == 1.0
+ assert summary.errors == 0
+ assert summary.benchmark == "test"
+ assert summary.model == "test-model"
+
+ def test_with_errors(self, tmp_path):
+ records = self._make_records(3)
+ output_path = tmp_path / "results.jsonl"
+
+ config = RunConfig(
+ benchmark="test",
+ backend="mock",
+ model="m",
+ max_workers=1,
+ output_path=str(output_path),
+ )
+
+ # Backend that raises on second call
+ class FailingBackend(MockBackend):
+ def __init__(self):
+ super().__init__()
+ self._fail_count = 0
+
+ def generate_full(self, prompt, **kw):
+ self._fail_count += 1
+ if self._fail_count == 2:
+ raise RuntimeError("test error")
+ return super().generate_full(prompt, **kw)
+
+ dataset = MockDataset(records)
+ backend = FailingBackend()
+ scorer = MockScorer(result=True)
+
+ runner = EvalRunner(config, dataset, backend, scorer)
+ summary = runner.run()
+
+ assert summary.total_samples == 3
+ assert summary.errors == 1
+
+ def test_per_subject_breakdown(self, tmp_path):
+ records = self._make_records(4)
+ output_path = tmp_path / "results.jsonl"
+
+ config = RunConfig(
+ benchmark="test",
+ backend="mock",
+ model="m",
+ max_workers=1,
+ output_path=str(output_path),
+ )
+
+ dataset = MockDataset(records)
+ backend = MockBackend()
+ scorer = MockScorer(result=True)
+
+ runner = EvalRunner(config, dataset, backend, scorer)
+ summary = runner.run()
+
+ assert "math" in summary.per_subject
+ assert "science" in summary.per_subject
+ assert summary.per_subject["math"]["accuracy"] == 1.0
+
+ def test_jsonl_output(self, tmp_path):
+ records = self._make_records(3)
+ output_path = tmp_path / "results.jsonl"
+
+ config = RunConfig(
+ benchmark="test",
+ backend="mock",
+ model="m",
+ max_workers=1,
+ output_path=str(output_path),
+ )
+
+ dataset = MockDataset(records)
+ backend = MockBackend()
+ scorer = MockScorer(result=True)
+
+ runner = EvalRunner(config, dataset, backend, scorer)
+ runner.run()
+
+ # Verify JSONL
+ lines = output_path.read_text().strip().split("\n")
+ assert len(lines) == 3
+ first = json.loads(lines[0])
+ assert "record_id" in first
+ assert "model_answer" in first
+ assert "is_correct" in first
+
+ # Verify summary JSON
+ summary_path = output_path.with_suffix(".summary.json")
+ assert summary_path.exists()
+ summary_data = json.loads(summary_path.read_text())
+ assert summary_data["total_samples"] == 3
+
+ def test_parallel_workers(self, tmp_path):
+ records = self._make_records(10)
+ output_path = tmp_path / "results.jsonl"
+
+ config = RunConfig(
+ benchmark="test",
+ backend="mock",
+ model="m",
+ max_workers=4,
+ output_path=str(output_path),
+ )
+
+ dataset = MockDataset(records)
+ backend = MockBackend()
+ scorer = MockScorer(result=True)
+
+ runner = EvalRunner(config, dataset, backend, scorer)
+ summary = runner.run()
+
+ assert summary.total_samples == 10
+ assert summary.correct == 10
+
+ def test_mixed_scoring(self, tmp_path):
+ records = self._make_records(4)
+ output_path = tmp_path / "results.jsonl"
+
+ config = RunConfig(
+ benchmark="test",
+ backend="mock",
+ model="m",
+ max_workers=1,
+ output_path=str(output_path),
+ )
+
+ # Scorer that alternates correct/incorrect
+ class AlternatingScorer(MockScorer):
+ def __init__(self):
+ super().__init__()
+ self._count = 0
+
+ def score(self, record, model_answer):
+ self._count += 1
+ return (self._count % 2 == 0), {"count": self._count}
+
+ dataset = MockDataset(records)
+ backend = MockBackend()
+ scorer = AlternatingScorer()
+
+ runner = EvalRunner(config, dataset, backend, scorer)
+ summary = runner.run()
+
+ assert summary.scored_samples == 4
+ assert summary.correct == 2
+ assert summary.accuracy == 0.5
diff --git a/evals/tests/test_supergpqa.py b/evals/tests/test_supergpqa.py
new file mode 100644
index 00000000..5dedb361
--- /dev/null
+++ b/evals/tests/test_supergpqa.py
@@ -0,0 +1,105 @@
+"""Tests for SuperGPQA scorer logic."""
+
+from __future__ import annotations
+
+from evals.core.types import EvalRecord
+from evals.tests.conftest import MockBackend
+
+
+class TestSuperGPQAScorer:
+ def _make_record(self, answer="B", options=None):
+ if options is None:
+ options = ["Option A", "Option B", "Option C", "Option D"]
+ return EvalRecord(
+ record_id="sgpqa-1",
+ problem=(
+ "What is X?\n\nOptions:\n"
+ "A. Option A\nB. Option B\n"
+ "C. Option C\nD. Option D\n\n"
+ "Respond with the correct letter only."
+ ),
+ reference=answer,
+ category="reasoning",
+ subject="math",
+ metadata={"options": options},
+ )
+
+ def test_correct_extraction(self):
+ from evals.scorers.supergpqa_mcq import SuperGPQAScorer
+
+ backend = MockBackend(responses={})
+ backend._default_response = "B"
+ scorer = SuperGPQAScorer(backend, "gpt-4o")
+
+ record = self._make_record(answer="B")
+ is_correct, meta = scorer.score(record, "The answer is B")
+
+ assert is_correct is True
+ assert meta["reference_letter"] == "B"
+ assert meta["candidate_letter"] == "B"
+
+ def test_incorrect_extraction(self):
+ from evals.scorers.supergpqa_mcq import SuperGPQAScorer
+
+ backend = MockBackend()
+ backend._default_response = "A"
+ scorer = SuperGPQAScorer(backend, "gpt-4o")
+
+ record = self._make_record(answer="B")
+ is_correct, meta = scorer.score(record, "I think A")
+
+ assert is_correct is False
+ assert meta["candidate_letter"] == "A"
+
+ def test_missing_reference(self):
+ from evals.scorers.supergpqa_mcq import SuperGPQAScorer
+
+ backend = MockBackend()
+ scorer = SuperGPQAScorer(backend, "gpt-4o")
+
+ record = self._make_record(answer="")
+ is_correct, meta = scorer.score(record, "B")
+
+ assert is_correct is None
+ assert meta["reason"] == "missing_reference_letter"
+
+ def test_no_extraction(self):
+ from evals.scorers.supergpqa_mcq import SuperGPQAScorer
+
+ backend = MockBackend()
+ backend._default_response = "NONE"
+ scorer = SuperGPQAScorer(backend, "gpt-4o")
+
+ record = self._make_record(answer="B")
+ is_correct, meta = scorer.score(record, "I don't know")
+
+ assert is_correct is None
+ assert meta["reason"] == "no_choice_letter_extracted"
+
+ def test_valid_letters_from_options(self):
+ from evals.scorers.supergpqa_mcq import SuperGPQAScorer
+
+ backend = MockBackend()
+ scorer = SuperGPQAScorer(backend, "gpt-4o")
+
+ # 5 options
+ metadata = {"options": ["A", "B", "C", "D", "E"]}
+ letters = scorer._valid_letters_from_options(metadata)
+ assert letters == "ABCDE"
+
+ # No options
+ letters = scorer._valid_letters_from_options({})
+ assert letters == "ABCD"
+
+ def test_extraction_with_verbose_response(self):
+ from evals.scorers.supergpqa_mcq import SuperGPQAScorer
+
+ backend = MockBackend()
+ backend._default_response = "THE ANSWER IS: C"
+ scorer = SuperGPQAScorer(backend, "gpt-4o")
+
+ record = self._make_record(answer="C")
+ is_correct, meta = scorer.score(record, "After analysis, C is correct")
+
+ assert is_correct is True
+ assert meta["candidate_letter"] == "C"
diff --git a/evals/tests/test_types.py b/evals/tests/test_types.py
new file mode 100644
index 00000000..64cfcfd6
--- /dev/null
+++ b/evals/tests/test_types.py
@@ -0,0 +1,88 @@
+"""Tests for core data types."""
+
+from __future__ import annotations
+
+from evals.core.types import EvalRecord, EvalResult, RunConfig, RunSummary
+
+
+class TestEvalRecord:
+ def test_creation(self):
+ r = EvalRecord(
+ record_id="r1", problem="What?", reference="42",
+ category="reasoning",
+ )
+ assert r.record_id == "r1"
+ assert r.problem == "What?"
+ assert r.reference == "42"
+ assert r.category == "reasoning"
+ assert r.subject == ""
+ assert r.metadata == {}
+
+ def test_with_subject_and_metadata(self):
+ r = EvalRecord(
+ record_id="r2", problem="Q", reference="A",
+ category="chat", subject="greet",
+ metadata={"key": "val"},
+ )
+ assert r.subject == "greet"
+ assert r.metadata == {"key": "val"}
+
+
+class TestEvalResult:
+ def test_defaults(self):
+ r = EvalResult(record_id="r1", model_answer="42")
+ assert r.is_correct is None
+ assert r.score is None
+ assert r.latency_seconds == 0.0
+ assert r.prompt_tokens == 0
+ assert r.completion_tokens == 0
+ assert r.cost_usd == 0.0
+ assert r.error is None
+ assert r.scoring_metadata == {}
+
+ def test_full(self):
+ r = EvalResult(
+ record_id="r1", model_answer="42", is_correct=True,
+ score=1.0, latency_seconds=1.5, prompt_tokens=100,
+ completion_tokens=50, cost_usd=0.01,
+ scoring_metadata={"match": "exact"},
+ )
+ assert r.is_correct is True
+ assert r.score == 1.0
+ assert r.cost_usd == 0.01
+
+
+class TestRunConfig:
+ def test_defaults(self):
+ c = RunConfig(benchmark="supergpqa", backend="jarvis-direct", model="qwen3:8b")
+ assert c.max_samples is None
+ assert c.max_workers == 4
+ assert c.temperature == 0.0
+ assert c.max_tokens == 2048
+ assert c.judge_model == "gpt-4o"
+ assert c.seed == 42
+ assert c.tools == []
+
+ def test_with_agent(self):
+ c = RunConfig(
+ benchmark="gaia", backend="jarvis-agent", model="gpt-4o",
+ engine_key="cloud", agent_name="orchestrator",
+ tools=["calculator", "think"],
+ )
+ assert c.agent_name == "orchestrator"
+ assert c.tools == ["calculator", "think"]
+
+
+class TestRunSummary:
+ def test_creation(self):
+ s = RunSummary(
+ benchmark="supergpqa", category="reasoning",
+ backend="jarvis-direct", model="qwen3:8b",
+ total_samples=100, scored_samples=95, correct=47,
+ accuracy=0.495, errors=5, mean_latency_seconds=2.1,
+ total_cost_usd=0.0,
+ per_subject={"math": {"accuracy": 0.5}},
+ )
+ assert s.accuracy == 0.495
+ assert s.per_subject["math"]["accuracy"] == 0.5
+ assert s.started_at == 0.0
diff --git a/evals/tests/test_wildchat.py b/evals/tests/test_wildchat.py
new file mode 100644
index 00000000..fae2c9d3
--- /dev/null
+++ b/evals/tests/test_wildchat.py
@@ -0,0 +1,117 @@
+"""Tests for WildChat scorer (verdict parsing and dual comparison)."""
+
+from __future__ import annotations
+
+from evals.core.types import EvalRecord
+from evals.scorers.wildchat_judge import WildChatScorer
+from evals.tests.conftest import MockBackend
+
+
+class TestVerdictParsing:
+ def test_verdict_to_bool_generated_is_a(self):
+ assert WildChatScorer._verdict_to_bool("A>>B", generated_is_a=True) is True
+ assert WildChatScorer._verdict_to_bool("A>B", generated_is_a=True) is True
+ assert WildChatScorer._verdict_to_bool("A=B", generated_is_a=True) is True
+ assert WildChatScorer._verdict_to_bool("B>A", generated_is_a=True) is False
+ assert WildChatScorer._verdict_to_bool("B>>A", generated_is_a=True) is False
+
+ def test_verdict_to_bool_generated_is_b(self):
+ assert WildChatScorer._verdict_to_bool("A>>B", generated_is_a=False) is False
+ assert WildChatScorer._verdict_to_bool("A>B", generated_is_a=False) is False
+ assert WildChatScorer._verdict_to_bool("A=B", generated_is_a=False) is True
+ assert WildChatScorer._verdict_to_bool("B>A", generated_is_a=False) is True
+ assert WildChatScorer._verdict_to_bool("B>>A", generated_is_a=False) is True
+
+ def test_verdict_none(self):
+ assert WildChatScorer._verdict_to_bool(None, generated_is_a=True) is None
+ assert WildChatScorer._verdict_to_bool("", generated_is_a=True) is None
+
+ def test_unknown_verdict(self):
+ assert WildChatScorer._verdict_to_bool("X>Y", generated_is_a=True) is None
+
+
+class TestWildChatScorer:
+ def _make_record(self, reference="I'm fine, thanks!"):
+ return EvalRecord(
+ record_id="wc-1",
+ problem="How are you?",
+ reference=reference,
+ category="chat",
+ subject="conversation",
+ )
+
+ def test_model_wins(self):
+ backend = MockBackend()
+ # First call: model as A, verdict A>>B (model better)
+ # Second call: reference as A, verdict A>>B (reference better → model loses)
+ # But since it's OR logic, model wins if either comparison says it's good
+ call_count = 0
+
+ def mock_generate(prompt, **kw):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ return '```json\n{"verdict": "[[A>>B]]"}\n```'
+ else:
+ return '```json\n{"verdict": "[[A>>B]]"}\n```'
+
+ backend.generate = mock_generate
+ scorer = WildChatScorer(backend, "gpt-4o")
+
+ record = self._make_record()
+ is_correct, meta = scorer.score(record, "I'm doing great!")
+
+ assert is_correct is True
+
+ def test_model_loses(self):
+ backend = MockBackend()
+ call_count = 0
+
+ def mock_generate(prompt, **kw):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ # model as A, reference as B → B wins
+ return '```json\n{"verdict": "[[B>>A]]"}\n```'
+ else:
+ # reference as A, model as B → A wins (reference better)
+ return '```json\n{"verdict": "[[A>>B]]"}\n```'
+
+ backend.generate = mock_generate
+ scorer = WildChatScorer(backend, "gpt-4o")
+
+ record = self._make_record()
+ is_correct, meta = scorer.score(record, "Bad response")
+
+ assert is_correct is False
+
+ def test_tie(self):
+ backend = MockBackend()
+ backend._default_response = '{"verdict": "[[A=B]]"}'
+ scorer = WildChatScorer(backend, "gpt-4o")
+
+ record = self._make_record()
+ is_correct, meta = scorer.score(record, "I'm fine, thanks!")
+
+ assert is_correct is True # Tie counts as correct
+
+ def test_empty_reference(self):
+ backend = MockBackend()
+ scorer = WildChatScorer(backend, "gpt-4o")
+
+ record = self._make_record("")
+ is_correct, meta = scorer.score(record, "Hello")
+
+ assert is_correct is None
+ assert meta["reason"] == "empty_reference"
+
+ def test_missing_verdict(self):
+ backend = MockBackend()
+ backend._default_response = "I cannot decide between them."
+ scorer = WildChatScorer(backend, "gpt-4o")
+
+ record = self._make_record()
+ is_correct, meta = scorer.score(record, "Hello")
+
+ assert is_correct is None
+ assert "missing_verdicts" in str(meta.get("reason", ""))
diff --git a/pyproject.toml b/pyproject.toml
index 3b00aed4..81a68204 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -85,3 +85,7 @@ src = ["src", "tests"]
[tool.ruff.lint]
select = ["E", "F", "I", "W"]
+
+[tool.ruff.lint.per-file-ignores]
+"evals/datasets/*.py" = ["E501"]
+"evals/scorers/*.py" = ["E501"]