mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
Add evaluation framework and center README logo SVGs
Evaluation framework (evals/): benchmarking system for measuring accuracy
across four categories — Chat (WildChat), Reasoning (SuperGPQA), RAG (FRAMES),
and Agentic (GAIA). Two backends: jarvis-direct (engine-level) and jarvis-agent
(agent-level with tool calling), both supporting local and cloud models.
Datasets adapted from IPW, scorers include exact match, LLM letter extraction,
and LLM-as-judge. Parallel execution via ThreadPoolExecutor with incremental
JSONL output. CLI: python -m evals {run,run-all,summarize,list}. 57 tests pass.
SVG fix: center logo content within viewBox by wrapping icon+text in a
translate(90,0) group, eliminating the left-shift visible in the README.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jon Saad-Falcon
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
6f91077ad4
commit
bd49383201
@@ -1,24 +1,26 @@
|
||||
<svg width="520" height="80" viewBox="0 0 520 80" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<!-- Circuit Node Mark -->
|
||||
<g transform="translate(0, 0)">
|
||||
<!-- Central hub -->
|
||||
<circle cx="40" cy="40" r="12" fill="#e0e0e8"/>
|
||||
<circle cx="40" cy="40" r="7.5" fill="#3b82f6"/>
|
||||
<!-- Trace lines -->
|
||||
<line x1="40" y1="28" x2="40" y2="10" stroke="#e0e0e8" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="40" y1="52" x2="40" y2="70" stroke="#e0e0e8" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="28" y1="40" x2="10" y2="40" stroke="#e0e0e8" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="52" y1="40" x2="70" y2="40" stroke="#e0e0e8" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="31.5" y1="31.5" x2="17" y2="17" stroke="#e0e0e8" stroke-width="2.5" stroke-linecap="round"/>
|
||||
<line x1="48.5" y1="31.5" x2="63" y2="17" stroke="#e0e0e8" stroke-width="2.5" stroke-linecap="round"/>
|
||||
<!-- Outer nodes -->
|
||||
<circle cx="40" cy="7" r="5" fill="#e0e0e8"/>
|
||||
<circle cx="40" cy="73" r="5" fill="#e0e0e8"/>
|
||||
<circle cx="7" cy="40" r="5" fill="#e0e0e8"/>
|
||||
<circle cx="73" cy="40" r="5" fill="#e0e0e8"/>
|
||||
<circle cx="14.5" cy="14.5" r="4.5" fill="#6366f1"/>
|
||||
<circle cx="65.5" cy="14.5" r="4.5" fill="#10b981"/>
|
||||
<g transform="translate(90, 0)">
|
||||
<!-- Circuit Node Mark -->
|
||||
<g>
|
||||
<!-- Central hub -->
|
||||
<circle cx="40" cy="40" r="12" fill="#e0e0e8"/>
|
||||
<circle cx="40" cy="40" r="7.5" fill="#3b82f6"/>
|
||||
<!-- Trace lines -->
|
||||
<line x1="40" y1="28" x2="40" y2="10" stroke="#e0e0e8" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="40" y1="52" x2="40" y2="70" stroke="#e0e0e8" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="28" y1="40" x2="10" y2="40" stroke="#e0e0e8" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="52" y1="40" x2="70" y2="40" stroke="#e0e0e8" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="31.5" y1="31.5" x2="17" y2="17" stroke="#e0e0e8" stroke-width="2.5" stroke-linecap="round"/>
|
||||
<line x1="48.5" y1="31.5" x2="63" y2="17" stroke="#e0e0e8" stroke-width="2.5" stroke-linecap="round"/>
|
||||
<!-- Outer nodes -->
|
||||
<circle cx="40" cy="7" r="5" fill="#e0e0e8"/>
|
||||
<circle cx="40" cy="73" r="5" fill="#e0e0e8"/>
|
||||
<circle cx="7" cy="40" r="5" fill="#e0e0e8"/>
|
||||
<circle cx="73" cy="40" r="5" fill="#e0e0e8"/>
|
||||
<circle cx="14.5" cy="14.5" r="4.5" fill="#6366f1"/>
|
||||
<circle cx="65.5" cy="14.5" r="4.5" fill="#10b981"/>
|
||||
</g>
|
||||
<!-- Text -->
|
||||
<text x="100" y="52" font-family="'IBM Plex Mono', 'SF Mono', 'Menlo', monospace" font-weight="600" font-size="42" fill="#eeeef4" letter-spacing="-1.5">openjarvis</text>
|
||||
</g>
|
||||
<!-- Text -->
|
||||
<text x="100" y="52" font-family="'IBM Plex Mono', 'SF Mono', 'Menlo', monospace" font-weight="600" font-size="42" fill="#eeeef4" letter-spacing="-1.5">openjarvis</text>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 1.4 KiB After Width: | Height: | Size: 1.5 KiB |
@@ -1,24 +1,26 @@
|
||||
<svg width="520" height="80" viewBox="0 0 520 80" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<!-- Circuit Node Mark -->
|
||||
<g transform="translate(0, 0)">
|
||||
<!-- Central hub -->
|
||||
<circle cx="40" cy="40" r="12" fill="#1a1a1e"/>
|
||||
<circle cx="40" cy="40" r="7.5" fill="#3b82f6"/>
|
||||
<!-- Trace lines -->
|
||||
<line x1="40" y1="28" x2="40" y2="10" stroke="#1a1a1e" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="40" y1="52" x2="40" y2="70" stroke="#1a1a1e" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="28" y1="40" x2="10" y2="40" stroke="#1a1a1e" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="52" y1="40" x2="70" y2="40" stroke="#1a1a1e" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="31.5" y1="31.5" x2="17" y2="17" stroke="#1a1a1e" stroke-width="2.5" stroke-linecap="round"/>
|
||||
<line x1="48.5" y1="31.5" x2="63" y2="17" stroke="#1a1a1e" stroke-width="2.5" stroke-linecap="round"/>
|
||||
<!-- Outer nodes -->
|
||||
<circle cx="40" cy="7" r="5" fill="#1a1a1e"/>
|
||||
<circle cx="40" cy="73" r="5" fill="#1a1a1e"/>
|
||||
<circle cx="7" cy="40" r="5" fill="#1a1a1e"/>
|
||||
<circle cx="73" cy="40" r="5" fill="#1a1a1e"/>
|
||||
<circle cx="14.5" cy="14.5" r="4.5" fill="#6366f1"/>
|
||||
<circle cx="65.5" cy="14.5" r="4.5" fill="#10b981"/>
|
||||
<g transform="translate(90, 0)">
|
||||
<!-- Circuit Node Mark -->
|
||||
<g>
|
||||
<!-- Central hub -->
|
||||
<circle cx="40" cy="40" r="12" fill="#1a1a1e"/>
|
||||
<circle cx="40" cy="40" r="7.5" fill="#3b82f6"/>
|
||||
<!-- Trace lines -->
|
||||
<line x1="40" y1="28" x2="40" y2="10" stroke="#1a1a1e" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="40" y1="52" x2="40" y2="70" stroke="#1a1a1e" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="28" y1="40" x2="10" y2="40" stroke="#1a1a1e" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="52" y1="40" x2="70" y2="40" stroke="#1a1a1e" stroke-width="3" stroke-linecap="round"/>
|
||||
<line x1="31.5" y1="31.5" x2="17" y2="17" stroke="#1a1a1e" stroke-width="2.5" stroke-linecap="round"/>
|
||||
<line x1="48.5" y1="31.5" x2="63" y2="17" stroke="#1a1a1e" stroke-width="2.5" stroke-linecap="round"/>
|
||||
<!-- Outer nodes -->
|
||||
<circle cx="40" cy="7" r="5" fill="#1a1a1e"/>
|
||||
<circle cx="40" cy="73" r="5" fill="#1a1a1e"/>
|
||||
<circle cx="7" cy="40" r="5" fill="#1a1a1e"/>
|
||||
<circle cx="73" cy="40" r="5" fill="#1a1a1e"/>
|
||||
<circle cx="14.5" cy="14.5" r="4.5" fill="#6366f1"/>
|
||||
<circle cx="65.5" cy="14.5" r="4.5" fill="#10b981"/>
|
||||
</g>
|
||||
<!-- Text -->
|
||||
<text x="100" y="52" font-family="'IBM Plex Mono', 'SF Mono', 'Menlo', monospace" font-weight="600" font-size="42" fill="#1a1a1e" letter-spacing="-1.5">openjarvis</text>
|
||||
</g>
|
||||
<!-- Text -->
|
||||
<text x="100" y="52" font-family="'IBM Plex Mono', 'SF Mono', 'Menlo', monospace" font-weight="600" font-size="42" fill="#1a1a1e" letter-spacing="-1.5">openjarvis</text>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 1.4 KiB After Width: | Height: | Size: 1.5 KiB |
@@ -0,0 +1 @@
|
||||
"""OpenJarvis Evaluation Framework."""
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Allow running as ``python -m evals``."""
|
||||
|
||||
from evals.cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Inference backends for evaluation."""
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Jarvis Agent backend — agent-level inference with tool calling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from evals.core.backend import InferenceBackend
|
||||
|
||||
|
||||
class JarvisAgentBackend(InferenceBackend):
|
||||
"""Agent-level inference via SystemBuilder + JarvisSystem.ask().
|
||||
|
||||
Supports tool calling via the agent harness. Works for both local
|
||||
and cloud models.
|
||||
"""
|
||||
|
||||
backend_id = "jarvis-agent"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_key: Optional[str] = None,
|
||||
agent_name: str = "orchestrator",
|
||||
tools: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
from openjarvis.system import SystemBuilder
|
||||
|
||||
self._agent_name = agent_name
|
||||
self._tools = tools or []
|
||||
|
||||
builder = SystemBuilder()
|
||||
if engine_key:
|
||||
builder.engine(engine_key)
|
||||
builder.agent(agent_name)
|
||||
if tools:
|
||||
builder.tools(tools)
|
||||
self._system = builder.telemetry(False).traces(False).build()
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
model: str,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> str:
|
||||
result = self.generate_full(
|
||||
prompt, model=model, system=system,
|
||||
temperature=temperature, max_tokens=max_tokens,
|
||||
)
|
||||
return result["content"]
|
||||
|
||||
def generate_full(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
model: str,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> Dict[str, Any]:
|
||||
t0 = time.monotonic()
|
||||
result = self._system.ask(
|
||||
prompt,
|
||||
agent=self._agent_name,
|
||||
tools=self._tools if self._tools else None,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
|
||||
usage = result.get("usage", {})
|
||||
return {
|
||||
"content": result.get("content", ""),
|
||||
"usage": usage,
|
||||
"model": result.get("model", model),
|
||||
"latency_seconds": elapsed,
|
||||
"cost_usd": result.get("cost_usd", 0.0),
|
||||
"turns": result.get("turns", 1),
|
||||
"tool_results": result.get("tool_results", []),
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
self._system.close()
|
||||
|
||||
|
||||
__all__ = ["JarvisAgentBackend"]
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Jarvis Direct backend — engine-level inference for local and cloud models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from evals.core.backend import InferenceBackend
|
||||
|
||||
|
||||
class JarvisDirectBackend(InferenceBackend):
|
||||
"""Direct engine inference via SystemBuilder.
|
||||
|
||||
Works for both local models (Ollama, vLLM, etc.) and cloud models
|
||||
(OpenAI, Anthropic, Google) via the CloudEngine.
|
||||
"""
|
||||
|
||||
backend_id = "jarvis-direct"
|
||||
|
||||
def __init__(self, engine_key: Optional[str] = None) -> None:
|
||||
from openjarvis.system import SystemBuilder
|
||||
|
||||
builder = SystemBuilder()
|
||||
if engine_key:
|
||||
builder.engine(engine_key)
|
||||
self._system = builder.telemetry(False).traces(False).build()
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
model: str,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> str:
|
||||
result = self.generate_full(
|
||||
prompt, model=model, system=system,
|
||||
temperature=temperature, max_tokens=max_tokens,
|
||||
)
|
||||
return result["content"]
|
||||
|
||||
def generate_full(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
model: str,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> Dict[str, Any]:
|
||||
from openjarvis.core.types import Message, Role
|
||||
|
||||
messages = []
|
||||
if system:
|
||||
messages.append(Message(role=Role.SYSTEM, content=system))
|
||||
messages.append(Message(role=Role.USER, content=prompt))
|
||||
|
||||
t0 = time.monotonic()
|
||||
result = self._system.engine.generate(
|
||||
messages, model=model,
|
||||
temperature=temperature, max_tokens=max_tokens,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
|
||||
usage = result.get("usage", {})
|
||||
return {
|
||||
"content": result.get("content", ""),
|
||||
"usage": usage,
|
||||
"model": result.get("model", model),
|
||||
"latency_seconds": elapsed,
|
||||
"cost_usd": result.get("cost_usd", 0.0),
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
self._system.close()
|
||||
|
||||
|
||||
__all__ = ["JarvisDirectBackend"]
|
||||
+301
@@ -0,0 +1,301 @@
|
||||
"""CLI for the OpenJarvis evaluation framework."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import click
|
||||
|
||||
# Registry of available benchmarks and their metadata
|
||||
BENCHMARKS = {
|
||||
"supergpqa": {"category": "reasoning", "description": "SuperGPQA multiple-choice"},
|
||||
"gaia": {"category": "agentic", "description": "GAIA agentic benchmark"},
|
||||
"frames": {"category": "rag", "description": "FRAMES multi-hop RAG"},
|
||||
"wildchat": {"category": "chat", "description": "WildChat conversation quality"},
|
||||
}
|
||||
|
||||
BACKENDS = {
|
||||
"jarvis-direct": "Engine-level inference (local or cloud)",
|
||||
"jarvis-agent": "Agent-level inference with tool calling",
|
||||
}
|
||||
|
||||
|
||||
def _setup_logging(verbose: bool) -> None:
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
def _build_backend(backend_name: str, engine_key: Optional[str],
|
||||
agent_name: str, tools: list[str]):
|
||||
"""Construct the appropriate backend."""
|
||||
if backend_name == "jarvis-agent":
|
||||
from evals.backends.jarvis_agent import JarvisAgentBackend
|
||||
return JarvisAgentBackend(
|
||||
engine_key=engine_key,
|
||||
agent_name=agent_name,
|
||||
tools=tools,
|
||||
)
|
||||
else:
|
||||
from evals.backends.jarvis_direct import JarvisDirectBackend
|
||||
return JarvisDirectBackend(engine_key=engine_key)
|
||||
|
||||
|
||||
def _build_dataset(benchmark: str):
|
||||
"""Construct the dataset provider for a benchmark."""
|
||||
if benchmark == "supergpqa":
|
||||
from evals.datasets.supergpqa import SuperGPQADataset
|
||||
return SuperGPQADataset()
|
||||
elif benchmark == "gaia":
|
||||
from evals.datasets.gaia import GAIADataset
|
||||
return GAIADataset()
|
||||
elif benchmark == "frames":
|
||||
from evals.datasets.frames import FRAMESDataset
|
||||
return FRAMESDataset()
|
||||
elif benchmark == "wildchat":
|
||||
from evals.datasets.wildchat import WildChatDataset
|
||||
return WildChatDataset()
|
||||
else:
|
||||
raise click.ClickException(f"Unknown benchmark: {benchmark}")
|
||||
|
||||
|
||||
def _build_scorer(benchmark: str, judge_backend, judge_model: str):
|
||||
"""Construct the scorer for a benchmark."""
|
||||
if benchmark == "supergpqa":
|
||||
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
||||
return SuperGPQAScorer(judge_backend, judge_model)
|
||||
elif benchmark == "gaia":
|
||||
from evals.scorers.gaia_exact import GAIAScorer
|
||||
return GAIAScorer(judge_backend, judge_model)
|
||||
elif benchmark == "frames":
|
||||
from evals.scorers.frames_judge import FRAMESScorer
|
||||
return FRAMESScorer(judge_backend, judge_model)
|
||||
elif benchmark == "wildchat":
|
||||
from evals.scorers.wildchat_judge import WildChatScorer
|
||||
return WildChatScorer(judge_backend, judge_model)
|
||||
else:
|
||||
raise click.ClickException(f"Unknown benchmark: {benchmark}")
|
||||
|
||||
|
||||
def _build_judge_backend(judge_model: str):
|
||||
"""Build the judge backend (always cloud for LLM-as-judge)."""
|
||||
from evals.backends.jarvis_direct import JarvisDirectBackend
|
||||
return JarvisDirectBackend(engine_key="cloud")
|
||||
|
||||
|
||||
@click.group()
|
||||
def main():
|
||||
"""OpenJarvis Evaluation Framework."""
|
||||
|
||||
|
||||
@main.command()
|
||||
@click.option("-b", "--benchmark", required=True,
|
||||
type=click.Choice(list(BENCHMARKS.keys())),
|
||||
help="Benchmark to run")
|
||||
@click.option("--backend", default="jarvis-direct",
|
||||
type=click.Choice(list(BACKENDS.keys())),
|
||||
help="Inference backend")
|
||||
@click.option("-m", "--model", required=True, help="Model identifier")
|
||||
@click.option("-e", "--engine", "engine_key", default=None,
|
||||
help="Engine key (ollama, vllm, cloud, ...)")
|
||||
@click.option("--agent", "agent_name", default="orchestrator",
|
||||
help="Agent name for jarvis-agent backend")
|
||||
@click.option("--tools", default="", help="Comma-separated tool names")
|
||||
@click.option("-n", "--max-samples", type=int, default=None,
|
||||
help="Maximum samples to evaluate")
|
||||
@click.option("-w", "--max-workers", type=int, default=4,
|
||||
help="Parallel workers")
|
||||
@click.option("--judge-model", default="gpt-4o",
|
||||
help="LLM judge model")
|
||||
@click.option("-o", "--output", "output_path", default=None,
|
||||
help="Output JSONL path")
|
||||
@click.option("--seed", type=int, default=42, help="Random seed")
|
||||
@click.option("--split", "dataset_split", default=None,
|
||||
help="Dataset split override")
|
||||
@click.option("--temperature", type=float, default=0.0,
|
||||
help="Generation temperature")
|
||||
@click.option("--max-tokens", type=int, default=2048,
|
||||
help="Max output tokens")
|
||||
@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
|
||||
def run(benchmark, backend, model, engine_key, agent_name, tools,
|
||||
max_samples, max_workers, judge_model, output_path, seed,
|
||||
dataset_split, temperature, max_tokens, verbose):
|
||||
"""Run a single benchmark evaluation."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
from evals.core.runner import EvalRunner
|
||||
from evals.core.types import RunConfig
|
||||
|
||||
tool_list = [t.strip() for t in tools.split(",") if t.strip()] if tools else []
|
||||
|
||||
config = RunConfig(
|
||||
benchmark=benchmark,
|
||||
backend=backend,
|
||||
model=model,
|
||||
max_samples=max_samples,
|
||||
max_workers=max_workers,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
judge_model=judge_model,
|
||||
engine_key=engine_key,
|
||||
agent_name=agent_name,
|
||||
tools=tool_list,
|
||||
output_path=output_path,
|
||||
seed=seed,
|
||||
dataset_split=dataset_split,
|
||||
)
|
||||
|
||||
eval_backend = _build_backend(backend, engine_key, agent_name, tool_list)
|
||||
dataset = _build_dataset(benchmark)
|
||||
judge_backend = _build_judge_backend(judge_model)
|
||||
scorer = _build_scorer(benchmark, judge_backend, judge_model)
|
||||
|
||||
runner = EvalRunner(config, dataset, eval_backend, scorer)
|
||||
|
||||
try:
|
||||
summary = runner.run()
|
||||
finally:
|
||||
eval_backend.close()
|
||||
judge_backend.close()
|
||||
|
||||
# Print summary
|
||||
click.echo(f"\n{'=' * 60}")
|
||||
click.echo(f"Benchmark: {summary.benchmark}")
|
||||
click.echo(f"Model: {summary.model}")
|
||||
click.echo(f"Backend: {summary.backend}")
|
||||
click.echo(f"Samples: {summary.total_samples}")
|
||||
click.echo(f"Scored: {summary.scored_samples}")
|
||||
click.echo(f"Correct: {summary.correct}")
|
||||
click.echo(f"Accuracy: {summary.accuracy:.4f}")
|
||||
click.echo(f"Errors: {summary.errors}")
|
||||
click.echo(f"Latency: {summary.mean_latency_seconds:.2f}s (mean)")
|
||||
click.echo(f"Cost: ${summary.total_cost_usd:.4f}")
|
||||
if summary.per_subject:
|
||||
click.echo("\nPer-subject breakdown:")
|
||||
for subj, stats in sorted(summary.per_subject.items()):
|
||||
click.echo(f" {subj}: {stats['accuracy']:.4f} "
|
||||
f"({int(stats['correct'])}/{int(stats['scored'])})")
|
||||
click.echo(f"{'=' * 60}")
|
||||
|
||||
|
||||
@main.command("run-all")
|
||||
@click.option("-m", "--model", required=True, help="Model identifier")
|
||||
@click.option("-e", "--engine", "engine_key", default=None,
|
||||
help="Engine key")
|
||||
@click.option("-n", "--max-samples", type=int, default=None,
|
||||
help="Max samples per benchmark")
|
||||
@click.option("-w", "--max-workers", type=int, default=4,
|
||||
help="Parallel workers")
|
||||
@click.option("--judge-model", default="gpt-4o", help="LLM judge model")
|
||||
@click.option("--output-dir", default="results/",
|
||||
help="Output directory for results")
|
||||
@click.option("--seed", type=int, default=42, help="Random seed")
|
||||
@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
|
||||
def run_all(model, engine_key, max_samples, max_workers, judge_model,
|
||||
output_dir, seed, verbose):
|
||||
"""Run all benchmarks."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
from evals.core.runner import EvalRunner
|
||||
from evals.core.types import RunConfig
|
||||
|
||||
output_dir_path = Path(output_dir)
|
||||
output_dir_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_slug = model.replace("/", "-").replace(":", "-")
|
||||
summaries = []
|
||||
|
||||
for bench_name in BENCHMARKS:
|
||||
click.echo(f"\n--- Running {bench_name} ---")
|
||||
output_path = output_dir_path / f"{bench_name}_{model_slug}.jsonl"
|
||||
|
||||
config = RunConfig(
|
||||
benchmark=bench_name,
|
||||
backend="jarvis-direct",
|
||||
model=model,
|
||||
max_samples=max_samples,
|
||||
max_workers=max_workers,
|
||||
judge_model=judge_model,
|
||||
engine_key=engine_key,
|
||||
output_path=str(output_path),
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
eval_backend = _build_backend("jarvis-direct", engine_key, "orchestrator", [])
|
||||
dataset = _build_dataset(bench_name)
|
||||
judge_backend = _build_judge_backend(judge_model)
|
||||
scorer = _build_scorer(bench_name, judge_backend, judge_model)
|
||||
|
||||
runner = EvalRunner(config, dataset, eval_backend, scorer)
|
||||
try:
|
||||
summary = runner.run()
|
||||
summaries.append(summary)
|
||||
click.echo(f" {bench_name}: {summary.accuracy:.4f} "
|
||||
f"({summary.correct}/{summary.scored_samples})")
|
||||
except Exception as exc:
|
||||
click.echo(f" {bench_name}: FAILED — {exc}", err=True)
|
||||
finally:
|
||||
eval_backend.close()
|
||||
judge_backend.close()
|
||||
|
||||
# Print overall summary
|
||||
if summaries:
|
||||
click.echo(f"\n{'=' * 60}")
|
||||
click.echo("Overall Results:")
|
||||
for s in summaries:
|
||||
click.echo(f" {s.benchmark:12s} {s.accuracy:.4f} "
|
||||
f"({s.correct}/{s.scored_samples})")
|
||||
click.echo(f"{'=' * 60}")
|
||||
|
||||
|
||||
@main.command()
|
||||
@click.argument("jsonl_path", type=click.Path(exists=True))
|
||||
def summarize(jsonl_path):
|
||||
"""Summarize results from a JSONL output file."""
|
||||
records = []
|
||||
with open(jsonl_path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
records.append(json.loads(line))
|
||||
|
||||
if not records:
|
||||
click.echo("No records found.")
|
||||
return
|
||||
|
||||
total = len(records)
|
||||
scored = [r for r in records if r.get("is_correct") is not None]
|
||||
correct = [r for r in scored if r["is_correct"]]
|
||||
errors = [r for r in records if r.get("error")]
|
||||
accuracy = len(correct) / len(scored) if scored else 0.0
|
||||
|
||||
click.echo(f"File: {jsonl_path}")
|
||||
click.echo(f"Benchmark: {records[0].get('benchmark', '?')}")
|
||||
click.echo(f"Model: {records[0].get('model', '?')}")
|
||||
click.echo(f"Total: {total}")
|
||||
click.echo(f"Scored: {len(scored)}")
|
||||
click.echo(f"Correct: {len(correct)}")
|
||||
click.echo(f"Accuracy: {accuracy:.4f}")
|
||||
click.echo(f"Errors: {len(errors)}")
|
||||
|
||||
|
||||
@main.command("list")
|
||||
def list_cmd():
|
||||
"""List available benchmarks and backends."""
|
||||
click.echo("Benchmarks:")
|
||||
for name, info in BENCHMARKS.items():
|
||||
click.echo(f" {name:12s} [{info['category']:10s}] {info['description']}")
|
||||
|
||||
click.echo("\nBackends:")
|
||||
for name, desc in BACKENDS.items():
|
||||
click.echo(f" {name:16s} {desc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Core evaluation types and ABCs."""
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Abstract base class for inference backends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
class InferenceBackend(ABC):
|
||||
"""Base class for all inference backends used in evaluation."""
|
||||
|
||||
backend_id: str
|
||||
|
||||
@abstractmethod
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
model: str,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> str:
|
||||
"""Generate a response and return just the text content."""
|
||||
|
||||
@abstractmethod
|
||||
def generate_full(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
model: str,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> Dict[str, Any]:
|
||||
"""Generate a response and return full details.
|
||||
|
||||
Returns dict with keys: content, usage, model, latency_seconds, cost_usd.
|
||||
"""
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release resources."""
|
||||
|
||||
|
||||
__all__ = ["InferenceBackend"]
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Abstract base class for dataset providers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
|
||||
class DatasetProvider(ABC):
|
||||
"""Base class for all evaluation dataset providers."""
|
||||
|
||||
dataset_id: str
|
||||
dataset_name: str
|
||||
|
||||
@abstractmethod
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
"""Load the dataset (possibly downloading from HuggingFace)."""
|
||||
|
||||
@abstractmethod
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
"""Iterate over loaded records."""
|
||||
|
||||
@abstractmethod
|
||||
def size(self) -> int:
|
||||
"""Return the number of loaded records."""
|
||||
|
||||
|
||||
__all__ = ["DatasetProvider"]
|
||||
@@ -0,0 +1,236 @@
|
||||
"""EvalRunner — parallel execution of evaluation samples."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from evals.core.backend import InferenceBackend
|
||||
from evals.core.dataset import DatasetProvider
|
||||
from evals.core.scorer import Scorer
|
||||
from evals.core.types import EvalRecord, EvalResult, RunConfig, RunSummary
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EvalRunner:
|
||||
"""Runs an evaluation benchmark with parallel sample execution."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: RunConfig,
|
||||
dataset: DatasetProvider,
|
||||
backend: InferenceBackend,
|
||||
scorer: Scorer,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._dataset = dataset
|
||||
self._backend = backend
|
||||
self._scorer = scorer
|
||||
self._results: List[EvalResult] = []
|
||||
self._output_file: Optional[Any] = None
|
||||
|
||||
def run(self) -> RunSummary:
|
||||
"""Execute the evaluation and return a summary."""
|
||||
cfg = self._config
|
||||
started_at = time.time()
|
||||
|
||||
self._dataset.load(
|
||||
max_samples=cfg.max_samples,
|
||||
split=cfg.dataset_split,
|
||||
seed=cfg.seed,
|
||||
)
|
||||
records = list(self._dataset.iter_records())
|
||||
LOGGER.info(
|
||||
"Running %s: %d samples, backend=%s, model=%s, workers=%d",
|
||||
cfg.benchmark, len(records), cfg.backend, cfg.model, cfg.max_workers,
|
||||
)
|
||||
|
||||
# Open output file for incremental JSONL writing
|
||||
output_path = self._resolve_output_path()
|
||||
if output_path:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._output_file = open(output_path, "w")
|
||||
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool:
|
||||
futures = {
|
||||
pool.submit(self._process_one, r): r for r in records
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
self._results.append(result)
|
||||
self._flush_result(result)
|
||||
finally:
|
||||
if self._output_file:
|
||||
self._output_file.close()
|
||||
self._output_file = None
|
||||
|
||||
ended_at = time.time()
|
||||
summary = self._compute_summary(records, started_at, ended_at)
|
||||
|
||||
# Write summary JSON alongside JSONL
|
||||
if output_path:
|
||||
summary_path = output_path.with_suffix(".summary.json")
|
||||
with open(summary_path, "w") as f:
|
||||
json.dump(_summary_to_dict(summary), f, indent=2)
|
||||
LOGGER.info("Results written to %s", output_path)
|
||||
LOGGER.info("Summary written to %s", summary_path)
|
||||
|
||||
return summary
|
||||
|
||||
def _process_one(self, record: EvalRecord) -> EvalResult:
|
||||
"""Process a single evaluation sample."""
|
||||
cfg = self._config
|
||||
try:
|
||||
full = self._backend.generate_full(
|
||||
record.problem,
|
||||
model=cfg.model,
|
||||
temperature=cfg.temperature,
|
||||
max_tokens=cfg.max_tokens,
|
||||
)
|
||||
content = full.get("content", "")
|
||||
usage = full.get("usage", {})
|
||||
latency = full.get("latency_seconds", 0.0)
|
||||
cost = full.get("cost_usd", 0.0)
|
||||
|
||||
is_correct, scoring_meta = self._scorer.score(record, content)
|
||||
|
||||
return EvalResult(
|
||||
record_id=record.record_id,
|
||||
model_answer=content,
|
||||
is_correct=is_correct,
|
||||
score=1.0 if is_correct else (0.0 if is_correct is not None else None),
|
||||
latency_seconds=latency,
|
||||
prompt_tokens=usage.get("prompt_tokens", 0),
|
||||
completion_tokens=usage.get("completion_tokens", 0),
|
||||
cost_usd=cost,
|
||||
scoring_metadata=scoring_meta,
|
||||
)
|
||||
except Exception as exc:
|
||||
LOGGER.error("Error processing %s: %s", record.record_id, exc)
|
||||
return EvalResult(
|
||||
record_id=record.record_id,
|
||||
model_answer="",
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
def _flush_result(self, result: EvalResult) -> None:
|
||||
"""Append a single result to the output JSONL file."""
|
||||
if not self._output_file:
|
||||
return
|
||||
record_dict = {
|
||||
"record_id": result.record_id,
|
||||
"benchmark": self._config.benchmark,
|
||||
"model": self._config.model,
|
||||
"backend": self._config.backend,
|
||||
"model_answer": result.model_answer,
|
||||
"is_correct": result.is_correct,
|
||||
"score": result.score,
|
||||
"latency_seconds": result.latency_seconds,
|
||||
"prompt_tokens": result.prompt_tokens,
|
||||
"completion_tokens": result.completion_tokens,
|
||||
"cost_usd": result.cost_usd,
|
||||
"error": result.error,
|
||||
"scoring_metadata": result.scoring_metadata,
|
||||
}
|
||||
self._output_file.write(json.dumps(record_dict) + "\n")
|
||||
self._output_file.flush()
|
||||
|
||||
def _resolve_output_path(self) -> Optional[Path]:
|
||||
"""Determine the output file path."""
|
||||
if self._config.output_path:
|
||||
return Path(self._config.output_path)
|
||||
# Auto-generate based on benchmark + model
|
||||
model_slug = self._config.model.replace("/", "-").replace(":", "-")
|
||||
name = f"{self._config.benchmark}_{model_slug}.jsonl"
|
||||
return Path(name)
|
||||
|
||||
def _compute_summary(
|
||||
self,
|
||||
records: List[EvalRecord],
|
||||
started_at: float,
|
||||
ended_at: float,
|
||||
) -> RunSummary:
|
||||
"""Compute aggregate statistics from results."""
|
||||
cfg = self._config
|
||||
results = self._results
|
||||
|
||||
scored = [r for r in results if r.is_correct is not None]
|
||||
correct = [r for r in scored if r.is_correct]
|
||||
errors = [r for r in results if r.error]
|
||||
|
||||
latencies = [r.latency_seconds for r in results if r.latency_seconds > 0]
|
||||
mean_latency = sum(latencies) / len(latencies) if latencies else 0.0
|
||||
total_cost = sum(r.cost_usd for r in results)
|
||||
|
||||
# Per-subject breakdown
|
||||
record_map = {r.record_id: r for r in records}
|
||||
subject_groups: Dict[str, List[EvalResult]] = defaultdict(list)
|
||||
for r in results:
|
||||
rec = record_map.get(r.record_id)
|
||||
subj = rec.subject if rec and rec.subject else "general"
|
||||
subject_groups[subj].append(r)
|
||||
|
||||
per_subject: Dict[str, Dict[str, float]] = {}
|
||||
for subj, subj_results in sorted(subject_groups.items()):
|
||||
subj_scored = [r for r in subj_results if r.is_correct is not None]
|
||||
subj_correct = [r for r in subj_scored if r.is_correct]
|
||||
subj_acc = len(subj_correct) / len(subj_scored) if subj_scored else 0.0
|
||||
per_subject[subj] = {
|
||||
"accuracy": round(subj_acc, 4),
|
||||
"total": float(len(subj_results)),
|
||||
"scored": float(len(subj_scored)),
|
||||
"correct": float(len(subj_correct)),
|
||||
}
|
||||
|
||||
# Determine category from records
|
||||
categories = {r.category for r in records}
|
||||
category = categories.pop() if len(categories) == 1 else cfg.benchmark
|
||||
|
||||
accuracy = len(correct) / len(scored) if scored else 0.0
|
||||
|
||||
return RunSummary(
|
||||
benchmark=cfg.benchmark,
|
||||
category=category,
|
||||
backend=cfg.backend,
|
||||
model=cfg.model,
|
||||
total_samples=len(results),
|
||||
scored_samples=len(scored),
|
||||
correct=len(correct),
|
||||
accuracy=round(accuracy, 4),
|
||||
errors=len(errors),
|
||||
mean_latency_seconds=round(mean_latency, 4),
|
||||
total_cost_usd=round(total_cost, 6),
|
||||
per_subject=per_subject,
|
||||
started_at=started_at,
|
||||
ended_at=ended_at,
|
||||
)
|
||||
|
||||
|
||||
def _summary_to_dict(s: RunSummary) -> Dict[str, Any]:
|
||||
"""Convert a RunSummary to a JSON-serializable dict."""
|
||||
return {
|
||||
"benchmark": s.benchmark,
|
||||
"category": s.category,
|
||||
"backend": s.backend,
|
||||
"model": s.model,
|
||||
"total_samples": s.total_samples,
|
||||
"scored_samples": s.scored_samples,
|
||||
"correct": s.correct,
|
||||
"accuracy": s.accuracy,
|
||||
"errors": s.errors,
|
||||
"mean_latency_seconds": s.mean_latency_seconds,
|
||||
"total_cost_usd": s.total_cost_usd,
|
||||
"per_subject": s.per_subject,
|
||||
"started_at": s.started_at,
|
||||
"ended_at": s.ended_at,
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["EvalRunner"]
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Abstract base classes for scoring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from evals.core.backend import InferenceBackend
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
|
||||
class Scorer(ABC):
|
||||
"""Base class for all scorers."""
|
||||
|
||||
scorer_id: str
|
||||
|
||||
@abstractmethod
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
"""Score a model answer against the reference.
|
||||
|
||||
Returns (is_correct, metadata) where is_correct may be None
|
||||
if scoring could not be determined.
|
||||
"""
|
||||
|
||||
|
||||
class LLMJudgeScorer(Scorer):
|
||||
"""Base for scorers that need an LLM to judge answers."""
|
||||
|
||||
def __init__(self, judge_backend: InferenceBackend, judge_model: str) -> None:
|
||||
self._judge_backend = judge_backend
|
||||
self._judge_model = judge_model
|
||||
|
||||
def _ask_judge(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 1024,
|
||||
) -> str:
|
||||
"""Send a prompt to the judge LLM and return the response text."""
|
||||
return self._judge_backend.generate(
|
||||
prompt,
|
||||
model=self._judge_model,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["LLMJudgeScorer", "Scorer"]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Core data types for the evaluation framework."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EvalRecord:
|
||||
"""A single evaluation sample."""
|
||||
|
||||
record_id: str
|
||||
problem: str
|
||||
reference: str
|
||||
category: str # "chat" | "reasoning" | "rag" | "agentic"
|
||||
subject: str = ""
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EvalResult:
|
||||
"""Result of evaluating a single sample."""
|
||||
|
||||
record_id: str
|
||||
model_answer: str
|
||||
is_correct: Optional[bool] = None
|
||||
score: Optional[float] = None
|
||||
latency_seconds: float = 0.0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cost_usd: float = 0.0
|
||||
error: Optional[str] = None
|
||||
scoring_metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RunConfig:
|
||||
"""Configuration for an evaluation run."""
|
||||
|
||||
benchmark: str
|
||||
backend: str
|
||||
model: str
|
||||
max_samples: Optional[int] = None
|
||||
max_workers: int = 4
|
||||
temperature: float = 0.0
|
||||
max_tokens: int = 2048
|
||||
judge_model: str = "gpt-4o"
|
||||
engine_key: Optional[str] = None
|
||||
agent_name: Optional[str] = None
|
||||
tools: List[str] = field(default_factory=list)
|
||||
output_path: Optional[str] = None
|
||||
seed: int = 42
|
||||
dataset_split: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RunSummary:
|
||||
"""Summary statistics for a completed evaluation run."""
|
||||
|
||||
benchmark: str
|
||||
category: str
|
||||
backend: str
|
||||
model: str
|
||||
total_samples: int
|
||||
scored_samples: int
|
||||
correct: int
|
||||
accuracy: float
|
||||
errors: int
|
||||
mean_latency_seconds: float
|
||||
total_cost_usd: float
|
||||
per_subject: Dict[str, Dict[str, float]] = field(default_factory=dict)
|
||||
started_at: float = 0.0
|
||||
ended_at: float = 0.0
|
||||
|
||||
|
||||
__all__ = ["EvalRecord", "EvalResult", "RunConfig", "RunSummary"]
|
||||
@@ -0,0 +1 @@
|
||||
"""Dataset providers for evaluation benchmarks."""
|
||||
@@ -0,0 +1,136 @@
|
||||
"""FRAMES benchmark dataset (google/frames-benchmark).
|
||||
|
||||
Adapted from IPW's frames.py dataset loader.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Iterable, List, MutableMapping, Optional, Sequence
|
||||
|
||||
from evals.core.dataset import DatasetProvider
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
_DEFAULT_INPUT_PROMPT = """Please answer the question below. You should:
|
||||
|
||||
- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings.
|
||||
- If the answer is a number, return only the number without any units unless specified otherwise.
|
||||
- If the answer is a string, don't include articles, and don't use abbreviations (e.g. for states).
|
||||
- If the answer is a comma separated list, apply the above rules to each element in the list.
|
||||
- This question may require multi-hop reasoning across multiple Wikipedia articles.
|
||||
{wiki_context}
|
||||
|
||||
Here is the question:
|
||||
|
||||
{question}"""
|
||||
|
||||
|
||||
class FRAMESDataset(DatasetProvider):
|
||||
"""FRAMES multi-hop factual retrieval benchmark."""
|
||||
|
||||
dataset_id = "frames"
|
||||
dataset_name = "FRAMES"
|
||||
|
||||
_hf_path = "google/frames-benchmark"
|
||||
_default_split = "test"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._records: List[EvalRecord] = []
|
||||
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
from datasets import load_dataset
|
||||
|
||||
use_split = split or self._default_split
|
||||
dataset = load_dataset(self._hf_path, split=use_split)
|
||||
|
||||
rows: Sequence[MutableMapping[str, object]]
|
||||
if hasattr(dataset, "to_list"):
|
||||
rows = dataset.to_list()
|
||||
else:
|
||||
rows = list(dataset)
|
||||
|
||||
if seed is not None:
|
||||
rng = random.Random(seed)
|
||||
rows = list(rows)
|
||||
rng.shuffle(rows)
|
||||
|
||||
if max_samples is not None:
|
||||
rows = rows[:max_samples]
|
||||
|
||||
self._records = []
|
||||
for idx, raw in enumerate(rows):
|
||||
record = self._convert_row(raw, idx)
|
||||
if record is not None:
|
||||
self._records.append(record)
|
||||
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
return iter(self._records)
|
||||
|
||||
def size(self) -> int:
|
||||
return len(self._records)
|
||||
|
||||
def _convert_row(
|
||||
self, raw: MutableMapping[str, object], idx: int,
|
||||
) -> Optional[EvalRecord]:
|
||||
question = str(
|
||||
raw.get("Prompt") or raw.get("prompt") or raw.get("question") or ""
|
||||
).strip()
|
||||
answer = str(
|
||||
raw.get("Answer") or raw.get("answer") or raw.get("gold_answer") or ""
|
||||
).strip()
|
||||
|
||||
if not question or not answer:
|
||||
return None
|
||||
|
||||
# Extract reasoning types
|
||||
reasoning = raw.get("reasoning_types", raw.get("reasoning_type", ""))
|
||||
if isinstance(reasoning, list):
|
||||
reasoning = ", ".join(str(r) for r in reasoning)
|
||||
reasoning = str(reasoning)
|
||||
|
||||
# Extract wiki links
|
||||
wiki_links_raw = raw.get("wiki_links", raw.get("wikipedia_links", []))
|
||||
if isinstance(wiki_links_raw, str):
|
||||
wiki_links = [link.strip() for link in wiki_links_raw.split(",") if link.strip()]
|
||||
elif isinstance(wiki_links_raw, list):
|
||||
wiki_links = [str(link) for link in wiki_links_raw]
|
||||
else:
|
||||
wiki_links = []
|
||||
|
||||
# Build wiki context
|
||||
wiki_context = ""
|
||||
if wiki_links:
|
||||
wiki_context = (
|
||||
"\n\nRelevant Wikipedia articles that may help answer this question:\n"
|
||||
+ "\n".join(f"- {link}" for link in wiki_links)
|
||||
)
|
||||
|
||||
problem = _DEFAULT_INPUT_PROMPT.format(
|
||||
question=question, wiki_context=wiki_context,
|
||||
)
|
||||
|
||||
subject = reasoning if reasoning else "general"
|
||||
|
||||
metadata = {
|
||||
"index": idx,
|
||||
"reasoning_types": reasoning,
|
||||
"wiki_links": wiki_links,
|
||||
}
|
||||
|
||||
return EvalRecord(
|
||||
record_id=f"frames-{idx}",
|
||||
problem=problem,
|
||||
reference=answer,
|
||||
category="rag",
|
||||
subject=subject,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["FRAMESDataset"]
|
||||
@@ -0,0 +1,168 @@
|
||||
"""GAIA benchmark dataset (gaia-benchmark/GAIA).
|
||||
|
||||
Adapted from IPW's gaia.py dataset loader.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, MutableMapping, Optional, Sequence
|
||||
|
||||
from evals.core.dataset import DatasetProvider
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
_DEFAULT_CACHE_DIR = Path.home() / ".cache" / "gaia_benchmark"
|
||||
|
||||
_DEFAULT_INPUT_PROMPT = """Please answer the question below. You should:
|
||||
|
||||
- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings.
|
||||
- If the answer is a number, return only the number without any units unless specified otherwise.
|
||||
- If the answer is a string, don't include articles, and don't use abbreviations (e.g. for states).
|
||||
- If the answer is a comma separated list, apply the above rules to each element in the list.
|
||||
|
||||
{file}
|
||||
|
||||
Here is the question:
|
||||
|
||||
{question}"""
|
||||
|
||||
|
||||
class GAIADataset(DatasetProvider):
|
||||
"""GAIA agentic benchmark dataset."""
|
||||
|
||||
dataset_id = "gaia"
|
||||
dataset_name = "GAIA"
|
||||
|
||||
_hf_path = "gaia-benchmark/GAIA"
|
||||
_default_subset = "2023_all"
|
||||
_default_split = "validation"
|
||||
|
||||
def __init__(self, cache_dir: Optional[str] = None) -> None:
|
||||
self._cache_dir = Path(cache_dir) if cache_dir else _DEFAULT_CACHE_DIR
|
||||
self._records: List[EvalRecord] = []
|
||||
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
from datasets import load_dataset
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
use_split = split or self._default_split
|
||||
|
||||
# Ensure dataset is downloaded
|
||||
dataset_location = self._cache_dir / "GAIA"
|
||||
if not dataset_location.exists():
|
||||
dataset_location.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
snapshot_download(
|
||||
repo_id=self._hf_path,
|
||||
repo_type="dataset",
|
||||
local_dir=str(dataset_location),
|
||||
)
|
||||
except Exception:
|
||||
shutil.rmtree(dataset_location, ignore_errors=True)
|
||||
raise
|
||||
|
||||
dataset = load_dataset(
|
||||
str(dataset_location),
|
||||
name=self._default_subset,
|
||||
split=use_split,
|
||||
)
|
||||
|
||||
rows: Sequence[MutableMapping[str, object]]
|
||||
if hasattr(dataset, "to_list"):
|
||||
rows = dataset.to_list()
|
||||
else:
|
||||
rows = list(dataset)
|
||||
|
||||
if seed is not None:
|
||||
rng = random.Random(seed)
|
||||
rows = list(rows)
|
||||
rng.shuffle(rows)
|
||||
|
||||
if max_samples is not None:
|
||||
rows = rows[:max_samples]
|
||||
|
||||
files_location = dataset_location / "2023" / use_split
|
||||
|
||||
self._records = []
|
||||
for idx, raw in enumerate(rows):
|
||||
record = self._convert_row(raw, files_location, idx)
|
||||
if record is not None:
|
||||
self._records.append(record)
|
||||
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
return iter(self._records)
|
||||
|
||||
def size(self) -> int:
|
||||
return len(self._records)
|
||||
|
||||
def _convert_row(
|
||||
self,
|
||||
raw: MutableMapping[str, object],
|
||||
files_location: Path,
|
||||
idx: int,
|
||||
) -> Optional[EvalRecord]:
|
||||
task_id = str(raw.get("task_id") or "")
|
||||
question = str(raw.get("Question") or "").strip()
|
||||
answer = str(raw.get("Final answer") or "").strip()
|
||||
level = raw.get("Level")
|
||||
|
||||
if not question or not answer:
|
||||
return None
|
||||
|
||||
# Discover associated files
|
||||
file_name: Optional[str] = None
|
||||
file_path: Optional[Path] = None
|
||||
if files_location.exists():
|
||||
files = [f for f in os.listdir(files_location) if task_id in f]
|
||||
if files:
|
||||
file_name = files[0]
|
||||
file_path = files_location / file_name
|
||||
|
||||
# Format the prompt
|
||||
if file_name and file_path:
|
||||
file_info = (
|
||||
f"The following file is referenced in the question below and you will "
|
||||
f"likely need to use it in order to find the correct answer.\n"
|
||||
f"File name: {file_name}\n"
|
||||
f"File path: {file_path}\n"
|
||||
f"Use the file reading tools to access this file."
|
||||
)
|
||||
elif file_name:
|
||||
file_info = (
|
||||
f"The following file is referenced in the question: {file_name}\n"
|
||||
f"(Note: File path not available)"
|
||||
)
|
||||
else:
|
||||
file_info = ""
|
||||
|
||||
problem = _DEFAULT_INPUT_PROMPT.format(file=file_info, question=question)
|
||||
|
||||
subject = f"level_{level}" if level else "general"
|
||||
|
||||
metadata = {
|
||||
"task_id": task_id,
|
||||
"level": level,
|
||||
"file_name": file_name,
|
||||
"file_path": str(file_path) if file_path else None,
|
||||
}
|
||||
|
||||
return EvalRecord(
|
||||
record_id=f"gaia-{task_id or idx}",
|
||||
problem=problem,
|
||||
reference=answer,
|
||||
category="agentic",
|
||||
subject=subject,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["GAIADataset"]
|
||||
@@ -0,0 +1,121 @@
|
||||
"""SuperGPQA dataset provider (m-a-p/SuperGPQA).
|
||||
|
||||
Adapted from IPW's supergpqa.py dataset loader.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Iterable, List, MutableMapping, Optional, Sequence
|
||||
|
||||
from evals.core.dataset import DatasetProvider
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
|
||||
def _format_options(options: Iterable[str]) -> str:
|
||||
rendered = []
|
||||
for idx, option in enumerate(options):
|
||||
letter = chr(ord("A") + idx)
|
||||
rendered.append(f"{letter}. {option}")
|
||||
return "\n".join(rendered)
|
||||
|
||||
|
||||
class SuperGPQADataset(DatasetProvider):
|
||||
"""SuperGPQA multiple-choice benchmark dataset."""
|
||||
|
||||
dataset_id = "supergpqa"
|
||||
dataset_name = "SuperGPQA"
|
||||
|
||||
_hf_path = "m-a-p/SuperGPQA"
|
||||
_default_split = "train"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._records: List[EvalRecord] = []
|
||||
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
from datasets import load_dataset
|
||||
|
||||
use_split = split or self._default_split
|
||||
dataset = load_dataset(self._hf_path, split=use_split)
|
||||
|
||||
rows: Sequence[MutableMapping[str, object]]
|
||||
if hasattr(dataset, "to_list"):
|
||||
rows = dataset.to_list()
|
||||
else:
|
||||
rows = list(dataset)
|
||||
|
||||
if seed is not None:
|
||||
rng = random.Random(seed)
|
||||
rows = list(rows)
|
||||
rng.shuffle(rows)
|
||||
|
||||
if max_samples is not None:
|
||||
rows = rows[:max_samples]
|
||||
|
||||
self._records = []
|
||||
for idx, raw in enumerate(rows):
|
||||
record = self._convert_row(raw, idx)
|
||||
if record is not None:
|
||||
self._records.append(record)
|
||||
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
return iter(self._records)
|
||||
|
||||
def size(self) -> int:
|
||||
return len(self._records)
|
||||
|
||||
def _convert_row(
|
||||
self, raw: MutableMapping[str, object], idx: int,
|
||||
) -> Optional[EvalRecord]:
|
||||
question = str(raw.get("question") or "").strip()
|
||||
options_raw = raw.get("options") or []
|
||||
options = [str(o).strip() for o in options_raw if str(o).strip()]
|
||||
answer_letter = str(raw.get("answer_letter") or "").strip().upper()
|
||||
answer_text = str(raw.get("answer") or "").strip()
|
||||
|
||||
subject = str(
|
||||
raw.get("subfield")
|
||||
or raw.get("field")
|
||||
or raw.get("discipline")
|
||||
or "general"
|
||||
).strip() or "general"
|
||||
|
||||
if not question or not options or not answer_letter:
|
||||
return None
|
||||
|
||||
prompt_parts = [
|
||||
question, "",
|
||||
"Options:",
|
||||
_format_options(options), "",
|
||||
"Respond with the correct letter only.",
|
||||
]
|
||||
problem = "\n".join(part for part in prompt_parts if part).strip()
|
||||
|
||||
metadata = {
|
||||
"uuid": raw.get("uuid"),
|
||||
"discipline": raw.get("discipline"),
|
||||
"field": raw.get("field"),
|
||||
"subfield": raw.get("subfield"),
|
||||
"difficulty": raw.get("difficulty"),
|
||||
"is_calculation": raw.get("is_calculation"),
|
||||
"answer_text": answer_text,
|
||||
"options": options,
|
||||
}
|
||||
|
||||
return EvalRecord(
|
||||
record_id=f"supergpqa-{idx}",
|
||||
problem=problem,
|
||||
reference=answer_letter,
|
||||
category="reasoning",
|
||||
subject=subject,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["SuperGPQADataset"]
|
||||
@@ -0,0 +1,111 @@
|
||||
"""WildChat dataset provider (allenai/WildChat-1M).
|
||||
|
||||
Filters to English single-turn conversations for chat quality evaluation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Iterable, List, MutableMapping, Optional, Sequence
|
||||
|
||||
from evals.core.dataset import DatasetProvider
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
|
||||
class WildChatDataset(DatasetProvider):
|
||||
"""WildChat conversation quality benchmark."""
|
||||
|
||||
dataset_id = "wildchat"
|
||||
dataset_name = "WildChat"
|
||||
|
||||
_hf_path = "allenai/WildChat-1M"
|
||||
_default_split = "train"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._records: List[EvalRecord] = []
|
||||
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
from datasets import load_dataset
|
||||
|
||||
use_split = split or self._default_split
|
||||
dataset = load_dataset(self._hf_path, split=use_split)
|
||||
|
||||
rows: Sequence[MutableMapping[str, object]]
|
||||
if hasattr(dataset, "to_list"):
|
||||
rows = dataset.to_list()
|
||||
else:
|
||||
rows = list(dataset)
|
||||
|
||||
# Filter to English single-turn conversations
|
||||
filtered: List[MutableMapping[str, object]] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, MutableMapping):
|
||||
row = dict(row)
|
||||
|
||||
language = str(row.get("language") or "").lower()
|
||||
if language != "english":
|
||||
continue
|
||||
|
||||
conversation = row.get("conversation")
|
||||
if not isinstance(conversation, list):
|
||||
continue
|
||||
|
||||
# Single-turn: exactly one user message and one assistant message
|
||||
if len(conversation) != 2:
|
||||
continue
|
||||
|
||||
user_msg = conversation[0]
|
||||
asst_msg = conversation[1]
|
||||
if (
|
||||
str(user_msg.get("role", "")) != "user"
|
||||
or str(asst_msg.get("role", "")) != "assistant"
|
||||
):
|
||||
continue
|
||||
|
||||
user_content = str(user_msg.get("content", "")).strip()
|
||||
asst_content = str(asst_msg.get("content", "")).strip()
|
||||
if not user_content or not asst_content:
|
||||
continue
|
||||
|
||||
row["_user_content"] = user_content
|
||||
row["_asst_content"] = asst_content
|
||||
filtered.append(row)
|
||||
|
||||
# Shuffle with seed
|
||||
if seed is not None:
|
||||
rng = random.Random(seed)
|
||||
rng.shuffle(filtered)
|
||||
|
||||
if max_samples is not None:
|
||||
filtered = filtered[:max_samples]
|
||||
|
||||
self._records = []
|
||||
for idx, raw in enumerate(filtered):
|
||||
self._records.append(
|
||||
EvalRecord(
|
||||
record_id=f"wildchat-{idx}",
|
||||
problem=raw["_user_content"],
|
||||
reference=raw["_asst_content"],
|
||||
category="chat",
|
||||
subject="conversation",
|
||||
metadata={
|
||||
"model": str(raw.get("model", "")),
|
||||
"language": "english",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
return iter(self._records)
|
||||
|
||||
def size(self) -> int:
|
||||
return len(self._records)
|
||||
|
||||
|
||||
__all__ = ["WildChatDataset"]
|
||||
@@ -0,0 +1,23 @@
|
||||
[project]
|
||||
name = "openjarvis-evals"
|
||||
version = "0.1.0"
|
||||
description = "Evaluation framework for OpenJarvis"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"openjarvis>=1.0.0",
|
||||
"click>=8",
|
||||
"datasets>=2.14",
|
||||
"huggingface-hub>=0.20",
|
||||
"tqdm>=4.65",
|
||||
"rich>=13",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = ["pytest>=8", "pytest-cov>=5"]
|
||||
|
||||
[project.scripts]
|
||||
openjarvis-eval = "evals.cli:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
@@ -0,0 +1 @@
|
||||
"""Scoring implementations for evaluation benchmarks."""
|
||||
@@ -0,0 +1,99 @@
|
||||
"""FRAMES scorer — LLM-as-judge for multi-hop factual retrieval.
|
||||
|
||||
Adapted from IPW's frames.py evaluation handler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from evals.core.scorer import LLMJudgeScorer
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
_GRADER_TEMPLATE = """You are evaluating an AI system's answer to a multi-hop factual question.
|
||||
|
||||
Compare the predicted answer against the Ground Truth Answer and determine if the prediction is correct.
|
||||
|
||||
## Evaluation Guidelines
|
||||
|
||||
1. **Focus on semantic meaning**: Look for equivalent information - exact wording is not required.
|
||||
2. **Assess factual accuracy**: Determine whether the essential facts from the Ground Truth are present in the answer.
|
||||
3. **Ignore minor differences**: Capitalization, punctuation, formatting, and word order don't matter.
|
||||
4. **Partial credit**: If the Ground Truth has multiple parts, all essential parts must be present for a correct rating.
|
||||
5. **Additional information**: Extra correct information in the prediction is acceptable, but extra incorrect information is not.
|
||||
|
||||
## Question
|
||||
{question}
|
||||
|
||||
## Ground Truth Answer
|
||||
{ground_truth}
|
||||
|
||||
## Predicted Answer
|
||||
{predicted_answer}
|
||||
|
||||
Your response MUST use exactly this format:
|
||||
extracted_final_answer: <the final answer extracted from the predicted answer, or 'None' if no answer is present>
|
||||
reasoning: <brief explanation of why the extracted answer is or is not correct>
|
||||
correct: <yes or no>"""
|
||||
|
||||
|
||||
class FRAMESScorer(LLMJudgeScorer):
|
||||
"""LLM-as-judge evaluation for FRAMES multi-hop factual retrieval."""
|
||||
|
||||
scorer_id = "frames"
|
||||
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
if not model_answer or not model_answer.strip():
|
||||
return False, {"reason": "empty_response"}
|
||||
|
||||
reference = record.reference
|
||||
if not reference or not reference.strip():
|
||||
return None, {"reason": "no_ground_truth"}
|
||||
|
||||
prompt = _GRADER_TEMPLATE.format(
|
||||
question=record.problem,
|
||||
ground_truth=reference,
|
||||
predicted_answer=model_answer,
|
||||
)
|
||||
|
||||
try:
|
||||
raw = self._ask_judge(prompt, temperature=0.0, max_tokens=1024)
|
||||
|
||||
structured_match = re.search(
|
||||
r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
|
||||
)
|
||||
if structured_match:
|
||||
is_correct = structured_match.group(1).lower() == "yes"
|
||||
else:
|
||||
response_upper = raw.upper().strip()
|
||||
if "TRUE" in response_upper:
|
||||
is_correct = True
|
||||
elif "FALSE" in response_upper:
|
||||
is_correct = False
|
||||
else:
|
||||
LOGGER.warning("Could not parse grade from response: %s", raw[:50])
|
||||
is_correct = False
|
||||
|
||||
meta: Dict[str, Any] = {
|
||||
"raw_judge_output": raw,
|
||||
}
|
||||
extracted = re.search(
|
||||
r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
|
||||
)
|
||||
if extracted:
|
||||
meta["extracted_answer"] = extracted.group(1).strip()
|
||||
|
||||
return is_correct, meta
|
||||
|
||||
except Exception as exc:
|
||||
LOGGER.error("FRAMES scoring failed: %s", exc)
|
||||
return None, {"error": str(exc)}
|
||||
|
||||
|
||||
__all__ = ["FRAMESScorer"]
|
||||
@@ -0,0 +1,165 @@
|
||||
"""GAIA scorer — normalized exact match with LLM fallback.
|
||||
|
||||
Adapted from IPW's gaia.py evaluation handler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import string
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from evals.core.scorer import LLMJudgeScorer
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Normalization helpers (ported from IPW)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _normalize_number_str(number_str: str) -> float:
|
||||
for char in ["$", "%", ","]:
|
||||
number_str = number_str.replace(char, "")
|
||||
try:
|
||||
return float(number_str)
|
||||
except ValueError:
|
||||
return float("inf")
|
||||
|
||||
|
||||
def _normalize_str(input_str: str, remove_punct: bool = True) -> str:
|
||||
no_spaces = re.sub(r"\s", "", input_str)
|
||||
if remove_punct:
|
||||
translator = str.maketrans("", "", string.punctuation)
|
||||
return no_spaces.lower().translate(translator)
|
||||
return no_spaces.lower()
|
||||
|
||||
|
||||
def _split_string(s: str, char_list: list[str] | None = None) -> list[str]:
|
||||
if char_list is None:
|
||||
char_list = [",", ";"]
|
||||
pattern = f"[{''.join(char_list)}]"
|
||||
return re.split(pattern, s)
|
||||
|
||||
|
||||
def _is_float(element: object) -> bool:
|
||||
try:
|
||||
float(element) # type: ignore[arg-type]
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def exact_match(model_answer: str, ground_truth: str) -> bool:
|
||||
"""GAIA exact-match scorer with normalization for numbers, lists, and strings."""
|
||||
if model_answer is None:
|
||||
model_answer = "None"
|
||||
|
||||
if _is_float(ground_truth):
|
||||
normalized = _normalize_number_str(model_answer)
|
||||
return normalized == float(ground_truth)
|
||||
|
||||
if any(char in ground_truth for char in [",", ";"]):
|
||||
gt_elems = _split_string(ground_truth)
|
||||
ma_elems = _split_string(model_answer)
|
||||
if len(gt_elems) != len(ma_elems):
|
||||
return False
|
||||
comparisons = []
|
||||
for ma_elem, gt_elem in zip(ma_elems, gt_elems):
|
||||
if _is_float(gt_elem):
|
||||
comparisons.append(
|
||||
_normalize_number_str(ma_elem) == float(gt_elem)
|
||||
)
|
||||
else:
|
||||
comparisons.append(
|
||||
_normalize_str(ma_elem, remove_punct=False)
|
||||
== _normalize_str(gt_elem, remove_punct=False)
|
||||
)
|
||||
return all(comparisons)
|
||||
|
||||
return _normalize_str(model_answer) == _normalize_str(ground_truth)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM fallback prompt
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_LLM_FALLBACK_PROMPT = """Your job is to determine if the predicted answer is semantically equivalent to the gold target.
|
||||
|
||||
Question: {question}
|
||||
Gold target: {ground_truth}
|
||||
Predicted answer: {response}
|
||||
|
||||
Consider the following:
|
||||
- Numerical answers should match exactly (accounting for different formats like $1,000 vs 1000)
|
||||
- List answers should contain all elements (order may vary)
|
||||
- String answers should have the same meaning (case and punctuation don't matter)
|
||||
|
||||
Your response MUST use exactly this format:
|
||||
extracted_final_answer: <the final answer extracted from the predicted answer, or 'None' if no answer is present>
|
||||
reasoning: <brief explanation of why the extracted answer matches or does not match the gold target>
|
||||
correct: <yes or no>"""
|
||||
|
||||
|
||||
class GAIAScorer(LLMJudgeScorer):
|
||||
"""GAIA evaluation: exact match with normalization + LLM fallback."""
|
||||
|
||||
scorer_id = "gaia"
|
||||
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
if not model_answer or not model_answer.strip():
|
||||
return False, {"reason": "empty_response"}
|
||||
|
||||
reference = record.reference
|
||||
if not reference or not reference.strip():
|
||||
return None, {"reason": "no_ground_truth"}
|
||||
|
||||
# Try exact match first (fast, no API call)
|
||||
if exact_match(model_answer, reference):
|
||||
return True, {"match_type": "exact"}
|
||||
|
||||
# LLM fallback for semantic comparison
|
||||
try:
|
||||
prompt = _LLM_FALLBACK_PROMPT.format(
|
||||
question=record.problem or "(No question provided)",
|
||||
response=model_answer,
|
||||
ground_truth=reference,
|
||||
)
|
||||
raw = self._ask_judge(prompt, temperature=0.0, max_tokens=1024)
|
||||
|
||||
structured_match = re.search(
|
||||
r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
|
||||
)
|
||||
if structured_match:
|
||||
is_correct = structured_match.group(1).lower() == "yes"
|
||||
else:
|
||||
is_correct = (
|
||||
"CORRECT" in raw.upper() and "INCORRECT" not in raw.upper()
|
||||
)
|
||||
|
||||
meta: Dict[str, Any] = {
|
||||
"match_type": "llm_fallback",
|
||||
"raw_judge_output": raw,
|
||||
}
|
||||
extracted_match = re.search(
|
||||
r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
|
||||
)
|
||||
if extracted_match:
|
||||
meta["extracted_answer"] = extracted_match.group(1).strip()
|
||||
|
||||
return is_correct, meta
|
||||
|
||||
except Exception as exc:
|
||||
LOGGER.error("GAIA LLM fallback failed: %s", exc)
|
||||
return False, {
|
||||
"match_type": "llm_fallback_error",
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["GAIAScorer", "exact_match"]
|
||||
@@ -0,0 +1,98 @@
|
||||
"""SuperGPQA MCQ scorer — LLM-based letter extraction + exact match.
|
||||
|
||||
Adapted from IPW's mcq.py and gpqa.py evaluation handlers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from evals.core.scorer import LLMJudgeScorer
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SuperGPQAScorer(LLMJudgeScorer):
|
||||
"""Score SuperGPQA responses by extracting answer letter via LLM."""
|
||||
|
||||
scorer_id = "supergpqa"
|
||||
|
||||
def _valid_letters_from_options(self, metadata: Dict[str, Any]) -> str:
|
||||
options = metadata.get("options")
|
||||
if isinstance(options, list) and options:
|
||||
n = len(options)
|
||||
return "".join(chr(ord("A") + i) for i in range(n))
|
||||
return "ABCD"
|
||||
|
||||
def _extract_answer_with_llm(
|
||||
self,
|
||||
problem: str,
|
||||
model_answer: str,
|
||||
valid_letters: str,
|
||||
) -> Optional[str]:
|
||||
"""Use the judge LLM to extract the answer letter from the response."""
|
||||
last_letter = valid_letters[-1] if valid_letters else "D"
|
||||
|
||||
system_prompt = (
|
||||
f"You are an answer extraction assistant. Extract the final multiple choice answer "
|
||||
f"from the response. Return ONLY a single letter (A-{last_letter}). "
|
||||
f"If no valid answer letter is found, return 'NONE'."
|
||||
)
|
||||
|
||||
user_prompt = (
|
||||
f"Problem: {problem}\nResponse: {model_answer}\n\n"
|
||||
f"Extract the final answer letter:"
|
||||
)
|
||||
|
||||
try:
|
||||
raw_response = self._ask_judge(
|
||||
user_prompt, system=system_prompt,
|
||||
temperature=0.0, max_tokens=5,
|
||||
)
|
||||
|
||||
extracted = raw_response.strip().upper()
|
||||
|
||||
# Handle "The answer is: A" etc.
|
||||
answer_match = re.search(
|
||||
r"(?:THE ANSWER IS:?\s*)?([A-Z])", extracted, re.IGNORECASE,
|
||||
)
|
||||
if answer_match:
|
||||
extracted = answer_match.group(1).upper()
|
||||
|
||||
if extracted in valid_letters:
|
||||
return extracted
|
||||
|
||||
return None
|
||||
|
||||
except Exception as exc:
|
||||
LOGGER.error("Error in LLM-based answer extraction: %s", exc)
|
||||
return None
|
||||
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
ref = record.reference.strip().upper()
|
||||
if not ref:
|
||||
return None, {"reason": "missing_reference_letter"}
|
||||
|
||||
valid_letters = self._valid_letters_from_options(record.metadata)
|
||||
|
||||
candidate = self._extract_answer_with_llm(
|
||||
record.problem, model_answer, valid_letters,
|
||||
)
|
||||
if not candidate:
|
||||
return None, {"reason": "no_choice_letter_extracted"}
|
||||
|
||||
is_correct = candidate == ref
|
||||
meta = {
|
||||
"reference_letter": ref,
|
||||
"candidate_letter": candidate,
|
||||
"valid_letters": valid_letters,
|
||||
}
|
||||
return is_correct, meta
|
||||
|
||||
|
||||
__all__ = ["SuperGPQAScorer"]
|
||||
@@ -0,0 +1,161 @@
|
||||
"""WildChat scorer — dual-comparison LLM-as-judge.
|
||||
|
||||
Adapted from IPW's wildchat.py evaluation handler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from evals.core.scorer import LLMJudgeScorer
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
SYSTEM_PROMPT = """You are an impartial judge evaluating the quality of two AI-assistant replies to the same user prompt.
|
||||
|
||||
Step 1 – Generate your own answer
|
||||
Write the response *you* would give to the user. Keep it separate from later analysis.
|
||||
|
||||
Step 2 – Decide the query type
|
||||
Classify the user prompt as either
|
||||
• **Subjective / open-ended** (creative writing, opinion, advice, brainstorming)
|
||||
• **Objective / technical** (code, math, logical derivations with a single correct outcome)
|
||||
If uncertain, default to "Subjective".
|
||||
|
||||
Step 3 – Score each assistant with the correct rubric
|
||||
|
||||
| Query type | Criteria |
|
||||
|------------|----------|
|
||||
| Subjective / open-ended | 1. Correctness / factual soundness 2. Helpfulness 3. Relevance 4. Conciseness 5. Creativity & novelty |
|
||||
| Objective / technical | 1. Correctness only |
|
||||
|
||||
When using the multi-criteria rubric, note strengths and weaknesses for **each** dimension.
|
||||
When using the single-criterion rubric, focus exclusively on factual / functional accuracy and ignore style or flair.
|
||||
|
||||
Step 4 – Compare & justify
|
||||
Explain which assistant is better and why, correcting any mistakes you find. Highlight missing but important details. **Be concise.**
|
||||
|
||||
Step 5 – Verdict
|
||||
1. Assistant A is significantly better: [[A>>B]]
|
||||
2. Assistant A is slightly better: [[A>B]]
|
||||
3. Tie, Assistant A is equal: [[A=B]]
|
||||
4. Assistant B is slightly better: [[B>A]]
|
||||
5. Assistant B is significantly better: [[B>>A]]
|
||||
|
||||
Choose exactly one token from: `[[A>>B]]`, `[[A>B]]`, `[[A=B]]`, `[[B>A]]`, `[[B>>A]]`.
|
||||
|
||||
---
|
||||
|
||||
### Output format (strict)
|
||||
Return **only** a JSON object that matches the provided schema:
|
||||
|
||||
<Your Response To The User Prompt>
|
||||
|
||||
```json
|
||||
{
|
||||
"query_type": "<query type>",
|
||||
"explanation": "<multi-criteria explanation> | <single-criteria explanation> (if query_type is \\"Objective / technical\\")",
|
||||
"verdict": "<one verdict token from: [[A>>B]], [[A>B]], [[A=B]], [[B>A]], [[B>>A]]>"
|
||||
}
|
||||
```"""
|
||||
|
||||
|
||||
class WildChatScorer(LLMJudgeScorer):
|
||||
"""Dual-comparison LLM-as-judge for chat quality."""
|
||||
|
||||
scorer_id = "wildchat"
|
||||
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
reference = record.reference
|
||||
if not reference or not reference.strip():
|
||||
return None, {"reason": "empty_reference"}
|
||||
|
||||
# Two comparisons: (model vs reference) and (reference vs model)
|
||||
verdict1, response1 = self._get_judge_verdict(
|
||||
record.problem, model_answer, reference,
|
||||
)
|
||||
verdict2, response2 = self._get_judge_verdict(
|
||||
record.problem, reference, model_answer,
|
||||
)
|
||||
|
||||
if verdict1 is None or verdict2 is None:
|
||||
return None, {
|
||||
"reason": "missing_verdicts",
|
||||
"verdict1": verdict1,
|
||||
"verdict2": verdict2,
|
||||
}
|
||||
|
||||
result1 = self._verdict_to_bool(verdict1, generated_is_a=True)
|
||||
result2 = self._verdict_to_bool(verdict2, generated_is_a=False)
|
||||
|
||||
meta: Dict[str, Any] = {
|
||||
"generated_as_a": {"verdict": verdict1, "response": response1},
|
||||
"generated_as_b": {"verdict": verdict2, "response": response2},
|
||||
}
|
||||
|
||||
if result1 is None or result2 is None:
|
||||
return None, meta
|
||||
|
||||
final_result = result1 or result2
|
||||
return final_result, meta
|
||||
|
||||
def _get_judge_verdict(
|
||||
self, problem: str, response_a: str, response_b: str,
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
prompt = (
|
||||
f"<|User Prompt|>\n{problem}\n\n"
|
||||
f"<|The Start of Assistant A's Answer|>\n{response_a}\n"
|
||||
f"<|The End of Assistant A's Answer|>\n\n"
|
||||
f"<|The Start of Assistant B's Answer|>\n{response_b}\n"
|
||||
f"<|The End of Assistant B's Answer|>"
|
||||
)
|
||||
|
||||
try:
|
||||
raw = self._ask_judge(
|
||||
prompt, system=SYSTEM_PROMPT,
|
||||
temperature=0.0, max_tokens=1024,
|
||||
)
|
||||
except Exception as exc:
|
||||
LOGGER.error("WildChat judge call failed: %s", exc)
|
||||
return None, None
|
||||
|
||||
content = raw.strip()
|
||||
verdict_match = re.search(r"\[\[([AB][><=]{1,2}[AB])\]\]", content)
|
||||
if verdict_match:
|
||||
return verdict_match.group(1), content
|
||||
|
||||
return None, content
|
||||
|
||||
@staticmethod
|
||||
def _verdict_to_bool(
|
||||
verdict: Optional[str], generated_is_a: bool,
|
||||
) -> Optional[bool]:
|
||||
if not verdict:
|
||||
return None
|
||||
|
||||
verdict_map_a = {
|
||||
"A>>B": True,
|
||||
"A>B": True,
|
||||
"A=B": True,
|
||||
"B>A": False,
|
||||
"B>>A": False,
|
||||
}
|
||||
|
||||
verdict_map_b = {
|
||||
"A>>B": False,
|
||||
"A>B": False,
|
||||
"A=B": True,
|
||||
"B>A": True,
|
||||
"B>>A": True,
|
||||
}
|
||||
|
||||
verdict_map = verdict_map_a if generated_is_a else verdict_map_b
|
||||
return verdict_map.get(verdict)
|
||||
|
||||
|
||||
__all__ = ["WildChatScorer"]
|
||||
@@ -0,0 +1 @@
|
||||
"""Tests for the evaluation framework."""
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Shared test fixtures for the evaluation framework."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure evals package is importable from the repo root
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
|
||||
|
||||
from evals.core.backend import InferenceBackend
|
||||
from evals.core.dataset import DatasetProvider
|
||||
from evals.core.scorer import Scorer
|
||||
from evals.core.types import EvalRecord
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock backend
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MockBackend(InferenceBackend):
|
||||
"""Backend that returns canned responses for testing."""
|
||||
|
||||
backend_id = "mock"
|
||||
|
||||
def __init__(self, responses: Optional[Dict[str, str]] = None) -> None:
|
||||
self._responses = responses or {}
|
||||
self._default_response = "Mock response"
|
||||
self._call_count = 0
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
model: str,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> str:
|
||||
self._call_count += 1
|
||||
return self._responses.get(prompt, self._default_response)
|
||||
|
||||
def generate_full(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
model: str,
|
||||
system: str = "",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> Dict[str, Any]:
|
||||
content = self.generate(
|
||||
prompt, model=model, system=system,
|
||||
temperature=temperature, max_tokens=max_tokens,
|
||||
)
|
||||
return {
|
||||
"content": content,
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"total_tokens": 150,
|
||||
},
|
||||
"model": model,
|
||||
"latency_seconds": 0.1,
|
||||
"cost_usd": 0.001,
|
||||
}
|
||||
|
||||
|
||||
class MockScorer(Scorer):
|
||||
"""Scorer that always returns a fixed result."""
|
||||
|
||||
scorer_id = "mock"
|
||||
|
||||
def __init__(self, result: bool = True) -> None:
|
||||
self._result = result
|
||||
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
return self._result, {"mock": True}
|
||||
|
||||
|
||||
class MockDataset(DatasetProvider):
|
||||
"""Dataset that yields fixed records."""
|
||||
|
||||
dataset_id = "mock"
|
||||
dataset_name = "Mock"
|
||||
|
||||
def __init__(self, records: Optional[list[EvalRecord]] = None) -> None:
|
||||
self._records = records or []
|
||||
|
||||
def load(self, *, max_samples=None, split=None, seed=None) -> None:
|
||||
pass
|
||||
|
||||
def iter_records(self):
|
||||
return iter(self._records)
|
||||
|
||||
def size(self) -> int:
|
||||
return len(self._records)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def mock_backend():
|
||||
return MockBackend()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def mock_scorer():
|
||||
return MockScorer()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def sample_records():
|
||||
return [
|
||||
EvalRecord(
|
||||
record_id="test-001",
|
||||
problem="What is 2+2?",
|
||||
reference="4",
|
||||
category="reasoning",
|
||||
subject="math",
|
||||
),
|
||||
EvalRecord(
|
||||
record_id="test-002",
|
||||
problem="What is the capital of France?",
|
||||
reference="Paris",
|
||||
category="reasoning",
|
||||
subject="geography",
|
||||
),
|
||||
EvalRecord(
|
||||
record_id="test-003",
|
||||
problem="Hello, how are you?",
|
||||
reference="I'm fine, thank you!",
|
||||
category="chat",
|
||||
subject="greeting",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def mock_dataset(sample_records):
|
||||
return MockDataset(sample_records)
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Tests for backend construction with mocks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestJarvisDirectBackend:
|
||||
@patch("openjarvis.system.SystemBuilder")
|
||||
def test_construction_default(self, mock_builder_cls):
|
||||
mock_builder = MagicMock()
|
||||
mock_builder.engine.return_value = mock_builder
|
||||
mock_builder.telemetry.return_value = mock_builder
|
||||
mock_builder.traces.return_value = mock_builder
|
||||
mock_system = MagicMock()
|
||||
mock_builder.build.return_value = mock_system
|
||||
mock_builder_cls.return_value = mock_builder
|
||||
|
||||
from evals.backends.jarvis_direct import JarvisDirectBackend
|
||||
|
||||
backend = JarvisDirectBackend()
|
||||
assert backend.backend_id == "jarvis-direct"
|
||||
mock_builder.telemetry.assert_called_with(False)
|
||||
mock_builder.traces.assert_called_with(False)
|
||||
mock_builder.build.assert_called_once()
|
||||
|
||||
@patch("openjarvis.system.SystemBuilder")
|
||||
def test_construction_with_engine_key(self, mock_builder_cls):
|
||||
mock_builder = MagicMock()
|
||||
mock_builder.engine.return_value = mock_builder
|
||||
mock_builder.telemetry.return_value = mock_builder
|
||||
mock_builder.traces.return_value = mock_builder
|
||||
mock_builder.build.return_value = MagicMock()
|
||||
mock_builder_cls.return_value = mock_builder
|
||||
|
||||
from evals.backends.jarvis_direct import JarvisDirectBackend
|
||||
|
||||
JarvisDirectBackend(engine_key="cloud")
|
||||
mock_builder.engine.assert_called_with("cloud")
|
||||
|
||||
@patch("openjarvis.system.SystemBuilder")
|
||||
def test_generate_full(self, mock_builder_cls):
|
||||
mock_builder = MagicMock()
|
||||
mock_builder.engine.return_value = mock_builder
|
||||
mock_builder.telemetry.return_value = mock_builder
|
||||
mock_builder.traces.return_value = mock_builder
|
||||
mock_system = MagicMock()
|
||||
mock_system.engine.generate.return_value = {
|
||||
"content": "42",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5},
|
||||
"model": "test-model",
|
||||
"cost_usd": 0.001,
|
||||
}
|
||||
mock_builder.build.return_value = mock_system
|
||||
mock_builder_cls.return_value = mock_builder
|
||||
|
||||
from evals.backends.jarvis_direct import JarvisDirectBackend
|
||||
|
||||
backend = JarvisDirectBackend()
|
||||
result = backend.generate_full("What is 2+2?", model="test-model")
|
||||
|
||||
assert result["content"] == "42"
|
||||
assert result["cost_usd"] == 0.001
|
||||
assert "latency_seconds" in result
|
||||
|
||||
@patch("openjarvis.system.SystemBuilder")
|
||||
def test_generate(self, mock_builder_cls):
|
||||
mock_builder = MagicMock()
|
||||
mock_builder.engine.return_value = mock_builder
|
||||
mock_builder.telemetry.return_value = mock_builder
|
||||
mock_builder.traces.return_value = mock_builder
|
||||
mock_system = MagicMock()
|
||||
mock_system.engine.generate.return_value = {
|
||||
"content": "Paris",
|
||||
"usage": {},
|
||||
}
|
||||
mock_builder.build.return_value = mock_system
|
||||
mock_builder_cls.return_value = mock_builder
|
||||
|
||||
from evals.backends.jarvis_direct import JarvisDirectBackend
|
||||
|
||||
backend = JarvisDirectBackend()
|
||||
text = backend.generate("Capital of France?", model="m")
|
||||
assert text == "Paris"
|
||||
|
||||
|
||||
class TestJarvisAgentBackend:
|
||||
@patch("openjarvis.system.SystemBuilder")
|
||||
def test_construction(self, mock_builder_cls):
|
||||
mock_builder = MagicMock()
|
||||
mock_builder.engine.return_value = mock_builder
|
||||
mock_builder.agent.return_value = mock_builder
|
||||
mock_builder.tools.return_value = mock_builder
|
||||
mock_builder.telemetry.return_value = mock_builder
|
||||
mock_builder.traces.return_value = mock_builder
|
||||
mock_builder.build.return_value = MagicMock()
|
||||
mock_builder_cls.return_value = mock_builder
|
||||
|
||||
from evals.backends.jarvis_agent import JarvisAgentBackend
|
||||
|
||||
backend = JarvisAgentBackend(
|
||||
engine_key="cloud", agent_name="orchestrator",
|
||||
tools=["calculator", "think"],
|
||||
)
|
||||
assert backend.backend_id == "jarvis-agent"
|
||||
mock_builder.engine.assert_called_with("cloud")
|
||||
mock_builder.agent.assert_called_with("orchestrator")
|
||||
mock_builder.tools.assert_called_with(["calculator", "think"])
|
||||
|
||||
@patch("openjarvis.system.SystemBuilder")
|
||||
def test_generate_full(self, mock_builder_cls):
|
||||
mock_builder = MagicMock()
|
||||
mock_builder.engine.return_value = mock_builder
|
||||
mock_builder.agent.return_value = mock_builder
|
||||
mock_builder.tools.return_value = mock_builder
|
||||
mock_builder.telemetry.return_value = mock_builder
|
||||
mock_builder.traces.return_value = mock_builder
|
||||
mock_system = MagicMock()
|
||||
mock_system.ask.return_value = {
|
||||
"content": "The answer is 4.",
|
||||
"usage": {"prompt_tokens": 50, "completion_tokens": 20},
|
||||
"model": "gpt-4o",
|
||||
"turns": 2,
|
||||
"tool_results": [
|
||||
{"tool_name": "calculator", "content": "4", "success": True},
|
||||
],
|
||||
}
|
||||
mock_builder.build.return_value = mock_system
|
||||
mock_builder_cls.return_value = mock_builder
|
||||
|
||||
from evals.backends.jarvis_agent import JarvisAgentBackend
|
||||
|
||||
backend = JarvisAgentBackend(agent_name="orchestrator")
|
||||
result = backend.generate_full("What is 2+2?", model="gpt-4o")
|
||||
|
||||
assert result["content"] == "The answer is 4."
|
||||
assert result["turns"] == 2
|
||||
assert len(result["tool_results"]) == 1
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Tests for FRAMES scorer (judge prompt formatting and verdict parsing)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.core.types import EvalRecord
|
||||
from evals.scorers.frames_judge import _GRADER_TEMPLATE, FRAMESScorer
|
||||
from evals.tests.conftest import MockBackend
|
||||
|
||||
|
||||
class TestGraderTemplate:
|
||||
def test_template_formatting(self):
|
||||
result = _GRADER_TEMPLATE.format(
|
||||
question="What is the capital?",
|
||||
ground_truth="Paris",
|
||||
predicted_answer="The capital is Paris",
|
||||
)
|
||||
assert "What is the capital?" in result
|
||||
assert "Paris" in result
|
||||
assert "The capital is Paris" in result
|
||||
assert "correct: <yes or no>" in result
|
||||
|
||||
def test_template_has_required_sections(self):
|
||||
assert "## Question" in _GRADER_TEMPLATE
|
||||
assert "## Ground Truth Answer" in _GRADER_TEMPLATE
|
||||
assert "## Predicted Answer" in _GRADER_TEMPLATE
|
||||
assert "extracted_final_answer:" in _GRADER_TEMPLATE
|
||||
|
||||
|
||||
class TestFRAMESScorer:
|
||||
def _make_record(self, reference="Paris"):
|
||||
return EvalRecord(
|
||||
record_id="frames-1",
|
||||
problem="What is the capital of France?",
|
||||
reference=reference,
|
||||
category="rag",
|
||||
subject="general",
|
||||
)
|
||||
|
||||
def test_correct_answer(self):
|
||||
backend = MockBackend()
|
||||
backend._default_response = (
|
||||
"extracted_final_answer: Paris\n"
|
||||
"reasoning: The answer correctly identifies Paris as the capital.\n"
|
||||
"correct: yes"
|
||||
)
|
||||
scorer = FRAMESScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("Paris")
|
||||
is_correct, meta = scorer.score(record, "The capital is Paris")
|
||||
|
||||
assert is_correct is True
|
||||
assert "raw_judge_output" in meta
|
||||
assert meta["extracted_answer"] == "Paris"
|
||||
|
||||
def test_incorrect_answer(self):
|
||||
backend = MockBackend()
|
||||
backend._default_response = (
|
||||
"extracted_final_answer: London\n"
|
||||
"reasoning: London is not the capital of France.\n"
|
||||
"correct: no"
|
||||
)
|
||||
scorer = FRAMESScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("Paris")
|
||||
is_correct, meta = scorer.score(record, "London")
|
||||
|
||||
assert is_correct is False
|
||||
|
||||
def test_empty_response(self):
|
||||
backend = MockBackend()
|
||||
scorer = FRAMESScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("Paris")
|
||||
is_correct, meta = scorer.score(record, "")
|
||||
|
||||
assert is_correct is False
|
||||
assert meta["reason"] == "empty_response"
|
||||
|
||||
def test_no_ground_truth(self):
|
||||
backend = MockBackend()
|
||||
scorer = FRAMESScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("")
|
||||
is_correct, meta = scorer.score(record, "Paris")
|
||||
|
||||
assert is_correct is None
|
||||
assert meta["reason"] == "no_ground_truth"
|
||||
|
||||
def test_fallback_true_false_parsing(self):
|
||||
backend = MockBackend()
|
||||
backend._default_response = "The prediction is TRUE"
|
||||
scorer = FRAMESScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("Paris")
|
||||
is_correct, _ = scorer.score(record, "Paris")
|
||||
|
||||
assert is_correct is True
|
||||
|
||||
def test_judge_error(self):
|
||||
class ErrorBackend(MockBackend):
|
||||
def generate(self, prompt, **kw):
|
||||
raise RuntimeError("API error")
|
||||
|
||||
backend = ErrorBackend()
|
||||
scorer = FRAMESScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("Paris")
|
||||
is_correct, meta = scorer.score(record, "Paris")
|
||||
|
||||
assert is_correct is None
|
||||
assert "error" in meta
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Tests for GAIA scorer logic (normalization and exact match)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.core.types import EvalRecord
|
||||
from evals.scorers.gaia_exact import (
|
||||
GAIAScorer,
|
||||
_is_float,
|
||||
_normalize_number_str,
|
||||
_normalize_str,
|
||||
_split_string,
|
||||
exact_match,
|
||||
)
|
||||
from evals.tests.conftest import MockBackend
|
||||
|
||||
|
||||
class TestNormalization:
|
||||
def test_normalize_number_str(self):
|
||||
assert _normalize_number_str("1000") == 1000.0
|
||||
assert _normalize_number_str("$1,000") == 1000.0
|
||||
assert _normalize_number_str("50%") == 50.0
|
||||
assert _normalize_number_str("abc") == float("inf")
|
||||
|
||||
def test_normalize_str(self):
|
||||
assert _normalize_str("Hello World") == "helloworld"
|
||||
assert _normalize_str("Hello, World!", remove_punct=True) == "helloworld"
|
||||
assert _normalize_str("Hello, World!", remove_punct=False) == "hello,world!"
|
||||
|
||||
def test_split_string(self):
|
||||
assert _split_string("a, b, c") == ["a", " b", " c"]
|
||||
assert _split_string("a; b") == ["a", " b"]
|
||||
|
||||
def test_is_float(self):
|
||||
assert _is_float("3.14") is True
|
||||
assert _is_float("42") is True
|
||||
assert _is_float("abc") is False
|
||||
assert _is_float(None) is False
|
||||
|
||||
|
||||
class TestExactMatch:
|
||||
def test_number_match(self):
|
||||
assert exact_match("42", "42") is True
|
||||
assert exact_match("$1,000", "1000") is True
|
||||
assert exact_match("43", "42") is False
|
||||
|
||||
def test_string_match(self):
|
||||
assert exact_match("Paris", "paris") is True
|
||||
assert exact_match(" Paris ", "paris") is True
|
||||
assert exact_match("London", "Paris") is False
|
||||
|
||||
def test_list_match(self):
|
||||
assert exact_match("1, 2, 3", "1, 2, 3") is True
|
||||
assert exact_match("1, 2", "1, 2, 3") is False
|
||||
|
||||
def test_none_answer(self):
|
||||
assert exact_match(None, "42") is False
|
||||
|
||||
def test_punctuation_handling(self):
|
||||
assert exact_match("Hello!", "Hello") is True
|
||||
assert exact_match("test.", "test") is True
|
||||
|
||||
|
||||
class TestGAIAScorer:
|
||||
def _make_record(self, reference="42"):
|
||||
return EvalRecord(
|
||||
record_id="gaia-001",
|
||||
problem="What is the answer?",
|
||||
reference=reference,
|
||||
category="agentic",
|
||||
subject="level_1",
|
||||
)
|
||||
|
||||
def test_exact_match_correct(self):
|
||||
backend = MockBackend()
|
||||
scorer = GAIAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("42")
|
||||
is_correct, meta = scorer.score(record, "42")
|
||||
|
||||
assert is_correct is True
|
||||
assert meta["match_type"] == "exact"
|
||||
|
||||
def test_exact_match_with_formatting(self):
|
||||
backend = MockBackend()
|
||||
scorer = GAIAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("1000")
|
||||
is_correct, meta = scorer.score(record, "$1,000")
|
||||
|
||||
assert is_correct is True
|
||||
assert meta["match_type"] == "exact"
|
||||
|
||||
def test_llm_fallback(self):
|
||||
backend = MockBackend()
|
||||
backend._default_response = (
|
||||
"extracted_final_answer: 42\n"
|
||||
"reasoning: The answer is semantically equivalent.\n"
|
||||
"correct: yes"
|
||||
)
|
||||
scorer = GAIAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("42")
|
||||
is_correct, meta = scorer.score(record, "The answer is forty-two")
|
||||
|
||||
assert is_correct is True
|
||||
assert meta["match_type"] == "llm_fallback"
|
||||
|
||||
def test_llm_fallback_incorrect(self):
|
||||
backend = MockBackend()
|
||||
backend._default_response = (
|
||||
"extracted_final_answer: 43\n"
|
||||
"reasoning: Different number.\n"
|
||||
"correct: no"
|
||||
)
|
||||
scorer = GAIAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("42")
|
||||
is_correct, meta = scorer.score(record, "The answer is 43")
|
||||
|
||||
assert is_correct is False
|
||||
|
||||
def test_empty_response(self):
|
||||
backend = MockBackend()
|
||||
scorer = GAIAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("42")
|
||||
is_correct, meta = scorer.score(record, "")
|
||||
|
||||
assert is_correct is False
|
||||
assert meta["reason"] == "empty_response"
|
||||
|
||||
def test_no_ground_truth(self):
|
||||
backend = MockBackend()
|
||||
scorer = GAIAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("")
|
||||
is_correct, meta = scorer.score(record, "42")
|
||||
|
||||
assert is_correct is None
|
||||
assert meta["reason"] == "no_ground_truth"
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Tests for the EvalRunner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from evals.core.runner import EvalRunner
|
||||
from evals.core.types import EvalRecord, RunConfig
|
||||
from evals.tests.conftest import MockBackend, MockDataset, MockScorer
|
||||
|
||||
|
||||
class TestEvalRunner:
|
||||
def _make_records(self, n=5):
|
||||
return [
|
||||
EvalRecord(
|
||||
record_id=f"r{i}",
|
||||
problem=f"Question {i}",
|
||||
reference=f"Answer {i}",
|
||||
category="reasoning",
|
||||
subject="math" if i % 2 == 0 else "science",
|
||||
)
|
||||
for i in range(n)
|
||||
]
|
||||
|
||||
def test_basic_run(self, tmp_path):
|
||||
records = self._make_records(5)
|
||||
output_path = tmp_path / "results.jsonl"
|
||||
|
||||
config = RunConfig(
|
||||
benchmark="test",
|
||||
backend="mock",
|
||||
model="test-model",
|
||||
max_workers=1,
|
||||
output_path=str(output_path),
|
||||
)
|
||||
|
||||
dataset = MockDataset(records)
|
||||
backend = MockBackend()
|
||||
scorer = MockScorer(result=True)
|
||||
|
||||
runner = EvalRunner(config, dataset, backend, scorer)
|
||||
summary = runner.run()
|
||||
|
||||
assert summary.total_samples == 5
|
||||
assert summary.scored_samples == 5
|
||||
assert summary.correct == 5
|
||||
assert summary.accuracy == 1.0
|
||||
assert summary.errors == 0
|
||||
assert summary.benchmark == "test"
|
||||
assert summary.model == "test-model"
|
||||
|
||||
def test_with_errors(self, tmp_path):
|
||||
records = self._make_records(3)
|
||||
output_path = tmp_path / "results.jsonl"
|
||||
|
||||
config = RunConfig(
|
||||
benchmark="test",
|
||||
backend="mock",
|
||||
model="m",
|
||||
max_workers=1,
|
||||
output_path=str(output_path),
|
||||
)
|
||||
|
||||
# Backend that raises on second call
|
||||
class FailingBackend(MockBackend):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._fail_count = 0
|
||||
|
||||
def generate_full(self, prompt, **kw):
|
||||
self._fail_count += 1
|
||||
if self._fail_count == 2:
|
||||
raise RuntimeError("test error")
|
||||
return super().generate_full(prompt, **kw)
|
||||
|
||||
dataset = MockDataset(records)
|
||||
backend = FailingBackend()
|
||||
scorer = MockScorer(result=True)
|
||||
|
||||
runner = EvalRunner(config, dataset, backend, scorer)
|
||||
summary = runner.run()
|
||||
|
||||
assert summary.total_samples == 3
|
||||
assert summary.errors == 1
|
||||
|
||||
def test_per_subject_breakdown(self, tmp_path):
|
||||
records = self._make_records(4)
|
||||
output_path = tmp_path / "results.jsonl"
|
||||
|
||||
config = RunConfig(
|
||||
benchmark="test",
|
||||
backend="mock",
|
||||
model="m",
|
||||
max_workers=1,
|
||||
output_path=str(output_path),
|
||||
)
|
||||
|
||||
dataset = MockDataset(records)
|
||||
backend = MockBackend()
|
||||
scorer = MockScorer(result=True)
|
||||
|
||||
runner = EvalRunner(config, dataset, backend, scorer)
|
||||
summary = runner.run()
|
||||
|
||||
assert "math" in summary.per_subject
|
||||
assert "science" in summary.per_subject
|
||||
assert summary.per_subject["math"]["accuracy"] == 1.0
|
||||
|
||||
def test_jsonl_output(self, tmp_path):
|
||||
records = self._make_records(3)
|
||||
output_path = tmp_path / "results.jsonl"
|
||||
|
||||
config = RunConfig(
|
||||
benchmark="test",
|
||||
backend="mock",
|
||||
model="m",
|
||||
max_workers=1,
|
||||
output_path=str(output_path),
|
||||
)
|
||||
|
||||
dataset = MockDataset(records)
|
||||
backend = MockBackend()
|
||||
scorer = MockScorer(result=True)
|
||||
|
||||
runner = EvalRunner(config, dataset, backend, scorer)
|
||||
runner.run()
|
||||
|
||||
# Verify JSONL
|
||||
lines = output_path.read_text().strip().split("\n")
|
||||
assert len(lines) == 3
|
||||
first = json.loads(lines[0])
|
||||
assert "record_id" in first
|
||||
assert "model_answer" in first
|
||||
assert "is_correct" in first
|
||||
|
||||
# Verify summary JSON
|
||||
summary_path = output_path.with_suffix(".summary.json")
|
||||
assert summary_path.exists()
|
||||
summary_data = json.loads(summary_path.read_text())
|
||||
assert summary_data["total_samples"] == 3
|
||||
|
||||
def test_parallel_workers(self, tmp_path):
|
||||
records = self._make_records(10)
|
||||
output_path = tmp_path / "results.jsonl"
|
||||
|
||||
config = RunConfig(
|
||||
benchmark="test",
|
||||
backend="mock",
|
||||
model="m",
|
||||
max_workers=4,
|
||||
output_path=str(output_path),
|
||||
)
|
||||
|
||||
dataset = MockDataset(records)
|
||||
backend = MockBackend()
|
||||
scorer = MockScorer(result=True)
|
||||
|
||||
runner = EvalRunner(config, dataset, backend, scorer)
|
||||
summary = runner.run()
|
||||
|
||||
assert summary.total_samples == 10
|
||||
assert summary.correct == 10
|
||||
|
||||
def test_mixed_scoring(self, tmp_path):
|
||||
records = self._make_records(4)
|
||||
output_path = tmp_path / "results.jsonl"
|
||||
|
||||
config = RunConfig(
|
||||
benchmark="test",
|
||||
backend="mock",
|
||||
model="m",
|
||||
max_workers=1,
|
||||
output_path=str(output_path),
|
||||
)
|
||||
|
||||
# Scorer that alternates correct/incorrect
|
||||
class AlternatingScorer(MockScorer):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._count = 0
|
||||
|
||||
def score(self, record, model_answer):
|
||||
self._count += 1
|
||||
return (self._count % 2 == 0), {"count": self._count}
|
||||
|
||||
dataset = MockDataset(records)
|
||||
backend = MockBackend()
|
||||
scorer = AlternatingScorer()
|
||||
|
||||
runner = EvalRunner(config, dataset, backend, scorer)
|
||||
summary = runner.run()
|
||||
|
||||
assert summary.scored_samples == 4
|
||||
assert summary.correct == 2
|
||||
assert summary.accuracy == 0.5
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Tests for SuperGPQA scorer logic."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.core.types import EvalRecord
|
||||
from evals.tests.conftest import MockBackend
|
||||
|
||||
|
||||
class TestSuperGPQAScorer:
|
||||
def _make_record(self, answer="B", options=None):
|
||||
if options is None:
|
||||
options = ["Option A", "Option B", "Option C", "Option D"]
|
||||
return EvalRecord(
|
||||
record_id="sgpqa-1",
|
||||
problem=(
|
||||
"What is X?\n\nOptions:\n"
|
||||
"A. Option A\nB. Option B\n"
|
||||
"C. Option C\nD. Option D\n\n"
|
||||
"Respond with the correct letter only."
|
||||
),
|
||||
reference=answer,
|
||||
category="reasoning",
|
||||
subject="math",
|
||||
metadata={"options": options},
|
||||
)
|
||||
|
||||
def test_correct_extraction(self):
|
||||
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
||||
|
||||
backend = MockBackend(responses={})
|
||||
backend._default_response = "B"
|
||||
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record(answer="B")
|
||||
is_correct, meta = scorer.score(record, "The answer is B")
|
||||
|
||||
assert is_correct is True
|
||||
assert meta["reference_letter"] == "B"
|
||||
assert meta["candidate_letter"] == "B"
|
||||
|
||||
def test_incorrect_extraction(self):
|
||||
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
||||
|
||||
backend = MockBackend()
|
||||
backend._default_response = "A"
|
||||
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record(answer="B")
|
||||
is_correct, meta = scorer.score(record, "I think A")
|
||||
|
||||
assert is_correct is False
|
||||
assert meta["candidate_letter"] == "A"
|
||||
|
||||
def test_missing_reference(self):
|
||||
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
||||
|
||||
backend = MockBackend()
|
||||
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record(answer="")
|
||||
is_correct, meta = scorer.score(record, "B")
|
||||
|
||||
assert is_correct is None
|
||||
assert meta["reason"] == "missing_reference_letter"
|
||||
|
||||
def test_no_extraction(self):
|
||||
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
||||
|
||||
backend = MockBackend()
|
||||
backend._default_response = "NONE"
|
||||
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record(answer="B")
|
||||
is_correct, meta = scorer.score(record, "I don't know")
|
||||
|
||||
assert is_correct is None
|
||||
assert meta["reason"] == "no_choice_letter_extracted"
|
||||
|
||||
def test_valid_letters_from_options(self):
|
||||
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
||||
|
||||
backend = MockBackend()
|
||||
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
||||
|
||||
# 5 options
|
||||
metadata = {"options": ["A", "B", "C", "D", "E"]}
|
||||
letters = scorer._valid_letters_from_options(metadata)
|
||||
assert letters == "ABCDE"
|
||||
|
||||
# No options
|
||||
letters = scorer._valid_letters_from_options({})
|
||||
assert letters == "ABCD"
|
||||
|
||||
def test_extraction_with_verbose_response(self):
|
||||
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
||||
|
||||
backend = MockBackend()
|
||||
backend._default_response = "THE ANSWER IS: C"
|
||||
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record(answer="C")
|
||||
is_correct, meta = scorer.score(record, "After analysis, C is correct")
|
||||
|
||||
assert is_correct is True
|
||||
assert meta["candidate_letter"] == "C"
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Tests for core data types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.core.types import EvalRecord, EvalResult, RunConfig, RunSummary
|
||||
|
||||
|
||||
class TestEvalRecord:
|
||||
def test_creation(self):
|
||||
r = EvalRecord(
|
||||
record_id="r1", problem="What?", reference="42",
|
||||
category="reasoning",
|
||||
)
|
||||
assert r.record_id == "r1"
|
||||
assert r.problem == "What?"
|
||||
assert r.reference == "42"
|
||||
assert r.category == "reasoning"
|
||||
assert r.subject == ""
|
||||
assert r.metadata == {}
|
||||
|
||||
def test_with_subject_and_metadata(self):
|
||||
r = EvalRecord(
|
||||
record_id="r2", problem="Q", reference="A",
|
||||
category="chat", subject="greet",
|
||||
metadata={"key": "val"},
|
||||
)
|
||||
assert r.subject == "greet"
|
||||
assert r.metadata == {"key": "val"}
|
||||
|
||||
|
||||
class TestEvalResult:
|
||||
def test_defaults(self):
|
||||
r = EvalResult(record_id="r1", model_answer="42")
|
||||
assert r.is_correct is None
|
||||
assert r.score is None
|
||||
assert r.latency_seconds == 0.0
|
||||
assert r.prompt_tokens == 0
|
||||
assert r.completion_tokens == 0
|
||||
assert r.cost_usd == 0.0
|
||||
assert r.error is None
|
||||
assert r.scoring_metadata == {}
|
||||
|
||||
def test_full(self):
|
||||
r = EvalResult(
|
||||
record_id="r1", model_answer="42", is_correct=True,
|
||||
score=1.0, latency_seconds=1.5, prompt_tokens=100,
|
||||
completion_tokens=50, cost_usd=0.01,
|
||||
scoring_metadata={"match": "exact"},
|
||||
)
|
||||
assert r.is_correct is True
|
||||
assert r.score == 1.0
|
||||
assert r.cost_usd == 0.01
|
||||
|
||||
|
||||
class TestRunConfig:
|
||||
def test_defaults(self):
|
||||
c = RunConfig(benchmark="supergpqa", backend="jarvis-direct", model="qwen3:8b")
|
||||
assert c.max_samples is None
|
||||
assert c.max_workers == 4
|
||||
assert c.temperature == 0.0
|
||||
assert c.max_tokens == 2048
|
||||
assert c.judge_model == "gpt-4o"
|
||||
assert c.seed == 42
|
||||
assert c.tools == []
|
||||
|
||||
def test_with_agent(self):
|
||||
c = RunConfig(
|
||||
benchmark="gaia", backend="jarvis-agent", model="gpt-4o",
|
||||
engine_key="cloud", agent_name="orchestrator",
|
||||
tools=["calculator", "think"],
|
||||
)
|
||||
assert c.agent_name == "orchestrator"
|
||||
assert c.tools == ["calculator", "think"]
|
||||
|
||||
|
||||
class TestRunSummary:
|
||||
def test_creation(self):
|
||||
s = RunSummary(
|
||||
benchmark="supergpqa", category="reasoning",
|
||||
backend="jarvis-direct", model="qwen3:8b",
|
||||
total_samples=100, scored_samples=95, correct=47,
|
||||
accuracy=0.495, errors=5, mean_latency_seconds=2.1,
|
||||
total_cost_usd=0.0,
|
||||
per_subject={"math": {"accuracy": 0.5}},
|
||||
)
|
||||
assert s.accuracy == 0.495
|
||||
assert s.per_subject["math"]["accuracy"] == 0.5
|
||||
assert s.started_at == 0.0
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Tests for WildChat scorer (verdict parsing and dual comparison)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.core.types import EvalRecord
|
||||
from evals.scorers.wildchat_judge import WildChatScorer
|
||||
from evals.tests.conftest import MockBackend
|
||||
|
||||
|
||||
class TestVerdictParsing:
|
||||
def test_verdict_to_bool_generated_is_a(self):
|
||||
assert WildChatScorer._verdict_to_bool("A>>B", generated_is_a=True) is True
|
||||
assert WildChatScorer._verdict_to_bool("A>B", generated_is_a=True) is True
|
||||
assert WildChatScorer._verdict_to_bool("A=B", generated_is_a=True) is True
|
||||
assert WildChatScorer._verdict_to_bool("B>A", generated_is_a=True) is False
|
||||
assert WildChatScorer._verdict_to_bool("B>>A", generated_is_a=True) is False
|
||||
|
||||
def test_verdict_to_bool_generated_is_b(self):
|
||||
assert WildChatScorer._verdict_to_bool("A>>B", generated_is_a=False) is False
|
||||
assert WildChatScorer._verdict_to_bool("A>B", generated_is_a=False) is False
|
||||
assert WildChatScorer._verdict_to_bool("A=B", generated_is_a=False) is True
|
||||
assert WildChatScorer._verdict_to_bool("B>A", generated_is_a=False) is True
|
||||
assert WildChatScorer._verdict_to_bool("B>>A", generated_is_a=False) is True
|
||||
|
||||
def test_verdict_none(self):
|
||||
assert WildChatScorer._verdict_to_bool(None, generated_is_a=True) is None
|
||||
assert WildChatScorer._verdict_to_bool("", generated_is_a=True) is None
|
||||
|
||||
def test_unknown_verdict(self):
|
||||
assert WildChatScorer._verdict_to_bool("X>Y", generated_is_a=True) is None
|
||||
|
||||
|
||||
class TestWildChatScorer:
|
||||
def _make_record(self, reference="I'm fine, thanks!"):
|
||||
return EvalRecord(
|
||||
record_id="wc-1",
|
||||
problem="How are you?",
|
||||
reference=reference,
|
||||
category="chat",
|
||||
subject="conversation",
|
||||
)
|
||||
|
||||
def test_model_wins(self):
|
||||
backend = MockBackend()
|
||||
# First call: model as A, verdict A>>B (model better)
|
||||
# Second call: reference as A, verdict A>>B (reference better → model loses)
|
||||
# But since it's OR logic, model wins if either comparison says it's good
|
||||
call_count = 0
|
||||
|
||||
def mock_generate(prompt, **kw):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return '```json\n{"verdict": "[[A>>B]]"}\n```'
|
||||
else:
|
||||
return '```json\n{"verdict": "[[A>>B]]"}\n```'
|
||||
|
||||
backend.generate = mock_generate
|
||||
scorer = WildChatScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record()
|
||||
is_correct, meta = scorer.score(record, "I'm doing great!")
|
||||
|
||||
assert is_correct is True
|
||||
|
||||
def test_model_loses(self):
|
||||
backend = MockBackend()
|
||||
call_count = 0
|
||||
|
||||
def mock_generate(prompt, **kw):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
# model as A, reference as B → B wins
|
||||
return '```json\n{"verdict": "[[B>>A]]"}\n```'
|
||||
else:
|
||||
# reference as A, model as B → A wins (reference better)
|
||||
return '```json\n{"verdict": "[[A>>B]]"}\n```'
|
||||
|
||||
backend.generate = mock_generate
|
||||
scorer = WildChatScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record()
|
||||
is_correct, meta = scorer.score(record, "Bad response")
|
||||
|
||||
assert is_correct is False
|
||||
|
||||
def test_tie(self):
|
||||
backend = MockBackend()
|
||||
backend._default_response = '{"verdict": "[[A=B]]"}'
|
||||
scorer = WildChatScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record()
|
||||
is_correct, meta = scorer.score(record, "I'm fine, thanks!")
|
||||
|
||||
assert is_correct is True # Tie counts as correct
|
||||
|
||||
def test_empty_reference(self):
|
||||
backend = MockBackend()
|
||||
scorer = WildChatScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record("")
|
||||
is_correct, meta = scorer.score(record, "Hello")
|
||||
|
||||
assert is_correct is None
|
||||
assert meta["reason"] == "empty_reference"
|
||||
|
||||
def test_missing_verdict(self):
|
||||
backend = MockBackend()
|
||||
backend._default_response = "I cannot decide between them."
|
||||
scorer = WildChatScorer(backend, "gpt-4o")
|
||||
|
||||
record = self._make_record()
|
||||
is_correct, meta = scorer.score(record, "Hello")
|
||||
|
||||
assert is_correct is None
|
||||
assert "missing_verdicts" in str(meta.get("reason", ""))
|
||||
@@ -85,3 +85,7 @@ src = ["src", "tests"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "W"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"evals/datasets/*.py" = ["E501"]
|
||||
"evals/scorers/*.py" = ["E501"]
|
||||
|
||||
Reference in New Issue
Block a user