mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
bd49383201
Evaluation framework (evals/): benchmarking system for measuring accuracy
across four categories — Chat (WildChat), Reasoning (SuperGPQA), RAG (FRAMES),
and Agentic (GAIA). Two backends: jarvis-direct (engine-level) and jarvis-agent
(agent-level with tool calling), both supporting local and cloud models.
Datasets adapted from IPW, scorers include exact match, LLM letter extraction,
and LLM-as-judge. Parallel execution via ThreadPoolExecutor with incremental
JSONL output. CLI: python -m evals {run,run-all,summarize,list}. 57 tests pass.
SVG fix: center logo content within viewBox by wrapping icon+text in a
translate(90,0) group, eliminating the left-shift visible in the README.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
112 lines
3.5 KiB
Python
112 lines
3.5 KiB
Python
"""Tests for FRAMES scorer (judge prompt formatting and verdict parsing)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.core.types import EvalRecord
|
|
from evals.scorers.frames_judge import _GRADER_TEMPLATE, FRAMESScorer
|
|
from evals.tests.conftest import MockBackend
|
|
|
|
|
|
class TestGraderTemplate:
|
|
def test_template_formatting(self):
|
|
result = _GRADER_TEMPLATE.format(
|
|
question="What is the capital?",
|
|
ground_truth="Paris",
|
|
predicted_answer="The capital is Paris",
|
|
)
|
|
assert "What is the capital?" in result
|
|
assert "Paris" in result
|
|
assert "The capital is Paris" in result
|
|
assert "correct: <yes or no>" in result
|
|
|
|
def test_template_has_required_sections(self):
|
|
assert "## Question" in _GRADER_TEMPLATE
|
|
assert "## Ground Truth Answer" in _GRADER_TEMPLATE
|
|
assert "## Predicted Answer" in _GRADER_TEMPLATE
|
|
assert "extracted_final_answer:" in _GRADER_TEMPLATE
|
|
|
|
|
|
class TestFRAMESScorer:
|
|
def _make_record(self, reference="Paris"):
|
|
return EvalRecord(
|
|
record_id="frames-1",
|
|
problem="What is the capital of France?",
|
|
reference=reference,
|
|
category="rag",
|
|
subject="general",
|
|
)
|
|
|
|
def test_correct_answer(self):
|
|
backend = MockBackend()
|
|
backend._default_response = (
|
|
"extracted_final_answer: Paris\n"
|
|
"reasoning: The answer correctly identifies Paris as the capital.\n"
|
|
"correct: yes"
|
|
)
|
|
scorer = FRAMESScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record("Paris")
|
|
is_correct, meta = scorer.score(record, "The capital is Paris")
|
|
|
|
assert is_correct is True
|
|
assert "raw_judge_output" in meta
|
|
assert meta["extracted_answer"] == "Paris"
|
|
|
|
def test_incorrect_answer(self):
|
|
backend = MockBackend()
|
|
backend._default_response = (
|
|
"extracted_final_answer: London\n"
|
|
"reasoning: London is not the capital of France.\n"
|
|
"correct: no"
|
|
)
|
|
scorer = FRAMESScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record("Paris")
|
|
is_correct, meta = scorer.score(record, "London")
|
|
|
|
assert is_correct is False
|
|
|
|
def test_empty_response(self):
|
|
backend = MockBackend()
|
|
scorer = FRAMESScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record("Paris")
|
|
is_correct, meta = scorer.score(record, "")
|
|
|
|
assert is_correct is False
|
|
assert meta["reason"] == "empty_response"
|
|
|
|
def test_no_ground_truth(self):
|
|
backend = MockBackend()
|
|
scorer = FRAMESScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record("")
|
|
is_correct, meta = scorer.score(record, "Paris")
|
|
|
|
assert is_correct is None
|
|
assert meta["reason"] == "no_ground_truth"
|
|
|
|
def test_fallback_true_false_parsing(self):
|
|
backend = MockBackend()
|
|
backend._default_response = "The prediction is TRUE"
|
|
scorer = FRAMESScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record("Paris")
|
|
is_correct, _ = scorer.score(record, "Paris")
|
|
|
|
assert is_correct is True
|
|
|
|
def test_judge_error(self):
|
|
class ErrorBackend(MockBackend):
|
|
def generate(self, prompt, **kw):
|
|
raise RuntimeError("API error")
|
|
|
|
backend = ErrorBackend()
|
|
scorer = FRAMESScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record("Paris")
|
|
is_correct, meta = scorer.score(record, "Paris")
|
|
|
|
assert is_correct is None
|
|
assert "error" in meta
|