mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
bd49383201
Evaluation framework (evals/): benchmarking system for measuring accuracy
across four categories — Chat (WildChat), Reasoning (SuperGPQA), RAG (FRAMES),
and Agentic (GAIA). Two backends: jarvis-direct (engine-level) and jarvis-agent
(agent-level with tool calling), both supporting local and cloud models.
Datasets adapted from IPW, scorers include exact match, LLM letter extraction,
and LLM-as-judge. Parallel execution via ThreadPoolExecutor with incremental
JSONL output. CLI: python -m evals {run,run-all,summarize,list}. 57 tests pass.
SVG fix: center logo content within viewBox by wrapping icon+text in a
translate(90,0) group, eliminating the left-shift visible in the README.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
106 lines
3.4 KiB
Python
106 lines
3.4 KiB
Python
"""Tests for SuperGPQA scorer logic."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.core.types import EvalRecord
|
|
from evals.tests.conftest import MockBackend
|
|
|
|
|
|
class TestSuperGPQAScorer:
|
|
def _make_record(self, answer="B", options=None):
|
|
if options is None:
|
|
options = ["Option A", "Option B", "Option C", "Option D"]
|
|
return EvalRecord(
|
|
record_id="sgpqa-1",
|
|
problem=(
|
|
"What is X?\n\nOptions:\n"
|
|
"A. Option A\nB. Option B\n"
|
|
"C. Option C\nD. Option D\n\n"
|
|
"Respond with the correct letter only."
|
|
),
|
|
reference=answer,
|
|
category="reasoning",
|
|
subject="math",
|
|
metadata={"options": options},
|
|
)
|
|
|
|
def test_correct_extraction(self):
|
|
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
|
|
|
backend = MockBackend(responses={})
|
|
backend._default_response = "B"
|
|
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record(answer="B")
|
|
is_correct, meta = scorer.score(record, "The answer is B")
|
|
|
|
assert is_correct is True
|
|
assert meta["reference_letter"] == "B"
|
|
assert meta["candidate_letter"] == "B"
|
|
|
|
def test_incorrect_extraction(self):
|
|
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
|
|
|
backend = MockBackend()
|
|
backend._default_response = "A"
|
|
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record(answer="B")
|
|
is_correct, meta = scorer.score(record, "I think A")
|
|
|
|
assert is_correct is False
|
|
assert meta["candidate_letter"] == "A"
|
|
|
|
def test_missing_reference(self):
|
|
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
|
|
|
backend = MockBackend()
|
|
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record(answer="")
|
|
is_correct, meta = scorer.score(record, "B")
|
|
|
|
assert is_correct is None
|
|
assert meta["reason"] == "missing_reference_letter"
|
|
|
|
def test_no_extraction(self):
|
|
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
|
|
|
backend = MockBackend()
|
|
backend._default_response = "NONE"
|
|
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record(answer="B")
|
|
is_correct, meta = scorer.score(record, "I don't know")
|
|
|
|
assert is_correct is None
|
|
assert meta["reason"] == "no_choice_letter_extracted"
|
|
|
|
def test_valid_letters_from_options(self):
|
|
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
|
|
|
backend = MockBackend()
|
|
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
|
|
|
# 5 options
|
|
metadata = {"options": ["A", "B", "C", "D", "E"]}
|
|
letters = scorer._valid_letters_from_options(metadata)
|
|
assert letters == "ABCDE"
|
|
|
|
# No options
|
|
letters = scorer._valid_letters_from_options({})
|
|
assert letters == "ABCD"
|
|
|
|
def test_extraction_with_verbose_response(self):
|
|
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
|
|
|
|
backend = MockBackend()
|
|
backend._default_response = "THE ANSWER IS: C"
|
|
scorer = SuperGPQAScorer(backend, "gpt-4o")
|
|
|
|
record = self._make_record(answer="C")
|
|
is_correct, meta = scorer.score(record, "After analysis, C is correct")
|
|
|
|
assert is_correct is True
|
|
assert meta["candidate_letter"] == "C"
|