mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-31 03:12:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
bd49383201
Evaluation framework (evals/): benchmarking system for measuring accuracy
across four categories — Chat (WildChat), Reasoning (SuperGPQA), RAG (FRAMES),
and Agentic (GAIA). Two backends: jarvis-direct (engine-level) and jarvis-agent
(agent-level with tool calling), both supporting local and cloud models.
Datasets adapted from IPW, scorers include exact match, LLM letter extraction,
and LLM-as-judge. Parallel execution via ThreadPoolExecutor with incremental
JSONL output. CLI: python -m evals {run,run-all,summarize,list}. 57 tests pass.
SVG fix: center logo content within viewBox by wrapping icon+text in a
translate(90,0) group, eliminating the left-shift visible in the README.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
37 lines
843 B
Python
37 lines
843 B
Python
"""Abstract base class for dataset providers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from typing import Iterable, Optional
|
|
|
|
from evals.core.types import EvalRecord
|
|
|
|
|
|
class DatasetProvider(ABC):
|
|
"""Base class for all evaluation dataset providers."""
|
|
|
|
dataset_id: str
|
|
dataset_name: str
|
|
|
|
@abstractmethod
|
|
def load(
|
|
self,
|
|
*,
|
|
max_samples: Optional[int] = None,
|
|
split: Optional[str] = None,
|
|
seed: Optional[int] = None,
|
|
) -> None:
|
|
"""Load the dataset (possibly downloading from HuggingFace)."""
|
|
|
|
@abstractmethod
|
|
def iter_records(self) -> Iterable[EvalRecord]:
|
|
"""Iterate over loaded records."""
|
|
|
|
@abstractmethod
|
|
def size(self) -> int:
|
|
"""Return the number of loaded records."""
|
|
|
|
|
|
__all__ = ["DatasetProvider"]
|