mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
bd49383201
Evaluation framework (evals/): benchmarking system for measuring accuracy
across four categories — Chat (WildChat), Reasoning (SuperGPQA), RAG (FRAMES),
and Agentic (GAIA). Two backends: jarvis-direct (engine-level) and jarvis-agent
(agent-level with tool calling), both supporting local and cloud models.
Datasets adapted from IPW, scorers include exact match, LLM letter extraction,
and LLM-as-judge. Parallel execution via ThreadPoolExecutor with incremental
JSONL output. CLI: python -m evals {run,run-all,summarize,list}. 57 tests pass.
SVG fix: center logo content within viewBox by wrapping icon+text in a
translate(90,0) group, eliminating the left-shift visible in the README.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
46 lines
1.0 KiB
Python
46 lines
1.0 KiB
Python
"""Abstract base class for inference backends."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from typing import Any, Dict
|
|
|
|
|
|
class InferenceBackend(ABC):
|
|
"""Base class for all inference backends used in evaluation."""
|
|
|
|
backend_id: str
|
|
|
|
@abstractmethod
|
|
def generate(
|
|
self,
|
|
prompt: str,
|
|
*,
|
|
model: str,
|
|
system: str = "",
|
|
temperature: float = 0.0,
|
|
max_tokens: int = 2048,
|
|
) -> str:
|
|
"""Generate a response and return just the text content."""
|
|
|
|
@abstractmethod
|
|
def generate_full(
|
|
self,
|
|
prompt: str,
|
|
*,
|
|
model: str,
|
|
system: str = "",
|
|
temperature: float = 0.0,
|
|
max_tokens: int = 2048,
|
|
) -> Dict[str, Any]:
|
|
"""Generate a response and return full details.
|
|
|
|
Returns dict with keys: content, usage, model, latency_seconds, cost_usd.
|
|
"""
|
|
|
|
def close(self) -> None:
|
|
"""Release resources."""
|
|
|
|
|
|
__all__ = ["InferenceBackend"]
|