mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
bd49383201
Evaluation framework (evals/): benchmarking system for measuring accuracy
across four categories — Chat (WildChat), Reasoning (SuperGPQA), RAG (FRAMES),
and Agentic (GAIA). Two backends: jarvis-direct (engine-level) and jarvis-agent
(agent-level with tool calling), both supporting local and cloud models.
Datasets adapted from IPW, scorers include exact match, LLM letter extraction,
and LLM-as-judge. Parallel execution via ThreadPoolExecutor with incremental
JSONL output. CLI: python -m evals {run,run-all,summarize,list}. 57 tests pass.
SVG fix: center logo content within viewBox by wrapping icon+text in a
translate(90,0) group, eliminating the left-shift visible in the README.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
24 lines
477 B
TOML
24 lines
477 B
TOML
[project]
|
|
name = "openjarvis-evals"
|
|
version = "0.1.0"
|
|
description = "Evaluation framework for OpenJarvis"
|
|
requires-python = ">=3.10"
|
|
dependencies = [
|
|
"openjarvis>=1.0.0",
|
|
"click>=8",
|
|
"datasets>=2.14",
|
|
"huggingface-hub>=0.20",
|
|
"tqdm>=4.65",
|
|
"rich>=13",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = ["pytest>=8", "pytest-cov>=5"]
|
|
|
|
[project.scripts]
|
|
openjarvis-eval = "evals.cli:main"
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|