mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
feat: Phase 23 — Differentiated functionalities
Trace-driven learning pipeline: - TrainingDataMiner: extract SFT/routing/agent pairs from traces - LoRATrainer: fine-tune local models from trace-derived data - AgentConfigEvolver: rewrite agent configs from trace analysis - LearningOrchestrator: coordinate mine→train→evolve cycle, wired into SystemBuilder Eval framework (15 real IPW benchmarks): - Datasets: SuperGPQA, GPQA, MMLU-Pro, MATH-500, Natural Reasoning, HLE, SimpleQA, WildChat, IPW, GAIA, FRAMES, SWE-bench, SWEfficiency, TerminalBench, TerminalBench Native - Scorers: MCQ extraction, LLM-judge, exact match, structural validation - CLI: jarvis eval list|run|compare|report Composable abstractions: - Recipe system: TOML composition of all 5 pillars (3 built-in recipes) - Agent templates: 15 pre-configured TOML manifests with system prompts - Bundled skills: 20 ready-to-use TOML skill manifests - Operator recipes: researcher (4h), correspondent (5min), sentinel (2h) 102 files changed, ~11,500 lines added. 3241 tests pass (44 skipped). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
2849f39cbc
commit
2aebcd7d77
@@ -0,0 +1,39 @@
|
||||
"""Tests for the ``jarvis eval`` CLI commands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from openjarvis.cli import cli
|
||||
|
||||
|
||||
class TestEvalCLI:
|
||||
"""Tests for the eval command group."""
|
||||
|
||||
def test_eval_group_exists(self):
|
||||
"""``jarvis eval --help`` shows run/compare/report/list subcommands."""
|
||||
result = CliRunner().invoke(cli, ["eval", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "run" in result.output
|
||||
assert "compare" in result.output
|
||||
assert "report" in result.output
|
||||
assert "list" in result.output
|
||||
|
||||
def test_eval_list_benchmarks(self):
|
||||
"""``jarvis eval list`` exits 0 and outputs benchmark names."""
|
||||
result = CliRunner().invoke(cli, ["eval", "list"])
|
||||
assert result.exit_code == 0
|
||||
assert "supergpqa" in result.output
|
||||
assert "gaia" in result.output
|
||||
assert "frames" in result.output
|
||||
assert "wildchat" in result.output
|
||||
# Should also show backends
|
||||
assert "jarvis-direct" in result.output
|
||||
assert "jarvis-agent" in result.output
|
||||
|
||||
def test_eval_run_missing_args(self):
|
||||
"""``jarvis eval run`` without required args fails gracefully."""
|
||||
result = CliRunner().invoke(cli, ["eval", "run"])
|
||||
# Should fail because neither --config nor --benchmark/--model given
|
||||
assert result.exit_code != 0
|
||||
assert "config" in result.output.lower() or "benchmark" in result.output.lower()
|
||||
Reference in New Issue
Block a user