mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
2aebcd7d77
Trace-driven learning pipeline: - TrainingDataMiner: extract SFT/routing/agent pairs from traces - LoRATrainer: fine-tune local models from trace-derived data - AgentConfigEvolver: rewrite agent configs from trace analysis - LearningOrchestrator: coordinate mine→train→evolve cycle, wired into SystemBuilder Eval framework (15 real IPW benchmarks): - Datasets: SuperGPQA, GPQA, MMLU-Pro, MATH-500, Natural Reasoning, HLE, SimpleQA, WildChat, IPW, GAIA, FRAMES, SWE-bench, SWEfficiency, TerminalBench, TerminalBench Native - Scorers: MCQ extraction, LLM-judge, exact match, structural validation - CLI: jarvis eval list|run|compare|report Composable abstractions: - Recipe system: TOML composition of all 5 pillars (3 built-in recipes) - Agent templates: 15 pre-configured TOML manifests with system prompts - Bundled skills: 20 ready-to-use TOML skill manifests - Operator recipes: researcher (4h), correspondent (5min), sentinel (2h) 102 files changed, ~11,500 lines added. 3241 tests pass (44 skipped). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
92 lines
3.0 KiB
Python
92 lines
3.0 KiB
Python
"""Tests for the 3 operator recipes: researcher, correspondent, sentinel."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from openjarvis.operators.loader import load_operator
|
|
|
|
_OPERATORS_DIR = Path(__file__).parent.parent.parent / "recipes" / "operators"
|
|
|
|
|
|
class TestResearcherOperator:
|
|
def test_loads_valid_manifest(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "researcher.toml")
|
|
assert manifest.name == "researcher"
|
|
assert manifest.max_turns >= 10
|
|
assert manifest.system_prompt # loaded from prompt file
|
|
|
|
def test_has_required_tools(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "researcher.toml")
|
|
required = {
|
|
"web_search",
|
|
"http_request",
|
|
"memory_store",
|
|
"memory_search",
|
|
"think",
|
|
"file_write",
|
|
}
|
|
assert required.issubset(set(manifest.tools))
|
|
|
|
def test_has_kg_tools(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "researcher.toml")
|
|
assert "kg_add_entity" in manifest.tools
|
|
assert "kg_add_relation" in manifest.tools
|
|
|
|
def test_has_schedule(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "researcher.toml")
|
|
assert manifest.schedule_type in ("cron", "interval")
|
|
|
|
|
|
class TestCorrespondentOperator:
|
|
def test_loads_valid_manifest(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "correspondent.toml")
|
|
assert manifest.name == "correspondent"
|
|
assert manifest.max_turns >= 10
|
|
assert manifest.system_prompt
|
|
|
|
def test_has_required_tools(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "correspondent.toml")
|
|
required = {"memory_store", "memory_search", "think", "llm_call"}
|
|
assert required.issubset(set(manifest.tools))
|
|
|
|
def test_interval_schedule(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "correspondent.toml")
|
|
assert manifest.schedule_type == "interval"
|
|
assert manifest.schedule_value == "300"
|
|
|
|
|
|
class TestSentinelOperator:
|
|
def test_loads_valid_manifest(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "sentinel.toml")
|
|
assert manifest.name == "sentinel"
|
|
assert manifest.max_turns >= 10
|
|
assert manifest.system_prompt
|
|
|
|
def test_has_required_tools(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "sentinel.toml")
|
|
required = {
|
|
"web_search",
|
|
"http_request",
|
|
"memory_store",
|
|
"memory_search",
|
|
"think",
|
|
}
|
|
assert required.issubset(set(manifest.tools))
|
|
|
|
def test_has_kg_tools(self):
|
|
manifest = load_operator(_OPERATORS_DIR / "sentinel.toml")
|
|
assert "kg_add_entity" in manifest.tools
|
|
|
|
|
|
class TestAllOperators:
|
|
@pytest.mark.parametrize(
|
|
"filename",
|
|
["researcher.toml", "correspondent.toml", "sentinel.toml"],
|
|
)
|
|
def test_all_load_without_error(self, filename):
|
|
manifest = load_operator(_OPERATORS_DIR / filename)
|
|
assert manifest.name
|
|
assert manifest.tools
|
|
assert manifest.system_prompt
|