mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 23:21:28 +00:00
472 lines
15 KiB
Python
472 lines
15 KiB
Python
"""Tests for the unified compose system — enhanced Recipe, bridges, and discovery."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import textwrap
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from openjarvis.recipes.loader import (
|
|
Recipe,
|
|
discover_recipes,
|
|
load_recipe,
|
|
resolve_recipe,
|
|
)
|
|
|
|
# -- Discrete recipe TOML -----------------------------------------------
|
|
|
|
DISCRETE_TOML = textwrap.dedent("""\
|
|
[recipe]
|
|
name = "bench-agent"
|
|
kind = "discrete"
|
|
description = "A discrete agent for benchmarking"
|
|
version = "0.1.0"
|
|
|
|
[intelligence]
|
|
model = "qwen3:8b"
|
|
quantization = "q4_K_M"
|
|
provider = "ollama"
|
|
|
|
[engine]
|
|
key = "ollama"
|
|
|
|
[agent]
|
|
type = "native_react"
|
|
max_turns = 20
|
|
temperature = 0.2
|
|
tools = ["shell_exec", "file_read", "think"]
|
|
system_prompt = "You are a benchmark agent."
|
|
|
|
[learning]
|
|
routing = "heuristic"
|
|
agent = "none"
|
|
|
|
[eval]
|
|
benchmarks = ["terminalbench", "gaia"]
|
|
backend = "jarvis-agent"
|
|
max_samples = 50
|
|
judge_model = "gpt-4o"
|
|
""")
|
|
|
|
# -- Operator recipe TOML -----------------------------------------------
|
|
|
|
OPERATOR_TOML = textwrap.dedent("""\
|
|
[recipe]
|
|
name = "my-operator"
|
|
kind = "operator"
|
|
description = "A test operator"
|
|
version = "2.0.0"
|
|
|
|
[intelligence]
|
|
model = "qwen3:8b"
|
|
|
|
[engine]
|
|
key = "ollama"
|
|
|
|
[agent]
|
|
type = "orchestrator"
|
|
max_turns = 15
|
|
temperature = 0.3
|
|
tools = ["web_search", "memory_store", "think"]
|
|
system_prompt = "You are a monitoring agent."
|
|
|
|
[schedule]
|
|
type = "interval"
|
|
value = "600"
|
|
|
|
[channels]
|
|
output = ["slack", "telegram"]
|
|
|
|
[learning]
|
|
routing = "heuristic"
|
|
""")
|
|
|
|
# -- Legacy operator TOML -----------------------------------------------
|
|
|
|
LEGACY_OPERATOR_TOML = textwrap.dedent("""\
|
|
[operator]
|
|
name = "legacy-op"
|
|
description = "A legacy operator manifest"
|
|
version = "0.1.0"
|
|
|
|
[operator.agent]
|
|
max_turns = 10
|
|
temperature = 0.4
|
|
tools = ["think", "web_search"]
|
|
system_prompt = "Legacy prompt."
|
|
|
|
[operator.schedule]
|
|
type = "cron"
|
|
value = "0 */2 * * *"
|
|
""")
|
|
|
|
|
|
# ========================================================================
|
|
# Recipe loading
|
|
# ========================================================================
|
|
|
|
|
|
class TestLoadDiscreteRecipe:
|
|
def test_load_discrete_fields(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "bench.toml"
|
|
p.write_text(DISCRETE_TOML)
|
|
r = load_recipe(p)
|
|
|
|
assert r.name == "bench-agent"
|
|
assert r.kind == "discrete"
|
|
assert r.model == "qwen3:8b"
|
|
assert r.quantization == "q4_K_M"
|
|
assert r.provider == "ollama"
|
|
assert r.engine_key == "ollama"
|
|
assert r.agent_type == "native_react"
|
|
assert r.max_turns == 20
|
|
assert r.temperature == pytest.approx(0.2)
|
|
assert r.tools == ["shell_exec", "file_read", "think"]
|
|
assert r.system_prompt == "You are a benchmark agent."
|
|
assert r.routing_policy == "heuristic"
|
|
assert r.eval_benchmarks == ["terminalbench", "gaia"]
|
|
assert r.eval_backend == "jarvis-agent"
|
|
assert r.eval_max_samples == 50
|
|
assert r.eval_judge_model == "gpt-4o"
|
|
|
|
def test_default_kind_is_discrete(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "min.toml"
|
|
p.write_text('[recipe]\nname = "min"\n[intelligence]\nmodel = "x"\n')
|
|
r = load_recipe(p)
|
|
assert r.kind == "discrete"
|
|
|
|
|
|
class TestLoadOperatorRecipe:
|
|
def test_load_operator_fields(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "op.toml"
|
|
p.write_text(OPERATOR_TOML)
|
|
r = load_recipe(p)
|
|
|
|
assert r.name == "my-operator"
|
|
assert r.kind == "operator"
|
|
assert r.schedule_type == "interval"
|
|
assert r.schedule_value == "600"
|
|
assert r.channels == ["slack", "telegram"]
|
|
assert r.tools == ["web_search", "memory_store", "think"]
|
|
|
|
def test_schedule_implies_operator_kind(self, tmp_path: Path) -> None:
|
|
toml = textwrap.dedent("""\
|
|
[recipe]
|
|
name = "auto-op"
|
|
|
|
[agent]
|
|
type = "orchestrator"
|
|
tools = ["think"]
|
|
|
|
[schedule]
|
|
type = "cron"
|
|
value = "0 8 * * *"
|
|
""")
|
|
p = tmp_path / "auto.toml"
|
|
p.write_text(toml)
|
|
r = load_recipe(p)
|
|
assert r.kind == "operator"
|
|
|
|
def test_external_prompt_file(self, tmp_path: Path) -> None:
|
|
prompt_file = tmp_path / "prompt.md"
|
|
prompt_file.write_text("External prompt content.")
|
|
|
|
toml = textwrap.dedent(f"""\
|
|
[recipe]
|
|
name = "ext-prompt"
|
|
|
|
[agent]
|
|
type = "simple"
|
|
system_prompt_path = "{prompt_file}"
|
|
""")
|
|
p = tmp_path / "ext.toml"
|
|
p.write_text(toml)
|
|
r = load_recipe(p)
|
|
assert r.system_prompt == "External prompt content."
|
|
|
|
|
|
class TestLoadLegacyOperator:
|
|
def test_legacy_operator_converted_to_recipe(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "legacy.toml"
|
|
p.write_text(LEGACY_OPERATOR_TOML)
|
|
r = load_recipe(p)
|
|
|
|
assert r.kind == "operator"
|
|
assert r.name == "legacy-op"
|
|
assert r.max_turns == 10
|
|
assert r.temperature == pytest.approx(0.4)
|
|
assert r.tools == ["think", "web_search"]
|
|
assert r.system_prompt == "Legacy prompt."
|
|
assert r.schedule_type == "cron"
|
|
assert r.schedule_value == "0 */2 * * *"
|
|
|
|
|
|
# ========================================================================
|
|
# Discovery
|
|
# ========================================================================
|
|
|
|
|
|
class TestDiscoverByKind:
|
|
def test_discover_all(self) -> None:
|
|
all_recipes = discover_recipes()
|
|
kinds = {r.kind for r in all_recipes}
|
|
assert "discrete" in kinds
|
|
assert len(all_recipes) >= 4 # at least original 3 + new ones
|
|
|
|
def test_discover_discrete_only(self) -> None:
|
|
discrete = discover_recipes(kind="discrete")
|
|
for r in discrete:
|
|
assert r.kind == "discrete"
|
|
|
|
def test_discover_operator_only(self) -> None:
|
|
operators = discover_recipes(kind="operator")
|
|
for r in operators:
|
|
assert r.kind == "operator"
|
|
|
|
def test_discover_operators_subdir(self) -> None:
|
|
"""Operator recipes in data/operators/ are discovered."""
|
|
all_recipes = discover_recipes()
|
|
names = {r.name for r in all_recipes}
|
|
# The new unified operator recipes should be found
|
|
assert "twitter-sentinel" in names or "correspondent" in names
|
|
|
|
def test_extra_dirs_discrete(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "custom.toml"
|
|
p.write_text(DISCRETE_TOML)
|
|
recipes = discover_recipes(extra_dirs=[tmp_path], kind="discrete")
|
|
names = {r.name for r in recipes}
|
|
assert "bench-agent" in names
|
|
|
|
def test_extra_dirs_operator(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "custom_op.toml"
|
|
p.write_text(OPERATOR_TOML)
|
|
recipes = discover_recipes(extra_dirs=[tmp_path], kind="operator")
|
|
names = {r.name for r in recipes}
|
|
assert "my-operator" in names
|
|
|
|
|
|
# ========================================================================
|
|
# Bridge: to_eval_suite
|
|
# ========================================================================
|
|
|
|
|
|
class TestRecipeToEvalSuite:
|
|
def test_basic_eval_suite(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "bench.toml"
|
|
p.write_text(DISCRETE_TOML)
|
|
r = load_recipe(p)
|
|
|
|
suite = r.to_eval_suite()
|
|
|
|
assert suite.meta.name == "bench-agent-eval"
|
|
assert len(suite.models) == 1
|
|
assert suite.models[0].name == "qwen3:8b"
|
|
assert suite.models[0].engine == "ollama"
|
|
assert len(suite.benchmarks) == 2
|
|
bench_names = {b.name for b in suite.benchmarks}
|
|
assert bench_names == {"terminalbench", "gaia"}
|
|
for b in suite.benchmarks:
|
|
assert b.backend == "jarvis-agent"
|
|
assert b.agent == "native_react"
|
|
assert b.tools == ["shell_exec", "file_read", "think"]
|
|
assert b.max_samples == 50
|
|
assert b.judge_model == "gpt-4o"
|
|
|
|
def test_eval_suite_benchmark_override(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "bench.toml"
|
|
p.write_text(DISCRETE_TOML)
|
|
r = load_recipe(p)
|
|
|
|
suite = r.to_eval_suite(benchmarks=["supergpqa"])
|
|
|
|
assert len(suite.benchmarks) == 1
|
|
assert suite.benchmarks[0].name == "supergpqa"
|
|
|
|
def test_eval_suite_max_samples_override(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "bench.toml"
|
|
p.write_text(DISCRETE_TOML)
|
|
r = load_recipe(p)
|
|
|
|
suite = r.to_eval_suite(max_samples=10)
|
|
|
|
for b in suite.benchmarks:
|
|
assert b.max_samples == 10
|
|
|
|
def test_eval_suite_judge_override(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "bench.toml"
|
|
p.write_text(DISCRETE_TOML)
|
|
r = load_recipe(p)
|
|
|
|
suite = r.to_eval_suite(judge_model="gpt-5")
|
|
|
|
assert suite.judge.model == "gpt-5"
|
|
for b in suite.benchmarks:
|
|
assert b.judge_model == "gpt-5"
|
|
|
|
def test_eval_suite_no_model_raises(self) -> None:
|
|
r = Recipe(name="no-model", eval_benchmarks=["gaia"])
|
|
with pytest.raises(ValueError, match="no model"):
|
|
r.to_eval_suite()
|
|
|
|
def test_eval_suite_no_benchmarks_raises(self) -> None:
|
|
r = Recipe(name="no-bench", model="qwen3:8b")
|
|
with pytest.raises(ValueError, match="no benchmarks"):
|
|
r.to_eval_suite()
|
|
|
|
def test_eval_suite_falls_back_to_suites(self) -> None:
|
|
r = Recipe(
|
|
name="suites-fallback",
|
|
model="qwen3:8b",
|
|
eval_suites=["coding"],
|
|
)
|
|
suite = r.to_eval_suite()
|
|
assert len(suite.benchmarks) == 1
|
|
assert suite.benchmarks[0].name == "coding"
|
|
|
|
def test_eval_suite_direct_backend_when_no_agent(self) -> None:
|
|
r = Recipe(
|
|
name="direct",
|
|
model="qwen3:8b",
|
|
eval_benchmarks=["supergpqa"],
|
|
)
|
|
suite = r.to_eval_suite()
|
|
assert suite.benchmarks[0].backend == "jarvis-direct"
|
|
assert suite.benchmarks[0].agent is None
|
|
assert suite.benchmarks[0].tools == []
|
|
|
|
|
|
# ========================================================================
|
|
# Bridge: to_operator_manifest
|
|
# ========================================================================
|
|
|
|
|
|
class TestRecipeToOperatorManifest:
|
|
def test_basic_operator_manifest(self, tmp_path: Path) -> None:
|
|
p = tmp_path / "op.toml"
|
|
p.write_text(OPERATOR_TOML)
|
|
r = load_recipe(p)
|
|
|
|
m = r.to_operator_manifest()
|
|
|
|
assert m.id == "my-operator"
|
|
assert m.name == "my-operator"
|
|
assert m.version == "2.0.0"
|
|
assert m.description == "A test operator"
|
|
assert m.tools == ["web_search", "memory_store", "think"]
|
|
assert m.system_prompt == "You are a monitoring agent."
|
|
assert m.max_turns == 15
|
|
assert m.temperature == pytest.approx(0.3)
|
|
assert m.schedule_type == "interval"
|
|
assert m.schedule_value == "600"
|
|
|
|
def test_operator_no_schedule_raises(self) -> None:
|
|
r = Recipe(name="no-sched", kind="operator")
|
|
with pytest.raises(ValueError, match="no \\[schedule\\]"):
|
|
r.to_operator_manifest()
|
|
|
|
def test_operator_defaults(self) -> None:
|
|
r = Recipe(
|
|
name="minimal-op",
|
|
kind="operator",
|
|
schedule_type="interval",
|
|
schedule_value="300",
|
|
)
|
|
m = r.to_operator_manifest()
|
|
assert m.max_turns == 20
|
|
assert m.temperature == pytest.approx(0.3)
|
|
assert m.schedule_value == "300"
|
|
|
|
|
|
# ========================================================================
|
|
# Builder kwargs with new fields
|
|
# ========================================================================
|
|
|
|
|
|
class TestBuilderKwargsNewFields:
|
|
def test_provider_in_kwargs(self) -> None:
|
|
r = Recipe(name="cloud", model="gpt-4o", provider="openai")
|
|
kw = r.to_builder_kwargs()
|
|
assert kw["provider"] == "openai"
|
|
|
|
def test_system_prompt_path_resolved(self, tmp_path: Path) -> None:
|
|
prompt = tmp_path / "prompt.txt"
|
|
prompt.write_text("Hello from file.")
|
|
r = Recipe(
|
|
name="ext",
|
|
system_prompt_path=str(prompt),
|
|
)
|
|
kw = r.to_builder_kwargs()
|
|
assert kw["system_prompt"] == "Hello from file."
|
|
|
|
def test_schedule_and_channels_not_in_kwargs(self) -> None:
|
|
"""Schedule/channel fields are operator-specific and not builder kwargs."""
|
|
r = Recipe(
|
|
name="op",
|
|
kind="operator",
|
|
schedule_type="cron",
|
|
schedule_value="0 * * * *",
|
|
channels=["slack"],
|
|
)
|
|
kw = r.to_builder_kwargs()
|
|
assert "schedule_type" not in kw
|
|
assert "channels" not in kw
|
|
|
|
|
|
# ========================================================================
|
|
# Built-in recipe loading
|
|
# ========================================================================
|
|
|
|
|
|
class TestBuiltinRecipes:
|
|
def test_terminalbench_react_loads(self) -> None:
|
|
r = resolve_recipe("terminalbench-react")
|
|
assert r is not None
|
|
assert r.kind == "discrete"
|
|
assert r.agent_type == "native_react"
|
|
assert "terminalbench" in r.eval_benchmarks
|
|
|
|
def test_gaia_orchestrator_loads(self) -> None:
|
|
r = resolve_recipe("gaia-orchestrator")
|
|
assert r is not None
|
|
assert r.kind == "discrete"
|
|
assert r.agent_type == "orchestrator"
|
|
assert "gaia" in r.eval_benchmarks
|
|
|
|
def test_swebench_openhands_loads(self) -> None:
|
|
r = resolve_recipe("swebench-openhands")
|
|
assert r is not None
|
|
assert r.kind == "discrete"
|
|
assert r.agent_type == "native_openhands"
|
|
|
|
def test_coding_benchmark_loads(self) -> None:
|
|
r = resolve_recipe("coding-benchmark")
|
|
assert r is not None
|
|
assert r.kind == "discrete"
|
|
assert "terminalbench" in r.eval_benchmarks
|
|
assert "swebench" in r.eval_benchmarks
|
|
|
|
def test_twitter_sentinel_loads(self) -> None:
|
|
r = resolve_recipe("twitter-sentinel")
|
|
assert r is not None
|
|
assert r.kind == "operator"
|
|
assert r.schedule_type == "interval"
|
|
|
|
def test_inbox_triage_loads(self) -> None:
|
|
r = resolve_recipe("inbox-triage")
|
|
assert r is not None
|
|
assert r.kind == "operator"
|
|
assert r.schedule_type == "interval"
|
|
|
|
def test_news_briefing_loads(self) -> None:
|
|
r = resolve_recipe("news-briefing")
|
|
assert r is not None
|
|
assert r.kind == "operator"
|
|
assert r.schedule_type == "cron"
|
|
|
|
def test_repo_watcher_loads(self) -> None:
|
|
r = resolve_recipe("repo-watcher")
|
|
assert r is not None
|
|
assert r.kind == "operator"
|
|
assert r.schedule_type == "interval"
|