From 2cd131847cb6b6c7bea196e3d24987327c28d11a Mon Sep 17 00:00:00 2001 From: krypticmouse Date: Wed, 4 Mar 2026 19:35:33 -0800 Subject: [PATCH] Add unified composition system --- src/openjarvis/cli/__init__.py | 2 + src/openjarvis/cli/compose_cmd.py | 457 +++++++++++++++++ src/openjarvis/recipes/__init__.py | 13 +- src/openjarvis/recipes/composer.py | 141 ++++++ .../recipes/data/coding_benchmark.toml | 36 ++ .../recipes/data/gaia_orchestrator.toml | 35 ++ .../recipes/data/operators/inbox_triage.toml | 57 +++ .../recipes/data/operators/news_briefing.toml | 57 +++ .../recipes/data/operators/repo_watcher.toml | 64 +++ .../data/operators/twitter_sentinel.toml | 50 ++ .../recipes/data/swebench_openhands.toml | 41 ++ .../recipes/data/terminalbench_react.toml | 35 ++ src/openjarvis/recipes/loader.py | 220 ++++++-- tests/recipes/test_compose.py | 471 ++++++++++++++++++ 14 files changed, 1638 insertions(+), 41 deletions(-) create mode 100644 src/openjarvis/cli/compose_cmd.py create mode 100644 src/openjarvis/recipes/composer.py create mode 100644 src/openjarvis/recipes/data/coding_benchmark.toml create mode 100644 src/openjarvis/recipes/data/gaia_orchestrator.toml create mode 100644 src/openjarvis/recipes/data/operators/inbox_triage.toml create mode 100644 src/openjarvis/recipes/data/operators/news_briefing.toml create mode 100644 src/openjarvis/recipes/data/operators/repo_watcher.toml create mode 100644 src/openjarvis/recipes/data/operators/twitter_sentinel.toml create mode 100644 src/openjarvis/recipes/data/swebench_openhands.toml create mode 100644 src/openjarvis/recipes/data/terminalbench_react.toml create mode 100644 tests/recipes/test_compose.py diff --git a/src/openjarvis/cli/__init__.py b/src/openjarvis/cli/__init__.py index d58c7d09..8bff4ecc 100644 --- a/src/openjarvis/cli/__init__.py +++ b/src/openjarvis/cli/__init__.py @@ -11,6 +11,7 @@ from openjarvis.cli.ask import ask from openjarvis.cli.bench_cmd import bench from openjarvis.cli.channel_cmd import channel from openjarvis.cli.chat_cmd import chat +from openjarvis.cli.compose_cmd import compose from openjarvis.cli.daemon_cmd import restart, start, status, stop from openjarvis.cli.doctor_cmd import doctor from openjarvis.cli.eval_cmd import eval_group @@ -71,6 +72,7 @@ cli.add_command(host, "host") cli.add_command(quickstart, "quickstart") cli.add_command(optimize_group, "optimize") cli.add_command(feedback_group, "feedback") +cli.add_command(compose, "compose") def main() -> None: diff --git a/src/openjarvis/cli/compose_cmd.py b/src/openjarvis/cli/compose_cmd.py new file mode 100644 index 00000000..9caba1c0 --- /dev/null +++ b/src/openjarvis/cli/compose_cmd.py @@ -0,0 +1,457 @@ +"""``jarvis compose`` — unified composition CLI for discrete agents and operators.""" + +from __future__ import annotations + +import sys +from typing import Optional + +import click +from rich.console import Console +from rich.table import Table + + +@click.group() +def compose() -> None: + """Compose, run, benchmark, and deploy OpenJarvis configurations. + + Recipes are unified TOML configs that wire all five pillars + (Intelligence, Engine, Agent, Tools, Learning). They come in two + kinds: + + \b + discrete One-shot or benchmark-oriented agents + operator Persistent, scheduled autonomous agents + """ + + +# ------------------------------------------------------------------ # +# jarvis compose list +# ------------------------------------------------------------------ # + + +@compose.command("list") +@click.option( + "-k", "--kind", "kind", default=None, + type=click.Choice(["discrete", "operator"]), + help="Filter by recipe kind.", +) +def compose_list(kind: Optional[str]) -> None: + """List all discovered compositions (recipes and operators).""" + console = Console(stderr=True) + try: + from openjarvis.recipes.loader import discover_recipes + + recipes = discover_recipes(kind=kind) + if not recipes: + console.print("[dim]No compositions found.[/dim]") + console.print( + "[dim]Place TOML recipes in src/openjarvis/recipes/data/ " + "or ~/.openjarvis/recipes/[/dim]" + ) + return + + table = Table(title="Compositions", border_style="bright_blue") + table.add_column("Name", style="cyan", no_wrap=True) + table.add_column("Kind", style="yellow") + table.add_column("Model", style="green") + table.add_column("Agent", style="magenta") + table.add_column("Tools", style="white") + table.add_column("Description") + + for r in sorted(recipes, key=lambda r: (r.kind, r.name)): + tools_str = ", ".join(r.tools[:3]) + if len(r.tools) > 3: + tools_str += f" (+{len(r.tools) - 3})" + table.add_row( + r.name, + r.kind, + r.model or "-", + r.agent_type or "-", + tools_str or "-", + r.description[:60] if r.description else "", + ) + + console.print(table) + except Exception as exc: + console.print(f"[red]Error: {exc}[/red]") + + +# ------------------------------------------------------------------ # +# jarvis compose show +# ------------------------------------------------------------------ # + + +@compose.command("show") +@click.argument("name") +def compose_show(name: str) -> None: + """Show detailed configuration of a composition.""" + console = Console(stderr=True) + try: + from openjarvis.recipes.loader import resolve_recipe + + recipe = resolve_recipe(name) + if recipe is None: + console.print(f"[red]Composition not found: {name}[/red]") + return + + console.print(f"[bold cyan]{recipe.name}[/bold cyan] ({recipe.kind})") + console.print(f" {recipe.description}\n") + + # Intelligence + console.print("[bold]Intelligence[/bold]") + console.print(f" model: {recipe.model or '-'}") + console.print(f" quantization: {recipe.quantization or '-'}") + console.print(f" provider: {recipe.provider or '-'}") + console.print() + + # Engine + console.print("[bold]Engine[/bold]") + console.print(f" key: {recipe.engine_key or '-'}") + console.print() + + # Agent + console.print("[bold]Agent[/bold]") + console.print(f" type: {recipe.agent_type or '-'}") + console.print(f" max_turns: {recipe.max_turns or '-'}") + console.print(f" temperature: {recipe.temperature or '-'}") + console.print(f" tools: {', '.join(recipe.tools) or '-'}") + if recipe.system_prompt: + preview = recipe.system_prompt[:120].replace("\n", " ") + console.print(f" prompt: {preview}...") + console.print() + + # Learning + console.print("[bold]Learning[/bold]") + console.print(f" routing: {recipe.routing_policy or '-'}") + console.print(f" agent: {recipe.agent_policy or '-'}") + console.print() + + # Kind-specific sections + if recipe.kind == "discrete": + benchmarks = recipe.eval_benchmarks or recipe.eval_suites + if benchmarks: + console.print("[bold]Eval[/bold]") + console.print(f" benchmarks: {', '.join(benchmarks)}") + console.print(f" backend: {recipe.eval_backend or 'auto'}") + console.print(f" max_samples: {recipe.eval_max_samples or 'all'}") + console.print(f" judge_model: {recipe.eval_judge_model or 'default'}") + elif recipe.kind == "operator": + if recipe.schedule_type: + console.print("[bold]Schedule[/bold]") + console.print(f" type: {recipe.schedule_type}") + console.print(f" value: {recipe.schedule_value}") + if recipe.channels: + console.print("[bold]Channels[/bold]") + console.print(f" output: {', '.join(recipe.channels)}") + except Exception as exc: + console.print(f"[red]Error: {exc}[/red]") + + +# ------------------------------------------------------------------ # +# jarvis compose run +# ------------------------------------------------------------------ # + + +@compose.command("run") +@click.argument("name") +@click.argument("query", nargs=-1, required=True) +@click.option("--json", "output_json", is_flag=True, help="Output raw JSON result.") +def compose_run(name: str, query: tuple[str, ...], output_json: bool) -> None: + """Run a composition against a single query.""" + console = Console(stderr=True) + query_text = " ".join(query) + + try: + from openjarvis.recipes.loader import resolve_recipe + + recipe = resolve_recipe(name) + if recipe is None: + console.print(f"[red]Composition not found: {name}[/red]") + sys.exit(1) + + kwargs = recipe.to_builder_kwargs() + console.print( + f"[dim]Running [cyan]{recipe.name}[/cyan] " + f"({recipe.agent_type or 'direct'} / " + f"{recipe.model or 'default'})...[/dim]" + ) + + from openjarvis.system import SystemBuilder + + builder = SystemBuilder() + if "engine_key" in kwargs: + builder = builder.engine(kwargs["engine_key"]) + if "model" in kwargs: + builder = builder.model(kwargs["model"]) + if "agent" in kwargs: + builder = builder.agent(kwargs["agent"]) + if "tools" in kwargs: + builder = builder.tools(kwargs["tools"]) + + system = builder.build() + + try: + agent_kwargs = {} + if kwargs.get("system_prompt"): + agent_kwargs["system_prompt"] = kwargs["system_prompt"] + if kwargs.get("max_turns"): + agent_kwargs["max_turns"] = kwargs["max_turns"] + if kwargs.get("temperature"): + agent_kwargs["temperature"] = kwargs["temperature"] + + result = system.ask(query_text, **agent_kwargs) + + if output_json: + import json as json_mod + if isinstance(result, str): + click.echo(json_mod.dumps({"content": result}, indent=2)) + else: + click.echo(json_mod.dumps({ + "content": result.content, + "turns": getattr(result, "turns", 1), + }, indent=2)) + else: + content = result if isinstance(result, str) else result.content + click.echo(content) + finally: + system.close() + except Exception as exc: + console.print(f"[red]Error: {exc}[/red]") + sys.exit(1) + + +# ------------------------------------------------------------------ # +# jarvis compose bench +# ------------------------------------------------------------------ # + + +@compose.command("bench") +@click.argument("name") +@click.option( + "-b", "--benchmark", "benchmark", default=None, multiple=True, + help="Override benchmarks (can specify multiple).", +) +@click.option( + "-n", "--max-samples", "max_samples", type=int, default=None, + help="Maximum samples per benchmark.", +) +@click.option( + "--judge", "judge_model", default=None, + help="LLM judge model override.", +) +@click.option( + "-v", "--verbose", "verbose", is_flag=True, default=False, + help="Verbose logging.", +) +def compose_bench( + name: str, + benchmark: tuple[str, ...], + max_samples: Optional[int], + judge_model: Optional[str], + verbose: bool, +) -> None: + """Benchmark a discrete composition against eval datasets. + + Uses the recipe's model, engine, agent, and tools to run the eval + framework against the benchmarks defined in the recipe (or overridden + via --benchmark). + """ + console = Console(stderr=True) + + try: + from openjarvis.recipes.loader import resolve_recipe + + recipe = resolve_recipe(name) + if recipe is None: + console.print(f"[red]Composition not found: {name}[/red]") + sys.exit(1) + + benchmarks_list = list(benchmark) if benchmark else None + suite = recipe.to_eval_suite( + benchmarks=benchmarks_list, + max_samples=max_samples, + judge_model=judge_model, + ) + + from openjarvis.evals.core.config import expand_suite + + run_configs = expand_suite(suite) + + console.print( + f"[cyan]Composition:[/cyan] {recipe.name}\n" + f"[cyan]Model:[/cyan] {recipe.model}\n" + f"[cyan]Agent:[/cyan] {recipe.agent_type or 'direct'}\n" + f"[cyan]Tools:[/cyan] {', '.join(recipe.tools) or 'none'}\n" + f"[cyan]Benchmarks:[/cyan] {len(run_configs)} run(s)" + ) + + try: + from openjarvis.evals.cli import _run_single + except ImportError: + console.print("[red]Eval CLI module not available.[/red]") + sys.exit(1) + + results_table = Table( + title="Benchmark Results", + border_style="bright_blue", + title_style="bold cyan", + ) + results_table.add_column("Benchmark", style="cyan") + results_table.add_column("Accuracy", justify="right", style="bold green") + results_table.add_column("Correct / Scored", justify="right") + results_table.add_column("Errors", justify="right", style="red") + + for i, rc in enumerate(run_configs, 1): + console.print( + f"\n[bold]Run {i}/{len(run_configs)}:[/bold] {rc.benchmark}" + ) + try: + summary = _run_single(rc, console=console) + results_table.add_row( + rc.benchmark, + f"{summary.accuracy:.4f}", + f"{summary.correct}/{summary.scored_samples}", + str(summary.errors), + ) + except Exception as exc: + console.print(f" [red bold]FAILED:[/red bold] {exc}") + results_table.add_row(rc.benchmark, "-", "-", str(exc)[:40]) + + console.print() + console.print(results_table) + except Exception as exc: + console.print(f"[red]Error: {exc}[/red]") + sys.exit(1) + + +# ------------------------------------------------------------------ # +# jarvis compose deploy +# ------------------------------------------------------------------ # + + +@compose.command("deploy") +@click.argument("name") +def compose_deploy(name: str) -> None: + """Deploy an operator composition (activate its scheduler task).""" + console = Console(stderr=True) + + try: + from openjarvis.recipes.loader import resolve_recipe + + recipe = resolve_recipe(name) + if recipe is None: + console.print(f"[red]Composition not found: {name}[/red]") + sys.exit(1) + + if recipe.kind != "operator": + console.print( + f"[red]Recipe '{name}' is a {recipe.kind} composition, " + f"not an operator. Only operators can be deployed.[/red]" + ) + sys.exit(1) + + manifest = recipe.to_operator_manifest() + + from openjarvis.operators.manager import OperatorManager + from openjarvis.system import SystemBuilder + + system = SystemBuilder().scheduler(True).sessions(True).build() + manager = OperatorManager(system) + system.operator_manager = manager + + manager.register(manifest) + task_id = manager.activate(manifest.id) + + console.print( + f"[green]Deployed operator [cyan]{name}[/cyan] " + f"(task: {task_id}, schedule: " + f"{recipe.schedule_type}={recipe.schedule_value})[/green]" + ) + except Exception as exc: + console.print(f"[red]Error: {exc}[/red]") + sys.exit(1) + + +# ------------------------------------------------------------------ # +# jarvis compose stop +# ------------------------------------------------------------------ # + + +@compose.command("stop") +@click.argument("name") +def compose_stop(name: str) -> None: + """Stop a deployed operator composition.""" + console = Console(stderr=True) + + try: + from openjarvis.operators.manager import OperatorManager + from openjarvis.system import SystemBuilder + + system = SystemBuilder().scheduler(True).sessions(True).build() + manager = OperatorManager(system) + system.operator_manager = manager + + # Discover all known operators so the manager knows about them + from openjarvis.core.config import DEFAULT_CONFIG_DIR + from openjarvis.recipes.loader import _PROJECT_OPERATORS_DIR + + for d in [DEFAULT_CONFIG_DIR / "operators", _PROJECT_OPERATORS_DIR]: + if d.is_dir(): + manager.discover(d) + + manager.deactivate(name) + console.print(f"[yellow]Stopped operator {name}[/yellow]") + except Exception as exc: + console.print(f"[red]Error: {exc}[/red]") + sys.exit(1) + + +# ------------------------------------------------------------------ # +# jarvis compose status +# ------------------------------------------------------------------ # + + +@compose.command("status") +def compose_status() -> None: + """Show status of all deployed operators.""" + console = Console(stderr=True) + + try: + from openjarvis.operators.manager import OperatorManager + from openjarvis.system import SystemBuilder + + system = SystemBuilder().scheduler(True).sessions(True).build() + manager = OperatorManager(system) + system.operator_manager = manager + + from openjarvis.core.config import DEFAULT_CONFIG_DIR + from openjarvis.recipes.loader import _PROJECT_OPERATORS_DIR + + for d in [DEFAULT_CONFIG_DIR / "operators", _PROJECT_OPERATORS_DIR]: + if d.is_dir(): + manager.discover(d) + + statuses = manager.status() + if not statuses: + console.print("[dim]No operators registered.[/dim]") + return + + table = Table(title="Operator Status", border_style="bright_blue") + table.add_column("Name", style="cyan") + table.add_column("State", style="yellow") + table.add_column("Schedule", style="white") + table.add_column("Last Run", style="dim") + + for s in statuses: + table.add_row( + s.get("id", "?"), + s.get("state", "unknown"), + s.get("schedule", ""), + s.get("last_run", "-"), + ) + console.print(table) + except Exception as exc: + console.print(f"[red]Error: {exc}[/red]") + + +__all__ = ["compose"] diff --git a/src/openjarvis/recipes/__init__.py b/src/openjarvis/recipes/__init__.py index fc31ebc3..5a8ffabb 100644 --- a/src/openjarvis/recipes/__init__.py +++ b/src/openjarvis/recipes/__init__.py @@ -1,5 +1,9 @@ """Recipe system — composable pillar configurations.""" +from openjarvis.recipes.composer import ( + recipe_to_eval_suite, + recipe_to_operator, +) from openjarvis.recipes.loader import ( Recipe, discover_recipes, @@ -7,4 +11,11 @@ from openjarvis.recipes.loader import ( resolve_recipe, ) -__all__ = ["Recipe", "discover_recipes", "load_recipe", "resolve_recipe"] +__all__ = [ + "Recipe", + "discover_recipes", + "load_recipe", + "recipe_to_eval_suite", + "recipe_to_operator", + "resolve_recipe", +] diff --git a/src/openjarvis/recipes/composer.py b/src/openjarvis/recipes/composer.py new file mode 100644 index 00000000..75b23b76 --- /dev/null +++ b/src/openjarvis/recipes/composer.py @@ -0,0 +1,141 @@ +"""Composer bridges — convert a Recipe into EvalSuiteConfig or OperatorManifest. + +These are pure-function transformations that let the unified Recipe format +drive both the eval framework and the operator system without those systems +needing to know about recipes directly. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional + +if TYPE_CHECKING: + from openjarvis.evals.core.types import EvalSuiteConfig + from openjarvis.operators.types import OperatorManifest + from openjarvis.recipes.loader import Recipe + + +def recipe_to_eval_suite( + recipe: Recipe, + benchmarks: Optional[List[str]] = None, + max_samples: Optional[int] = None, + judge_model: Optional[str] = None, +) -> EvalSuiteConfig: + """Build an ``EvalSuiteConfig`` from a recipe. + + The recipe's model / engine become the single ``[[models]]`` entry. + The recipe's ``eval_benchmarks`` (or the *benchmarks* override) become + ``[[benchmarks]]`` entries. Agent type and tools are inherited so the + eval runner constructs the right backend automatically. + + Args: + recipe: Source recipe. + benchmarks: Override benchmark list (defaults to ``recipe.eval_benchmarks``). + max_samples: Override per-benchmark sample cap. + judge_model: Override LLM judge model. + + Raises: + ValueError: If no model or benchmarks can be resolved. + """ + from openjarvis.evals.core.types import ( + BenchmarkConfig, + DefaultsConfig, + EvalSuiteConfig, + ExecutionConfig, + JudgeConfig, + MetaConfig, + ModelConfig, + ) + + bench_names = benchmarks or list(recipe.eval_benchmarks) + if not bench_names: + bench_names = list(recipe.eval_suites) + if not bench_names: + raise ValueError( + f"Recipe '{recipe.name}' has no benchmarks defined and none were " + "provided. Set [eval] benchmarks in the TOML or pass benchmarks=." + ) + + model_name = recipe.model + if not model_name: + raise ValueError( + f"Recipe '{recipe.name}' has no model defined. " + "Set [intelligence] model in the TOML." + ) + + has_agent = recipe.agent_type is not None + backend = recipe.eval_backend or ("jarvis-agent" if has_agent else "jarvis-direct") + + model_cfg = ModelConfig( + name=model_name, + engine=recipe.engine_key, + provider=recipe.provider, + temperature=recipe.temperature, + ) + + bench_cfgs: list[BenchmarkConfig] = [] + for bname in bench_names: + bench_cfgs.append(BenchmarkConfig( + name=bname, + backend=backend, + max_samples=max_samples or recipe.eval_max_samples, + agent=recipe.agent_type if has_agent else None, + tools=list(recipe.tools) if has_agent else [], + judge_model=judge_model or recipe.eval_judge_model, + )) + + return EvalSuiteConfig( + meta=MetaConfig( + name=f"{recipe.name}-eval", + description=f"Auto-generated eval suite from recipe '{recipe.name}'", + ), + defaults=DefaultsConfig( + temperature=recipe.temperature or 0.0, + max_tokens=2048, + ), + judge=JudgeConfig( + model=judge_model or recipe.eval_judge_model or "gpt-5-mini-2025-08-07", + ), + run=ExecutionConfig(), + models=[model_cfg], + benchmarks=bench_cfgs, + ) + + +def recipe_to_operator(recipe: Recipe) -> OperatorManifest: + """Build an ``OperatorManifest`` from a recipe. + + Maps the recipe's agent, schedule, and channel fields into the + operator manifest format used by ``OperatorManager``. + + Raises: + ValueError: If schedule information is missing. + """ + from openjarvis.operators.types import OperatorManifest + + if not recipe.schedule_type: + raise ValueError( + f"Recipe '{recipe.name}' has no [schedule] section. " + "Operator recipes must define schedule_type and schedule_value." + ) + + prompt = recipe.system_prompt or "" + prompt_path = recipe.system_prompt_path or "" + + return OperatorManifest( + id=recipe.name, + name=recipe.name, + version=recipe.version, + description=recipe.description, + tools=list(recipe.tools), + system_prompt=prompt, + system_prompt_path=prompt_path, + max_turns=recipe.max_turns or 20, + temperature=recipe.temperature or 0.3, + schedule_type=recipe.schedule_type, + schedule_value=recipe.schedule_value or "300", + required_capabilities=list(recipe.required_capabilities), + ) + + +__all__ = ["recipe_to_eval_suite", "recipe_to_operator"] diff --git a/src/openjarvis/recipes/data/coding_benchmark.toml b/src/openjarvis/recipes/data/coding_benchmark.toml new file mode 100644 index 00000000..873625a9 --- /dev/null +++ b/src/openjarvis/recipes/data/coding_benchmark.toml @@ -0,0 +1,36 @@ +[recipe] +name = "coding-benchmark" +kind = "discrete" +description = "ReAct coding agent benchmarked against TerminalBench and SWE-bench" +version = "1.0.0" + +[intelligence] +model = "qwen3:8b" +quantization = "q4_K_M" + +[engine] +key = "ollama" + +[agent] +type = "native_react" +max_turns = 20 +temperature = 0.2 +tools = ["file_read", "file_write", "shell_exec", "code_interpreter", "apply_patch", "think"] +system_prompt = """You are an expert programmer and systems engineer. You solve coding tasks by reasoning through problems step-by-step, reading code, writing solutions, and verifying them with tests. + +Approach each task methodically: +1. Understand what is being asked +2. Explore relevant files and code +3. Plan your solution +4. Implement changes carefully +5. Test and verify your work + +Use the tools available to read files, write code, execute commands, and apply patches. Always verify that your changes work correctly before concluding.""" + +[learning] +routing = "heuristic" + +[eval] +benchmarks = ["terminalbench", "swebench"] +backend = "jarvis-agent" +max_samples = 50 diff --git a/src/openjarvis/recipes/data/gaia_orchestrator.toml b/src/openjarvis/recipes/data/gaia_orchestrator.toml new file mode 100644 index 00000000..834d6cd0 --- /dev/null +++ b/src/openjarvis/recipes/data/gaia_orchestrator.toml @@ -0,0 +1,35 @@ +[recipe] +name = "gaia-orchestrator" +kind = "discrete" +description = "Orchestrator agent with web search and memory for GAIA agentic benchmark" +version = "1.0.0" + +[intelligence] +model = "qwen3:8b" + +[engine] +key = "ollama" + +[agent] +type = "orchestrator" +max_turns = 15 +temperature = 0.3 +tools = ["web_search", "http_request", "memory_store", "memory_search", "think", "calculator", "file_read"] +system_prompt = """You are a highly capable research assistant that solves complex, multi-step questions requiring web research, reasoning, and tool use. + +When answering questions: +1. Break complex questions into sub-tasks +2. Use web search to find relevant information +3. Store important findings in memory for reference +4. Cross-reference multiple sources when possible +5. Provide precise, well-supported answers + +Always think step-by-step before acting. Verify your answers against the evidence you've gathered.""" + +[learning] +routing = "heuristic" + +[eval] +benchmarks = ["gaia"] +backend = "jarvis-agent" +max_samples = 100 diff --git a/src/openjarvis/recipes/data/operators/inbox_triage.toml b/src/openjarvis/recipes/data/operators/inbox_triage.toml new file mode 100644 index 00000000..e6487a4a --- /dev/null +++ b/src/openjarvis/recipes/data/operators/inbox_triage.toml @@ -0,0 +1,57 @@ +[recipe] +name = "inbox-triage" +kind = "operator" +description = "Monitors Slack and Gmail for important messages, classifies by urgency, and drafts responses" +version = "1.0.0" + +[intelligence] +model = "qwen3:8b" + +[engine] +key = "ollama" + +[agent] +type = "orchestrator" +max_turns = 15 +temperature = 0.4 +tools = ["memory_store", "memory_search", "think", "http_request"] +system_prompt = """You are an inbox triage agent. Your job is to periodically check messaging channels (Slack, email) for important messages, classify them by urgency, and produce a prioritized summary with optional draft responses. + +Each triage cycle: + +1. GATHER — Check all configured messaging sources for new or unread messages since the last cycle. + +2. CLASSIFY — For each message, determine: + - Urgency: CRITICAL / HIGH / MEDIUM / LOW / FYI + - Category: action-required / decision-needed / informational / social / automated + - Sender importance: direct-report / peer / manager / external / system + +3. PRIORITIZE — Rank messages by combined urgency and sender importance. Group related threads. + +4. DRAFT — For CRITICAL and HIGH urgency action-required messages, draft a brief response or suggest next steps. + +5. SUMMARIZE — Produce a triage digest: + + CRITICAL (act now): + - [source] sender: subject — | Suggested action: + + HIGH (act today): + - [source] sender: subject — + + MEDIUM (this week): + - [source] sender: subject — + + LOW/FYI (when convenient): + - messages from + +Store message metadata in memory to track response patterns and avoid re-triaging.""" + +[schedule] +type = "interval" +value = "300" + +[channels] +output = ["slack"] + +[learning] +routing = "heuristic" diff --git a/src/openjarvis/recipes/data/operators/news_briefing.toml b/src/openjarvis/recipes/data/operators/news_briefing.toml new file mode 100644 index 00000000..f34a34a1 --- /dev/null +++ b/src/openjarvis/recipes/data/operators/news_briefing.toml @@ -0,0 +1,57 @@ +[recipe] +name = "news-briefing" +kind = "operator" +description = "Daily news digest — searches configured topics, synthesizes key stories, and delivers a morning briefing" +version = "1.0.0" + +[intelligence] +model = "qwen3:8b" + +[engine] +key = "ollama" + +[agent] +type = "orchestrator" +max_turns = 20 +temperature = 0.4 +tools = ["web_search", "http_request", "memory_store", "memory_search", "think"] +system_prompt = """You are a news briefing agent. Each cycle, you research configured topics across multiple sources and produce a concise, well-structured news digest. + +Briefing workflow: + +1. RESEARCH — Search the web for each configured topic. Cover multiple angles: + - Major news outlets for headline stories + - Industry/trade sources for domain-specific developments + - Social media for emerging trends and public reaction + +2. CROSS-REFERENCE — Verify key claims across at least 2 sources. Flag single-source stories as unconfirmed. + +3. CONTEXTUALIZE — Check memory for related previous stories. Note developments, trend changes, and follow-ups to earlier news. + +4. SYNTHESIZE — Write a briefing with these sections: + + TOP STORIES (3-5 most important) + Each: headline, 2-3 sentence summary, significance, sources + + SECTOR UPDATES (grouped by topic) + Each: brief summary with source + + TRENDS & SIGNALS + Emerging patterns, sentiment shifts, early indicators + + FOLLOW-UPS + Updates on previously reported stories (from memory) + +5. STORE — Save today's briefing to memory for future context and trend tracking. + +Write in a clear, journalistic style. Lead with the most consequential stories. Be factual and cite sources.""" + +[schedule] +type = "cron" +value = "0 8 * * *" + +[channels] +output = ["slack"] + +[learning] +routing = "heuristic" diff --git a/src/openjarvis/recipes/data/operators/repo_watcher.toml b/src/openjarvis/recipes/data/operators/repo_watcher.toml new file mode 100644 index 00000000..29f6ac8b --- /dev/null +++ b/src/openjarvis/recipes/data/operators/repo_watcher.toml @@ -0,0 +1,64 @@ +[recipe] +name = "repo-watcher" +kind = "operator" +description = "Monitors GitHub repositories for new issues, PRs, and releases — summarizes activity with priority scoring" +version = "1.0.0" + +[intelligence] +model = "qwen3:8b" + +[engine] +key = "ollama" + +[agent] +type = "orchestrator" +max_turns = 15 +temperature = 0.3 +tools = ["web_search", "http_request", "memory_store", "memory_search", "think"] +system_prompt = """You are a GitHub repository monitoring agent. You track configured repositories for important activity and produce structured reports. + +Each monitoring cycle: + +1. CHECK — Query the GitHub API for recent activity across configured repositories: + - New issues (opened since last check) + - New/updated pull requests + - New releases and tags + - Notable discussions and comments + +2. ANALYZE — For each item, assess: + - Priority: P0 (critical bug/security) / P1 (important) / P2 (normal) / P3 (low) + - Type: bug / feature / enhancement / docs / ci / security + - Impact: breaking / significant / minor / cosmetic + +3. CORRELATE — Check memory for related items. Link PRs to issues. Note patterns (recurring bugs, stale PRs, active contributors). + +4. REPORT — Produce a structured activity report: + + ATTENTION NEEDED + - Security advisories, critical bugs, breaking changes + + NEW ISSUES (by priority) + - [repo] #number: title — labels, assignee + + PULL REQUESTS + - [repo] #number: title — status (draft/review/approved/merged), author + + RELEASES + - [repo] vX.Y.Z: key changes summary + + TRENDS + - Activity level, response times, contributor patterns + +5. STORE — Save current state to memory for next-cycle comparison. + +Be concise. Focus on actionable items. Skip routine CI/bot activity.""" + +[schedule] +type = "interval" +value = "1800" + +[channels] +output = ["slack"] + +[learning] +routing = "heuristic" diff --git a/src/openjarvis/recipes/data/operators/twitter_sentinel.toml b/src/openjarvis/recipes/data/operators/twitter_sentinel.toml new file mode 100644 index 00000000..3de27ddc --- /dev/null +++ b/src/openjarvis/recipes/data/operators/twitter_sentinel.toml @@ -0,0 +1,50 @@ +[recipe] +name = "twitter-sentinel" +kind = "operator" +description = "Monitors Twitter/X for specified keywords and surfaces relevant posts with urgency scoring" +version = "1.0.0" + +[intelligence] +model = "qwen3:8b" + +[engine] +key = "ollama" + +[agent] +type = "orchestrator" +max_turns = 15 +temperature = 0.3 +tools = ["web_search", "http_request", "memory_store", "memory_search", "think"] +system_prompt = """You are a social media monitoring agent focused on Twitter/X. Your job is to periodically search for tweets matching specific keywords and topics, assess their relevance and urgency, and produce structured alerts. + +Each monitoring cycle: + +1. SCAN — Search Twitter/X for the configured keywords and topics using web search. Look for recent posts, trending discussions, and notable accounts. + +2. FILTER — Evaluate each result for relevance. Discard noise, spam, and off-topic content. Focus on substantive posts with real signal. + +3. SCORE — Rate each relevant finding on urgency (1-5): + - 5: Breaking/critical (requires immediate attention) + - 4: Important (notable development, should be seen today) + - 3: Interesting (worth tracking, no rush) + - 2: Background (context, low priority) + - 1: Marginal (barely relevant) + +4. STORE — Save significant findings to memory with urgency score, source URL, timestamp, and a brief summary. + +5. REPORT — Produce a structured digest of findings sorted by urgency. Include only items scoring 3 or higher unless it's a quiet period. + +Output format for each finding: +[URGENCY X/5] — +Source: +Context: <2-3 sentences of analysis>""" + +[schedule] +type = "interval" +value = "600" + +[channels] +output = ["slack"] + +[learning] +routing = "heuristic" diff --git a/src/openjarvis/recipes/data/swebench_openhands.toml b/src/openjarvis/recipes/data/swebench_openhands.toml new file mode 100644 index 00000000..6286df6c --- /dev/null +++ b/src/openjarvis/recipes/data/swebench_openhands.toml @@ -0,0 +1,41 @@ +[recipe] +name = "swebench-openhands" +kind = "discrete" +description = "CodeAct agent (OpenHands-style) for SWE-bench software engineering tasks" +version = "1.0.0" + +[intelligence] +model = "qwen3:8b" +quantization = "q4_K_M" + +[engine] +key = "ollama" + +[agent] +type = "native_openhands" +max_turns = 25 +temperature = 0.2 +tools = ["file_read", "file_write", "shell_exec", "apply_patch", "think", "code_interpreter"] +system_prompt = """You are an expert software engineer working on real-world GitHub issues. You use the CodeAct approach: think carefully, then act by writing and executing code. + +Your workflow: +1. Read the issue description and understand what needs to be fixed +2. Explore the repository structure to find relevant files +3. Read the relevant source code to understand the problem +4. Write a fix and apply it +5. Run tests to verify the fix works +6. Iterate if tests fail + +Key principles: +- Always read the failing test or error output carefully +- Make minimal, targeted changes +- Don't break existing functionality +- Write clean, idiomatic code matching the project's style""" + +[learning] +routing = "heuristic" + +[eval] +benchmarks = ["swebench"] +backend = "jarvis-agent" +max_samples = 50 diff --git a/src/openjarvis/recipes/data/terminalbench_react.toml b/src/openjarvis/recipes/data/terminalbench_react.toml new file mode 100644 index 00000000..b74d85af --- /dev/null +++ b/src/openjarvis/recipes/data/terminalbench_react.toml @@ -0,0 +1,35 @@ +[recipe] +name = "terminalbench-react" +kind = "discrete" +description = "ReAct agent tuned for terminal and shell tasks — benchmarks against TerminalBench" +version = "1.0.0" + +[intelligence] +model = "qwen3:8b" +quantization = "q4_K_M" + +[engine] +key = "ollama" + +[agent] +type = "native_react" +max_turns = 20 +temperature = 0.2 +tools = ["shell_exec", "file_read", "file_write", "think"] +system_prompt = """You are an expert at terminal and shell operations. You solve tasks by reasoning step-by-step and executing shell commands. + +When given a task: +1. Analyze what needs to be done +2. Plan your approach +3. Execute commands one at a time, observing the output +4. Verify the result before concluding + +You have access to a full Linux shell environment. Use standard Unix tools (grep, sed, awk, find, etc.) and scripting when appropriate. Always check command exit codes and handle errors gracefully.""" + +[learning] +routing = "heuristic" + +[eval] +benchmarks = ["terminalbench"] +backend = "jarvis-agent" +max_samples = 50 diff --git a/src/openjarvis/recipes/loader.py b/src/openjarvis/recipes/loader.py index c92d7357..46210039 100644 --- a/src/openjarvis/recipes/loader.py +++ b/src/openjarvis/recipes/loader.py @@ -1,4 +1,12 @@ -"""Recipe loader — load and resolve TOML recipe files.""" +"""Recipe loader — load and resolve TOML recipe files. + +Recipes are the universal composition format for OpenJarvis. Each recipe +specifies all five pillars (Intelligence, Engine, Agent, Tools, Learning) +and carries a ``kind`` that determines its lifecycle: + +* ``"discrete"`` — one-shot or benchmark-oriented agents +* ``"operator"`` — persistent, scheduled agents +""" from __future__ import annotations @@ -14,21 +22,29 @@ except ModuleNotFoundError: # Built-in recipes directory (package data) _PROJECT_RECIPES_DIR = Path(__file__).resolve().parent / "data" -# User-level recipes directory +_PROJECT_OPERATORS_DIR = _PROJECT_RECIPES_DIR / "operators" +# User-level directories _USER_RECIPES_DIR = Path.home() / ".openjarvis" / "recipes" +_USER_OPERATORS_DIR = Path.home() / ".openjarvis" / "operators" @dataclass(slots=True) class Recipe: - """A composable pillar configuration loaded from TOML.""" + """A composable pillar configuration loaded from TOML. + + Covers both *discrete* agents (benchmarking / one-shot) and *operator* + agents (persistent / scheduled) through the ``kind`` field. + """ name: str description: str = "" version: str = "1.0.0" + kind: str = "discrete" # "discrete" | "operator" # Intelligence model: Optional[str] = None quantization: Optional[str] = None + provider: Optional[str] = None # Engine engine_key: Optional[str] = None @@ -39,17 +55,36 @@ class Recipe: temperature: Optional[float] = None tools: List[str] = field(default_factory=list) system_prompt: Optional[str] = None + system_prompt_path: Optional[str] = None # Learning routing_policy: Optional[str] = None agent_policy: Optional[str] = None - # Eval + # Eval (discrete agents) eval_suites: List[str] = field(default_factory=list) + eval_benchmarks: List[str] = field(default_factory=list) + eval_backend: Optional[str] = None + eval_max_samples: Optional[int] = None + eval_judge_model: Optional[str] = None + + # Schedule (operators) + schedule_type: Optional[str] = None + schedule_value: Optional[str] = None + + # Channels (operators) + channels: List[str] = field(default_factory=list) + + # Security + required_capabilities: List[str] = field(default_factory=list) # Raw TOML data for forward-compat raw: Dict[str, Any] = field(default_factory=dict) + # ------------------------------------------------------------------ # + # Conversion helpers + # ------------------------------------------------------------------ # + def to_builder_kwargs(self) -> Dict[str, Any]: """Convert recipe fields to kwargs for SystemBuilder/Jarvis. @@ -69,49 +104,65 @@ class Recipe: kwargs["temperature"] = self.temperature if self.max_turns is not None: kwargs["max_turns"] = self.max_turns - if self.system_prompt is not None: - kwargs["system_prompt"] = self.system_prompt + prompt = self.system_prompt + if prompt is None and self.system_prompt_path is not None: + p = Path(self.system_prompt_path) + if p.exists(): + prompt = p.read_text(encoding="utf-8") + if prompt is not None: + kwargs["system_prompt"] = prompt if self.routing_policy is not None: kwargs["routing_policy"] = self.routing_policy if self.agent_policy is not None: kwargs["agent_policy"] = self.agent_policy if self.quantization is not None: kwargs["quantization"] = self.quantization + if self.provider is not None: + kwargs["provider"] = self.provider if self.eval_suites: kwargs["eval_suites"] = self.eval_suites return kwargs + def to_eval_suite( + self, + benchmarks: Optional[List[str]] = None, + max_samples: Optional[int] = None, + judge_model: Optional[str] = None, + ) -> Any: + """Convert this recipe into an ``EvalSuiteConfig``. + + Uses the recipe's model/engine as the single ``[[models]]`` entry + and the recipe's benchmarks (or *benchmarks* override) as + ``[[benchmarks]]``, inheriting agent type and tools. + """ + from openjarvis.recipes.composer import recipe_to_eval_suite + + return recipe_to_eval_suite( + self, + benchmarks=benchmarks, + max_samples=max_samples, + judge_model=judge_model, + ) + + def to_operator_manifest(self) -> Any: + """Convert this recipe into an ``OperatorManifest``.""" + from openjarvis.recipes.composer import recipe_to_operator + + return recipe_to_operator(self) + + +# ------------------------------------------------------------------ # +# TOML loader +# ------------------------------------------------------------------ # + def load_recipe(path: str | Path) -> Recipe: """Load a recipe from a TOML file. - Expected TOML format:: - - [recipe] - name = "coding_assistant" - description = "..." - version = "1.0.0" - - [intelligence] - model = "qwen3:8b" - quantization = "q4_K_M" - - [engine] - key = "ollama" - - [agent] - type = "native_react" - max_turns = 10 - temperature = 0.3 - tools = ["file_read", "file_write", "code_interpreter", "think"] - system_prompt = "You are a coding assistant..." - - [learning] - routing = "grpo" - agent = "icl_updater" - - [eval] - suites = ["coding", "reasoning"] + Supports the unified format with ``[recipe]``, ``[intelligence]``, + ``[engine]``, ``[agent]``, ``[learning]``, ``[eval]``, ``[schedule]``, + and ``[channels]`` sections. Also auto-detects legacy operator manifests + that use ``[operator]`` as the top-level key. Raises: FileNotFoundError: If *path* does not exist. @@ -123,43 +174,132 @@ def load_recipe(path: str | Path) -> Recipe: with open(path, "rb") as fh: data = tomllib.load(fh) + # Auto-detect legacy operator manifests ([operator] key) + if "operator" in data and "recipe" not in data: + return _load_operator_as_recipe(path, data) + recipe_sec = data.get("recipe", {}) intel_sec = data.get("intelligence", {}) engine_sec = data.get("engine", {}) agent_sec = data.get("agent", {}) learning_sec = data.get("learning", {}) eval_sec = data.get("eval", {}) + schedule_sec = data.get("schedule", {}) + channels_sec = data.get("channels", {}) + + system_prompt = agent_sec.get("system_prompt") + system_prompt_path = agent_sec.get("system_prompt_path") + + # Resolve external prompt relative to TOML file + if not system_prompt and system_prompt_path: + prompt_p = Path(system_prompt_path) + if not prompt_p.is_absolute(): + prompt_p = path.parent / prompt_p + if prompt_p.exists(): + system_prompt = prompt_p.read_text(encoding="utf-8") + system_prompt_path = str(prompt_p) + + kind = recipe_sec.get("kind", "discrete") + if schedule_sec and kind == "discrete": + kind = "operator" return Recipe( name=recipe_sec.get("name", path.stem), description=recipe_sec.get("description", ""), version=recipe_sec.get("version", "1.0.0"), + kind=kind, model=intel_sec.get("model"), quantization=intel_sec.get("quantization"), + provider=intel_sec.get("provider") or engine_sec.get("provider"), engine_key=engine_sec.get("key"), agent_type=agent_sec.get("type"), max_turns=agent_sec.get("max_turns"), temperature=agent_sec.get("temperature"), tools=agent_sec.get("tools", []), - system_prompt=agent_sec.get("system_prompt"), + system_prompt=system_prompt, + system_prompt_path=system_prompt_path, routing_policy=learning_sec.get("routing"), agent_policy=learning_sec.get("agent"), eval_suites=eval_sec.get("suites", []), + eval_benchmarks=eval_sec.get("benchmarks", []), + eval_backend=eval_sec.get("backend"), + eval_max_samples=eval_sec.get("max_samples"), + eval_judge_model=eval_sec.get("judge_model"), + schedule_type=schedule_sec.get("type"), + schedule_value=str(schedule_sec["value"]) if "value" in schedule_sec else None, + channels=channels_sec.get("output", []), + required_capabilities=recipe_sec.get("required_capabilities", []), raw=data, ) +def _load_operator_as_recipe(path: Path, data: Dict[str, Any]) -> Recipe: + """Convert a legacy ``[operator]`` manifest into a Recipe.""" + op = data["operator"] + agent_data = op.get("agent", {}) + schedule = op.get("schedule", {}) + + system_prompt = agent_data.get("system_prompt", op.get("system_prompt", "")) + system_prompt_path = agent_data.get( + "system_prompt_path", op.get("system_prompt_path", ""), + ) + if not system_prompt and system_prompt_path: + prompt_p = Path(system_prompt_path) + if not prompt_p.is_absolute(): + prompt_p = path.parent / prompt_p + if prompt_p.exists(): + system_prompt = prompt_p.read_text(encoding="utf-8") + system_prompt_path = str(prompt_p) + + sched_type = schedule.get("type", op.get("schedule_type", "interval")) + sched_value = schedule.get("value", op.get("schedule_value", "300")) + + return Recipe( + name=op.get("name", path.stem), + description=op.get("description", ""), + version=op.get("version", "1.0.0"), + kind="operator", + tools=agent_data.get("tools", op.get("tools", [])), + system_prompt=system_prompt or None, + system_prompt_path=system_prompt_path or None, + max_turns=agent_data.get("max_turns", op.get("max_turns", 20)), + temperature=agent_data.get("temperature", op.get("temperature", 0.3)), + schedule_type=sched_type, + schedule_value=str(sched_value), + required_capabilities=op.get("required_capabilities", []), + raw=data, + ) + + +# ------------------------------------------------------------------ # +# Discovery +# ------------------------------------------------------------------ # + + def discover_recipes( extra_dirs: Optional[List[str | Path]] = None, + *, + kind: Optional[str] = None, ) -> List[Recipe]: """Discover all TOML recipes from known directories. Search order (later entries override earlier ones by name): - 1. Project ``recipes/`` directory - 2. User ``~/.openjarvis/recipes/`` directory - 3. Any additional directories in *extra_dirs* + 1. Project ``recipes/data/`` directory (discrete recipes) + 2. Project ``recipes/data/operators/`` directory (operator recipes) + 3. User ``~/.openjarvis/recipes/`` directory + 4. User ``~/.openjarvis/operators/`` directory + 5. Any additional directories in *extra_dirs* + + Args: + extra_dirs: Additional directories to scan. + kind: If set, filter to only "discrete" or "operator" recipes. """ - dirs: List[Path] = [_PROJECT_RECIPES_DIR, _USER_RECIPES_DIR] + dirs: List[Path] = [ + _PROJECT_RECIPES_DIR, + _PROJECT_OPERATORS_DIR, + _USER_RECIPES_DIR, + _USER_OPERATORS_DIR, + ] if extra_dirs: dirs.extend(Path(d) for d in extra_dirs) @@ -170,9 +310,9 @@ def discover_recipes( for toml_path in sorted(d.glob("*.toml")): try: recipe = load_recipe(toml_path) - recipes[recipe.name] = recipe + if kind is None or recipe.kind == kind: + recipes[recipe.name] = recipe except Exception: - # Skip malformed recipe files continue return list(recipes.values()) diff --git a/tests/recipes/test_compose.py b/tests/recipes/test_compose.py new file mode 100644 index 00000000..0e69bed3 --- /dev/null +++ b/tests/recipes/test_compose.py @@ -0,0 +1,471 @@ +"""Tests for the unified compose system — enhanced Recipe, bridges, and discovery.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from openjarvis.recipes.loader import ( + Recipe, + discover_recipes, + load_recipe, + resolve_recipe, +) + +# -- Discrete recipe TOML ----------------------------------------------- + +DISCRETE_TOML = textwrap.dedent("""\ + [recipe] + name = "bench-agent" + kind = "discrete" + description = "A discrete agent for benchmarking" + version = "1.0.0" + + [intelligence] + model = "qwen3:8b" + quantization = "q4_K_M" + provider = "ollama" + + [engine] + key = "ollama" + + [agent] + type = "native_react" + max_turns = 20 + temperature = 0.2 + tools = ["shell_exec", "file_read", "think"] + system_prompt = "You are a benchmark agent." + + [learning] + routing = "heuristic" + agent = "none" + + [eval] + benchmarks = ["terminalbench", "gaia"] + backend = "jarvis-agent" + max_samples = 50 + judge_model = "gpt-4o" +""") + +# -- Operator recipe TOML ----------------------------------------------- + +OPERATOR_TOML = textwrap.dedent("""\ + [recipe] + name = "my-operator" + kind = "operator" + description = "A test operator" + version = "2.0.0" + + [intelligence] + model = "qwen3:8b" + + [engine] + key = "ollama" + + [agent] + type = "orchestrator" + max_turns = 15 + temperature = 0.3 + tools = ["web_search", "memory_store", "think"] + system_prompt = "You are a monitoring agent." + + [schedule] + type = "interval" + value = "600" + + [channels] + output = ["slack", "telegram"] + + [learning] + routing = "heuristic" +""") + +# -- Legacy operator TOML ----------------------------------------------- + +LEGACY_OPERATOR_TOML = textwrap.dedent("""\ + [operator] + name = "legacy-op" + description = "A legacy operator manifest" + version = "1.0.0" + + [operator.agent] + max_turns = 10 + temperature = 0.4 + tools = ["think", "web_search"] + system_prompt = "Legacy prompt." + + [operator.schedule] + type = "cron" + value = "0 */2 * * *" +""") + + +# ======================================================================== +# Recipe loading +# ======================================================================== + + +class TestLoadDiscreteRecipe: + def test_load_discrete_fields(self, tmp_path: Path) -> None: + p = tmp_path / "bench.toml" + p.write_text(DISCRETE_TOML) + r = load_recipe(p) + + assert r.name == "bench-agent" + assert r.kind == "discrete" + assert r.model == "qwen3:8b" + assert r.quantization == "q4_K_M" + assert r.provider == "ollama" + assert r.engine_key == "ollama" + assert r.agent_type == "native_react" + assert r.max_turns == 20 + assert r.temperature == pytest.approx(0.2) + assert r.tools == ["shell_exec", "file_read", "think"] + assert r.system_prompt == "You are a benchmark agent." + assert r.routing_policy == "heuristic" + assert r.eval_benchmarks == ["terminalbench", "gaia"] + assert r.eval_backend == "jarvis-agent" + assert r.eval_max_samples == 50 + assert r.eval_judge_model == "gpt-4o" + + def test_default_kind_is_discrete(self, tmp_path: Path) -> None: + p = tmp_path / "min.toml" + p.write_text('[recipe]\nname = "min"\n[intelligence]\nmodel = "x"\n') + r = load_recipe(p) + assert r.kind == "discrete" + + +class TestLoadOperatorRecipe: + def test_load_operator_fields(self, tmp_path: Path) -> None: + p = tmp_path / "op.toml" + p.write_text(OPERATOR_TOML) + r = load_recipe(p) + + assert r.name == "my-operator" + assert r.kind == "operator" + assert r.schedule_type == "interval" + assert r.schedule_value == "600" + assert r.channels == ["slack", "telegram"] + assert r.tools == ["web_search", "memory_store", "think"] + + def test_schedule_implies_operator_kind(self, tmp_path: Path) -> None: + toml = textwrap.dedent("""\ + [recipe] + name = "auto-op" + + [agent] + type = "orchestrator" + tools = ["think"] + + [schedule] + type = "cron" + value = "0 8 * * *" + """) + p = tmp_path / "auto.toml" + p.write_text(toml) + r = load_recipe(p) + assert r.kind == "operator" + + def test_external_prompt_file(self, tmp_path: Path) -> None: + prompt_file = tmp_path / "prompt.md" + prompt_file.write_text("External prompt content.") + + toml = textwrap.dedent(f"""\ + [recipe] + name = "ext-prompt" + + [agent] + type = "simple" + system_prompt_path = "{prompt_file}" + """) + p = tmp_path / "ext.toml" + p.write_text(toml) + r = load_recipe(p) + assert r.system_prompt == "External prompt content." + + +class TestLoadLegacyOperator: + def test_legacy_operator_converted_to_recipe(self, tmp_path: Path) -> None: + p = tmp_path / "legacy.toml" + p.write_text(LEGACY_OPERATOR_TOML) + r = load_recipe(p) + + assert r.kind == "operator" + assert r.name == "legacy-op" + assert r.max_turns == 10 + assert r.temperature == pytest.approx(0.4) + assert r.tools == ["think", "web_search"] + assert r.system_prompt == "Legacy prompt." + assert r.schedule_type == "cron" + assert r.schedule_value == "0 */2 * * *" + + +# ======================================================================== +# Discovery +# ======================================================================== + + +class TestDiscoverByKind: + def test_discover_all(self) -> None: + all_recipes = discover_recipes() + kinds = {r.kind for r in all_recipes} + assert "discrete" in kinds + assert len(all_recipes) >= 4 # at least original 3 + new ones + + def test_discover_discrete_only(self) -> None: + discrete = discover_recipes(kind="discrete") + for r in discrete: + assert r.kind == "discrete" + + def test_discover_operator_only(self) -> None: + operators = discover_recipes(kind="operator") + for r in operators: + assert r.kind == "operator" + + def test_discover_operators_subdir(self) -> None: + """Operator recipes in data/operators/ are discovered.""" + all_recipes = discover_recipes() + names = {r.name for r in all_recipes} + # The new unified operator recipes should be found + assert "twitter-sentinel" in names or "correspondent" in names + + def test_extra_dirs_discrete(self, tmp_path: Path) -> None: + p = tmp_path / "custom.toml" + p.write_text(DISCRETE_TOML) + recipes = discover_recipes(extra_dirs=[tmp_path], kind="discrete") + names = {r.name for r in recipes} + assert "bench-agent" in names + + def test_extra_dirs_operator(self, tmp_path: Path) -> None: + p = tmp_path / "custom_op.toml" + p.write_text(OPERATOR_TOML) + recipes = discover_recipes(extra_dirs=[tmp_path], kind="operator") + names = {r.name for r in recipes} + assert "my-operator" in names + + +# ======================================================================== +# Bridge: to_eval_suite +# ======================================================================== + + +class TestRecipeToEvalSuite: + def test_basic_eval_suite(self, tmp_path: Path) -> None: + p = tmp_path / "bench.toml" + p.write_text(DISCRETE_TOML) + r = load_recipe(p) + + suite = r.to_eval_suite() + + assert suite.meta.name == "bench-agent-eval" + assert len(suite.models) == 1 + assert suite.models[0].name == "qwen3:8b" + assert suite.models[0].engine == "ollama" + assert len(suite.benchmarks) == 2 + bench_names = {b.name for b in suite.benchmarks} + assert bench_names == {"terminalbench", "gaia"} + for b in suite.benchmarks: + assert b.backend == "jarvis-agent" + assert b.agent == "native_react" + assert b.tools == ["shell_exec", "file_read", "think"] + assert b.max_samples == 50 + assert b.judge_model == "gpt-4o" + + def test_eval_suite_benchmark_override(self, tmp_path: Path) -> None: + p = tmp_path / "bench.toml" + p.write_text(DISCRETE_TOML) + r = load_recipe(p) + + suite = r.to_eval_suite(benchmarks=["supergpqa"]) + + assert len(suite.benchmarks) == 1 + assert suite.benchmarks[0].name == "supergpqa" + + def test_eval_suite_max_samples_override(self, tmp_path: Path) -> None: + p = tmp_path / "bench.toml" + p.write_text(DISCRETE_TOML) + r = load_recipe(p) + + suite = r.to_eval_suite(max_samples=10) + + for b in suite.benchmarks: + assert b.max_samples == 10 + + def test_eval_suite_judge_override(self, tmp_path: Path) -> None: + p = tmp_path / "bench.toml" + p.write_text(DISCRETE_TOML) + r = load_recipe(p) + + suite = r.to_eval_suite(judge_model="gpt-5") + + assert suite.judge.model == "gpt-5" + for b in suite.benchmarks: + assert b.judge_model == "gpt-5" + + def test_eval_suite_no_model_raises(self) -> None: + r = Recipe(name="no-model", eval_benchmarks=["gaia"]) + with pytest.raises(ValueError, match="no model"): + r.to_eval_suite() + + def test_eval_suite_no_benchmarks_raises(self) -> None: + r = Recipe(name="no-bench", model="qwen3:8b") + with pytest.raises(ValueError, match="no benchmarks"): + r.to_eval_suite() + + def test_eval_suite_falls_back_to_suites(self) -> None: + r = Recipe( + name="suites-fallback", + model="qwen3:8b", + eval_suites=["coding"], + ) + suite = r.to_eval_suite() + assert len(suite.benchmarks) == 1 + assert suite.benchmarks[0].name == "coding" + + def test_eval_suite_direct_backend_when_no_agent(self) -> None: + r = Recipe( + name="direct", + model="qwen3:8b", + eval_benchmarks=["supergpqa"], + ) + suite = r.to_eval_suite() + assert suite.benchmarks[0].backend == "jarvis-direct" + assert suite.benchmarks[0].agent is None + assert suite.benchmarks[0].tools == [] + + +# ======================================================================== +# Bridge: to_operator_manifest +# ======================================================================== + + +class TestRecipeToOperatorManifest: + def test_basic_operator_manifest(self, tmp_path: Path) -> None: + p = tmp_path / "op.toml" + p.write_text(OPERATOR_TOML) + r = load_recipe(p) + + m = r.to_operator_manifest() + + assert m.id == "my-operator" + assert m.name == "my-operator" + assert m.version == "2.0.0" + assert m.description == "A test operator" + assert m.tools == ["web_search", "memory_store", "think"] + assert m.system_prompt == "You are a monitoring agent." + assert m.max_turns == 15 + assert m.temperature == pytest.approx(0.3) + assert m.schedule_type == "interval" + assert m.schedule_value == "600" + + def test_operator_no_schedule_raises(self) -> None: + r = Recipe(name="no-sched", kind="operator") + with pytest.raises(ValueError, match="no \\[schedule\\]"): + r.to_operator_manifest() + + def test_operator_defaults(self) -> None: + r = Recipe( + name="minimal-op", + kind="operator", + schedule_type="interval", + schedule_value="300", + ) + m = r.to_operator_manifest() + assert m.max_turns == 20 + assert m.temperature == pytest.approx(0.3) + assert m.schedule_value == "300" + + +# ======================================================================== +# Builder kwargs with new fields +# ======================================================================== + + +class TestBuilderKwargsNewFields: + def test_provider_in_kwargs(self) -> None: + r = Recipe(name="cloud", model="gpt-4o", provider="openai") + kw = r.to_builder_kwargs() + assert kw["provider"] == "openai" + + def test_system_prompt_path_resolved(self, tmp_path: Path) -> None: + prompt = tmp_path / "prompt.txt" + prompt.write_text("Hello from file.") + r = Recipe( + name="ext", + system_prompt_path=str(prompt), + ) + kw = r.to_builder_kwargs() + assert kw["system_prompt"] == "Hello from file." + + def test_schedule_and_channels_not_in_kwargs(self) -> None: + """Schedule/channel fields are operator-specific and not builder kwargs.""" + r = Recipe( + name="op", + kind="operator", + schedule_type="cron", + schedule_value="0 * * * *", + channels=["slack"], + ) + kw = r.to_builder_kwargs() + assert "schedule_type" not in kw + assert "channels" not in kw + + +# ======================================================================== +# Built-in recipe loading +# ======================================================================== + + +class TestBuiltinRecipes: + def test_terminalbench_react_loads(self) -> None: + r = resolve_recipe("terminalbench-react") + assert r is not None + assert r.kind == "discrete" + assert r.agent_type == "native_react" + assert "terminalbench" in r.eval_benchmarks + + def test_gaia_orchestrator_loads(self) -> None: + r = resolve_recipe("gaia-orchestrator") + assert r is not None + assert r.kind == "discrete" + assert r.agent_type == "orchestrator" + assert "gaia" in r.eval_benchmarks + + def test_swebench_openhands_loads(self) -> None: + r = resolve_recipe("swebench-openhands") + assert r is not None + assert r.kind == "discrete" + assert r.agent_type == "native_openhands" + + def test_coding_benchmark_loads(self) -> None: + r = resolve_recipe("coding-benchmark") + assert r is not None + assert r.kind == "discrete" + assert "terminalbench" in r.eval_benchmarks + assert "swebench" in r.eval_benchmarks + + def test_twitter_sentinel_loads(self) -> None: + r = resolve_recipe("twitter-sentinel") + assert r is not None + assert r.kind == "operator" + assert r.schedule_type == "interval" + + def test_inbox_triage_loads(self) -> None: + r = resolve_recipe("inbox-triage") + assert r is not None + assert r.kind == "operator" + assert r.schedule_type == "interval" + + def test_news_briefing_loads(self) -> None: + r = resolve_recipe("news-briefing") + assert r is not None + assert r.kind == "operator" + assert r.schedule_type == "cron" + + def test_repo_watcher_loads(self) -> None: + r = resolve_recipe("repo-watcher") + assert r is not None + assert r.kind == "operator" + assert r.schedule_type == "interval"