mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 10:52:15 +00:00
Merge pull request #10 from HazyResearch/composition
Add unified composition system
This commit is contained in:
Jon Saad-Falcon
@@ -11,6 +11,7 @@ from openjarvis.cli.ask import ask
|
||||
from openjarvis.cli.bench_cmd import bench
|
||||
from openjarvis.cli.channel_cmd import channel
|
||||
from openjarvis.cli.chat_cmd import chat
|
||||
from openjarvis.cli.compose_cmd import compose
|
||||
from openjarvis.cli.daemon_cmd import restart, start, status, stop
|
||||
from openjarvis.cli.doctor_cmd import doctor
|
||||
from openjarvis.cli.eval_cmd import eval_group
|
||||
@@ -71,6 +72,7 @@ cli.add_command(host, "host")
|
||||
cli.add_command(quickstart, "quickstart")
|
||||
cli.add_command(optimize_group, "optimize")
|
||||
cli.add_command(feedback_group, "feedback")
|
||||
cli.add_command(compose, "compose")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
"""``jarvis compose`` — unified composition CLI for discrete agents and operators."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
|
||||
@click.group()
|
||||
def compose() -> None:
|
||||
"""Compose, run, benchmark, and deploy OpenJarvis configurations.
|
||||
|
||||
Recipes are unified TOML configs that wire all five pillars
|
||||
(Intelligence, Engine, Agent, Tools, Learning). They come in two
|
||||
kinds:
|
||||
|
||||
\b
|
||||
discrete One-shot or benchmark-oriented agents
|
||||
operator Persistent, scheduled autonomous agents
|
||||
"""
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# jarvis compose list
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
@compose.command("list")
|
||||
@click.option(
|
||||
"-k", "--kind", "kind", default=None,
|
||||
type=click.Choice(["discrete", "operator"]),
|
||||
help="Filter by recipe kind.",
|
||||
)
|
||||
def compose_list(kind: Optional[str]) -> None:
|
||||
"""List all discovered compositions (recipes and operators)."""
|
||||
console = Console(stderr=True)
|
||||
try:
|
||||
from openjarvis.recipes.loader import discover_recipes
|
||||
|
||||
recipes = discover_recipes(kind=kind)
|
||||
if not recipes:
|
||||
console.print("[dim]No compositions found.[/dim]")
|
||||
console.print(
|
||||
"[dim]Place TOML recipes in src/openjarvis/recipes/data/ "
|
||||
"or ~/.openjarvis/recipes/[/dim]"
|
||||
)
|
||||
return
|
||||
|
||||
table = Table(title="Compositions", border_style="bright_blue")
|
||||
table.add_column("Name", style="cyan", no_wrap=True)
|
||||
table.add_column("Kind", style="yellow")
|
||||
table.add_column("Model", style="green")
|
||||
table.add_column("Agent", style="magenta")
|
||||
table.add_column("Tools", style="white")
|
||||
table.add_column("Description")
|
||||
|
||||
for r in sorted(recipes, key=lambda r: (r.kind, r.name)):
|
||||
tools_str = ", ".join(r.tools[:3])
|
||||
if len(r.tools) > 3:
|
||||
tools_str += f" (+{len(r.tools) - 3})"
|
||||
table.add_row(
|
||||
r.name,
|
||||
r.kind,
|
||||
r.model or "-",
|
||||
r.agent_type or "-",
|
||||
tools_str or "-",
|
||||
r.description[:60] if r.description else "",
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# jarvis compose show
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
@compose.command("show")
|
||||
@click.argument("name")
|
||||
def compose_show(name: str) -> None:
|
||||
"""Show detailed configuration of a composition."""
|
||||
console = Console(stderr=True)
|
||||
try:
|
||||
from openjarvis.recipes.loader import resolve_recipe
|
||||
|
||||
recipe = resolve_recipe(name)
|
||||
if recipe is None:
|
||||
console.print(f"[red]Composition not found: {name}[/red]")
|
||||
return
|
||||
|
||||
console.print(f"[bold cyan]{recipe.name}[/bold cyan] ({recipe.kind})")
|
||||
console.print(f" {recipe.description}\n")
|
||||
|
||||
# Intelligence
|
||||
console.print("[bold]Intelligence[/bold]")
|
||||
console.print(f" model: {recipe.model or '-'}")
|
||||
console.print(f" quantization: {recipe.quantization or '-'}")
|
||||
console.print(f" provider: {recipe.provider or '-'}")
|
||||
console.print()
|
||||
|
||||
# Engine
|
||||
console.print("[bold]Engine[/bold]")
|
||||
console.print(f" key: {recipe.engine_key or '-'}")
|
||||
console.print()
|
||||
|
||||
# Agent
|
||||
console.print("[bold]Agent[/bold]")
|
||||
console.print(f" type: {recipe.agent_type or '-'}")
|
||||
console.print(f" max_turns: {recipe.max_turns or '-'}")
|
||||
console.print(f" temperature: {recipe.temperature or '-'}")
|
||||
console.print(f" tools: {', '.join(recipe.tools) or '-'}")
|
||||
if recipe.system_prompt:
|
||||
preview = recipe.system_prompt[:120].replace("\n", " ")
|
||||
console.print(f" prompt: {preview}...")
|
||||
console.print()
|
||||
|
||||
# Learning
|
||||
console.print("[bold]Learning[/bold]")
|
||||
console.print(f" routing: {recipe.routing_policy or '-'}")
|
||||
console.print(f" agent: {recipe.agent_policy or '-'}")
|
||||
console.print()
|
||||
|
||||
# Kind-specific sections
|
||||
if recipe.kind == "discrete":
|
||||
benchmarks = recipe.eval_benchmarks or recipe.eval_suites
|
||||
if benchmarks:
|
||||
console.print("[bold]Eval[/bold]")
|
||||
console.print(f" benchmarks: {', '.join(benchmarks)}")
|
||||
console.print(f" backend: {recipe.eval_backend or 'auto'}")
|
||||
console.print(f" max_samples: {recipe.eval_max_samples or 'all'}")
|
||||
console.print(f" judge_model: {recipe.eval_judge_model or 'default'}")
|
||||
elif recipe.kind == "operator":
|
||||
if recipe.schedule_type:
|
||||
console.print("[bold]Schedule[/bold]")
|
||||
console.print(f" type: {recipe.schedule_type}")
|
||||
console.print(f" value: {recipe.schedule_value}")
|
||||
if recipe.channels:
|
||||
console.print("[bold]Channels[/bold]")
|
||||
console.print(f" output: {', '.join(recipe.channels)}")
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# jarvis compose run
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
@compose.command("run")
|
||||
@click.argument("name")
|
||||
@click.argument("query", nargs=-1, required=True)
|
||||
@click.option("--json", "output_json", is_flag=True, help="Output raw JSON result.")
|
||||
def compose_run(name: str, query: tuple[str, ...], output_json: bool) -> None:
|
||||
"""Run a composition against a single query."""
|
||||
console = Console(stderr=True)
|
||||
query_text = " ".join(query)
|
||||
|
||||
try:
|
||||
from openjarvis.recipes.loader import resolve_recipe
|
||||
|
||||
recipe = resolve_recipe(name)
|
||||
if recipe is None:
|
||||
console.print(f"[red]Composition not found: {name}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
kwargs = recipe.to_builder_kwargs()
|
||||
console.print(
|
||||
f"[dim]Running [cyan]{recipe.name}[/cyan] "
|
||||
f"({recipe.agent_type or 'direct'} / "
|
||||
f"{recipe.model or 'default'})...[/dim]"
|
||||
)
|
||||
|
||||
from openjarvis.system import SystemBuilder
|
||||
|
||||
builder = SystemBuilder()
|
||||
if "engine_key" in kwargs:
|
||||
builder = builder.engine(kwargs["engine_key"])
|
||||
if "model" in kwargs:
|
||||
builder = builder.model(kwargs["model"])
|
||||
if "agent" in kwargs:
|
||||
builder = builder.agent(kwargs["agent"])
|
||||
if "tools" in kwargs:
|
||||
builder = builder.tools(kwargs["tools"])
|
||||
|
||||
system = builder.build()
|
||||
|
||||
try:
|
||||
agent_kwargs = {}
|
||||
if kwargs.get("system_prompt"):
|
||||
agent_kwargs["system_prompt"] = kwargs["system_prompt"]
|
||||
if kwargs.get("max_turns"):
|
||||
agent_kwargs["max_turns"] = kwargs["max_turns"]
|
||||
if kwargs.get("temperature"):
|
||||
agent_kwargs["temperature"] = kwargs["temperature"]
|
||||
|
||||
result = system.ask(query_text, **agent_kwargs)
|
||||
|
||||
if output_json:
|
||||
import json as json_mod
|
||||
if isinstance(result, str):
|
||||
click.echo(json_mod.dumps({"content": result}, indent=2))
|
||||
else:
|
||||
click.echo(json_mod.dumps({
|
||||
"content": result.content,
|
||||
"turns": getattr(result, "turns", 1),
|
||||
}, indent=2))
|
||||
else:
|
||||
content = result if isinstance(result, str) else result.content
|
||||
click.echo(content)
|
||||
finally:
|
||||
system.close()
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# jarvis compose bench
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
@compose.command("bench")
|
||||
@click.argument("name")
|
||||
@click.option(
|
||||
"-b", "--benchmark", "benchmark", default=None, multiple=True,
|
||||
help="Override benchmarks (can specify multiple).",
|
||||
)
|
||||
@click.option(
|
||||
"-n", "--max-samples", "max_samples", type=int, default=None,
|
||||
help="Maximum samples per benchmark.",
|
||||
)
|
||||
@click.option(
|
||||
"--judge", "judge_model", default=None,
|
||||
help="LLM judge model override.",
|
||||
)
|
||||
@click.option(
|
||||
"-v", "--verbose", "verbose", is_flag=True, default=False,
|
||||
help="Verbose logging.",
|
||||
)
|
||||
def compose_bench(
|
||||
name: str,
|
||||
benchmark: tuple[str, ...],
|
||||
max_samples: Optional[int],
|
||||
judge_model: Optional[str],
|
||||
verbose: bool,
|
||||
) -> None:
|
||||
"""Benchmark a discrete composition against eval datasets.
|
||||
|
||||
Uses the recipe's model, engine, agent, and tools to run the eval
|
||||
framework against the benchmarks defined in the recipe (or overridden
|
||||
via --benchmark).
|
||||
"""
|
||||
console = Console(stderr=True)
|
||||
|
||||
try:
|
||||
from openjarvis.recipes.loader import resolve_recipe
|
||||
|
||||
recipe = resolve_recipe(name)
|
||||
if recipe is None:
|
||||
console.print(f"[red]Composition not found: {name}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
benchmarks_list = list(benchmark) if benchmark else None
|
||||
suite = recipe.to_eval_suite(
|
||||
benchmarks=benchmarks_list,
|
||||
max_samples=max_samples,
|
||||
judge_model=judge_model,
|
||||
)
|
||||
|
||||
from openjarvis.evals.core.config import expand_suite
|
||||
|
||||
run_configs = expand_suite(suite)
|
||||
|
||||
console.print(
|
||||
f"[cyan]Composition:[/cyan] {recipe.name}\n"
|
||||
f"[cyan]Model:[/cyan] {recipe.model}\n"
|
||||
f"[cyan]Agent:[/cyan] {recipe.agent_type or 'direct'}\n"
|
||||
f"[cyan]Tools:[/cyan] {', '.join(recipe.tools) or 'none'}\n"
|
||||
f"[cyan]Benchmarks:[/cyan] {len(run_configs)} run(s)"
|
||||
)
|
||||
|
||||
try:
|
||||
from openjarvis.evals.cli import _run_single
|
||||
except ImportError:
|
||||
console.print("[red]Eval CLI module not available.[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
results_table = Table(
|
||||
title="Benchmark Results",
|
||||
border_style="bright_blue",
|
||||
title_style="bold cyan",
|
||||
)
|
||||
results_table.add_column("Benchmark", style="cyan")
|
||||
results_table.add_column("Accuracy", justify="right", style="bold green")
|
||||
results_table.add_column("Correct / Scored", justify="right")
|
||||
results_table.add_column("Errors", justify="right", style="red")
|
||||
|
||||
for i, rc in enumerate(run_configs, 1):
|
||||
console.print(
|
||||
f"\n[bold]Run {i}/{len(run_configs)}:[/bold] {rc.benchmark}"
|
||||
)
|
||||
try:
|
||||
summary = _run_single(rc, console=console)
|
||||
results_table.add_row(
|
||||
rc.benchmark,
|
||||
f"{summary.accuracy:.4f}",
|
||||
f"{summary.correct}/{summary.scored_samples}",
|
||||
str(summary.errors),
|
||||
)
|
||||
except Exception as exc:
|
||||
console.print(f" [red bold]FAILED:[/red bold] {exc}")
|
||||
results_table.add_row(rc.benchmark, "-", "-", str(exc)[:40])
|
||||
|
||||
console.print()
|
||||
console.print(results_table)
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# jarvis compose deploy
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
@compose.command("deploy")
|
||||
@click.argument("name")
|
||||
def compose_deploy(name: str) -> None:
|
||||
"""Deploy an operator composition (activate its scheduler task)."""
|
||||
console = Console(stderr=True)
|
||||
|
||||
try:
|
||||
from openjarvis.recipes.loader import resolve_recipe
|
||||
|
||||
recipe = resolve_recipe(name)
|
||||
if recipe is None:
|
||||
console.print(f"[red]Composition not found: {name}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
if recipe.kind != "operator":
|
||||
console.print(
|
||||
f"[red]Recipe '{name}' is a {recipe.kind} composition, "
|
||||
f"not an operator. Only operators can be deployed.[/red]"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
manifest = recipe.to_operator_manifest()
|
||||
|
||||
from openjarvis.operators.manager import OperatorManager
|
||||
from openjarvis.system import SystemBuilder
|
||||
|
||||
system = SystemBuilder().scheduler(True).sessions(True).build()
|
||||
manager = OperatorManager(system)
|
||||
system.operator_manager = manager
|
||||
|
||||
manager.register(manifest)
|
||||
task_id = manager.activate(manifest.id)
|
||||
|
||||
console.print(
|
||||
f"[green]Deployed operator [cyan]{name}[/cyan] "
|
||||
f"(task: {task_id}, schedule: "
|
||||
f"{recipe.schedule_type}={recipe.schedule_value})[/green]"
|
||||
)
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# jarvis compose stop
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
@compose.command("stop")
|
||||
@click.argument("name")
|
||||
def compose_stop(name: str) -> None:
|
||||
"""Stop a deployed operator composition."""
|
||||
console = Console(stderr=True)
|
||||
|
||||
try:
|
||||
from openjarvis.operators.manager import OperatorManager
|
||||
from openjarvis.system import SystemBuilder
|
||||
|
||||
system = SystemBuilder().scheduler(True).sessions(True).build()
|
||||
manager = OperatorManager(system)
|
||||
system.operator_manager = manager
|
||||
|
||||
# Discover all known operators so the manager knows about them
|
||||
from openjarvis.core.config import DEFAULT_CONFIG_DIR
|
||||
from openjarvis.recipes.loader import _PROJECT_OPERATORS_DIR
|
||||
|
||||
for d in [DEFAULT_CONFIG_DIR / "operators", _PROJECT_OPERATORS_DIR]:
|
||||
if d.is_dir():
|
||||
manager.discover(d)
|
||||
|
||||
manager.deactivate(name)
|
||||
console.print(f"[yellow]Stopped operator {name}[/yellow]")
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# jarvis compose status
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
@compose.command("status")
|
||||
def compose_status() -> None:
|
||||
"""Show status of all deployed operators."""
|
||||
console = Console(stderr=True)
|
||||
|
||||
try:
|
||||
from openjarvis.operators.manager import OperatorManager
|
||||
from openjarvis.system import SystemBuilder
|
||||
|
||||
system = SystemBuilder().scheduler(True).sessions(True).build()
|
||||
manager = OperatorManager(system)
|
||||
system.operator_manager = manager
|
||||
|
||||
from openjarvis.core.config import DEFAULT_CONFIG_DIR
|
||||
from openjarvis.recipes.loader import _PROJECT_OPERATORS_DIR
|
||||
|
||||
for d in [DEFAULT_CONFIG_DIR / "operators", _PROJECT_OPERATORS_DIR]:
|
||||
if d.is_dir():
|
||||
manager.discover(d)
|
||||
|
||||
statuses = manager.status()
|
||||
if not statuses:
|
||||
console.print("[dim]No operators registered.[/dim]")
|
||||
return
|
||||
|
||||
table = Table(title="Operator Status", border_style="bright_blue")
|
||||
table.add_column("Name", style="cyan")
|
||||
table.add_column("State", style="yellow")
|
||||
table.add_column("Schedule", style="white")
|
||||
table.add_column("Last Run", style="dim")
|
||||
|
||||
for s in statuses:
|
||||
table.add_row(
|
||||
s.get("id", "?"),
|
||||
s.get("state", "unknown"),
|
||||
s.get("schedule", ""),
|
||||
s.get("last_run", "-"),
|
||||
)
|
||||
console.print(table)
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
|
||||
|
||||
__all__ = ["compose"]
|
||||
@@ -1,5 +1,9 @@
|
||||
"""Recipe system — composable pillar configurations."""
|
||||
|
||||
from openjarvis.recipes.composer import (
|
||||
recipe_to_eval_suite,
|
||||
recipe_to_operator,
|
||||
)
|
||||
from openjarvis.recipes.loader import (
|
||||
Recipe,
|
||||
discover_recipes,
|
||||
@@ -7,4 +11,11 @@ from openjarvis.recipes.loader import (
|
||||
resolve_recipe,
|
||||
)
|
||||
|
||||
__all__ = ["Recipe", "discover_recipes", "load_recipe", "resolve_recipe"]
|
||||
__all__ = [
|
||||
"Recipe",
|
||||
"discover_recipes",
|
||||
"load_recipe",
|
||||
"recipe_to_eval_suite",
|
||||
"recipe_to_operator",
|
||||
"resolve_recipe",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Composer bridges — convert a Recipe into EvalSuiteConfig or OperatorManifest.
|
||||
|
||||
These are pure-function transformations that let the unified Recipe format
|
||||
drive both the eval framework and the operator system without those systems
|
||||
needing to know about recipes directly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openjarvis.evals.core.types import EvalSuiteConfig
|
||||
from openjarvis.operators.types import OperatorManifest
|
||||
from openjarvis.recipes.loader import Recipe
|
||||
|
||||
|
||||
def recipe_to_eval_suite(
|
||||
recipe: Recipe,
|
||||
benchmarks: Optional[List[str]] = None,
|
||||
max_samples: Optional[int] = None,
|
||||
judge_model: Optional[str] = None,
|
||||
) -> EvalSuiteConfig:
|
||||
"""Build an ``EvalSuiteConfig`` from a recipe.
|
||||
|
||||
The recipe's model / engine become the single ``[[models]]`` entry.
|
||||
The recipe's ``eval_benchmarks`` (or the *benchmarks* override) become
|
||||
``[[benchmarks]]`` entries. Agent type and tools are inherited so the
|
||||
eval runner constructs the right backend automatically.
|
||||
|
||||
Args:
|
||||
recipe: Source recipe.
|
||||
benchmarks: Override benchmark list (defaults to ``recipe.eval_benchmarks``).
|
||||
max_samples: Override per-benchmark sample cap.
|
||||
judge_model: Override LLM judge model.
|
||||
|
||||
Raises:
|
||||
ValueError: If no model or benchmarks can be resolved.
|
||||
"""
|
||||
from openjarvis.evals.core.types import (
|
||||
BenchmarkConfig,
|
||||
DefaultsConfig,
|
||||
EvalSuiteConfig,
|
||||
ExecutionConfig,
|
||||
JudgeConfig,
|
||||
MetaConfig,
|
||||
ModelConfig,
|
||||
)
|
||||
|
||||
bench_names = benchmarks or list(recipe.eval_benchmarks)
|
||||
if not bench_names:
|
||||
bench_names = list(recipe.eval_suites)
|
||||
if not bench_names:
|
||||
raise ValueError(
|
||||
f"Recipe '{recipe.name}' has no benchmarks defined and none were "
|
||||
"provided. Set [eval] benchmarks in the TOML or pass benchmarks=."
|
||||
)
|
||||
|
||||
model_name = recipe.model
|
||||
if not model_name:
|
||||
raise ValueError(
|
||||
f"Recipe '{recipe.name}' has no model defined. "
|
||||
"Set [intelligence] model in the TOML."
|
||||
)
|
||||
|
||||
has_agent = recipe.agent_type is not None
|
||||
backend = recipe.eval_backend or ("jarvis-agent" if has_agent else "jarvis-direct")
|
||||
|
||||
model_cfg = ModelConfig(
|
||||
name=model_name,
|
||||
engine=recipe.engine_key,
|
||||
provider=recipe.provider,
|
||||
temperature=recipe.temperature,
|
||||
)
|
||||
|
||||
bench_cfgs: list[BenchmarkConfig] = []
|
||||
for bname in bench_names:
|
||||
bench_cfgs.append(BenchmarkConfig(
|
||||
name=bname,
|
||||
backend=backend,
|
||||
max_samples=max_samples or recipe.eval_max_samples,
|
||||
agent=recipe.agent_type if has_agent else None,
|
||||
tools=list(recipe.tools) if has_agent else [],
|
||||
judge_model=judge_model or recipe.eval_judge_model,
|
||||
))
|
||||
|
||||
return EvalSuiteConfig(
|
||||
meta=MetaConfig(
|
||||
name=f"{recipe.name}-eval",
|
||||
description=f"Auto-generated eval suite from recipe '{recipe.name}'",
|
||||
),
|
||||
defaults=DefaultsConfig(
|
||||
temperature=recipe.temperature or 0.0,
|
||||
max_tokens=2048,
|
||||
),
|
||||
judge=JudgeConfig(
|
||||
model=judge_model or recipe.eval_judge_model or "gpt-5-mini-2025-08-07",
|
||||
),
|
||||
run=ExecutionConfig(),
|
||||
models=[model_cfg],
|
||||
benchmarks=bench_cfgs,
|
||||
)
|
||||
|
||||
|
||||
def recipe_to_operator(recipe: Recipe) -> OperatorManifest:
|
||||
"""Build an ``OperatorManifest`` from a recipe.
|
||||
|
||||
Maps the recipe's agent, schedule, and channel fields into the
|
||||
operator manifest format used by ``OperatorManager``.
|
||||
|
||||
Raises:
|
||||
ValueError: If schedule information is missing.
|
||||
"""
|
||||
from openjarvis.operators.types import OperatorManifest
|
||||
|
||||
if not recipe.schedule_type:
|
||||
raise ValueError(
|
||||
f"Recipe '{recipe.name}' has no [schedule] section. "
|
||||
"Operator recipes must define schedule_type and schedule_value."
|
||||
)
|
||||
|
||||
prompt = recipe.system_prompt or ""
|
||||
prompt_path = recipe.system_prompt_path or ""
|
||||
|
||||
return OperatorManifest(
|
||||
id=recipe.name,
|
||||
name=recipe.name,
|
||||
version=recipe.version,
|
||||
description=recipe.description,
|
||||
tools=list(recipe.tools),
|
||||
system_prompt=prompt,
|
||||
system_prompt_path=prompt_path,
|
||||
max_turns=recipe.max_turns or 20,
|
||||
temperature=recipe.temperature or 0.3,
|
||||
schedule_type=recipe.schedule_type,
|
||||
schedule_value=recipe.schedule_value or "300",
|
||||
required_capabilities=list(recipe.required_capabilities),
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["recipe_to_eval_suite", "recipe_to_operator"]
|
||||
@@ -0,0 +1,36 @@
|
||||
[recipe]
|
||||
name = "coding-benchmark"
|
||||
kind = "discrete"
|
||||
description = "ReAct coding agent benchmarked against TerminalBench and SWE-bench"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
quantization = "q4_K_M"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "native_react"
|
||||
max_turns = 20
|
||||
temperature = 0.2
|
||||
tools = ["file_read", "file_write", "shell_exec", "code_interpreter", "apply_patch", "think"]
|
||||
system_prompt = """You are an expert programmer and systems engineer. You solve coding tasks by reasoning through problems step-by-step, reading code, writing solutions, and verifying them with tests.
|
||||
|
||||
Approach each task methodically:
|
||||
1. Understand what is being asked
|
||||
2. Explore relevant files and code
|
||||
3. Plan your solution
|
||||
4. Implement changes carefully
|
||||
5. Test and verify your work
|
||||
|
||||
Use the tools available to read files, write code, execute commands, and apply patches. Always verify that your changes work correctly before concluding."""
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
|
||||
[eval]
|
||||
benchmarks = ["terminalbench", "swebench"]
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
@@ -0,0 +1,35 @@
|
||||
[recipe]
|
||||
name = "gaia-orchestrator"
|
||||
kind = "discrete"
|
||||
description = "Orchestrator agent with web search and memory for GAIA agentic benchmark"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "orchestrator"
|
||||
max_turns = 15
|
||||
temperature = 0.3
|
||||
tools = ["web_search", "http_request", "memory_store", "memory_search", "think", "calculator", "file_read"]
|
||||
system_prompt = """You are a highly capable research assistant that solves complex, multi-step questions requiring web research, reasoning, and tool use.
|
||||
|
||||
When answering questions:
|
||||
1. Break complex questions into sub-tasks
|
||||
2. Use web search to find relevant information
|
||||
3. Store important findings in memory for reference
|
||||
4. Cross-reference multiple sources when possible
|
||||
5. Provide precise, well-supported answers
|
||||
|
||||
Always think step-by-step before acting. Verify your answers against the evidence you've gathered."""
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
|
||||
[eval]
|
||||
benchmarks = ["gaia"]
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 100
|
||||
@@ -0,0 +1,57 @@
|
||||
[recipe]
|
||||
name = "inbox-triage"
|
||||
kind = "operator"
|
||||
description = "Monitors Slack and Gmail for important messages, classifies by urgency, and drafts responses"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "orchestrator"
|
||||
max_turns = 15
|
||||
temperature = 0.4
|
||||
tools = ["memory_store", "memory_search", "think", "http_request"]
|
||||
system_prompt = """You are an inbox triage agent. Your job is to periodically check messaging channels (Slack, email) for important messages, classify them by urgency, and produce a prioritized summary with optional draft responses.
|
||||
|
||||
Each triage cycle:
|
||||
|
||||
1. GATHER — Check all configured messaging sources for new or unread messages since the last cycle.
|
||||
|
||||
2. CLASSIFY — For each message, determine:
|
||||
- Urgency: CRITICAL / HIGH / MEDIUM / LOW / FYI
|
||||
- Category: action-required / decision-needed / informational / social / automated
|
||||
- Sender importance: direct-report / peer / manager / external / system
|
||||
|
||||
3. PRIORITIZE — Rank messages by combined urgency and sender importance. Group related threads.
|
||||
|
||||
4. DRAFT — For CRITICAL and HIGH urgency action-required messages, draft a brief response or suggest next steps.
|
||||
|
||||
5. SUMMARIZE — Produce a triage digest:
|
||||
|
||||
CRITICAL (act now):
|
||||
- [source] sender: subject — <summary> | Suggested action: <action>
|
||||
|
||||
HIGH (act today):
|
||||
- [source] sender: subject — <summary>
|
||||
|
||||
MEDIUM (this week):
|
||||
- [source] sender: subject — <summary>
|
||||
|
||||
LOW/FYI (when convenient):
|
||||
- <count> messages from <sources>
|
||||
|
||||
Store message metadata in memory to track response patterns and avoid re-triaging."""
|
||||
|
||||
[schedule]
|
||||
type = "interval"
|
||||
value = "300"
|
||||
|
||||
[channels]
|
||||
output = ["slack"]
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
@@ -0,0 +1,57 @@
|
||||
[recipe]
|
||||
name = "news-briefing"
|
||||
kind = "operator"
|
||||
description = "Daily news digest — searches configured topics, synthesizes key stories, and delivers a morning briefing"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "orchestrator"
|
||||
max_turns = 20
|
||||
temperature = 0.4
|
||||
tools = ["web_search", "http_request", "memory_store", "memory_search", "think"]
|
||||
system_prompt = """You are a news briefing agent. Each cycle, you research configured topics across multiple sources and produce a concise, well-structured news digest.
|
||||
|
||||
Briefing workflow:
|
||||
|
||||
1. RESEARCH — Search the web for each configured topic. Cover multiple angles:
|
||||
- Major news outlets for headline stories
|
||||
- Industry/trade sources for domain-specific developments
|
||||
- Social media for emerging trends and public reaction
|
||||
|
||||
2. CROSS-REFERENCE — Verify key claims across at least 2 sources. Flag single-source stories as unconfirmed.
|
||||
|
||||
3. CONTEXTUALIZE — Check memory for related previous stories. Note developments, trend changes, and follow-ups to earlier news.
|
||||
|
||||
4. SYNTHESIZE — Write a briefing with these sections:
|
||||
|
||||
TOP STORIES (3-5 most important)
|
||||
Each: headline, 2-3 sentence summary, significance, sources
|
||||
|
||||
SECTOR UPDATES (grouped by topic)
|
||||
Each: brief summary with source
|
||||
|
||||
TRENDS & SIGNALS
|
||||
Emerging patterns, sentiment shifts, early indicators
|
||||
|
||||
FOLLOW-UPS
|
||||
Updates on previously reported stories (from memory)
|
||||
|
||||
5. STORE — Save today's briefing to memory for future context and trend tracking.
|
||||
|
||||
Write in a clear, journalistic style. Lead with the most consequential stories. Be factual and cite sources."""
|
||||
|
||||
[schedule]
|
||||
type = "cron"
|
||||
value = "0 8 * * *"
|
||||
|
||||
[channels]
|
||||
output = ["slack"]
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
@@ -0,0 +1,64 @@
|
||||
[recipe]
|
||||
name = "repo-watcher"
|
||||
kind = "operator"
|
||||
description = "Monitors GitHub repositories for new issues, PRs, and releases — summarizes activity with priority scoring"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "orchestrator"
|
||||
max_turns = 15
|
||||
temperature = 0.3
|
||||
tools = ["web_search", "http_request", "memory_store", "memory_search", "think"]
|
||||
system_prompt = """You are a GitHub repository monitoring agent. You track configured repositories for important activity and produce structured reports.
|
||||
|
||||
Each monitoring cycle:
|
||||
|
||||
1. CHECK — Query the GitHub API for recent activity across configured repositories:
|
||||
- New issues (opened since last check)
|
||||
- New/updated pull requests
|
||||
- New releases and tags
|
||||
- Notable discussions and comments
|
||||
|
||||
2. ANALYZE — For each item, assess:
|
||||
- Priority: P0 (critical bug/security) / P1 (important) / P2 (normal) / P3 (low)
|
||||
- Type: bug / feature / enhancement / docs / ci / security
|
||||
- Impact: breaking / significant / minor / cosmetic
|
||||
|
||||
3. CORRELATE — Check memory for related items. Link PRs to issues. Note patterns (recurring bugs, stale PRs, active contributors).
|
||||
|
||||
4. REPORT — Produce a structured activity report:
|
||||
|
||||
ATTENTION NEEDED
|
||||
- Security advisories, critical bugs, breaking changes
|
||||
|
||||
NEW ISSUES (by priority)
|
||||
- [repo] #number: title — labels, assignee
|
||||
|
||||
PULL REQUESTS
|
||||
- [repo] #number: title — status (draft/review/approved/merged), author
|
||||
|
||||
RELEASES
|
||||
- [repo] vX.Y.Z: key changes summary
|
||||
|
||||
TRENDS
|
||||
- Activity level, response times, contributor patterns
|
||||
|
||||
5. STORE — Save current state to memory for next-cycle comparison.
|
||||
|
||||
Be concise. Focus on actionable items. Skip routine CI/bot activity."""
|
||||
|
||||
[schedule]
|
||||
type = "interval"
|
||||
value = "1800"
|
||||
|
||||
[channels]
|
||||
output = ["slack"]
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
@@ -0,0 +1,50 @@
|
||||
[recipe]
|
||||
name = "twitter-sentinel"
|
||||
kind = "operator"
|
||||
description = "Monitors Twitter/X for specified keywords and surfaces relevant posts with urgency scoring"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "orchestrator"
|
||||
max_turns = 15
|
||||
temperature = 0.3
|
||||
tools = ["web_search", "http_request", "memory_store", "memory_search", "think"]
|
||||
system_prompt = """You are a social media monitoring agent focused on Twitter/X. Your job is to periodically search for tweets matching specific keywords and topics, assess their relevance and urgency, and produce structured alerts.
|
||||
|
||||
Each monitoring cycle:
|
||||
|
||||
1. SCAN — Search Twitter/X for the configured keywords and topics using web search. Look for recent posts, trending discussions, and notable accounts.
|
||||
|
||||
2. FILTER — Evaluate each result for relevance. Discard noise, spam, and off-topic content. Focus on substantive posts with real signal.
|
||||
|
||||
3. SCORE — Rate each relevant finding on urgency (1-5):
|
||||
- 5: Breaking/critical (requires immediate attention)
|
||||
- 4: Important (notable development, should be seen today)
|
||||
- 3: Interesting (worth tracking, no rush)
|
||||
- 2: Background (context, low priority)
|
||||
- 1: Marginal (barely relevant)
|
||||
|
||||
4. STORE — Save significant findings to memory with urgency score, source URL, timestamp, and a brief summary.
|
||||
|
||||
5. REPORT — Produce a structured digest of findings sorted by urgency. Include only items scoring 3 or higher unless it's a quiet period.
|
||||
|
||||
Output format for each finding:
|
||||
[URGENCY X/5] <topic> — <one-line summary>
|
||||
Source: <URL or account>
|
||||
Context: <2-3 sentences of analysis>"""
|
||||
|
||||
[schedule]
|
||||
type = "interval"
|
||||
value = "600"
|
||||
|
||||
[channels]
|
||||
output = ["slack"]
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
@@ -0,0 +1,41 @@
|
||||
[recipe]
|
||||
name = "swebench-openhands"
|
||||
kind = "discrete"
|
||||
description = "CodeAct agent (OpenHands-style) for SWE-bench software engineering tasks"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
quantization = "q4_K_M"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "native_openhands"
|
||||
max_turns = 25
|
||||
temperature = 0.2
|
||||
tools = ["file_read", "file_write", "shell_exec", "apply_patch", "think", "code_interpreter"]
|
||||
system_prompt = """You are an expert software engineer working on real-world GitHub issues. You use the CodeAct approach: think carefully, then act by writing and executing code.
|
||||
|
||||
Your workflow:
|
||||
1. Read the issue description and understand what needs to be fixed
|
||||
2. Explore the repository structure to find relevant files
|
||||
3. Read the relevant source code to understand the problem
|
||||
4. Write a fix and apply it
|
||||
5. Run tests to verify the fix works
|
||||
6. Iterate if tests fail
|
||||
|
||||
Key principles:
|
||||
- Always read the failing test or error output carefully
|
||||
- Make minimal, targeted changes
|
||||
- Don't break existing functionality
|
||||
- Write clean, idiomatic code matching the project's style"""
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
|
||||
[eval]
|
||||
benchmarks = ["swebench"]
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
@@ -0,0 +1,35 @@
|
||||
[recipe]
|
||||
name = "terminalbench-react"
|
||||
kind = "discrete"
|
||||
description = "ReAct agent tuned for terminal and shell tasks — benchmarks against TerminalBench"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
quantization = "q4_K_M"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "native_react"
|
||||
max_turns = 20
|
||||
temperature = 0.2
|
||||
tools = ["shell_exec", "file_read", "file_write", "think"]
|
||||
system_prompt = """You are an expert at terminal and shell operations. You solve tasks by reasoning step-by-step and executing shell commands.
|
||||
|
||||
When given a task:
|
||||
1. Analyze what needs to be done
|
||||
2. Plan your approach
|
||||
3. Execute commands one at a time, observing the output
|
||||
4. Verify the result before concluding
|
||||
|
||||
You have access to a full Linux shell environment. Use standard Unix tools (grep, sed, awk, find, etc.) and scripting when appropriate. Always check command exit codes and handle errors gracefully."""
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
|
||||
[eval]
|
||||
benchmarks = ["terminalbench"]
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
@@ -1,4 +1,12 @@
|
||||
"""Recipe loader — load and resolve TOML recipe files."""
|
||||
"""Recipe loader — load and resolve TOML recipe files.
|
||||
|
||||
Recipes are the universal composition format for OpenJarvis. Each recipe
|
||||
specifies all five pillars (Intelligence, Engine, Agent, Tools, Learning)
|
||||
and carries a ``kind`` that determines its lifecycle:
|
||||
|
||||
* ``"discrete"`` — one-shot or benchmark-oriented agents
|
||||
* ``"operator"`` — persistent, scheduled agents
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -14,21 +22,29 @@ except ModuleNotFoundError:
|
||||
|
||||
# Built-in recipes directory (package data)
|
||||
_PROJECT_RECIPES_DIR = Path(__file__).resolve().parent / "data"
|
||||
# User-level recipes directory
|
||||
_PROJECT_OPERATORS_DIR = _PROJECT_RECIPES_DIR / "operators"
|
||||
# User-level directories
|
||||
_USER_RECIPES_DIR = Path.home() / ".openjarvis" / "recipes"
|
||||
_USER_OPERATORS_DIR = Path.home() / ".openjarvis" / "operators"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Recipe:
|
||||
"""A composable pillar configuration loaded from TOML."""
|
||||
"""A composable pillar configuration loaded from TOML.
|
||||
|
||||
Covers both *discrete* agents (benchmarking / one-shot) and *operator*
|
||||
agents (persistent / scheduled) through the ``kind`` field.
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str = ""
|
||||
version: str = "1.0.0"
|
||||
kind: str = "discrete" # "discrete" | "operator"
|
||||
|
||||
# Intelligence
|
||||
model: Optional[str] = None
|
||||
quantization: Optional[str] = None
|
||||
provider: Optional[str] = None
|
||||
|
||||
# Engine
|
||||
engine_key: Optional[str] = None
|
||||
@@ -39,17 +55,36 @@ class Recipe:
|
||||
temperature: Optional[float] = None
|
||||
tools: List[str] = field(default_factory=list)
|
||||
system_prompt: Optional[str] = None
|
||||
system_prompt_path: Optional[str] = None
|
||||
|
||||
# Learning
|
||||
routing_policy: Optional[str] = None
|
||||
agent_policy: Optional[str] = None
|
||||
|
||||
# Eval
|
||||
# Eval (discrete agents)
|
||||
eval_suites: List[str] = field(default_factory=list)
|
||||
eval_benchmarks: List[str] = field(default_factory=list)
|
||||
eval_backend: Optional[str] = None
|
||||
eval_max_samples: Optional[int] = None
|
||||
eval_judge_model: Optional[str] = None
|
||||
|
||||
# Schedule (operators)
|
||||
schedule_type: Optional[str] = None
|
||||
schedule_value: Optional[str] = None
|
||||
|
||||
# Channels (operators)
|
||||
channels: List[str] = field(default_factory=list)
|
||||
|
||||
# Security
|
||||
required_capabilities: List[str] = field(default_factory=list)
|
||||
|
||||
# Raw TOML data for forward-compat
|
||||
raw: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Conversion helpers
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def to_builder_kwargs(self) -> Dict[str, Any]:
|
||||
"""Convert recipe fields to kwargs for SystemBuilder/Jarvis.
|
||||
|
||||
@@ -69,49 +104,65 @@ class Recipe:
|
||||
kwargs["temperature"] = self.temperature
|
||||
if self.max_turns is not None:
|
||||
kwargs["max_turns"] = self.max_turns
|
||||
if self.system_prompt is not None:
|
||||
kwargs["system_prompt"] = self.system_prompt
|
||||
prompt = self.system_prompt
|
||||
if prompt is None and self.system_prompt_path is not None:
|
||||
p = Path(self.system_prompt_path)
|
||||
if p.exists():
|
||||
prompt = p.read_text(encoding="utf-8")
|
||||
if prompt is not None:
|
||||
kwargs["system_prompt"] = prompt
|
||||
if self.routing_policy is not None:
|
||||
kwargs["routing_policy"] = self.routing_policy
|
||||
if self.agent_policy is not None:
|
||||
kwargs["agent_policy"] = self.agent_policy
|
||||
if self.quantization is not None:
|
||||
kwargs["quantization"] = self.quantization
|
||||
if self.provider is not None:
|
||||
kwargs["provider"] = self.provider
|
||||
if self.eval_suites:
|
||||
kwargs["eval_suites"] = self.eval_suites
|
||||
return kwargs
|
||||
|
||||
def to_eval_suite(
|
||||
self,
|
||||
benchmarks: Optional[List[str]] = None,
|
||||
max_samples: Optional[int] = None,
|
||||
judge_model: Optional[str] = None,
|
||||
) -> Any:
|
||||
"""Convert this recipe into an ``EvalSuiteConfig``.
|
||||
|
||||
Uses the recipe's model/engine as the single ``[[models]]`` entry
|
||||
and the recipe's benchmarks (or *benchmarks* override) as
|
||||
``[[benchmarks]]``, inheriting agent type and tools.
|
||||
"""
|
||||
from openjarvis.recipes.composer import recipe_to_eval_suite
|
||||
|
||||
return recipe_to_eval_suite(
|
||||
self,
|
||||
benchmarks=benchmarks,
|
||||
max_samples=max_samples,
|
||||
judge_model=judge_model,
|
||||
)
|
||||
|
||||
def to_operator_manifest(self) -> Any:
|
||||
"""Convert this recipe into an ``OperatorManifest``."""
|
||||
from openjarvis.recipes.composer import recipe_to_operator
|
||||
|
||||
return recipe_to_operator(self)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# TOML loader
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
def load_recipe(path: str | Path) -> Recipe:
|
||||
"""Load a recipe from a TOML file.
|
||||
|
||||
Expected TOML format::
|
||||
|
||||
[recipe]
|
||||
name = "coding_assistant"
|
||||
description = "..."
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
quantization = "q4_K_M"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "native_react"
|
||||
max_turns = 10
|
||||
temperature = 0.3
|
||||
tools = ["file_read", "file_write", "code_interpreter", "think"]
|
||||
system_prompt = "You are a coding assistant..."
|
||||
|
||||
[learning]
|
||||
routing = "grpo"
|
||||
agent = "icl_updater"
|
||||
|
||||
[eval]
|
||||
suites = ["coding", "reasoning"]
|
||||
Supports the unified format with ``[recipe]``, ``[intelligence]``,
|
||||
``[engine]``, ``[agent]``, ``[learning]``, ``[eval]``, ``[schedule]``,
|
||||
and ``[channels]`` sections. Also auto-detects legacy operator manifests
|
||||
that use ``[operator]`` as the top-level key.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If *path* does not exist.
|
||||
@@ -123,43 +174,132 @@ def load_recipe(path: str | Path) -> Recipe:
|
||||
with open(path, "rb") as fh:
|
||||
data = tomllib.load(fh)
|
||||
|
||||
# Auto-detect legacy operator manifests ([operator] key)
|
||||
if "operator" in data and "recipe" not in data:
|
||||
return _load_operator_as_recipe(path, data)
|
||||
|
||||
recipe_sec = data.get("recipe", {})
|
||||
intel_sec = data.get("intelligence", {})
|
||||
engine_sec = data.get("engine", {})
|
||||
agent_sec = data.get("agent", {})
|
||||
learning_sec = data.get("learning", {})
|
||||
eval_sec = data.get("eval", {})
|
||||
schedule_sec = data.get("schedule", {})
|
||||
channels_sec = data.get("channels", {})
|
||||
|
||||
system_prompt = agent_sec.get("system_prompt")
|
||||
system_prompt_path = agent_sec.get("system_prompt_path")
|
||||
|
||||
# Resolve external prompt relative to TOML file
|
||||
if not system_prompt and system_prompt_path:
|
||||
prompt_p = Path(system_prompt_path)
|
||||
if not prompt_p.is_absolute():
|
||||
prompt_p = path.parent / prompt_p
|
||||
if prompt_p.exists():
|
||||
system_prompt = prompt_p.read_text(encoding="utf-8")
|
||||
system_prompt_path = str(prompt_p)
|
||||
|
||||
kind = recipe_sec.get("kind", "discrete")
|
||||
if schedule_sec and kind == "discrete":
|
||||
kind = "operator"
|
||||
|
||||
return Recipe(
|
||||
name=recipe_sec.get("name", path.stem),
|
||||
description=recipe_sec.get("description", ""),
|
||||
version=recipe_sec.get("version", "1.0.0"),
|
||||
kind=kind,
|
||||
model=intel_sec.get("model"),
|
||||
quantization=intel_sec.get("quantization"),
|
||||
provider=intel_sec.get("provider") or engine_sec.get("provider"),
|
||||
engine_key=engine_sec.get("key"),
|
||||
agent_type=agent_sec.get("type"),
|
||||
max_turns=agent_sec.get("max_turns"),
|
||||
temperature=agent_sec.get("temperature"),
|
||||
tools=agent_sec.get("tools", []),
|
||||
system_prompt=agent_sec.get("system_prompt"),
|
||||
system_prompt=system_prompt,
|
||||
system_prompt_path=system_prompt_path,
|
||||
routing_policy=learning_sec.get("routing"),
|
||||
agent_policy=learning_sec.get("agent"),
|
||||
eval_suites=eval_sec.get("suites", []),
|
||||
eval_benchmarks=eval_sec.get("benchmarks", []),
|
||||
eval_backend=eval_sec.get("backend"),
|
||||
eval_max_samples=eval_sec.get("max_samples"),
|
||||
eval_judge_model=eval_sec.get("judge_model"),
|
||||
schedule_type=schedule_sec.get("type"),
|
||||
schedule_value=str(schedule_sec["value"]) if "value" in schedule_sec else None,
|
||||
channels=channels_sec.get("output", []),
|
||||
required_capabilities=recipe_sec.get("required_capabilities", []),
|
||||
raw=data,
|
||||
)
|
||||
|
||||
|
||||
def _load_operator_as_recipe(path: Path, data: Dict[str, Any]) -> Recipe:
|
||||
"""Convert a legacy ``[operator]`` manifest into a Recipe."""
|
||||
op = data["operator"]
|
||||
agent_data = op.get("agent", {})
|
||||
schedule = op.get("schedule", {})
|
||||
|
||||
system_prompt = agent_data.get("system_prompt", op.get("system_prompt", ""))
|
||||
system_prompt_path = agent_data.get(
|
||||
"system_prompt_path", op.get("system_prompt_path", ""),
|
||||
)
|
||||
if not system_prompt and system_prompt_path:
|
||||
prompt_p = Path(system_prompt_path)
|
||||
if not prompt_p.is_absolute():
|
||||
prompt_p = path.parent / prompt_p
|
||||
if prompt_p.exists():
|
||||
system_prompt = prompt_p.read_text(encoding="utf-8")
|
||||
system_prompt_path = str(prompt_p)
|
||||
|
||||
sched_type = schedule.get("type", op.get("schedule_type", "interval"))
|
||||
sched_value = schedule.get("value", op.get("schedule_value", "300"))
|
||||
|
||||
return Recipe(
|
||||
name=op.get("name", path.stem),
|
||||
description=op.get("description", ""),
|
||||
version=op.get("version", "1.0.0"),
|
||||
kind="operator",
|
||||
tools=agent_data.get("tools", op.get("tools", [])),
|
||||
system_prompt=system_prompt or None,
|
||||
system_prompt_path=system_prompt_path or None,
|
||||
max_turns=agent_data.get("max_turns", op.get("max_turns", 20)),
|
||||
temperature=agent_data.get("temperature", op.get("temperature", 0.3)),
|
||||
schedule_type=sched_type,
|
||||
schedule_value=str(sched_value),
|
||||
required_capabilities=op.get("required_capabilities", []),
|
||||
raw=data,
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Discovery
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
def discover_recipes(
|
||||
extra_dirs: Optional[List[str | Path]] = None,
|
||||
*,
|
||||
kind: Optional[str] = None,
|
||||
) -> List[Recipe]:
|
||||
"""Discover all TOML recipes from known directories.
|
||||
|
||||
Search order (later entries override earlier ones by name):
|
||||
1. Project ``recipes/`` directory
|
||||
2. User ``~/.openjarvis/recipes/`` directory
|
||||
3. Any additional directories in *extra_dirs*
|
||||
1. Project ``recipes/data/`` directory (discrete recipes)
|
||||
2. Project ``recipes/data/operators/`` directory (operator recipes)
|
||||
3. User ``~/.openjarvis/recipes/`` directory
|
||||
4. User ``~/.openjarvis/operators/`` directory
|
||||
5. Any additional directories in *extra_dirs*
|
||||
|
||||
Args:
|
||||
extra_dirs: Additional directories to scan.
|
||||
kind: If set, filter to only "discrete" or "operator" recipes.
|
||||
"""
|
||||
dirs: List[Path] = [_PROJECT_RECIPES_DIR, _USER_RECIPES_DIR]
|
||||
dirs: List[Path] = [
|
||||
_PROJECT_RECIPES_DIR,
|
||||
_PROJECT_OPERATORS_DIR,
|
||||
_USER_RECIPES_DIR,
|
||||
_USER_OPERATORS_DIR,
|
||||
]
|
||||
if extra_dirs:
|
||||
dirs.extend(Path(d) for d in extra_dirs)
|
||||
|
||||
@@ -170,9 +310,9 @@ def discover_recipes(
|
||||
for toml_path in sorted(d.glob("*.toml")):
|
||||
try:
|
||||
recipe = load_recipe(toml_path)
|
||||
recipes[recipe.name] = recipe
|
||||
if kind is None or recipe.kind == kind:
|
||||
recipes[recipe.name] = recipe
|
||||
except Exception:
|
||||
# Skip malformed recipe files
|
||||
continue
|
||||
|
||||
return list(recipes.values())
|
||||
|
||||
@@ -0,0 +1,471 @@
|
||||
"""Tests for the unified compose system — enhanced Recipe, bridges, and discovery."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.recipes.loader import (
|
||||
Recipe,
|
||||
discover_recipes,
|
||||
load_recipe,
|
||||
resolve_recipe,
|
||||
)
|
||||
|
||||
# -- Discrete recipe TOML -----------------------------------------------
|
||||
|
||||
DISCRETE_TOML = textwrap.dedent("""\
|
||||
[recipe]
|
||||
name = "bench-agent"
|
||||
kind = "discrete"
|
||||
description = "A discrete agent for benchmarking"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
quantization = "q4_K_M"
|
||||
provider = "ollama"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "native_react"
|
||||
max_turns = 20
|
||||
temperature = 0.2
|
||||
tools = ["shell_exec", "file_read", "think"]
|
||||
system_prompt = "You are a benchmark agent."
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
agent = "none"
|
||||
|
||||
[eval]
|
||||
benchmarks = ["terminalbench", "gaia"]
|
||||
backend = "jarvis-agent"
|
||||
max_samples = 50
|
||||
judge_model = "gpt-4o"
|
||||
""")
|
||||
|
||||
# -- Operator recipe TOML -----------------------------------------------
|
||||
|
||||
OPERATOR_TOML = textwrap.dedent("""\
|
||||
[recipe]
|
||||
name = "my-operator"
|
||||
kind = "operator"
|
||||
description = "A test operator"
|
||||
version = "2.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "orchestrator"
|
||||
max_turns = 15
|
||||
temperature = 0.3
|
||||
tools = ["web_search", "memory_store", "think"]
|
||||
system_prompt = "You are a monitoring agent."
|
||||
|
||||
[schedule]
|
||||
type = "interval"
|
||||
value = "600"
|
||||
|
||||
[channels]
|
||||
output = ["slack", "telegram"]
|
||||
|
||||
[learning]
|
||||
routing = "heuristic"
|
||||
""")
|
||||
|
||||
# -- Legacy operator TOML -----------------------------------------------
|
||||
|
||||
LEGACY_OPERATOR_TOML = textwrap.dedent("""\
|
||||
[operator]
|
||||
name = "legacy-op"
|
||||
description = "A legacy operator manifest"
|
||||
version = "1.0.0"
|
||||
|
||||
[operator.agent]
|
||||
max_turns = 10
|
||||
temperature = 0.4
|
||||
tools = ["think", "web_search"]
|
||||
system_prompt = "Legacy prompt."
|
||||
|
||||
[operator.schedule]
|
||||
type = "cron"
|
||||
value = "0 */2 * * *"
|
||||
""")
|
||||
|
||||
|
||||
# ========================================================================
|
||||
# Recipe loading
|
||||
# ========================================================================
|
||||
|
||||
|
||||
class TestLoadDiscreteRecipe:
|
||||
def test_load_discrete_fields(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "bench.toml"
|
||||
p.write_text(DISCRETE_TOML)
|
||||
r = load_recipe(p)
|
||||
|
||||
assert r.name == "bench-agent"
|
||||
assert r.kind == "discrete"
|
||||
assert r.model == "qwen3:8b"
|
||||
assert r.quantization == "q4_K_M"
|
||||
assert r.provider == "ollama"
|
||||
assert r.engine_key == "ollama"
|
||||
assert r.agent_type == "native_react"
|
||||
assert r.max_turns == 20
|
||||
assert r.temperature == pytest.approx(0.2)
|
||||
assert r.tools == ["shell_exec", "file_read", "think"]
|
||||
assert r.system_prompt == "You are a benchmark agent."
|
||||
assert r.routing_policy == "heuristic"
|
||||
assert r.eval_benchmarks == ["terminalbench", "gaia"]
|
||||
assert r.eval_backend == "jarvis-agent"
|
||||
assert r.eval_max_samples == 50
|
||||
assert r.eval_judge_model == "gpt-4o"
|
||||
|
||||
def test_default_kind_is_discrete(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "min.toml"
|
||||
p.write_text('[recipe]\nname = "min"\n[intelligence]\nmodel = "x"\n')
|
||||
r = load_recipe(p)
|
||||
assert r.kind == "discrete"
|
||||
|
||||
|
||||
class TestLoadOperatorRecipe:
|
||||
def test_load_operator_fields(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "op.toml"
|
||||
p.write_text(OPERATOR_TOML)
|
||||
r = load_recipe(p)
|
||||
|
||||
assert r.name == "my-operator"
|
||||
assert r.kind == "operator"
|
||||
assert r.schedule_type == "interval"
|
||||
assert r.schedule_value == "600"
|
||||
assert r.channels == ["slack", "telegram"]
|
||||
assert r.tools == ["web_search", "memory_store", "think"]
|
||||
|
||||
def test_schedule_implies_operator_kind(self, tmp_path: Path) -> None:
|
||||
toml = textwrap.dedent("""\
|
||||
[recipe]
|
||||
name = "auto-op"
|
||||
|
||||
[agent]
|
||||
type = "orchestrator"
|
||||
tools = ["think"]
|
||||
|
||||
[schedule]
|
||||
type = "cron"
|
||||
value = "0 8 * * *"
|
||||
""")
|
||||
p = tmp_path / "auto.toml"
|
||||
p.write_text(toml)
|
||||
r = load_recipe(p)
|
||||
assert r.kind == "operator"
|
||||
|
||||
def test_external_prompt_file(self, tmp_path: Path) -> None:
|
||||
prompt_file = tmp_path / "prompt.md"
|
||||
prompt_file.write_text("External prompt content.")
|
||||
|
||||
toml = textwrap.dedent(f"""\
|
||||
[recipe]
|
||||
name = "ext-prompt"
|
||||
|
||||
[agent]
|
||||
type = "simple"
|
||||
system_prompt_path = "{prompt_file}"
|
||||
""")
|
||||
p = tmp_path / "ext.toml"
|
||||
p.write_text(toml)
|
||||
r = load_recipe(p)
|
||||
assert r.system_prompt == "External prompt content."
|
||||
|
||||
|
||||
class TestLoadLegacyOperator:
|
||||
def test_legacy_operator_converted_to_recipe(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "legacy.toml"
|
||||
p.write_text(LEGACY_OPERATOR_TOML)
|
||||
r = load_recipe(p)
|
||||
|
||||
assert r.kind == "operator"
|
||||
assert r.name == "legacy-op"
|
||||
assert r.max_turns == 10
|
||||
assert r.temperature == pytest.approx(0.4)
|
||||
assert r.tools == ["think", "web_search"]
|
||||
assert r.system_prompt == "Legacy prompt."
|
||||
assert r.schedule_type == "cron"
|
||||
assert r.schedule_value == "0 */2 * * *"
|
||||
|
||||
|
||||
# ========================================================================
|
||||
# Discovery
|
||||
# ========================================================================
|
||||
|
||||
|
||||
class TestDiscoverByKind:
|
||||
def test_discover_all(self) -> None:
|
||||
all_recipes = discover_recipes()
|
||||
kinds = {r.kind for r in all_recipes}
|
||||
assert "discrete" in kinds
|
||||
assert len(all_recipes) >= 4 # at least original 3 + new ones
|
||||
|
||||
def test_discover_discrete_only(self) -> None:
|
||||
discrete = discover_recipes(kind="discrete")
|
||||
for r in discrete:
|
||||
assert r.kind == "discrete"
|
||||
|
||||
def test_discover_operator_only(self) -> None:
|
||||
operators = discover_recipes(kind="operator")
|
||||
for r in operators:
|
||||
assert r.kind == "operator"
|
||||
|
||||
def test_discover_operators_subdir(self) -> None:
|
||||
"""Operator recipes in data/operators/ are discovered."""
|
||||
all_recipes = discover_recipes()
|
||||
names = {r.name for r in all_recipes}
|
||||
# The new unified operator recipes should be found
|
||||
assert "twitter-sentinel" in names or "correspondent" in names
|
||||
|
||||
def test_extra_dirs_discrete(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "custom.toml"
|
||||
p.write_text(DISCRETE_TOML)
|
||||
recipes = discover_recipes(extra_dirs=[tmp_path], kind="discrete")
|
||||
names = {r.name for r in recipes}
|
||||
assert "bench-agent" in names
|
||||
|
||||
def test_extra_dirs_operator(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "custom_op.toml"
|
||||
p.write_text(OPERATOR_TOML)
|
||||
recipes = discover_recipes(extra_dirs=[tmp_path], kind="operator")
|
||||
names = {r.name for r in recipes}
|
||||
assert "my-operator" in names
|
||||
|
||||
|
||||
# ========================================================================
|
||||
# Bridge: to_eval_suite
|
||||
# ========================================================================
|
||||
|
||||
|
||||
class TestRecipeToEvalSuite:
|
||||
def test_basic_eval_suite(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "bench.toml"
|
||||
p.write_text(DISCRETE_TOML)
|
||||
r = load_recipe(p)
|
||||
|
||||
suite = r.to_eval_suite()
|
||||
|
||||
assert suite.meta.name == "bench-agent-eval"
|
||||
assert len(suite.models) == 1
|
||||
assert suite.models[0].name == "qwen3:8b"
|
||||
assert suite.models[0].engine == "ollama"
|
||||
assert len(suite.benchmarks) == 2
|
||||
bench_names = {b.name for b in suite.benchmarks}
|
||||
assert bench_names == {"terminalbench", "gaia"}
|
||||
for b in suite.benchmarks:
|
||||
assert b.backend == "jarvis-agent"
|
||||
assert b.agent == "native_react"
|
||||
assert b.tools == ["shell_exec", "file_read", "think"]
|
||||
assert b.max_samples == 50
|
||||
assert b.judge_model == "gpt-4o"
|
||||
|
||||
def test_eval_suite_benchmark_override(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "bench.toml"
|
||||
p.write_text(DISCRETE_TOML)
|
||||
r = load_recipe(p)
|
||||
|
||||
suite = r.to_eval_suite(benchmarks=["supergpqa"])
|
||||
|
||||
assert len(suite.benchmarks) == 1
|
||||
assert suite.benchmarks[0].name == "supergpqa"
|
||||
|
||||
def test_eval_suite_max_samples_override(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "bench.toml"
|
||||
p.write_text(DISCRETE_TOML)
|
||||
r = load_recipe(p)
|
||||
|
||||
suite = r.to_eval_suite(max_samples=10)
|
||||
|
||||
for b in suite.benchmarks:
|
||||
assert b.max_samples == 10
|
||||
|
||||
def test_eval_suite_judge_override(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "bench.toml"
|
||||
p.write_text(DISCRETE_TOML)
|
||||
r = load_recipe(p)
|
||||
|
||||
suite = r.to_eval_suite(judge_model="gpt-5")
|
||||
|
||||
assert suite.judge.model == "gpt-5"
|
||||
for b in suite.benchmarks:
|
||||
assert b.judge_model == "gpt-5"
|
||||
|
||||
def test_eval_suite_no_model_raises(self) -> None:
|
||||
r = Recipe(name="no-model", eval_benchmarks=["gaia"])
|
||||
with pytest.raises(ValueError, match="no model"):
|
||||
r.to_eval_suite()
|
||||
|
||||
def test_eval_suite_no_benchmarks_raises(self) -> None:
|
||||
r = Recipe(name="no-bench", model="qwen3:8b")
|
||||
with pytest.raises(ValueError, match="no benchmarks"):
|
||||
r.to_eval_suite()
|
||||
|
||||
def test_eval_suite_falls_back_to_suites(self) -> None:
|
||||
r = Recipe(
|
||||
name="suites-fallback",
|
||||
model="qwen3:8b",
|
||||
eval_suites=["coding"],
|
||||
)
|
||||
suite = r.to_eval_suite()
|
||||
assert len(suite.benchmarks) == 1
|
||||
assert suite.benchmarks[0].name == "coding"
|
||||
|
||||
def test_eval_suite_direct_backend_when_no_agent(self) -> None:
|
||||
r = Recipe(
|
||||
name="direct",
|
||||
model="qwen3:8b",
|
||||
eval_benchmarks=["supergpqa"],
|
||||
)
|
||||
suite = r.to_eval_suite()
|
||||
assert suite.benchmarks[0].backend == "jarvis-direct"
|
||||
assert suite.benchmarks[0].agent is None
|
||||
assert suite.benchmarks[0].tools == []
|
||||
|
||||
|
||||
# ========================================================================
|
||||
# Bridge: to_operator_manifest
|
||||
# ========================================================================
|
||||
|
||||
|
||||
class TestRecipeToOperatorManifest:
|
||||
def test_basic_operator_manifest(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "op.toml"
|
||||
p.write_text(OPERATOR_TOML)
|
||||
r = load_recipe(p)
|
||||
|
||||
m = r.to_operator_manifest()
|
||||
|
||||
assert m.id == "my-operator"
|
||||
assert m.name == "my-operator"
|
||||
assert m.version == "2.0.0"
|
||||
assert m.description == "A test operator"
|
||||
assert m.tools == ["web_search", "memory_store", "think"]
|
||||
assert m.system_prompt == "You are a monitoring agent."
|
||||
assert m.max_turns == 15
|
||||
assert m.temperature == pytest.approx(0.3)
|
||||
assert m.schedule_type == "interval"
|
||||
assert m.schedule_value == "600"
|
||||
|
||||
def test_operator_no_schedule_raises(self) -> None:
|
||||
r = Recipe(name="no-sched", kind="operator")
|
||||
with pytest.raises(ValueError, match="no \\[schedule\\]"):
|
||||
r.to_operator_manifest()
|
||||
|
||||
def test_operator_defaults(self) -> None:
|
||||
r = Recipe(
|
||||
name="minimal-op",
|
||||
kind="operator",
|
||||
schedule_type="interval",
|
||||
schedule_value="300",
|
||||
)
|
||||
m = r.to_operator_manifest()
|
||||
assert m.max_turns == 20
|
||||
assert m.temperature == pytest.approx(0.3)
|
||||
assert m.schedule_value == "300"
|
||||
|
||||
|
||||
# ========================================================================
|
||||
# Builder kwargs with new fields
|
||||
# ========================================================================
|
||||
|
||||
|
||||
class TestBuilderKwargsNewFields:
|
||||
def test_provider_in_kwargs(self) -> None:
|
||||
r = Recipe(name="cloud", model="gpt-4o", provider="openai")
|
||||
kw = r.to_builder_kwargs()
|
||||
assert kw["provider"] == "openai"
|
||||
|
||||
def test_system_prompt_path_resolved(self, tmp_path: Path) -> None:
|
||||
prompt = tmp_path / "prompt.txt"
|
||||
prompt.write_text("Hello from file.")
|
||||
r = Recipe(
|
||||
name="ext",
|
||||
system_prompt_path=str(prompt),
|
||||
)
|
||||
kw = r.to_builder_kwargs()
|
||||
assert kw["system_prompt"] == "Hello from file."
|
||||
|
||||
def test_schedule_and_channels_not_in_kwargs(self) -> None:
|
||||
"""Schedule/channel fields are operator-specific and not builder kwargs."""
|
||||
r = Recipe(
|
||||
name="op",
|
||||
kind="operator",
|
||||
schedule_type="cron",
|
||||
schedule_value="0 * * * *",
|
||||
channels=["slack"],
|
||||
)
|
||||
kw = r.to_builder_kwargs()
|
||||
assert "schedule_type" not in kw
|
||||
assert "channels" not in kw
|
||||
|
||||
|
||||
# ========================================================================
|
||||
# Built-in recipe loading
|
||||
# ========================================================================
|
||||
|
||||
|
||||
class TestBuiltinRecipes:
|
||||
def test_terminalbench_react_loads(self) -> None:
|
||||
r = resolve_recipe("terminalbench-react")
|
||||
assert r is not None
|
||||
assert r.kind == "discrete"
|
||||
assert r.agent_type == "native_react"
|
||||
assert "terminalbench" in r.eval_benchmarks
|
||||
|
||||
def test_gaia_orchestrator_loads(self) -> None:
|
||||
r = resolve_recipe("gaia-orchestrator")
|
||||
assert r is not None
|
||||
assert r.kind == "discrete"
|
||||
assert r.agent_type == "orchestrator"
|
||||
assert "gaia" in r.eval_benchmarks
|
||||
|
||||
def test_swebench_openhands_loads(self) -> None:
|
||||
r = resolve_recipe("swebench-openhands")
|
||||
assert r is not None
|
||||
assert r.kind == "discrete"
|
||||
assert r.agent_type == "native_openhands"
|
||||
|
||||
def test_coding_benchmark_loads(self) -> None:
|
||||
r = resolve_recipe("coding-benchmark")
|
||||
assert r is not None
|
||||
assert r.kind == "discrete"
|
||||
assert "terminalbench" in r.eval_benchmarks
|
||||
assert "swebench" in r.eval_benchmarks
|
||||
|
||||
def test_twitter_sentinel_loads(self) -> None:
|
||||
r = resolve_recipe("twitter-sentinel")
|
||||
assert r is not None
|
||||
assert r.kind == "operator"
|
||||
assert r.schedule_type == "interval"
|
||||
|
||||
def test_inbox_triage_loads(self) -> None:
|
||||
r = resolve_recipe("inbox-triage")
|
||||
assert r is not None
|
||||
assert r.kind == "operator"
|
||||
assert r.schedule_type == "interval"
|
||||
|
||||
def test_news_briefing_loads(self) -> None:
|
||||
r = resolve_recipe("news-briefing")
|
||||
assert r is not None
|
||||
assert r.kind == "operator"
|
||||
assert r.schedule_type == "cron"
|
||||
|
||||
def test_repo_watcher_loads(self) -> None:
|
||||
r = resolve_recipe("repo-watcher")
|
||||
assert r is not None
|
||||
assert r.kind == "operator"
|
||||
assert r.schedule_type == "interval"
|
||||
Reference in New Issue
Block a user