mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-31 03:12:16 +00:00
feat: Phase 23 — Differentiated functionalities
Trace-driven learning pipeline: - TrainingDataMiner: extract SFT/routing/agent pairs from traces - LoRATrainer: fine-tune local models from trace-derived data - AgentConfigEvolver: rewrite agent configs from trace analysis - LearningOrchestrator: coordinate mine→train→evolve cycle, wired into SystemBuilder Eval framework (15 real IPW benchmarks): - Datasets: SuperGPQA, GPQA, MMLU-Pro, MATH-500, Natural Reasoning, HLE, SimpleQA, WildChat, IPW, GAIA, FRAMES, SWE-bench, SWEfficiency, TerminalBench, TerminalBench Native - Scorers: MCQ extraction, LLM-judge, exact match, structural validation - CLI: jarvis eval list|run|compare|report Composable abstractions: - Recipe system: TOML composition of all 5 pillars (3 built-in recipes) - Agent templates: 15 pre-configured TOML manifests with system prompts - Bundled skills: 20 ready-to-use TOML skill manifests - Operator recipes: researcher (4h), correspondent (5min), sentinel (2h) 102 files changed, ~11,500 lines added. 3241 tests pass (44 skipped). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jon Saad-Falcon
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
2849f39cbc
commit
2aebcd7d77
@@ -7,16 +7,17 @@ import click
|
||||
import openjarvis
|
||||
from openjarvis.cli.add_cmd import add
|
||||
from openjarvis.cli.agent_cmd import agent
|
||||
from openjarvis.cli.operators_cmd import operators
|
||||
from openjarvis.cli.ask import ask
|
||||
from openjarvis.cli.bench_cmd import bench
|
||||
from openjarvis.cli.channel_cmd import channel
|
||||
from openjarvis.cli.chat_cmd import chat
|
||||
from openjarvis.cli.daemon_cmd import restart, start, status, stop
|
||||
from openjarvis.cli.doctor_cmd import doctor
|
||||
from openjarvis.cli.eval_cmd import eval_group
|
||||
from openjarvis.cli.init_cmd import init
|
||||
from openjarvis.cli.memory_cmd import memory
|
||||
from openjarvis.cli.model import model
|
||||
from openjarvis.cli.operators_cmd import operators
|
||||
from openjarvis.cli.scheduler_cmd import scheduler
|
||||
from openjarvis.cli.serve import serve
|
||||
from openjarvis.cli.skill_cmd import skill
|
||||
@@ -52,6 +53,7 @@ cli.add_command(status, "status")
|
||||
cli.add_command(vault, "vault")
|
||||
cli.add_command(add, "add")
|
||||
cli.add_command(operators, "operators")
|
||||
cli.add_command(eval_group, "eval")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
"""``jarvis eval`` — evaluation framework CLI commands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
# Known benchmarks and backends — mirrored from the evals framework so the
|
||||
# CLI can display them even when the (optional) evals package is not installed.
|
||||
KNOWN_BENCHMARKS = {
|
||||
"supergpqa": {"category": "reasoning", "description": "SuperGPQA multiple-choice"},
|
||||
"gpqa": {"category": "reasoning", "description": "GPQA graduate-level MCQ"},
|
||||
"mmlu-pro": {"category": "reasoning", "description": "MMLU-Pro multiple-choice"},
|
||||
"math500": {"category": "reasoning", "description": "MATH-500 math problems"},
|
||||
"natural-reasoning": {"category": "reasoning", "description": "Natural Reasoning"},
|
||||
"hle": {"category": "reasoning", "description": "HLE hard challenges"},
|
||||
"simpleqa": {"category": "chat", "description": "SimpleQA factual QA"},
|
||||
"wildchat": {"category": "chat", "description": "WildChat conversation quality"},
|
||||
"ipw": {"category": "chat", "description": "IPW mixed benchmark"},
|
||||
"gaia": {"category": "agentic", "description": "GAIA agentic benchmark"},
|
||||
"frames": {"category": "rag", "description": "FRAMES multi-hop RAG"},
|
||||
"swebench": {"category": "agentic", "description": "SWE-bench code patches"},
|
||||
"swefficiency": {"category": "agentic", "description": "SWEfficiency optimization"},
|
||||
"terminalbench": {
|
||||
"category": "agentic", "description": "TerminalBench terminal tasks",
|
||||
},
|
||||
"terminalbench-native": {
|
||||
"category": "agentic",
|
||||
"description": "TerminalBench Native (Docker)",
|
||||
},
|
||||
}
|
||||
|
||||
KNOWN_BACKENDS = {
|
||||
"jarvis-direct": "Engine-level inference (local or cloud)",
|
||||
"jarvis-agent": "Agent-level inference with tool calling",
|
||||
}
|
||||
|
||||
|
||||
@click.group("eval")
|
||||
def eval_group() -> None:
|
||||
"""Evaluation framework — benchmark models, agents, and learning."""
|
||||
|
||||
|
||||
@eval_group.command("list")
|
||||
def eval_list() -> None:
|
||||
"""List available benchmarks and backends."""
|
||||
console = Console()
|
||||
|
||||
bench_table = Table(
|
||||
title="[bold]Available Benchmarks[/bold]",
|
||||
border_style="bright_blue",
|
||||
title_style="bold cyan",
|
||||
)
|
||||
bench_table.add_column("Name", style="cyan", no_wrap=True)
|
||||
bench_table.add_column("Category", style="white")
|
||||
bench_table.add_column("Description")
|
||||
for name, info in KNOWN_BENCHMARKS.items():
|
||||
bench_table.add_row(name, info["category"], info["description"])
|
||||
console.print(bench_table)
|
||||
|
||||
backend_table = Table(
|
||||
title="[bold]Available Backends[/bold]",
|
||||
border_style="bright_blue",
|
||||
title_style="bold cyan",
|
||||
)
|
||||
backend_table.add_column("Name", style="cyan", no_wrap=True)
|
||||
backend_table.add_column("Description")
|
||||
for name, desc in KNOWN_BACKENDS.items():
|
||||
backend_table.add_row(name, desc)
|
||||
console.print(backend_table)
|
||||
|
||||
|
||||
@eval_group.command("run")
|
||||
@click.option(
|
||||
"-c", "--config", "config_path", default=None, type=click.Path(),
|
||||
help="TOML config file for suite runs.",
|
||||
)
|
||||
@click.option(
|
||||
"-b", "--benchmark", "benchmark", default=None,
|
||||
help="Benchmark to run (e.g. supergpqa, gaia, frames, wildchat).",
|
||||
)
|
||||
@click.option(
|
||||
"-m", "--model", "model", default=None,
|
||||
help="Model identifier.",
|
||||
)
|
||||
@click.option(
|
||||
"-n", "--max-samples", "max_samples", type=int, default=None,
|
||||
help="Maximum samples to evaluate.",
|
||||
)
|
||||
@click.option(
|
||||
"--backend", "backend", default="jarvis-direct",
|
||||
help="Inference backend (jarvis-direct or jarvis-agent).",
|
||||
)
|
||||
@click.option(
|
||||
"--agent", "agent_name", default=None,
|
||||
help="Agent name for jarvis-agent backend.",
|
||||
)
|
||||
@click.option(
|
||||
"--tools", "tools", default="",
|
||||
help="Comma-separated tool names.",
|
||||
)
|
||||
@click.option(
|
||||
"--telemetry/--no-telemetry", "telemetry", default=False,
|
||||
help="Enable telemetry collection during eval.",
|
||||
)
|
||||
@click.option(
|
||||
"-o", "--output", "output_path", default=None, type=click.Path(),
|
||||
help="Output JSONL path.",
|
||||
)
|
||||
@click.option(
|
||||
"-v", "--verbose", "verbose", is_flag=True, default=False,
|
||||
help="Verbose logging.",
|
||||
)
|
||||
def eval_run(
|
||||
config_path: Optional[str],
|
||||
benchmark: Optional[str],
|
||||
model: Optional[str],
|
||||
max_samples: Optional[int],
|
||||
backend: str,
|
||||
agent_name: Optional[str],
|
||||
tools: str,
|
||||
telemetry: bool,
|
||||
output_path: Optional[str],
|
||||
verbose: bool,
|
||||
) -> None:
|
||||
"""Run evaluation benchmarks."""
|
||||
console = Console(stderr=True)
|
||||
|
||||
# Config-driven mode: load TOML suite, expand, run all
|
||||
if config_path is not None:
|
||||
try:
|
||||
from evals.core.config import expand_suite, load_eval_config
|
||||
except ImportError:
|
||||
console.print(
|
||||
"[red]Eval framework not available. "
|
||||
"Ensure the evals package is importable.[/red]"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
suite = load_eval_config(config_path)
|
||||
run_configs = expand_suite(suite)
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error loading config: {exc}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
console.print(
|
||||
f"[cyan]Suite:[/cyan] {suite.meta.name or Path(config_path).stem}"
|
||||
)
|
||||
console.print(
|
||||
f"[cyan]Matrix:[/cyan] {len(suite.models)} model(s) x "
|
||||
f"{len(suite.benchmarks)} benchmark(s) = {len(run_configs)} run(s)"
|
||||
)
|
||||
|
||||
try:
|
||||
from evals.cli import _run_single
|
||||
except ImportError:
|
||||
console.print(
|
||||
"[red]Eval CLI module not available.[/red]"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
for i, rc in enumerate(run_configs, 1):
|
||||
console.print(
|
||||
f"\n[bold]Run {i}/{len(run_configs)}:[/bold] "
|
||||
f"{rc.benchmark} / {rc.model}"
|
||||
)
|
||||
try:
|
||||
summary = _run_single(rc, console=console)
|
||||
console.print(
|
||||
f" [green]{summary.accuracy:.4f}[/green] "
|
||||
f"({summary.correct}/{summary.scored_samples})"
|
||||
)
|
||||
except Exception as exc:
|
||||
console.print(f" [red bold]FAILED:[/red bold] {exc}")
|
||||
|
||||
return
|
||||
|
||||
# CLI-driven mode: require --benchmark and --model
|
||||
if benchmark is None or model is None:
|
||||
raise click.UsageError(
|
||||
"Provide either --config/-c for suite mode, "
|
||||
"or both --benchmark/-b and --model/-m for single-run mode."
|
||||
)
|
||||
|
||||
if benchmark not in KNOWN_BENCHMARKS:
|
||||
console.print(
|
||||
f"[yellow]Warning: unknown benchmark '{benchmark}'[/yellow]"
|
||||
)
|
||||
|
||||
try:
|
||||
from evals.core.types import RunConfig
|
||||
except ImportError:
|
||||
console.print(
|
||||
"[red]Eval framework not available. "
|
||||
"Ensure the evals package is importable.[/red]"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
tool_list = (
|
||||
[t.strip() for t in tools.split(",") if t.strip()] if tools else []
|
||||
)
|
||||
|
||||
config = RunConfig(
|
||||
benchmark=benchmark,
|
||||
backend=backend,
|
||||
model=model,
|
||||
max_samples=max_samples,
|
||||
agent_name=agent_name,
|
||||
tools=tool_list,
|
||||
output_path=output_path,
|
||||
telemetry=telemetry,
|
||||
)
|
||||
|
||||
try:
|
||||
from evals.cli import _run_single
|
||||
|
||||
console.print(
|
||||
f"[cyan]Benchmark:[/cyan] {benchmark}\n"
|
||||
f"[cyan]Model:[/cyan] {model}\n"
|
||||
f"[cyan]Backend:[/cyan] {backend}"
|
||||
)
|
||||
summary = _run_single(config, console=console)
|
||||
console.print(
|
||||
f"\n[green]Accuracy: {summary.accuracy:.4f}[/green] "
|
||||
f"({summary.correct}/{summary.scored_samples})"
|
||||
)
|
||||
except ImportError:
|
||||
console.print(
|
||||
"[red]Eval CLI module not available.[/red]"
|
||||
)
|
||||
sys.exit(1)
|
||||
except Exception as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@eval_group.command("compare")
|
||||
@click.argument("result_files", nargs=-1, required=True)
|
||||
@click.option(
|
||||
"--metric", "metric", default="accuracy",
|
||||
help="Metric to compare across runs (default: accuracy).",
|
||||
)
|
||||
def eval_compare(result_files: tuple[str, ...], metric: str) -> None:
|
||||
"""Compare results from multiple eval runs."""
|
||||
console = Console()
|
||||
|
||||
rows: list[dict] = []
|
||||
for path_str in result_files:
|
||||
path = Path(path_str)
|
||||
summary_path = path.with_suffix(".summary.json")
|
||||
|
||||
if summary_path.exists():
|
||||
with open(summary_path) as f:
|
||||
data = json.load(f)
|
||||
rows.append({
|
||||
"file": path.name,
|
||||
"benchmark": data.get("benchmark", "?"),
|
||||
"model": data.get("model", "?"),
|
||||
"value": data.get(metric, "N/A"),
|
||||
"samples": data.get("total_samples", 0),
|
||||
})
|
||||
elif path.exists() and path.suffix == ".jsonl":
|
||||
# Fall back to reading JSONL and computing metric on-the-fly
|
||||
records = []
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
records.append(json.loads(line))
|
||||
if records:
|
||||
if metric == "accuracy":
|
||||
scored = [
|
||||
r for r in records
|
||||
if r.get("is_correct") is not None
|
||||
]
|
||||
correct = [r for r in scored if r["is_correct"]]
|
||||
value = (
|
||||
len(correct) / len(scored) if scored else 0.0
|
||||
)
|
||||
else:
|
||||
values = [
|
||||
r[metric] for r in records
|
||||
if metric in r and r[metric] is not None
|
||||
]
|
||||
value = (
|
||||
sum(values) / len(values) if values else "N/A"
|
||||
)
|
||||
rows.append({
|
||||
"file": path.name,
|
||||
"benchmark": records[0].get("benchmark", "?"),
|
||||
"model": records[0].get("model", "?"),
|
||||
"value": value,
|
||||
"samples": len(records),
|
||||
})
|
||||
else:
|
||||
console.print(f"[yellow]Skipping missing file: {path_str}[/yellow]")
|
||||
|
||||
if not rows:
|
||||
console.print("[red]No valid result files found.[/red]")
|
||||
return
|
||||
|
||||
table = Table(
|
||||
title=f"[bold]Comparison — {metric}[/bold]",
|
||||
border_style="bright_blue",
|
||||
title_style="bold cyan",
|
||||
)
|
||||
table.add_column("File", style="dim")
|
||||
table.add_column("Benchmark", style="cyan")
|
||||
table.add_column("Model", style="green")
|
||||
table.add_column(metric.capitalize(), justify="right", style="bold")
|
||||
table.add_column("Samples", justify="right", style="dim")
|
||||
|
||||
for row in rows:
|
||||
val = row["value"]
|
||||
val_str = f"{val:.4f}" if isinstance(val, float) else str(val)
|
||||
table.add_row(
|
||||
row["file"], row["benchmark"], row["model"],
|
||||
val_str, str(row["samples"]),
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
@eval_group.command("report")
|
||||
@click.argument("result_file")
|
||||
def eval_report(result_file: str) -> None:
|
||||
"""Generate detailed report from eval results."""
|
||||
console = Console()
|
||||
path = Path(result_file)
|
||||
|
||||
# Try summary JSON first
|
||||
summary_path = path.with_suffix(".summary.json")
|
||||
if summary_path.exists():
|
||||
with open(summary_path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
console.print("[bold cyan]Evaluation Report[/bold cyan]")
|
||||
console.print(f" [cyan]Benchmark:[/cyan] {data.get('benchmark', '?')}")
|
||||
console.print(f" [cyan]Model:[/cyan] {data.get('model', '?')}")
|
||||
console.print(f" [cyan]Backend:[/cyan] {data.get('backend', '?')}")
|
||||
console.print(
|
||||
f" [cyan]Accuracy:[/cyan] "
|
||||
f"[bold]{data.get('accuracy', 0.0):.4f}[/bold]"
|
||||
)
|
||||
console.print(f" [cyan]Samples:[/cyan] {data.get('total_samples', 0)}")
|
||||
console.print(f" [cyan]Scored:[/cyan] {data.get('scored_samples', 0)}")
|
||||
console.print(f" [cyan]Correct:[/cyan] {data.get('correct', 0)}")
|
||||
console.print(f" [cyan]Errors:[/cyan] {data.get('errors', 0)}")
|
||||
console.print(
|
||||
f" [cyan]Latency:[/cyan] "
|
||||
f"{data.get('mean_latency_seconds', 0.0):.4f}s (mean)"
|
||||
)
|
||||
console.print(
|
||||
f" [cyan]Cost:[/cyan] "
|
||||
f"${data.get('total_cost_usd', 0.0):.6f}"
|
||||
)
|
||||
|
||||
# Per-subject breakdown
|
||||
per_subject = data.get("per_subject", {})
|
||||
if per_subject and len(per_subject) > 1:
|
||||
sub_table = Table(
|
||||
title="[bold]Per-Subject Breakdown[/bold]",
|
||||
border_style="bright_blue",
|
||||
)
|
||||
sub_table.add_column("Subject", style="cyan")
|
||||
sub_table.add_column("Accuracy", justify="right", style="bold")
|
||||
sub_table.add_column("Total", justify="right")
|
||||
sub_table.add_column("Correct", justify="right", style="green")
|
||||
|
||||
for subj, stats in sorted(per_subject.items()):
|
||||
sub_table.add_row(
|
||||
subj,
|
||||
f"{stats.get('accuracy', 0.0):.4f}",
|
||||
str(int(stats.get("total", 0))),
|
||||
str(int(stats.get("correct", 0))),
|
||||
)
|
||||
console.print(sub_table)
|
||||
|
||||
return
|
||||
|
||||
# Fall back to JSONL file
|
||||
if not path.exists():
|
||||
console.print(f"[red]File not found: {result_file}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
records: list[dict] = []
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
records.append(json.loads(line))
|
||||
|
||||
if not records:
|
||||
console.print("[yellow]No records found in file.[/yellow]")
|
||||
return
|
||||
|
||||
scored = [r for r in records if r.get("is_correct") is not None]
|
||||
correct = [r for r in scored if r["is_correct"]]
|
||||
errors = [r for r in records if r.get("error")]
|
||||
accuracy = len(correct) / len(scored) if scored else 0.0
|
||||
|
||||
console.print("[bold cyan]Evaluation Report[/bold cyan]")
|
||||
console.print(f" [cyan]File:[/cyan] {result_file}")
|
||||
console.print(f" [cyan]Benchmark:[/cyan] {records[0].get('benchmark', '?')}")
|
||||
console.print(f" [cyan]Model:[/cyan] {records[0].get('model', '?')}")
|
||||
console.print(f" [cyan]Total:[/cyan] {len(records)}")
|
||||
console.print(f" [cyan]Scored:[/cyan] {len(scored)}")
|
||||
console.print(f" [cyan]Correct:[/cyan] {len(correct)}")
|
||||
console.print(
|
||||
f" [cyan]Accuracy:[/cyan] [bold]{accuracy:.4f}[/bold]"
|
||||
)
|
||||
console.print(f" [cyan]Errors:[/cyan] {len(errors)}")
|
||||
|
||||
|
||||
__all__ = ["eval_group"]
|
||||
@@ -402,6 +402,14 @@ class LearningConfig:
|
||||
agent: AgentLearningConfig = field(default_factory=AgentLearningConfig)
|
||||
metrics: MetricsConfig = field(default_factory=MetricsConfig)
|
||||
|
||||
# Training pipeline
|
||||
training_enabled: bool = False
|
||||
training_schedule: str = "" # cron expression or empty for on-demand
|
||||
lora_rank: int = 16
|
||||
lora_alpha: int = 32
|
||||
min_sft_pairs: int = 50
|
||||
min_improvement: float = 0.02
|
||||
|
||||
# Backward-compat properties for old flat field names
|
||||
@property
|
||||
def default_policy(self) -> str:
|
||||
|
||||
@@ -8,11 +8,15 @@ from openjarvis.learning._stubs import (
|
||||
RouterPolicy,
|
||||
RoutingContext,
|
||||
)
|
||||
from openjarvis.learning.agent_evolver import AgentConfigEvolver
|
||||
from openjarvis.learning.heuristic_reward import HeuristicRewardFunction
|
||||
from openjarvis.learning.learning_orchestrator import LearningOrchestrator
|
||||
from openjarvis.learning.router import (
|
||||
HeuristicRouter,
|
||||
build_routing_context,
|
||||
)
|
||||
from openjarvis.learning.training.data import TrainingDataMiner
|
||||
from openjarvis.learning.training.lora import HAS_TORCH, LoRATrainer, LoRATrainingConfig
|
||||
|
||||
|
||||
def ensure_registered() -> None:
|
||||
@@ -73,12 +77,18 @@ def ensure_registered() -> None:
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AgentConfigEvolver",
|
||||
"HAS_TORCH",
|
||||
"HeuristicRewardFunction",
|
||||
"HeuristicRouter",
|
||||
"LearningOrchestrator",
|
||||
"LoRATrainer",
|
||||
"LoRATrainingConfig",
|
||||
"QueryAnalyzer",
|
||||
"RewardFunction",
|
||||
"RouterPolicy",
|
||||
"RoutingContext",
|
||||
"TrainingDataMiner",
|
||||
"build_routing_context",
|
||||
"ensure_registered",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,361 @@
|
||||
"""AgentConfigEvolver — analyze traces to evolve agent TOML configs.
|
||||
|
||||
Reads interaction traces to determine which agent/tool/parameter
|
||||
combinations perform best for different query classes, then writes
|
||||
updated TOML config files with automatic versioning and rollback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from openjarvis.core.types import StepType, Trace
|
||||
from openjarvis.learning.trace_policy import classify_query
|
||||
from openjarvis.traces.store import TraceStore
|
||||
|
||||
|
||||
def _format_toml_value(value: Any) -> str:
|
||||
"""Format a Python value as a TOML literal."""
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
if isinstance(value, float):
|
||||
return str(value)
|
||||
if isinstance(value, str):
|
||||
# Escape backslashes and quotes for TOML basic strings
|
||||
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return f'"{escaped}"'
|
||||
if isinstance(value, list):
|
||||
items = ", ".join(_format_toml_value(v) for v in value)
|
||||
return f"[{items}]"
|
||||
return repr(value)
|
||||
|
||||
|
||||
def _write_toml(path: Path, data: Dict[str, Any]) -> None:
|
||||
"""Write a dict as TOML to *path* using manual formatting."""
|
||||
lines: List[str] = []
|
||||
for section_name, section_data in data.items():
|
||||
if isinstance(section_data, dict):
|
||||
lines.append(f"[{section_name}]")
|
||||
for key, value in section_data.items():
|
||||
lines.append(f"{key} = {_format_toml_value(value)}")
|
||||
lines.append("")
|
||||
else:
|
||||
lines.append(f"{section_name} = {_format_toml_value(section_data)}")
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
class AgentConfigEvolver:
|
||||
"""Analyze traces to evolve agent TOML configs with versioning.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
trace_store:
|
||||
A :class:`TraceStore` used to fetch historical traces.
|
||||
config_dir:
|
||||
Directory where agent TOML configs are written.
|
||||
min_quality:
|
||||
Minimum average feedback score for a recommendation to be emitted.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
trace_store: TraceStore,
|
||||
*,
|
||||
config_dir: Union[str, Path],
|
||||
min_quality: float = 0.5,
|
||||
) -> None:
|
||||
self._store = trace_store
|
||||
self._config_dir = Path(config_dir)
|
||||
self._history_dir = self._config_dir / ".history"
|
||||
self._min_quality = min_quality
|
||||
|
||||
self._config_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._history_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# analyze
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def analyze(self) -> List[Dict[str, Any]]:
|
||||
"""Analyze traces, return recommendations per query class.
|
||||
|
||||
Returns a list of dicts, each containing:
|
||||
- ``query_class``: the classified query category
|
||||
- ``recommended_tools``: list of tool names sorted by frequency
|
||||
- ``recommended_agent``: the best-performing agent for this class
|
||||
- ``recommended_max_turns``: suggested max_turns value
|
||||
- ``sample_count``: number of traces analyzed for this class
|
||||
"""
|
||||
traces = self._store.list_traces(limit=10_000)
|
||||
if not traces:
|
||||
return []
|
||||
|
||||
# Group traces by query class
|
||||
groups: Dict[str, List[Trace]] = defaultdict(list)
|
||||
for trace in traces:
|
||||
qclass = classify_query(trace.query)
|
||||
groups[qclass].append(trace)
|
||||
|
||||
recommendations: List[Dict[str, Any]] = []
|
||||
for qclass, class_traces in sorted(groups.items()):
|
||||
rec = self._analyze_class(qclass, class_traces)
|
||||
if rec is not None:
|
||||
recommendations.append(rec)
|
||||
|
||||
return recommendations
|
||||
|
||||
def _analyze_class(
|
||||
self, qclass: str, traces: List[Trace]
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Build a recommendation for a single query class."""
|
||||
# Collect tool usage, agent performance, and turn counts
|
||||
tool_scores: Dict[str, _ToolScore] = defaultdict(lambda: _ToolScore())
|
||||
agent_scores: Dict[str, _AgentScore] = defaultdict(lambda: _AgentScore())
|
||||
turn_counts: List[int] = []
|
||||
|
||||
for trace in traces:
|
||||
feedback = trace.feedback if trace.feedback is not None else 0.0
|
||||
is_success = trace.outcome == "success"
|
||||
|
||||
# Count tools used in this trace
|
||||
trace_tools: List[str] = []
|
||||
tool_call_count = 0
|
||||
for step in trace.steps:
|
||||
step_type = (
|
||||
step.step_type.value
|
||||
if isinstance(step.step_type, StepType)
|
||||
else str(step.step_type)
|
||||
)
|
||||
if step_type == "tool_call":
|
||||
tool_call_count += 1
|
||||
tool_name = step.input.get("tool", "")
|
||||
if tool_name:
|
||||
trace_tools.append(tool_name)
|
||||
ts = tool_scores[tool_name]
|
||||
ts.count += 1
|
||||
ts.feedback_sum += feedback
|
||||
if is_success:
|
||||
ts.successes += 1
|
||||
|
||||
turn_counts.append(tool_call_count)
|
||||
|
||||
# Track agent performance
|
||||
if trace.agent:
|
||||
ag = agent_scores[trace.agent]
|
||||
ag.count += 1
|
||||
ag.feedback_sum += feedback
|
||||
if is_success:
|
||||
ag.successes += 1
|
||||
|
||||
if not agent_scores:
|
||||
return None
|
||||
|
||||
# Pick best agent by composite score
|
||||
best_agent = max(
|
||||
agent_scores.items(), key=lambda kv: kv[1].composite_score()
|
||||
)[0]
|
||||
|
||||
# Rank tools by composite score (frequency-weighted quality)
|
||||
ranked_tools = sorted(
|
||||
tool_scores.items(),
|
||||
key=lambda kv: kv[1].composite_score(),
|
||||
reverse=True,
|
||||
)
|
||||
recommended_tools = [name for name, _ in ranked_tools]
|
||||
|
||||
# Recommended max_turns: use the 75th percentile of observed tool calls
|
||||
# plus a small buffer, minimum 5
|
||||
if turn_counts:
|
||||
sorted_turns = sorted(turn_counts)
|
||||
p75_idx = int(len(sorted_turns) * 0.75)
|
||||
p75 = sorted_turns[min(p75_idx, len(sorted_turns) - 1)]
|
||||
recommended_max_turns = max(p75 + 2, 5)
|
||||
else:
|
||||
recommended_max_turns = 10
|
||||
|
||||
return {
|
||||
"query_class": qclass,
|
||||
"recommended_tools": recommended_tools,
|
||||
"recommended_agent": best_agent,
|
||||
"recommended_max_turns": recommended_max_turns,
|
||||
"sample_count": len(traces),
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# write_config
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def write_config(
|
||||
self,
|
||||
agent_name: str,
|
||||
*,
|
||||
tools: List[str],
|
||||
max_turns: int = 10,
|
||||
temperature: float = 0.3,
|
||||
system_prompt: str = "",
|
||||
) -> Path:
|
||||
"""Write agent TOML config, archiving previous version first.
|
||||
|
||||
Returns the :class:`Path` to the written config file.
|
||||
"""
|
||||
config_path = self._config_dir / f"{agent_name}.toml"
|
||||
|
||||
# Archive the existing config before overwriting
|
||||
if config_path.exists():
|
||||
self._archive(agent_name, config_path)
|
||||
|
||||
# Build the TOML data
|
||||
data = {
|
||||
"agent": {
|
||||
"name": agent_name,
|
||||
"tools": tools,
|
||||
"max_turns": max_turns,
|
||||
"temperature": temperature,
|
||||
"system_prompt": system_prompt,
|
||||
}
|
||||
}
|
||||
|
||||
_write_toml(config_path, data)
|
||||
return config_path
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# list_versions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def list_versions(self, agent_name: str) -> List[Dict[str, Any]]:
|
||||
"""List all versions (including current) for *agent_name*.
|
||||
|
||||
Returns a list of dicts with ``version``, ``path``, and ``modified``.
|
||||
Versions are numbered starting from 1 (oldest archived) through to
|
||||
the current (highest version number).
|
||||
"""
|
||||
versions: List[Dict[str, Any]] = []
|
||||
|
||||
# Collect archived versions from .history/
|
||||
pattern = f"{agent_name}.v*.toml"
|
||||
archived = sorted(self._history_dir.glob(pattern))
|
||||
for idx, archived_path in enumerate(archived, start=1):
|
||||
versions.append({
|
||||
"version": idx,
|
||||
"path": str(archived_path),
|
||||
"modified": archived_path.stat().st_mtime,
|
||||
})
|
||||
|
||||
# Current version
|
||||
current = self._config_dir / f"{agent_name}.toml"
|
||||
if current.exists():
|
||||
versions.append({
|
||||
"version": len(versions) + 1,
|
||||
"path": str(current),
|
||||
"modified": current.stat().st_mtime,
|
||||
})
|
||||
|
||||
return versions
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# rollback
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def rollback(self, agent_name: str, version: int) -> None:
|
||||
"""Rollback to a specific version.
|
||||
|
||||
Raises :class:`ValueError` if the requested version does not exist.
|
||||
"""
|
||||
versions = self.list_versions(agent_name)
|
||||
target = None
|
||||
for v in versions:
|
||||
if v["version"] == version:
|
||||
target = v
|
||||
break
|
||||
|
||||
if target is None:
|
||||
raise ValueError(
|
||||
f"Version {version} not found for agent '{agent_name}'. "
|
||||
f"Available versions: {[v['version'] for v in versions]}"
|
||||
)
|
||||
|
||||
target_path = Path(target["path"])
|
||||
config_path = self._config_dir / f"{agent_name}.toml"
|
||||
|
||||
# If the target is already the current file, nothing to do
|
||||
if target_path == config_path:
|
||||
return
|
||||
|
||||
# Archive current before rollback
|
||||
if config_path.exists():
|
||||
self._archive(agent_name, config_path)
|
||||
|
||||
# Copy the target version to become the current config
|
||||
shutil.copy2(str(target_path), str(config_path))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _archive(self, agent_name: str, config_path: Path) -> Path:
|
||||
"""Copy *config_path* into ``.history/`` with a version suffix."""
|
||||
# Determine next version number in .history/
|
||||
pattern = f"{agent_name}.v*.toml"
|
||||
existing = list(self._history_dir.glob(pattern))
|
||||
version_nums = []
|
||||
for p in existing:
|
||||
# Extract version number from name like "my_agent.v3.toml"
|
||||
stem = p.stem # "my_agent.v3"
|
||||
parts = stem.rsplit(".v", 1)
|
||||
if len(parts) == 2 and parts[1].isdigit():
|
||||
version_nums.append(int(parts[1]))
|
||||
next_ver = max(version_nums, default=0) + 1
|
||||
|
||||
dest = self._history_dir / f"{agent_name}.v{next_ver}.toml"
|
||||
shutil.copy2(str(config_path), str(dest))
|
||||
return dest
|
||||
|
||||
|
||||
class _ToolScore:
|
||||
"""Accumulator for per-tool scoring."""
|
||||
|
||||
__slots__ = ("count", "successes", "feedback_sum")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.count = 0
|
||||
self.successes = 0
|
||||
self.feedback_sum = 0.0
|
||||
|
||||
def composite_score(self) -> float:
|
||||
"""Weighted score combining success rate, feedback, and frequency."""
|
||||
if self.count == 0:
|
||||
return 0.0
|
||||
sr = self.successes / self.count
|
||||
fb = self.feedback_sum / self.count
|
||||
# Weight: 40% success + 40% feedback + 20% log-frequency
|
||||
import math
|
||||
|
||||
freq_bonus = math.log1p(self.count) / 10.0
|
||||
return 0.4 * sr + 0.4 * fb + 0.2 * min(freq_bonus, 1.0)
|
||||
|
||||
|
||||
class _AgentScore:
|
||||
"""Accumulator for per-agent scoring."""
|
||||
|
||||
__slots__ = ("count", "successes", "feedback_sum")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.count = 0
|
||||
self.successes = 0
|
||||
self.feedback_sum = 0.0
|
||||
|
||||
def composite_score(self) -> float:
|
||||
"""Weighted score combining success rate and feedback."""
|
||||
if self.count == 0:
|
||||
return 0.0
|
||||
sr = self.successes / self.count
|
||||
fb = self.feedback_sum / self.count
|
||||
return 0.6 * sr + 0.4 * fb
|
||||
|
||||
|
||||
__all__ = ["AgentConfigEvolver"]
|
||||
@@ -0,0 +1,202 @@
|
||||
"""LearningOrchestrator — coordinate the full trace->learn->eval loop.
|
||||
|
||||
Pulls traces from a :class:`TraceStore`, mines training data via
|
||||
:class:`TrainingDataMiner`, evolves agent configs via
|
||||
:class:`AgentConfigEvolver`, optionally runs LoRA fine-tuning, and
|
||||
gates acceptance on an evaluation function.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Optional, Union
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LearningOrchestrator:
|
||||
"""Orchestrate a single trace->learn->eval cycle.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
trace_store:
|
||||
Object with ``list_traces(limit=...)`` returning ``List[Trace]``
|
||||
(typically a :class:`TraceStore`).
|
||||
config_dir:
|
||||
Directory where agent TOML configs are written / evolved.
|
||||
eval_fn:
|
||||
Optional callable returning a float score (higher = better).
|
||||
Called before and after learning to gate acceptance.
|
||||
min_improvement:
|
||||
Minimum improvement in eval score required to accept the update.
|
||||
min_sft_pairs:
|
||||
Minimum number of SFT pairs required to trigger LoRA training.
|
||||
min_quality:
|
||||
Minimum feedback quality threshold for :class:`TrainingDataMiner`.
|
||||
lora_config:
|
||||
Optional :class:`LoRATrainingConfig`. When provided (and enough
|
||||
SFT pairs exist and ``torch`` is available), LoRA training runs.
|
||||
model_name:
|
||||
Model name for LoRA training (passed to :class:`LoRATrainer`).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
trace_store: Any,
|
||||
config_dir: Union[str, Path],
|
||||
eval_fn: Optional[Callable[[], float]] = None,
|
||||
min_improvement: float = 0.02,
|
||||
min_sft_pairs: int = 10,
|
||||
min_quality: float = 0.7,
|
||||
lora_config: Optional[Any] = None,
|
||||
model_name: Optional[str] = None,
|
||||
) -> None:
|
||||
from openjarvis.learning.agent_evolver import AgentConfigEvolver
|
||||
from openjarvis.learning.training.data import TrainingDataMiner
|
||||
|
||||
self._trace_store = trace_store
|
||||
self._config_dir = Path(config_dir)
|
||||
self._eval_fn = eval_fn
|
||||
self._min_improvement = min_improvement
|
||||
self._min_sft_pairs = min_sft_pairs
|
||||
self._lora_config = lora_config
|
||||
self._model_name = model_name
|
||||
|
||||
self._miner = TrainingDataMiner(trace_store, min_quality=min_quality)
|
||||
self._evolver = AgentConfigEvolver(
|
||||
trace_store, config_dir=self._config_dir
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run(self) -> Dict[str, Any]:
|
||||
"""Execute one learning cycle.
|
||||
|
||||
Returns a dict with at least ``timestamp`` and ``status`` keys.
|
||||
|
||||
Steps
|
||||
-----
|
||||
1. Mine traces: extract sft_pairs, routing_pairs, agent_pairs
|
||||
2. If no data: return skipped
|
||||
3. Run baseline eval (if eval_fn provided)
|
||||
4. Update routing recommendations
|
||||
5. Evolve agent configs
|
||||
6. Run LoRA training (if lora_config provided AND enough pairs
|
||||
AND torch available)
|
||||
7. Run post-learning eval (if eval_fn provided)
|
||||
8. Accept/reject based on improvement threshold
|
||||
"""
|
||||
result: Dict[str, Any] = {
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
|
||||
# 1. Mine training data from traces
|
||||
sft_pairs = self._miner.extract_sft_pairs()
|
||||
routing_pairs = self._miner.extract_routing_pairs()
|
||||
agent_pairs = self._miner.extract_agent_config_pairs()
|
||||
|
||||
result["sft_pairs"] = len(sft_pairs)
|
||||
result["routing_classes"] = len(routing_pairs)
|
||||
result["agent_classes"] = len(agent_pairs)
|
||||
|
||||
# 2. Check if there is any data at all
|
||||
total_data = len(sft_pairs) + len(routing_pairs) + len(agent_pairs)
|
||||
if total_data == 0:
|
||||
result["status"] = "skipped"
|
||||
result["reason"] = "no training data available"
|
||||
return result
|
||||
|
||||
# 3. Run baseline eval
|
||||
baseline_score: Optional[float] = None
|
||||
if self._eval_fn is not None:
|
||||
baseline_score = self._eval_fn()
|
||||
result["baseline_score"] = baseline_score
|
||||
|
||||
# 4. Update routing recommendations
|
||||
result["routing_updated"] = len(routing_pairs) > 0
|
||||
|
||||
# 5. Evolve agent configs
|
||||
recommendations = self._evolver.analyze()
|
||||
result["agent_configs_evolved"] = len(recommendations) > 0
|
||||
for rec in recommendations:
|
||||
agent_name = rec.get("recommended_agent", "default")
|
||||
tools = rec.get("recommended_tools", [])
|
||||
max_turns = rec.get("recommended_max_turns", 10)
|
||||
self._evolver.write_config(
|
||||
agent_name, tools=tools, max_turns=max_turns
|
||||
)
|
||||
|
||||
# 6. LoRA training (optional)
|
||||
result["lora_training"] = None
|
||||
if (
|
||||
self._lora_config is not None
|
||||
and len(sft_pairs) >= self._min_sft_pairs
|
||||
):
|
||||
lora_result = self._try_lora_training(sft_pairs)
|
||||
result["lora_training"] = lora_result
|
||||
|
||||
# 7. Post-learning eval
|
||||
post_score: Optional[float] = None
|
||||
if self._eval_fn is not None:
|
||||
post_score = self._eval_fn()
|
||||
result["post_score"] = post_score
|
||||
|
||||
# 8. Accept/reject based on improvement
|
||||
if baseline_score is not None and post_score is not None:
|
||||
improvement = post_score - baseline_score
|
||||
result["improvement"] = improvement
|
||||
if improvement >= self._min_improvement:
|
||||
result["accepted"] = True
|
||||
result["status"] = "completed"
|
||||
else:
|
||||
result["accepted"] = False
|
||||
result["status"] = "rejected"
|
||||
result["reason"] = (
|
||||
f"eval improvement {improvement:.4f} below "
|
||||
f"threshold {self._min_improvement}"
|
||||
)
|
||||
else:
|
||||
# No eval gate — always accept
|
||||
result["accepted"] = True
|
||||
result["status"] = "completed"
|
||||
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _try_lora_training(
|
||||
self, sft_pairs: list[Dict[str, Any]]
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Attempt LoRA training, returning result or None on failure."""
|
||||
try:
|
||||
from openjarvis.learning.training.lora import (
|
||||
HAS_TORCH,
|
||||
LoRATrainer,
|
||||
)
|
||||
except ImportError:
|
||||
logger.info("LoRA training skipped: training.lora not importable")
|
||||
return {"status": "skipped", "reason": "lora module unavailable"}
|
||||
|
||||
if not HAS_TORCH:
|
||||
logger.info("LoRA training skipped: torch not available")
|
||||
return {"status": "skipped", "reason": "torch not available"}
|
||||
|
||||
try:
|
||||
model_name = self._model_name or "Qwen/Qwen3-0.6B"
|
||||
trainer = LoRATrainer(
|
||||
self._lora_config, model_name=model_name
|
||||
)
|
||||
return trainer.train(sft_pairs)
|
||||
except Exception as exc:
|
||||
logger.warning("LoRA training failed: %s", exc)
|
||||
return {"status": "error", "reason": str(exc)}
|
||||
|
||||
|
||||
__all__ = ["LearningOrchestrator"]
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Training data extraction and fine-tuning pipelines for trace-driven learning."""
|
||||
|
||||
from openjarvis.learning.training.data import TrainingDataMiner
|
||||
from openjarvis.learning.training.lora import (
|
||||
HAS_TORCH,
|
||||
LoRATrainer,
|
||||
LoRATrainingConfig,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"HAS_TORCH",
|
||||
"LoRATrainer",
|
||||
"LoRATrainingConfig",
|
||||
"TrainingDataMiner",
|
||||
]
|
||||
@@ -0,0 +1,214 @@
|
||||
"""TrainingDataMiner — extract supervised training pairs from the TraceStore.
|
||||
|
||||
Provides three extraction modes:
|
||||
|
||||
* **SFT pairs** — (input, output) pairs from high-quality traces for
|
||||
supervised fine-tuning.
|
||||
* **Routing pairs** — per-query-class statistics identifying the best
|
||||
model for each class.
|
||||
* **Agent config pairs** — per-query-class statistics identifying the
|
||||
best agent and tool combination.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from openjarvis.core.types import StepType, Trace
|
||||
from openjarvis.learning.trace_policy import classify_query
|
||||
|
||||
|
||||
class TrainingDataMiner:
|
||||
"""Extract supervised training pairs from stored traces.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
trace_store:
|
||||
Any object with a ``list_traces(limit=...)`` method returning
|
||||
``List[Trace]`` (typically a :class:`TraceStore`).
|
||||
min_quality:
|
||||
Minimum ``feedback`` score for a trace to be included.
|
||||
min_samples_per_class:
|
||||
Minimum number of samples a query class must have to appear in
|
||||
routing/agent-config results.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
trace_store: Any,
|
||||
*,
|
||||
min_quality: float = 0.7,
|
||||
min_samples_per_class: int = 1,
|
||||
) -> None:
|
||||
self._store = trace_store
|
||||
self._min_quality = min_quality
|
||||
self._min_samples_per_class = min_samples_per_class
|
||||
|
||||
# -- helpers ----------------------------------------------------------------
|
||||
|
||||
def _quality_traces(self) -> List[Trace]:
|
||||
"""Return traces whose feedback meets the quality threshold."""
|
||||
all_traces = self._store.list_traces(limit=10000)
|
||||
return [
|
||||
t
|
||||
for t in all_traces
|
||||
if t.feedback is not None
|
||||
and t.feedback >= self._min_quality
|
||||
and t.outcome == "success"
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _tools_from_trace(trace: Trace) -> List[str]:
|
||||
"""Extract tool names from TOOL_CALL steps in a trace."""
|
||||
tools: List[str] = []
|
||||
for step in trace.steps:
|
||||
if step.step_type == StepType.TOOL_CALL:
|
||||
tool_name = step.input.get("tool")
|
||||
if tool_name:
|
||||
tools.append(tool_name)
|
||||
return tools
|
||||
|
||||
# -- public API -------------------------------------------------------------
|
||||
|
||||
def extract_sft_pairs(self) -> List[Dict[str, Any]]:
|
||||
"""Return SFT training pairs from high-quality traces.
|
||||
|
||||
Each entry is a dict with keys: ``input``, ``output``,
|
||||
``query_class``, ``model``, ``feedback``.
|
||||
|
||||
Duplicate ``(input, output)`` pairs are collapsed; the first
|
||||
occurrence is kept.
|
||||
"""
|
||||
traces = self._quality_traces()
|
||||
seen: set[tuple[str, str]] = set()
|
||||
pairs: List[Dict[str, Any]] = []
|
||||
|
||||
for t in traces:
|
||||
key = (t.query, t.result)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
pairs.append(
|
||||
{
|
||||
"input": t.query,
|
||||
"output": t.result,
|
||||
"query_class": classify_query(t.query),
|
||||
"model": t.model,
|
||||
"feedback": t.feedback,
|
||||
}
|
||||
)
|
||||
|
||||
return pairs
|
||||
|
||||
def extract_routing_pairs(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Return per-query-class routing recommendations.
|
||||
|
||||
Returns a dict mapping query class to:
|
||||
|
||||
* ``best_model`` — model with highest average feedback for the class.
|
||||
* ``avg_feedback`` — average feedback across all models for the class.
|
||||
* ``sample_count`` — total number of qualifying traces in the class.
|
||||
* ``all_models`` — dict of ``{model: {"avg_feedback": float, "count": int}}``.
|
||||
"""
|
||||
traces = self._quality_traces()
|
||||
|
||||
# Accumulate per (query_class, model) feedback scores
|
||||
class_model_scores: Dict[str, Dict[str, List[float]]] = defaultdict(
|
||||
lambda: defaultdict(list)
|
||||
)
|
||||
for t in traces:
|
||||
qc = classify_query(t.query)
|
||||
class_model_scores[qc][t.model].append(t.feedback) # type: ignore[arg-type]
|
||||
|
||||
result: Dict[str, Dict[str, Any]] = {}
|
||||
for qc, model_scores in class_model_scores.items():
|
||||
total_count = sum(len(scores) for scores in model_scores.values())
|
||||
if total_count < self._min_samples_per_class:
|
||||
continue
|
||||
|
||||
all_models: Dict[str, Dict[str, Any]] = {}
|
||||
best_model = ""
|
||||
best_avg = -1.0
|
||||
|
||||
for model, scores in model_scores.items():
|
||||
avg = sum(scores) / len(scores)
|
||||
all_models[model] = {"avg_feedback": avg, "count": len(scores)}
|
||||
if avg > best_avg:
|
||||
best_avg = avg
|
||||
best_model = model
|
||||
|
||||
total_scores = [s for scores in model_scores.values() for s in scores]
|
||||
overall_avg = sum(total_scores) / len(total_scores) if total_scores else 0.0
|
||||
|
||||
result[qc] = {
|
||||
"best_model": best_model,
|
||||
"avg_feedback": overall_avg,
|
||||
"sample_count": total_count,
|
||||
"all_models": all_models,
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
def extract_agent_config_pairs(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Return per-query-class agent and tool recommendations.
|
||||
|
||||
Returns a dict mapping query class to:
|
||||
|
||||
* ``best_agent`` — agent with the highest average feedback.
|
||||
* ``best_tools`` — most frequently used tools by the best agent.
|
||||
* ``avg_feedback`` — average feedback across all agents for the class.
|
||||
* ``sample_count`` — total number of qualifying traces in the class.
|
||||
"""
|
||||
traces = self._quality_traces()
|
||||
|
||||
# Accumulate per (query_class, agent) feedback and tools
|
||||
class_agent_scores: Dict[str, Dict[str, List[float]]] = defaultdict(
|
||||
lambda: defaultdict(list)
|
||||
)
|
||||
class_agent_tools: Dict[str, Dict[str, List[List[str]]]] = defaultdict(
|
||||
lambda: defaultdict(list)
|
||||
)
|
||||
|
||||
for t in traces:
|
||||
qc = classify_query(t.query)
|
||||
class_agent_scores[qc][t.agent].append(t.feedback) # type: ignore[arg-type]
|
||||
tools = self._tools_from_trace(t)
|
||||
class_agent_tools[qc][t.agent].append(tools)
|
||||
|
||||
result: Dict[str, Dict[str, Any]] = {}
|
||||
for qc, agent_scores in class_agent_scores.items():
|
||||
total_count = sum(len(scores) for scores in agent_scores.values())
|
||||
if total_count < self._min_samples_per_class:
|
||||
continue
|
||||
|
||||
best_agent = ""
|
||||
best_avg = -1.0
|
||||
for agent, scores in agent_scores.items():
|
||||
avg = sum(scores) / len(scores)
|
||||
if avg > best_avg:
|
||||
best_avg = avg
|
||||
best_agent = agent
|
||||
|
||||
# Collect tool frequency for best agent
|
||||
tool_freq: Dict[str, int] = defaultdict(int)
|
||||
for tool_list in class_agent_tools[qc].get(best_agent, []):
|
||||
for tool in tool_list:
|
||||
tool_freq[tool] += 1
|
||||
|
||||
best_tools = sorted(tool_freq, key=tool_freq.get, reverse=True) # type: ignore[arg-type]
|
||||
|
||||
total_scores = [s for scores in agent_scores.values() for s in scores]
|
||||
overall_avg = sum(total_scores) / len(total_scores) if total_scores else 0.0
|
||||
|
||||
result[qc] = {
|
||||
"best_agent": best_agent,
|
||||
"best_tools": best_tools,
|
||||
"avg_feedback": overall_avg,
|
||||
"sample_count": total_count,
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
__all__ = ["TrainingDataMiner"]
|
||||
@@ -0,0 +1,438 @@
|
||||
"""LoRATrainer — fine-tune local models via LoRA/QLoRA from trace-derived SFT pairs.
|
||||
|
||||
All ``torch``, ``transformers``, and ``peft`` imports are guarded so the
|
||||
module can be imported without GPU dependencies. The :class:`LoRATrainingConfig`
|
||||
dataclass works without any optional deps; :class:`LoRATrainer` raises
|
||||
``ImportError`` at construction time when ``torch`` is unavailable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Optional imports -----------------------------------------------------------
|
||||
try:
|
||||
import torch
|
||||
|
||||
HAS_TORCH = True
|
||||
except ImportError:
|
||||
HAS_TORCH = False
|
||||
torch = None # type: ignore[assignment]
|
||||
|
||||
try:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HAS_TRANSFORMERS = True
|
||||
except ImportError:
|
||||
HAS_TRANSFORMERS = False
|
||||
AutoModelForCausalLM = None # type: ignore[assignment,misc]
|
||||
AutoTokenizer = None # type: ignore[assignment,misc]
|
||||
|
||||
try:
|
||||
from peft import LoraConfig, TaskType, get_peft_model
|
||||
|
||||
HAS_PEFT = True
|
||||
except ImportError:
|
||||
HAS_PEFT = False
|
||||
LoraConfig = None # type: ignore[assignment,misc]
|
||||
TaskType = None # type: ignore[assignment,misc]
|
||||
get_peft_model = None # type: ignore[assignment,misc]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Device selection (shared with orchestrator sft_trainer)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _select_device(hint: Optional[str] = None) -> str:
|
||||
"""Select the best available PyTorch device.
|
||||
|
||||
Priority: explicit *hint* > cuda > mps > cpu.
|
||||
"""
|
||||
if hint is not None:
|
||||
return hint
|
||||
if not HAS_TORCH or torch is None:
|
||||
return "cpu"
|
||||
if torch.cuda.is_available():
|
||||
return "cuda"
|
||||
if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
|
||||
return "mps"
|
||||
return "cpu"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoRATrainingConfig:
|
||||
"""Configuration for LoRA / QLoRA fine-tuning."""
|
||||
|
||||
# LoRA params
|
||||
lora_rank: int = 16
|
||||
lora_alpha: int = 32
|
||||
lora_dropout: float = 0.05
|
||||
target_modules: List[str] = field(
|
||||
default_factory=lambda: ["q_proj", "v_proj"]
|
||||
)
|
||||
|
||||
# Training params
|
||||
num_epochs: int = 3
|
||||
batch_size: int = 4
|
||||
learning_rate: float = 2e-5
|
||||
weight_decay: float = 0.01
|
||||
warmup_ratio: float = 0.1
|
||||
max_grad_norm: float = 1.0
|
||||
max_seq_length: int = 2048
|
||||
|
||||
# QLoRA
|
||||
use_4bit: bool = False
|
||||
|
||||
# Output
|
||||
output_dir: str = "checkpoints/lora"
|
||||
save_every_n_epochs: int = 1
|
||||
|
||||
# Memory
|
||||
gradient_checkpointing: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.lora_rank < 1:
|
||||
raise ValueError(
|
||||
f"lora_rank must be >= 1, got {self.lora_rank}"
|
||||
)
|
||||
if self.num_epochs < 1:
|
||||
raise ValueError(
|
||||
f"num_epochs must be >= 1, got {self.num_epochs}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Trainer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LoRATrainer:
|
||||
"""Fine-tune a local causal LM with LoRA (or QLoRA) adapters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config:
|
||||
LoRA training configuration.
|
||||
model_name:
|
||||
HuggingFace model identifier or local path.
|
||||
device:
|
||||
PyTorch device string. ``None`` auto-detects (cuda > mps > cpu).
|
||||
|
||||
Raises
|
||||
------
|
||||
ImportError
|
||||
If ``torch`` is not installed.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: LoRATrainingConfig,
|
||||
*,
|
||||
model_name: str = "Qwen/Qwen3-0.6B",
|
||||
device: Optional[str] = None,
|
||||
) -> None:
|
||||
if not HAS_TORCH:
|
||||
raise ImportError(
|
||||
"torch is required for LoRATrainer. "
|
||||
"Install with: pip install torch transformers peft"
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.model_name = model_name
|
||||
self.device = _select_device(device)
|
||||
self.tokenizer: Any = None
|
||||
self.model: Any = None
|
||||
|
||||
# -- Public API ----------------------------------------------------------
|
||||
|
||||
def prepare_dataset(
|
||||
self, pairs: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Convert SFT pairs to tokenized examples.
|
||||
|
||||
Each returned dict contains ``input_ids``, ``attention_mask``,
|
||||
and ``text`` (the raw formatted string before tokenization).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pairs:
|
||||
List of dicts with at least ``input`` and ``output`` keys,
|
||||
as produced by :class:`TrainingDataMiner.extract_sft_pairs`.
|
||||
"""
|
||||
self._ensure_tokenizer()
|
||||
|
||||
dataset: List[Dict[str, Any]] = []
|
||||
for pair in pairs:
|
||||
text = self._format_pair(pair)
|
||||
encoding = self.tokenizer(
|
||||
text,
|
||||
truncation=True,
|
||||
max_length=self.config.max_seq_length,
|
||||
padding="max_length",
|
||||
return_tensors="pt",
|
||||
)
|
||||
dataset.append({
|
||||
"input_ids": encoding["input_ids"].squeeze(0),
|
||||
"attention_mask": encoding["attention_mask"].squeeze(0),
|
||||
"text": text,
|
||||
})
|
||||
|
||||
return dataset
|
||||
|
||||
def train(self, pairs: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Run LoRA fine-tuning on the given SFT pairs.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pairs:
|
||||
List of dicts with at least ``input`` and ``output`` keys.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
Training summary with keys: ``status``, ``epochs``,
|
||||
``total_steps``, ``avg_loss``, ``adapter_path``,
|
||||
``training_samples``.
|
||||
"""
|
||||
if not pairs:
|
||||
return {"status": "skipped", "reason": "no training data"}
|
||||
|
||||
dataset = self.prepare_dataset(pairs)
|
||||
self._load_model()
|
||||
self._apply_lora()
|
||||
|
||||
optimizer = torch.optim.AdamW(
|
||||
self.model.parameters(),
|
||||
lr=self.config.learning_rate,
|
||||
weight_decay=self.config.weight_decay,
|
||||
)
|
||||
|
||||
total_steps = 0
|
||||
cumulative_loss = 0.0
|
||||
num_loss_steps = 0
|
||||
|
||||
self.model.train()
|
||||
|
||||
for epoch in range(self.config.num_epochs):
|
||||
epoch_loss = self._train_epoch(dataset, optimizer)
|
||||
steps_in_epoch = max(
|
||||
1, (len(dataset) + self.config.batch_size - 1) // self.config.batch_size
|
||||
)
|
||||
total_steps += steps_in_epoch
|
||||
cumulative_loss += epoch_loss * steps_in_epoch
|
||||
num_loss_steps += steps_in_epoch
|
||||
|
||||
logger.info(
|
||||
"Epoch %d/%d loss=%.4f",
|
||||
epoch + 1,
|
||||
self.config.num_epochs,
|
||||
epoch_loss,
|
||||
)
|
||||
|
||||
if (epoch + 1) % self.config.save_every_n_epochs == 0:
|
||||
self._save_adapter(epoch + 1)
|
||||
|
||||
avg_loss = cumulative_loss / num_loss_steps if num_loss_steps else 0.0
|
||||
adapter_path = str(Path(self.config.output_dir) / "final")
|
||||
self._save_adapter_to(adapter_path)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"epochs": self.config.num_epochs,
|
||||
"total_steps": total_steps,
|
||||
"avg_loss": avg_loss,
|
||||
"adapter_path": adapter_path,
|
||||
"training_samples": len(pairs),
|
||||
}
|
||||
|
||||
# -- Internal helpers ----------------------------------------------------
|
||||
|
||||
def _ensure_tokenizer(self) -> None:
|
||||
"""Lazily load the tokenizer."""
|
||||
if self.tokenizer is not None:
|
||||
return
|
||||
|
||||
if not HAS_TRANSFORMERS:
|
||||
raise ImportError(
|
||||
"transformers is required for LoRATrainer. "
|
||||
"Install with: pip install transformers"
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
|
||||
if self.tokenizer.pad_token is None:
|
||||
self.tokenizer.pad_token = self.tokenizer.eos_token
|
||||
|
||||
def _load_model(self) -> None:
|
||||
"""Load the base model for fine-tuning."""
|
||||
if self.model is not None:
|
||||
return
|
||||
|
||||
if not HAS_TRANSFORMERS:
|
||||
raise ImportError(
|
||||
"transformers is required for LoRATrainer. "
|
||||
"Install with: pip install transformers"
|
||||
)
|
||||
|
||||
self._ensure_tokenizer()
|
||||
|
||||
model_kwargs: Dict[str, Any] = {"torch_dtype": torch.bfloat16}
|
||||
|
||||
if self.config.use_4bit:
|
||||
try:
|
||||
from transformers import BitsAndBytesConfig
|
||||
|
||||
model_kwargs["quantization_config"] = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_compute_dtype=torch.bfloat16,
|
||||
bnb_4bit_use_double_quant=True,
|
||||
bnb_4bit_quant_type="nf4",
|
||||
)
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"bitsandbytes not installed; falling back to bf16 "
|
||||
"(QLoRA disabled)"
|
||||
)
|
||||
|
||||
if self.device == "cuda" or self.device == "auto":
|
||||
model_kwargs["device_map"] = "auto"
|
||||
else:
|
||||
model_kwargs["device_map"] = {"": self.device}
|
||||
|
||||
self.model = AutoModelForCausalLM.from_pretrained(
|
||||
self.model_name, **model_kwargs
|
||||
)
|
||||
|
||||
if self.config.gradient_checkpointing and hasattr(
|
||||
self.model, "gradient_checkpointing_enable"
|
||||
):
|
||||
self.model.gradient_checkpointing_enable(
|
||||
gradient_checkpointing_kwargs={"use_reentrant": False}
|
||||
)
|
||||
|
||||
def _apply_lora(self) -> None:
|
||||
"""Wrap the loaded model with LoRA adapters via peft."""
|
||||
if not HAS_PEFT:
|
||||
raise ImportError(
|
||||
"peft is required for LoRA training. "
|
||||
"Install with: pip install peft"
|
||||
)
|
||||
|
||||
lora_config = LoraConfig(
|
||||
task_type=TaskType.CAUSAL_LM,
|
||||
r=self.config.lora_rank,
|
||||
lora_alpha=self.config.lora_alpha,
|
||||
lora_dropout=self.config.lora_dropout,
|
||||
target_modules=self.config.target_modules,
|
||||
)
|
||||
self.model = get_peft_model(self.model, lora_config)
|
||||
logger.info(
|
||||
"LoRA applied: rank=%d, alpha=%d, targets=%s",
|
||||
self.config.lora_rank,
|
||||
self.config.lora_alpha,
|
||||
self.config.target_modules,
|
||||
)
|
||||
|
||||
def _format_pair(self, pair: Dict[str, Any]) -> str:
|
||||
"""Format an SFT pair as a chat-style training string."""
|
||||
user_input = pair.get("input", "")
|
||||
assistant_output = pair.get("output", "")
|
||||
|
||||
if self.tokenizer is not None and hasattr(
|
||||
self.tokenizer, "apply_chat_template"
|
||||
):
|
||||
try:
|
||||
messages = [
|
||||
{"role": "user", "content": user_input},
|
||||
{"role": "assistant", "content": assistant_output},
|
||||
]
|
||||
return self.tokenizer.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=False
|
||||
)
|
||||
except Exception:
|
||||
pass # fall through to manual format
|
||||
|
||||
eos = ""
|
||||
if self.tokenizer is not None:
|
||||
eos = getattr(self.tokenizer, "eos_token", "") or ""
|
||||
return f"<|user|>\n{user_input}\n<|assistant|>\n{assistant_output}{eos}"
|
||||
|
||||
def _train_epoch(
|
||||
self,
|
||||
dataset: List[Dict[str, Any]],
|
||||
optimizer: Any,
|
||||
) -> float:
|
||||
"""Train one epoch over the dataset. Returns average loss."""
|
||||
total_loss = 0.0
|
||||
num_batches = 0
|
||||
|
||||
for i in range(0, len(dataset), self.config.batch_size):
|
||||
batch_items = dataset[i : i + self.config.batch_size]
|
||||
loss = self._train_step(batch_items, optimizer)
|
||||
total_loss += loss
|
||||
num_batches += 1
|
||||
|
||||
return total_loss / num_batches if num_batches else 0.0
|
||||
|
||||
def _train_step(
|
||||
self,
|
||||
batch_items: List[Dict[str, Any]],
|
||||
optimizer: Any,
|
||||
) -> float:
|
||||
"""Execute a single training step on a micro-batch."""
|
||||
input_ids = torch.stack(
|
||||
[item["input_ids"] for item in batch_items]
|
||||
).to(self.device)
|
||||
attention_mask = torch.stack(
|
||||
[item["attention_mask"] for item in batch_items]
|
||||
).to(self.device)
|
||||
|
||||
outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask,
|
||||
labels=input_ids,
|
||||
)
|
||||
loss = outputs.loss
|
||||
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm_(
|
||||
self.model.parameters(), self.config.max_grad_norm
|
||||
)
|
||||
optimizer.step()
|
||||
|
||||
return loss.item()
|
||||
|
||||
def _save_adapter(self, epoch: int) -> None:
|
||||
"""Save adapter checkpoint for the given epoch."""
|
||||
path = str(Path(self.config.output_dir) / f"epoch_{epoch}")
|
||||
self._save_adapter_to(path)
|
||||
|
||||
def _save_adapter_to(self, path: str) -> None:
|
||||
"""Save the LoRA adapter (and tokenizer) to *path*."""
|
||||
out = Path(path)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if self.model is not None:
|
||||
self.model.save_pretrained(path)
|
||||
if self.tokenizer is not None:
|
||||
self.tokenizer.save_pretrained(path)
|
||||
|
||||
logger.info("Adapter saved to %s", path)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"HAS_TORCH",
|
||||
"LoRATrainer",
|
||||
"LoRATrainingConfig",
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Recipe system — composable pillar configurations."""
|
||||
|
||||
from openjarvis.recipes.loader import (
|
||||
Recipe,
|
||||
discover_recipes,
|
||||
load_recipe,
|
||||
resolve_recipe,
|
||||
)
|
||||
|
||||
__all__ = ["Recipe", "discover_recipes", "load_recipe", "resolve_recipe"]
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Recipe loader — load and resolve TOML recipe files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib # type: ignore[no-redef]
|
||||
|
||||
|
||||
# Project-level recipes directory (sibling of src/)
|
||||
_PROJECT_RECIPES_DIR = Path(__file__).resolve().parents[3] / "recipes"
|
||||
# User-level recipes directory
|
||||
_USER_RECIPES_DIR = Path.home() / ".openjarvis" / "recipes"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Recipe:
|
||||
"""A composable pillar configuration loaded from TOML."""
|
||||
|
||||
name: str
|
||||
description: str = ""
|
||||
version: str = "1.0.0"
|
||||
|
||||
# Intelligence
|
||||
model: Optional[str] = None
|
||||
quantization: Optional[str] = None
|
||||
|
||||
# Engine
|
||||
engine_key: Optional[str] = None
|
||||
|
||||
# Agent
|
||||
agent_type: Optional[str] = None
|
||||
max_turns: Optional[int] = None
|
||||
temperature: Optional[float] = None
|
||||
tools: List[str] = field(default_factory=list)
|
||||
system_prompt: Optional[str] = None
|
||||
|
||||
# Learning
|
||||
routing_policy: Optional[str] = None
|
||||
agent_policy: Optional[str] = None
|
||||
|
||||
# Eval
|
||||
eval_suites: List[str] = field(default_factory=list)
|
||||
|
||||
# Raw TOML data for forward-compat
|
||||
raw: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_builder_kwargs(self) -> Dict[str, Any]:
|
||||
"""Convert recipe fields to kwargs for SystemBuilder/Jarvis.
|
||||
|
||||
Returns a dict with only the non-None fields, keyed to match
|
||||
the SystemBuilder fluent API or Jarvis constructor parameters.
|
||||
"""
|
||||
kwargs: Dict[str, Any] = {}
|
||||
if self.model is not None:
|
||||
kwargs["model"] = self.model
|
||||
if self.engine_key is not None:
|
||||
kwargs["engine_key"] = self.engine_key
|
||||
if self.agent_type is not None:
|
||||
kwargs["agent"] = self.agent_type
|
||||
if self.tools:
|
||||
kwargs["tools"] = self.tools
|
||||
if self.temperature is not None:
|
||||
kwargs["temperature"] = self.temperature
|
||||
if self.max_turns is not None:
|
||||
kwargs["max_turns"] = self.max_turns
|
||||
if self.system_prompt is not None:
|
||||
kwargs["system_prompt"] = self.system_prompt
|
||||
if self.routing_policy is not None:
|
||||
kwargs["routing_policy"] = self.routing_policy
|
||||
if self.agent_policy is not None:
|
||||
kwargs["agent_policy"] = self.agent_policy
|
||||
if self.quantization is not None:
|
||||
kwargs["quantization"] = self.quantization
|
||||
if self.eval_suites:
|
||||
kwargs["eval_suites"] = self.eval_suites
|
||||
return kwargs
|
||||
|
||||
|
||||
def load_recipe(path: str | Path) -> Recipe:
|
||||
"""Load a recipe from a TOML file.
|
||||
|
||||
Expected TOML format::
|
||||
|
||||
[recipe]
|
||||
name = "coding_assistant"
|
||||
description = "..."
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:8b"
|
||||
quantization = "q4_K_M"
|
||||
|
||||
[engine]
|
||||
key = "ollama"
|
||||
|
||||
[agent]
|
||||
type = "native_react"
|
||||
max_turns = 10
|
||||
temperature = 0.3
|
||||
tools = ["file_read", "file_write", "code_interpreter", "think"]
|
||||
system_prompt = "You are a coding assistant..."
|
||||
|
||||
[learning]
|
||||
routing = "grpo"
|
||||
agent = "icl_updater"
|
||||
|
||||
[eval]
|
||||
suites = ["coding", "reasoning"]
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If *path* does not exist.
|
||||
"""
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Recipe file not found: {path}")
|
||||
|
||||
with open(path, "rb") as fh:
|
||||
data = tomllib.load(fh)
|
||||
|
||||
recipe_sec = data.get("recipe", {})
|
||||
intel_sec = data.get("intelligence", {})
|
||||
engine_sec = data.get("engine", {})
|
||||
agent_sec = data.get("agent", {})
|
||||
learning_sec = data.get("learning", {})
|
||||
eval_sec = data.get("eval", {})
|
||||
|
||||
return Recipe(
|
||||
name=recipe_sec.get("name", path.stem),
|
||||
description=recipe_sec.get("description", ""),
|
||||
version=recipe_sec.get("version", "1.0.0"),
|
||||
model=intel_sec.get("model"),
|
||||
quantization=intel_sec.get("quantization"),
|
||||
engine_key=engine_sec.get("key"),
|
||||
agent_type=agent_sec.get("type"),
|
||||
max_turns=agent_sec.get("max_turns"),
|
||||
temperature=agent_sec.get("temperature"),
|
||||
tools=agent_sec.get("tools", []),
|
||||
system_prompt=agent_sec.get("system_prompt"),
|
||||
routing_policy=learning_sec.get("routing"),
|
||||
agent_policy=learning_sec.get("agent"),
|
||||
eval_suites=eval_sec.get("suites", []),
|
||||
raw=data,
|
||||
)
|
||||
|
||||
|
||||
def discover_recipes(
|
||||
extra_dirs: Optional[List[str | Path]] = None,
|
||||
) -> List[Recipe]:
|
||||
"""Discover all TOML recipes from known directories.
|
||||
|
||||
Search order (later entries override earlier ones by name):
|
||||
1. Project ``recipes/`` directory
|
||||
2. User ``~/.openjarvis/recipes/`` directory
|
||||
3. Any additional directories in *extra_dirs*
|
||||
"""
|
||||
dirs: List[Path] = [_PROJECT_RECIPES_DIR, _USER_RECIPES_DIR]
|
||||
if extra_dirs:
|
||||
dirs.extend(Path(d) for d in extra_dirs)
|
||||
|
||||
recipes: Dict[str, Recipe] = {}
|
||||
for d in dirs:
|
||||
if not d.is_dir():
|
||||
continue
|
||||
for toml_path in sorted(d.glob("*.toml")):
|
||||
try:
|
||||
recipe = load_recipe(toml_path)
|
||||
recipes[recipe.name] = recipe
|
||||
except Exception:
|
||||
# Skip malformed recipe files
|
||||
continue
|
||||
|
||||
return list(recipes.values())
|
||||
|
||||
|
||||
def resolve_recipe(name: str) -> Optional[Recipe]:
|
||||
"""Find a recipe by name from all known directories.
|
||||
|
||||
Returns ``None`` if no recipe with the given name is found.
|
||||
"""
|
||||
for recipe in discover_recipes():
|
||||
if recipe.name == name:
|
||||
return recipe
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ["Recipe", "discover_recipes", "load_recipe", "resolve_recipe"]
|
||||
@@ -40,6 +40,7 @@ class JarvisSystem:
|
||||
session_store: Optional[Any] = None # SessionStore
|
||||
capability_policy: Optional[Any] = None # CapabilityPolicy
|
||||
operator_manager: Optional[Any] = None # OperatorManager
|
||||
_learning_orchestrator: Optional[Any] = None # LearningOrchestrator
|
||||
|
||||
def ask(
|
||||
self,
|
||||
@@ -443,7 +444,10 @@ class SystemBuilder:
|
||||
# Set up capability policy
|
||||
capability_policy = self._setup_capabilities(config)
|
||||
|
||||
return JarvisSystem(
|
||||
# Set up learning orchestrator (when training is enabled)
|
||||
learning_orchestrator = self._setup_learning_orchestrator(config)
|
||||
|
||||
system = JarvisSystem(
|
||||
config=config,
|
||||
bus=bus,
|
||||
engine=engine,
|
||||
@@ -463,6 +467,8 @@ class SystemBuilder:
|
||||
session_store=session_store,
|
||||
capability_policy=capability_policy,
|
||||
)
|
||||
system._learning_orchestrator = learning_orchestrator
|
||||
return system
|
||||
|
||||
def _resolve_engine(self, config: JarvisConfig):
|
||||
"""Resolve the inference engine."""
|
||||
@@ -853,6 +859,37 @@ class SystemBuilder:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _setup_learning_orchestrator(config: JarvisConfig):
|
||||
"""Set up LearningOrchestrator when training is enabled."""
|
||||
if not config.learning.training_enabled:
|
||||
return None
|
||||
try:
|
||||
from openjarvis.core.config import DEFAULT_CONFIG_DIR
|
||||
from openjarvis.learning.learning_orchestrator import (
|
||||
LearningOrchestrator,
|
||||
)
|
||||
from openjarvis.learning.training.lora import LoRATrainingConfig
|
||||
from openjarvis.traces.store import TraceStore
|
||||
|
||||
trace_store = TraceStore(db_path=config.traces.db_path)
|
||||
config_dir = DEFAULT_CONFIG_DIR / "agent_configs"
|
||||
|
||||
lora_config = LoRATrainingConfig(
|
||||
lora_rank=config.learning.lora_rank,
|
||||
lora_alpha=config.learning.lora_alpha,
|
||||
)
|
||||
|
||||
return LearningOrchestrator(
|
||||
trace_store=trace_store,
|
||||
config_dir=config_dir,
|
||||
min_improvement=config.learning.min_improvement,
|
||||
min_sft_pairs=config.learning.min_sft_pairs,
|
||||
lora_config=lora_config,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _discover_external_mcp(server_cfg) -> List[BaseTool]:
|
||||
"""Discover tools from an external MCP server configuration."""
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Agent template system — pre-configured agent manifests."""
|
||||
|
||||
from openjarvis.templates.agent_templates import (
|
||||
AgentTemplate,
|
||||
discover_templates,
|
||||
load_template,
|
||||
)
|
||||
|
||||
__all__ = ["AgentTemplate", "discover_templates", "load_template"]
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Agent template loader — load pre-configured agent manifests from TOML files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib # type: ignore[no-redef]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AgentTemplate:
|
||||
"""A pre-configured agent manifest loaded from a TOML template."""
|
||||
|
||||
name: str
|
||||
description: str = ""
|
||||
system_prompt: str = ""
|
||||
agent_type: str = "simple"
|
||||
tools: List[str] = field(default_factory=list)
|
||||
max_turns: int = 10
|
||||
temperature: float = 0.7
|
||||
|
||||
|
||||
def load_template(path: str | Path) -> AgentTemplate:
|
||||
"""Load an agent template from a TOML file.
|
||||
|
||||
Expected format::
|
||||
|
||||
[template]
|
||||
name = "code-reviewer"
|
||||
description = "Reviews code for bugs, style, and best practices"
|
||||
|
||||
[agent]
|
||||
type = "native_react"
|
||||
max_turns = 8
|
||||
temperature = 0.3
|
||||
tools = ["file_read", "think"]
|
||||
system_prompt = \"\"\"You are a code reviewer...\"\"\"
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If *path* does not exist.
|
||||
"""
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Template file not found: {path}")
|
||||
|
||||
with open(path, "rb") as fh:
|
||||
data = tomllib.load(fh)
|
||||
|
||||
template_data: Dict = data.get("template", {})
|
||||
agent_data: Dict = data.get("agent", {})
|
||||
|
||||
return AgentTemplate(
|
||||
name=template_data.get("name", path.stem),
|
||||
description=template_data.get("description", ""),
|
||||
system_prompt=agent_data.get("system_prompt", ""),
|
||||
agent_type=agent_data.get("type", "simple"),
|
||||
tools=agent_data.get("tools", []),
|
||||
max_turns=agent_data.get("max_turns", 10),
|
||||
temperature=agent_data.get("temperature", 0.7),
|
||||
)
|
||||
|
||||
|
||||
def _builtin_templates_dir() -> Path:
|
||||
"""Return the path to the built-in templates shipped with the package."""
|
||||
# templates/agents/ at project root, 3 levels above this file
|
||||
return Path(__file__).resolve().parents[3] / "templates" / "agents"
|
||||
|
||||
|
||||
def _user_templates_dir() -> Path:
|
||||
"""Return the path to user-defined templates (~/.openjarvis/templates/agents/)."""
|
||||
return Path.home() / ".openjarvis" / "templates" / "agents"
|
||||
|
||||
|
||||
def discover_templates(
|
||||
extra_dirs: Optional[List[str | Path]] = None,
|
||||
) -> List[AgentTemplate]:
|
||||
"""Discover and load all agent templates from known directories.
|
||||
|
||||
Search order:
|
||||
1. Built-in templates shipped with the package (``templates/agents/``).
|
||||
2. User templates at ``~/.openjarvis/templates/agents/``.
|
||||
3. Any additional directories supplied via *extra_dirs*.
|
||||
|
||||
Returns a list of :class:`AgentTemplate` instances sorted by name.
|
||||
"""
|
||||
dirs: List[Path] = [_builtin_templates_dir(), _user_templates_dir()]
|
||||
if extra_dirs:
|
||||
dirs.extend(Path(d) for d in extra_dirs)
|
||||
|
||||
seen: Dict[str, AgentTemplate] = {}
|
||||
for directory in dirs:
|
||||
if not directory.is_dir():
|
||||
continue
|
||||
for toml_path in sorted(directory.glob("*.toml")):
|
||||
tpl = load_template(toml_path)
|
||||
# Later directories override earlier ones (user overrides builtin).
|
||||
seen[tpl.name] = tpl
|
||||
|
||||
return sorted(seen.values(), key=lambda t: t.name)
|
||||
|
||||
|
||||
__all__ = ["AgentTemplate", "discover_templates", "load_template"]
|
||||
Reference in New Issue
Block a user