diff --git a/.gitignore b/.gitignore
index 6d75fd5f..15351b0e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,3 +56,6 @@ src/openjarvis/server/static/
desktop/node_modules/
desktop/dist/
desktop/src-tauri/target/
+
+# Worktrees
+.worktrees/
diff --git a/CLAUDE.md b/CLAUDE.md
index 90cdcc3e..e780c153 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -4,13 +4,13 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
## Project Status
-OpenJarvis is a research framework for studying on-device AI systems. Phase 21 complete. Five composable pillars: Intelligence, Engine, Agents, Tools (with storage + MCP), and Learning — with trace-driven learning as a cross-cutting concern. ~2940 tests pass (~51 skipped for optional deps). Python SDK (`Jarvis` class), composition layer (`SystemBuilder`/`JarvisSystem`), benchmarking framework, Docker deployment, Tauri desktop app, 40+ tools, 20+ CLI commands, 40+ API endpoints all ready.
+OpenJarvis is a research framework for studying on-device AI systems. Phase 23 complete. Five composable pillars: Intelligence, Engine, Agents, Tools (with storage + MCP), and Learning — with trace-driven learning as a cross-cutting concern. ~3240 tests pass (~44 skipped for optional deps). Python SDK (`Jarvis` class), composition layer (`SystemBuilder`/`JarvisSystem`), eval framework (15 real benchmarks), composable recipes, agent templates, bundled skills, operator recipes, trace-driven learning pipeline, Docker deployment, Tauri desktop app, 40+ tools, 20+ CLI commands, 40+ API endpoints all ready.
## Build & Development Commands
```bash
uv sync --extra dev # Install deps + dev tools
-uv run pytest tests/ -v # Run ~2997 tests (~42 skipped if optional deps missing)
+uv run pytest tests/ -v # Run ~3241 tests (~44 skipped if optional deps missing)
uv run ruff check src/ tests/ # Lint
uv run jarvis --version # 1.0.0
uv run jarvis ask "Hello" # Query via discovered engine (direct mode)
@@ -59,9 +59,14 @@ uv run jarvis vault set MY_KEY # Store encrypted credential
uv run jarvis vault get MY_KEY # Retrieve credential
uv run jarvis vault list # List stored keys
uv run jarvis add github # Quick-add MCP server (github, slack, postgres, etc.)
+uv run jarvis eval list # List 15 benchmarks and backends
+uv run jarvis eval run -b supergpqa -m "qwen3:8b" # Single benchmark run
+uv run jarvis eval run -c evals/configs/suite.toml # Suite mode (models x benchmarks)
+uv run jarvis eval compare result1.jsonl result2.jsonl # Compare runs side-by-side
+uv run jarvis eval report result.jsonl # Detailed report with per-subject breakdown
uv run jarvis --help # Show all subcommands
uv run jarvis init --force # Detect hardware, write ~/.openjarvis/config.toml
-# Eval framework
+# Eval framework (direct module invocation)
source .env # Load API keys before running evals
uv run python -m evals run -c evals/configs/glm-4.7-flash-openhands.toml -v # Run eval suite from TOML config
uv run python -m evals run -b supergpqa -m "qwen3:8b" -n 50 # Run single benchmark
@@ -99,7 +104,7 @@ j.close() # Release resources
- **Package manager:** `uv` with `hatchling` build backend
- **Config:** `pyproject.toml` with extras for optional backends (e.g., `openjarvis[inference-vllm]`, `openjarvis[inference-mlx]`, `openjarvis[memory-colbert]`, `openjarvis[server]`, `openjarvis[openclaw]`, `openjarvis[energy-amd]`, `openjarvis[energy-apple]`, `openjarvis[energy-all]`, `openjarvis[security-signing]`, `openjarvis[sandbox-wasm]`, `openjarvis[dashboard]`, `openjarvis[browser]`, `openjarvis[media]`, `openjarvis[pdf]`, `openjarvis[channel-line]`, `openjarvis[channel-viber]`, `openjarvis[channel-reddit]`, `openjarvis[channel-mastodon]`, `openjarvis[channel-xmpp]`, `openjarvis[channel-rocketchat]`, `openjarvis[channel-zulip]`, `openjarvis[channel-twitch]`, `openjarvis[channel-nostr]`)
-- **CLI entry point:** `jarvis` (Click-based) — subcommands: `init`, `ask`, `serve`, `start`, `stop`, `restart`, `status`, `chat`, `model`, `memory`, `telemetry`, `bench`, `channel`, `scheduler`, `doctor`, `agent`, `workflow`, `skill`, `vault`, `add`
+- **CLI entry point:** `jarvis` (Click-based) — subcommands: `init`, `ask`, `serve`, `start`, `stop`, `restart`, `status`, `chat`, `model`, `memory`, `telemetry`, `bench`, `eval`, `channel`, `scheduler`, `doctor`, `agent`, `workflow`, `skill`, `vault`, `add`
- **Python:** 3.10+ required
- **Node.js:** 22+ required only for OpenClaw agent
@@ -126,7 +131,7 @@ OpenJarvis is a research framework for on-device AI organized around **five comp
- **MCP templates** (`tools/templates/`): `ToolTemplate` dynamically constructs tools from TOML specs. 10 builtin templates. `discover_templates()` auto-discovers.
- **`ToolExecutor`**: dispatch with RBAC check + taint check, `timeout_seconds` on `ToolSpec` (default 30s via `ThreadPoolExecutor`), event bus integration
- All registered via `@ToolRegistry.register("name")` decorator
-5. **Learning** (`src/openjarvis/learning/`) — Structured learning with nested per-pillar sub-policies. `LearningConfig` sections: `routing` (heuristic/learned/grpo/bandit), `intelligence` (none/sft), `agent` (none/agent_advisor/icl_updater), `metrics` (accuracy/latency/cost/efficiency weights). Policies: `SFTRouterPolicy` (query→model from traces), `AgentAdvisorPolicy` (LM-guided), `ICLUpdaterPolicy` (in-context with example DB, versioning, rollback, quality gates), `GRPORouterPolicy` (softmax sampling, group relative advantage, per-query-class weights), `BanditRouterPolicy` (Thompson Sampling / UCB1, per-arm stats). `SkillDiscovery` mines tool subsequences from traces to auto-generate skill manifests. Router policies: `HeuristicRouter`, `TraceDrivenPolicy`. Orchestrator training subpackage provides SFT and GRPO pipelines.
+5. **Learning** (`src/openjarvis/learning/`) — Structured learning with nested per-pillar sub-policies. `LearningConfig` sections: `routing` (heuristic/learned/grpo/bandit), `intelligence` (none/sft), `agent` (none/agent_advisor/icl_updater), `metrics` (accuracy/latency/cost/efficiency weights). Policies: `SFTRouterPolicy` (query→model from traces), `AgentAdvisorPolicy` (LM-guided), `ICLUpdaterPolicy` (in-context with example DB, versioning, rollback, quality gates), `GRPORouterPolicy` (softmax sampling, group relative advantage, per-query-class weights), `BanditRouterPolicy` (Thompson Sampling / UCB1, per-arm stats). `SkillDiscovery` mines tool subsequences from traces to auto-generate skill manifests. Router policies: `HeuristicRouter`, `TraceDrivenPolicy`. Orchestrator training subpackage provides SFT and GRPO pipelines. **Trace-driven learning pipeline**: `TrainingDataMiner` (extracts SFT pairs from traces with quality filters), `LoRATrainer` (LoRA fine-tuning with configurable rank/alpha, requires torch), `AgentConfigEvolver` (LM-guided agent config recommendations from trace patterns), `LearningOrchestrator` (wired into `SystemBuilder`, orchestrates mine→train→evolve cycle on schedule).
### Cross-cutting Systems
@@ -139,6 +144,10 @@ OpenJarvis is a research framework for on-device AI organized around **five comp
- **Composition Layer** (`system.py`) — `SystemBuilder` fluent builder → `JarvisSystem` with `ask()`, `close()`. Wires engine, model, agent, tools, telemetry, traces, workflow, sessions, capability policy.
- **SDK** (`sdk.py`) — `Jarvis` class: high-level sync API with `ask()`/`ask_full()`, `MemoryHandle`, lazy init, telemetry. Also exports `JarvisSystem`/`SystemBuilder`.
- **Benchmarks** (`bench/`) — `LatencyBenchmark`, `ThroughputBenchmark`, `EnergyBenchmark`. All registered via `BenchmarkRegistry`. CLI: `jarvis bench run`.
+- **Eval Framework** (`evals/`) — 15 real benchmark datasets from IPW: SuperGPQA, GPQA, MMLU-Pro, MATH-500, Natural Reasoning, HLE, SimpleQA, WildChat, IPW, GAIA, FRAMES, SWE-bench, SWEfficiency, TerminalBench, TerminalBench Native. Scorer types: MCQ letter extraction, LLM-judge, exact match, structural validation. `EvalRunner` with parallel execution. CLI: `jarvis eval list|run|compare|report`.
+- **Recipes** (`recipes/`, `src/openjarvis/recipes/`) — Composable TOML configs that wire all 5 pillars. `Recipe` dataclass with `to_builder_kwargs()`. `load_recipe()`, `discover_recipes()`, `resolve_recipe()`. 3 built-in recipes: coding_assistant, research_assistant, general_assistant. Operator recipes (`recipes/operators/`): researcher (4h cycle), correspondent (5min interval), sentinel (2h cycle).
+- **Agent Templates** (`templates/agents/`, `src/openjarvis/templates/`) — Pre-configured TOML manifests with system prompts, tool sets, behavioral parameters. `AgentTemplate` dataclass, `load_template()`, `discover_templates()`. 15 built-in templates (code-reviewer, debugger, architect, deep-researcher, fact-checker, summarizer, etc.).
+- **Bundled Skills** (`skills/builtin/`) — 20 ready-to-use TOML skill manifests. Categories: file management (organizer, deduplicator, backup), research (web-summarize, topic-research, knowledge-extract), code quality (lint, test-gen, security-scan, dependency-audit), productivity (email-draft, meeting-notes, daily-digest), document processing (compare, translate, data-analyze).
- **OpenClaw** (`agents/openclaw*.py`) — `OpenClawAgent` with `HttpTransport`/`SubprocessTransport`, JSON-line protocol, `ProviderPlugin`, `MemorySearchManager`.
- **API Server** (`server/`) — OpenAI-compatible via `jarvis serve` (FastAPI + uvicorn). Endpoints: `POST /v1/chat/completions`, `GET /v1/models`, `GET /health`, channel endpoints. SSE streaming.
- **Channels** (`channels/`) — `BaseChannel` ABC. `OpenClawChannelBridge` (WebSocket/HTTP to OpenClaw gateway). `WhatsAppBaileysChannel` (Baileys protocol, Node.js bridge, QR auth). Phase 21 channels: `LINEChannel`, `ViberChannel`, `MessengerChannel`, `RedditChannel`, `MastodonChannel`, `XMPPChannel`, `RocketChatChannel`, `ZulipChannel`, `TwitchChannel`, `NostrChannel`. All follow `BaseChannel` ABC with env var fallbacks, `@ChannelRegistry.register()`, `EventBus` integration.
@@ -227,3 +236,5 @@ OpenAI-compatible server via `jarvis serve`:
| v2.4 | 19 | Learning productionization: GRPO (softmax/advantage), BanditRouter (Thompson/UCB1), SkillDiscovery (trace mining), ICL updates (versioning/rollback/quality gates) |
| v2.5 | 20 | Tauri 2.0 desktop app: energy dashboard, trace debugger, learning curve visualization, memory browser, admin panel. CI for Linux/macOS/Windows |
| v2.6 | 21 | 10 new channels: LINE, Viber, Messenger, Reddit, Mastodon, XMPP, Rocket.Chat, Zulip, Twitch, Nostr |
+| v2.7 | 22 | Operators: persistent, scheduled autonomous agents with recipe + schedule + channel output |
+| v2.8 | 23 | Differentiated functionalities: trace-driven learning pipeline (TrainingDataMiner, LoRATrainer, AgentConfigEvolver, LearningOrchestrator), 15 real IPW benchmarks, composable recipes, 15 agent templates, 20 bundled skills, 3 operator recipes |
diff --git a/docs/plans/2026-02-28-differentiated-functionalities-design.md b/docs/plans/2026-02-28-differentiated-functionalities-design.md
new file mode 100644
index 00000000..7584cb2d
--- /dev/null
+++ b/docs/plans/2026-02-28-differentiated-functionalities-design.md
@@ -0,0 +1,177 @@
+# Differentiated Functionalities Design
+
+**Date:** 2026-02-28
+**Approach:** Learning Flywheel (Approach A)
+**Status:** Approved
+
+## Vision
+
+Make OpenJarvis's functionalities genuinely differentiated through four pillars of uniqueness:
+
+1. **Fully open-source on-device stack** — every component runs locally, user controls data
+2. **Native on-device intelligence** — make local LMs punch above their weight via trace-driven learning
+3. **Latency, privacy, energy as first-class** — alongside accuracy, not afterthoughts
+4. **Composable, programmable abstractions** — Intelligence, Agent, Tools, Engine, Learning as optimized components
+
+**Priority:** On-device depth first. Trace-driven learning is the core differentiator.
+
+## Design Decisions
+
+- **Approach A selected:** Build the trace-to-learn-to-eval loop as the backbone, hang user-facing features off it
+- **Learning targets:** Full stack — routing, agent logic, AND model weights (LoRA/QLoRA)
+- **Eval purpose:** Research platform — rigorous before/after measurement with reproducible configs
+- **Composition layers:** Python SDK + TOML configs + pre-built recipes
+- **Operators:** 3 recipes (researcher, correspondent, sentinel) — not a new abstraction, just recipe + schedule + channel
+- **Interleave:** Wire up real learning AND build new user-facing features simultaneously
+
+## Section 1: Trace-Driven Learning Pipeline
+
+The core differentiator. Makes the learning pipeline real, not just infrastructure.
+
+### Current State
+
+`TraceStore` captures traces, `TraceAnalyzer` computes stats, GRPO/Bandit policies update internal routing state, `SFTRouterPolicy` and `AgentAdvisorPolicy` exist but don't produce real training jobs, `ICLUpdaterPolicy` updates in-context examples with versioning/rollback.
+
+### New Components
+
+**Trace-to-Training Data pipeline** (`learning/training/data.py`): Mines the `TraceStore` for supervised pairs. For successful traces (user rated positively or task completed), extracts `(input, preferred_output)` pairs. For routing, extracts `(query_class, best_model)` pairs. For agent logic, extracts `(query_type, best_agent_config)` tuples. Filters: minimum trace quality score, deduplication, privacy scanning (strip PII before training).
+
+**LoRA/QLoRA fine-tuning** (`learning/training/lora.py`): Wraps `transformers` + `peft` + `trl` for actual weight updates on local models. Takes training data from the pipeline, runs LoRA fine-tuning with configurable rank/alpha/dropout. Produces adapter weights saved alongside the base model. Energy-aware via existing `EnergyMonitor`. Guarded by hardware detection — only runs if sufficient VRAM/RAM.
+
+**Agent config evolution** (`learning/agent_learning.py`): Makes `AgentAdvisorPolicy` actually rewrite agent TOML configs. Analyzes traces to identify: which tools were useful vs. never called, which system prompt patterns led to success, optimal `max_turns` per query class. Writes updated configs with version control (git-tracked, rollback via `ICLUpdaterPolicy` pattern).
+
+**Learning orchestrator** (`learning/orchestrator.py`): Coordinates all learning. Runs on a schedule (e.g., nightly) or on-demand. Steps: collect new traces, mine training data, run evals (baseline), fine-tune model / update agent configs, run evals (after), accept/reject based on improvement threshold. Atomic: if evals don't improve, rollback automatically.
+
+### Data Flow
+
+```
+User queries -> Traces recorded -> TraceStore (SQLite)
+ |
+ LearningOrchestrator (scheduled/on-demand)
+ | | |
+ TrainingDataMiner AgentConfigEvolver RoutingPolicyUpdater
+ | | |
+ LoRA fine-tune TOML config update GRPO/Bandit update
+ | | |
+ EvalHarness (before/after comparison)
+ |
+ Accept (deploy) or Reject (rollback)
+```
+
+## Section 2: Eval Framework
+
+Proves the learning flywheel works and serves as the research platform.
+
+### Five Workload-Type Eval Suites
+
+- **Chat:** Multi-turn conversation quality (MT-Bench style, coherence, helpfulness)
+- **Reasoning:** Logic/math/science (GSM8K, ARC, MMLU subsets that run locally)
+- **RAG:** Retrieval-augmented accuracy (measures whether memory retrieval improves answers vs. without, uses OpenJarvis's own memory backends)
+- **Agentic:** Task completion rate for multi-step tool-use workflows (custom scenarios: research a topic, triage inbox, etc.)
+- **Coding:** Code generation correctness (HumanEval, MBPP subsets)
+
+### On-Device Metrics as First-Class
+
+Every eval records: accuracy, latency (TTFT + total), energy consumption (via `EnergyMonitor`), peak memory, tokens/second. Results are multi-dimensional, not a single number. Configurable weighting (e.g., 40% accuracy, 30% latency, 20% energy, 10% memory).
+
+### Before/After Measurement
+
+`LearningOrchestrator` calls the eval harness before learning (baseline) and after (candidate). Improvement measured per-dimension. Configurable acceptance threshold (e.g., accept if accuracy improves >2% without latency regressing >10%).
+
+### Reproducible Configs
+
+Every eval run fully specified by TOML (model, agent, tools, dataset, metrics, hardware). Results are JSONL with full provenance. CLI: `jarvis eval run`, `jarvis eval compare`, `jarvis eval report`.
+
+### Ablation Support
+
+Run the same eval with one variable changed (with vs. without LoRA, with vs. without a tool, with vs. without memory). Built-in diffing and statistical significance testing.
+
+## Section 3: Composable Abstractions and Recipes
+
+The user-facing layer for interacting with the pillars.
+
+### Recipe System (`recipes/`)
+
+A recipe is a curated composition of all 5 pillars in a single TOML file: model selection, engine preference, agent type + system prompt, tool set, learning policy, and eval config. Recipes are opinionated defaults that work well together.
+
+Users load recipes via `jarvis ask --recipe coding_assistant "Fix this bug"` or `Jarvis(recipe="coding_assistant")` in Python.
+
+### Agent Templates (15-20)
+
+Pre-configured agent TOML manifests with system prompts, tool sets, and behavioral parameters.
+
+Categories:
+- **Coding:** code-reviewer, debugger, architect
+- **Research:** deep-researcher, fact-checker, summarizer
+- **Productivity:** inbox-triager, meeting-prep, note-taker
+- **General:** assistant, tutor, translator, writer
+
+### Bundled Skills (20-30)
+
+Focused on on-device use cases where local execution matters.
+
+Categories:
+- **File management:** organize, deduplicate, backup
+- **Personal knowledge:** extract, summarize, index
+- **Development:** lint, test, review
+- **Productivity:** calendar prep, email draft, todo management
+
+### Three Composition Layers
+
+- **CLI flags:** `--recipe`, `--agent`, `--tools`, `--model` for quick use
+- **TOML configs:** Full pillar configuration for persistent setups
+- **Python SDK:** `SystemBuilder` for programmatic composition, research scripts, custom pipelines
+
+## Section 4: Operator Recipes
+
+OpenJarvis's answer to autonomous task agents. Operators are NOT a new abstraction — they are recipe + schedule + channel output, composed via TOML.
+
+### Deep Researcher (`recipes/operators/researcher.toml`)
+
+Autonomous research agent. Given a topic, searches the web, cross-references sources, evaluates credibility, builds a knowledge graph entry, produces a cited report. Runs on-demand or scheduled.
+
+Tools: `web_search`, `http_request`, `memory_store`, `memory_search`, `kg_add_entity`, `kg_add_relation`, `file_write`.
+
+Learning: traces which search strategies yield useful results, evolves search query patterns over time.
+
+### Correspondent (`recipes/operators/correspondent.toml`)
+
+Messaging triage across Slack/Gmail/Discord. Monitors incoming messages, classifies urgency (urgent/normal/low/ignore), drafts responses for high-priority items, summarizes the rest into a daily digest.
+
+Tools: `memory_store`, `memory_search`, `think`, `llm_call`.
+
+Channels: Slack, email, Discord.
+
+Learning: adapts to user's triage preferences over time.
+
+### Sentinel (`recipes/operators/sentinel.toml`)
+
+Monitors Twitter, Reddit, Mastodon, Google Trends, RSS feeds, and specified URLs for changes relevant to user-defined topics. Produces alerts when something significant surfaces — trending discussions, sentiment shifts, breaking news, competitor activity.
+
+Tools: `web_search`, `http_request`, `memory_store`, `memory_search`, `kg_add_entity`.
+
+Channels: Twitter, Reddit, Mastodon (read via APIs), Google Trends.
+
+Learning: refines what "significant" means based on which alerts the user acts on vs. dismisses.
+
+## Section 5: Testing and Validation
+
+### Learning Pipeline Tests
+
+Unit tests for `TrainingDataMiner` (mock traces, verify correct pairs), `LoRATrainer` (mock training, verify adapter saved), `AgentConfigEvolver` (mock traces, verify TOML rewritten), `LearningOrchestrator` (mock all steps, verify accept/reject logic). Integration test: synthetic traces, actual LoRA training on tiny model, verify loss decreases.
+
+### Eval Framework Tests
+
+Each of the 5 eval suites has a smoke mode (5-10 examples, <30 seconds). Verify multi-dimensional scoring. Verify before/after comparison and acceptance thresholds. Verify ablation diffing.
+
+### Recipe and Template Tests
+
+Each recipe loads without error. Each agent template produces a valid agent. Each skill executes its steps. Composition tests: `--recipe` flag wires all pillars correctly.
+
+### Operator Tests
+
+Each operator recipe loads and schedules correctly. Mock tool execution to verify agent loop completes. Verify learning feedback loop (trace, config update, re-eval).
+
+### Regression
+
+All ~2940 existing tests continue to pass. Target ~200-300 additional tests for new functionality.
diff --git a/docs/plans/2026-02-28-differentiated-functionalities.md b/docs/plans/2026-02-28-differentiated-functionalities.md
new file mode 100644
index 00000000..c8d2abde
--- /dev/null
+++ b/docs/plans/2026-02-28-differentiated-functionalities.md
@@ -0,0 +1,2553 @@
+# Differentiated Functionalities Implementation Plan
+
+> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
+
+**Goal:** Make OpenJarvis's functionalities genuinely differentiated: real trace-driven learning (LoRA from traces, agent config evolution), research-grade eval framework, composable recipe system, and 3 Operator recipes.
+
+**Architecture:** Build the trace→learn→eval loop as the backbone (Section 1+2), then compose user-facing features on top (Section 3+4). The learning pipeline mines TraceStore for training data, runs LoRA fine-tuning via transformers/peft, evolves agent TOML configs, and validates improvements via the eval harness. Recipes, templates, and operators are TOML compositions of the 5 pillars.
+
+**Tech Stack:** Python 3.10+, transformers, peft, trl (optional deps for training), existing SQLite TraceStore, existing eval framework (extended), TOML configs.
+
+---
+
+## Section 1: Trace-Driven Learning Pipeline
+
+### Task 1: TrainingDataMiner — Extract training pairs from traces
+
+**Files:**
+- Create: `src/openjarvis/learning/training/__init__.py`
+- Create: `src/openjarvis/learning/training/data.py`
+- Test: `tests/learning/training/test_data.py`
+
+**Context:** TraceStore (`src/openjarvis/traces/store.py:63-214`) stores complete interaction traces in SQLite with query, agent, model, engine, result, outcome, feedback, and steps (route/retrieve/generate/tool_call/respond). We need to mine these for three types of supervised pairs: (input, preferred_output) for SFT, (query_class, best_model) for routing, and (query_type, best_agent_config) for agent learning. The existing `classify_query()` in `learning/trace_policy.py:31-42` classifies queries as code/math/short/long/general.
+
+**Step 1: Write the failing test for TrainingDataMiner**
+
+```python
+# tests/learning/training/test_data.py
+"""Tests for TrainingDataMiner — extracts training pairs from trace history."""
+
+import time
+from pathlib import Path
+from openjarvis.core.types import Trace, TraceStep, StepType
+from openjarvis.traces.store import TraceStore
+
+
+def _make_trace(
+ query: str,
+ result: str,
+ model: str = "qwen3:8b",
+ agent: str = "orchestrator",
+ outcome: str = "success",
+ feedback: float = 0.9,
+ tools_used: list[str] | None = None,
+) -> Trace:
+ """Helper to build a realistic trace."""
+ now = time.time()
+ steps = [
+ TraceStep(
+ step_type=StepType.GENERATE,
+ timestamp=now,
+ duration_seconds=1.2,
+ input={"query": query},
+ output={"content": result, "tokens": 150},
+ ),
+ ]
+ if tools_used:
+ for tool in tools_used:
+ steps.append(
+ TraceStep(
+ step_type=StepType.TOOL_CALL,
+ timestamp=now + 0.5,
+ duration_seconds=0.3,
+ input={"tool": tool, "args": "{}"},
+ output={"result": "ok"},
+ )
+ )
+ steps.append(
+ TraceStep(
+ step_type=StepType.RESPOND,
+ timestamp=now + 2.0,
+ duration_seconds=0.0,
+ input={},
+ output={"content": result},
+ )
+ )
+ return Trace(
+ query=query,
+ agent=agent,
+ model=model,
+ engine="ollama",
+ steps=steps,
+ result=result,
+ outcome=outcome,
+ feedback=feedback,
+ started_at=now,
+ ended_at=now + 2.0,
+ total_tokens=150,
+ total_latency_seconds=2.0,
+ )
+
+
+class TestTrainingDataMiner:
+ def setup_method(self, tmp_path_factory=None):
+ import tempfile
+ self._tmp = tempfile.mkdtemp()
+ self.store = TraceStore(Path(self._tmp) / "traces.db")
+
+ def teardown_method(self):
+ self.store.close()
+
+ def test_extract_sft_pairs_from_successful_traces(self):
+ """Successful traces with high feedback produce (input, output) pairs."""
+ from openjarvis.learning.training.data import TrainingDataMiner
+
+ self.store.save(_make_trace("Write hello world in Python", "print('hello world')", feedback=0.95))
+ self.store.save(_make_trace("Bad query", "Bad result", outcome="failure", feedback=0.2))
+
+ miner = TrainingDataMiner(self.store, min_quality=0.7)
+ pairs = miner.extract_sft_pairs()
+
+ assert len(pairs) == 1
+ assert pairs[0]["input"] == "Write hello world in Python"
+ assert pairs[0]["output"] == "print('hello world')"
+ assert pairs[0]["query_class"] == "code"
+
+ def test_extract_routing_pairs(self):
+ """Traces grouped by query class yield best-model recommendations."""
+ from openjarvis.learning.training.data import TrainingDataMiner
+
+ # Two models, same query class — model_b has better feedback
+ self.store.save(_make_trace("Solve 2+2", "4", model="model_a", feedback=0.6))
+ self.store.save(_make_trace("Solve 3+3", "6", model="model_b", feedback=0.95))
+ self.store.save(_make_trace("Solve 5*5", "25", model="model_b", feedback=0.9))
+
+ miner = TrainingDataMiner(self.store, min_quality=0.5)
+ routing = miner.extract_routing_pairs()
+
+ assert "math" in routing
+ assert routing["math"]["best_model"] == "model_b"
+ assert routing["math"]["sample_count"] >= 2
+
+ def test_extract_agent_config_pairs(self):
+ """Traces grouped by query class yield best agent+tools combos."""
+ from openjarvis.learning.training.data import TrainingDataMiner
+
+ self.store.save(_make_trace(
+ "Find info about Python",
+ "Python is a language",
+ agent="orchestrator",
+ tools_used=["web_search", "think"],
+ feedback=0.9,
+ ))
+ self.store.save(_make_trace(
+ "Search for Rust docs",
+ "Rust is a systems language",
+ agent="simple",
+ tools_used=[],
+ feedback=0.4,
+ ))
+
+ miner = TrainingDataMiner(self.store, min_quality=0.5)
+ agent_pairs = miner.extract_agent_config_pairs()
+
+ assert "general" in agent_pairs
+ best = agent_pairs["general"]
+ assert best["best_agent"] == "orchestrator"
+ assert "web_search" in best["best_tools"]
+
+ def test_empty_store_returns_empty(self):
+ """No traces → no training data."""
+ from openjarvis.learning.training.data import TrainingDataMiner
+
+ miner = TrainingDataMiner(self.store)
+ assert miner.extract_sft_pairs() == []
+ assert miner.extract_routing_pairs() == {}
+ assert miner.extract_agent_config_pairs() == {}
+
+ def test_min_quality_filter(self):
+ """Traces below min_quality are excluded."""
+ from openjarvis.learning.training.data import TrainingDataMiner
+
+ self.store.save(_make_trace("Hello", "Hi", feedback=0.3, outcome="success"))
+
+ miner = TrainingDataMiner(self.store, min_quality=0.5)
+ assert miner.extract_sft_pairs() == []
+
+ def test_deduplication(self):
+ """Identical (input, output) pairs are deduplicated."""
+ from openjarvis.learning.training.data import TrainingDataMiner
+
+ self.store.save(_make_trace("Hello", "Hi", feedback=0.9))
+ self.store.save(_make_trace("Hello", "Hi", feedback=0.85))
+
+ miner = TrainingDataMiner(self.store, min_quality=0.5)
+ pairs = miner.extract_sft_pairs()
+ assert len(pairs) == 1
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/learning/training/test_data.py -v`
+Expected: FAIL with `ModuleNotFoundError: No module named 'openjarvis.learning.training'`
+
+**Step 3: Write minimal implementation**
+
+```python
+# src/openjarvis/learning/training/__init__.py
+"""Training data extraction and model fine-tuning from traces."""
+
+# src/openjarvis/learning/training/data.py
+"""Extract training data from the TraceStore for supervised learning."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any, Dict, List, Optional
+
+from openjarvis.core.types import StepType
+
+# Reuse query classification from trace_policy
+_CODE_PAT = __import__("re").compile(
+ r"(```|import |def |class |function |const |let |var )", __import__("re").IGNORECASE
+)
+_MATH_PAT = __import__("re").compile(
+ r"\b(solve|integral|equation|derivative|sum|product|matrix|factor|simplify|calculate)\b",
+ __import__("re").IGNORECASE,
+)
+
+
+def _classify_query(query: str) -> str:
+ if _CODE_PAT.search(query):
+ return "code"
+ if _MATH_PAT.search(query):
+ return "math"
+ if len(query) < 50:
+ return "short"
+ if len(query) > 500:
+ return "long"
+ return "general"
+
+
+class TrainingDataMiner:
+ """Mine TraceStore for supervised training pairs."""
+
+ def __init__(
+ self,
+ trace_store: Any,
+ *,
+ min_quality: float = 0.7,
+ min_samples_per_class: int = 1,
+ ) -> None:
+ self._store = trace_store
+ self._min_quality = min_quality
+ self._min_samples = min_samples_per_class
+
+ def _good_traces(self) -> list:
+ """Return traces that meet quality threshold."""
+ all_traces = self._store.list_traces(limit=10000)
+ return [
+ t
+ for t in all_traces
+ if t.outcome == "success"
+ and t.feedback is not None
+ and t.feedback >= self._min_quality
+ ]
+
+ def extract_sft_pairs(self) -> List[Dict[str, Any]]:
+ """Extract (input, output) pairs for supervised fine-tuning."""
+ traces = self._good_traces()
+ seen: set[tuple[str, str]] = set()
+ pairs: list[dict[str, Any]] = []
+ for t in traces:
+ key = (t.query, t.result)
+ if key in seen:
+ continue
+ seen.add(key)
+ pairs.append(
+ {
+ "input": t.query,
+ "output": t.result,
+ "query_class": _classify_query(t.query),
+ "model": t.model,
+ "feedback": t.feedback,
+ }
+ )
+ return pairs
+
+ def extract_routing_pairs(self) -> Dict[str, Dict[str, Any]]:
+ """Extract best model per query class from traces."""
+ traces = self._good_traces()
+ # Group by (query_class, model) → list of feedbacks
+ groups: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
+ for t in traces:
+ qclass = _classify_query(t.query)
+ groups[qclass][t.model].append(t.feedback or 0.0)
+
+ result: dict[str, dict[str, Any]] = {}
+ for qclass, models in groups.items():
+ best_model = max(models, key=lambda m: sum(models[m]) / len(models[m]))
+ scores = models[best_model]
+ result[qclass] = {
+ "best_model": best_model,
+ "avg_feedback": sum(scores) / len(scores),
+ "sample_count": len(scores),
+ "all_models": {
+ m: {"avg_feedback": sum(s) / len(s), "count": len(s)}
+ for m, s in models.items()
+ },
+ }
+ return result
+
+ def extract_agent_config_pairs(self) -> Dict[str, Dict[str, Any]]:
+ """Extract best agent+tools combo per query class."""
+ traces = self._good_traces()
+ # Group by query_class → list of (agent, tools, feedback)
+ groups: dict[str, list[tuple[str, list[str], float]]] = defaultdict(list)
+ for t in traces:
+ qclass = _classify_query(t.query)
+ tools_used = [
+ s.input.get("tool", "")
+ for s in t.steps
+ if s.step_type == StepType.TOOL_CALL and s.input.get("tool")
+ ]
+ groups[qclass].append((t.agent, tools_used, t.feedback or 0.0))
+
+ result: dict[str, dict[str, Any]] = {}
+ for qclass, entries in groups.items():
+ # Score each agent by average feedback
+ agent_scores: dict[str, list[float]] = defaultdict(list)
+ agent_tools: dict[str, list[list[str]]] = defaultdict(list)
+ for agent, tools, fb in entries:
+ agent_scores[agent].append(fb)
+ agent_tools[agent].append(tools)
+
+ best_agent = max(agent_scores, key=lambda a: sum(agent_scores[a]) / len(agent_scores[a]))
+ # Most common tool set for best agent
+ all_tools: list[str] = []
+ for tl in agent_tools[best_agent]:
+ all_tools.extend(tl)
+ # Deduplicate preserving frequency order
+ tool_counts: dict[str, int] = defaultdict(int)
+ for tool in all_tools:
+ tool_counts[tool] += 1
+ best_tools = sorted(tool_counts, key=tool_counts.get, reverse=True)
+
+ result[qclass] = {
+ "best_agent": best_agent,
+ "best_tools": best_tools,
+ "avg_feedback": sum(agent_scores[best_agent]) / len(agent_scores[best_agent]),
+ "sample_count": len(agent_scores[best_agent]),
+ }
+ return result
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/learning/training/test_data.py -v`
+Expected: All 6 tests PASS
+
+**Step 5: Commit**
+
+```bash
+git add src/openjarvis/learning/training/__init__.py src/openjarvis/learning/training/data.py tests/learning/training/test_data.py
+git commit -m "feat: add TrainingDataMiner — extract SFT/routing/agent pairs from traces"
+```
+
+---
+
+### Task 2: LoRATrainer — Fine-tune local models from trace data
+
+**Files:**
+- Create: `src/openjarvis/learning/training/lora.py`
+- Test: `tests/learning/training/test_lora.py`
+
+**Context:** The orchestrator training (`learning/orchestrator/sft_trainer.py`) already wraps transformers for SFT training, but it's specialized for orchestrator episodes. We need a general-purpose LoRA trainer that takes the SFT pairs from TrainingDataMiner and fine-tunes any local model. The existing `OrchestratorPolicyModel.from_pretrained()` in `learning/orchestrator/policy_model.py:55-102` shows the pattern for loading HuggingFace models. Training deps (torch, transformers, peft) are optional — guard with `try/except`.
+
+**Step 1: Write the failing test**
+
+```python
+# tests/learning/training/test_lora.py
+"""Tests for LoRATrainer — fine-tune local models via LoRA/QLoRA."""
+
+import pytest
+
+
+class TestLoRATrainerConfig:
+ def test_default_config(self):
+ from openjarvis.learning.training.lora import LoRATrainingConfig
+
+ cfg = LoRATrainingConfig()
+ assert cfg.lora_rank == 16
+ assert cfg.lora_alpha == 32
+ assert cfg.lora_dropout == 0.05
+ assert cfg.num_epochs == 3
+ assert cfg.batch_size == 4
+ assert cfg.learning_rate == 2e-5
+ assert cfg.output_dir == "checkpoints/lora"
+
+ def test_custom_config(self):
+ from openjarvis.learning.training.lora import LoRATrainingConfig
+
+ cfg = LoRATrainingConfig(lora_rank=8, num_epochs=1, output_dir="/tmp/test")
+ assert cfg.lora_rank == 8
+ assert cfg.num_epochs == 1
+
+
+class TestLoRATrainer:
+ def test_init_without_torch_raises(self):
+ """LoRATrainer should raise ImportError if torch is unavailable."""
+ from openjarvis.learning.training.lora import LoRATrainer, LoRATrainingConfig, HAS_TORCH
+
+ if not HAS_TORCH:
+ with pytest.raises(ImportError, match="torch|transformers|peft"):
+ LoRATrainer(LoRATrainingConfig(), model_name="test")
+
+ def test_prepare_dataset_from_pairs(self):
+ """SFT pairs from TrainingDataMiner should convert to a training dataset."""
+ from openjarvis.learning.training.lora import LoRATrainer, LoRATrainingConfig, HAS_TORCH
+
+ if not HAS_TORCH:
+ pytest.skip("torch/transformers/peft not installed")
+
+ pairs = [
+ {"input": "What is 2+2?", "output": "4", "query_class": "math"},
+ {"input": "Hello", "output": "Hi there!", "query_class": "short"},
+ ]
+
+ trainer = LoRATrainer(LoRATrainingConfig(output_dir="/tmp/test_lora"), model_name="Qwen/Qwen3-0.6B")
+ dataset = trainer.prepare_dataset(pairs)
+ assert len(dataset) == 2
+ assert "input_ids" in dataset[0] or "text" in dataset[0]
+
+ def test_training_config_validates(self):
+ """Invalid configs should raise."""
+ from openjarvis.learning.training.lora import LoRATrainingConfig
+
+ with pytest.raises(ValueError):
+ LoRATrainingConfig(lora_rank=0)
+ with pytest.raises(ValueError):
+ LoRATrainingConfig(num_epochs=0)
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/learning/training/test_lora.py -v`
+Expected: FAIL with `ModuleNotFoundError`
+
+**Step 3: Write minimal implementation**
+
+```python
+# src/openjarvis/learning/training/lora.py
+"""LoRA/QLoRA fine-tuning from trace-derived training pairs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+try:
+ import torch
+ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+ from peft import LoraConfig, get_peft_model, TaskType
+
+ HAS_TORCH = True
+except ImportError:
+ HAS_TORCH = False
+
+
+@dataclass
+class LoRATrainingConfig:
+ """Configuration for LoRA fine-tuning."""
+
+ # LoRA
+ lora_rank: int = 16
+ lora_alpha: int = 32
+ lora_dropout: float = 0.05
+ target_modules: List[str] = field(default_factory=lambda: ["q_proj", "v_proj"])
+
+ # Training
+ num_epochs: int = 3
+ batch_size: int = 4
+ learning_rate: float = 2e-5
+ weight_decay: float = 0.01
+ warmup_ratio: float = 0.1
+ max_grad_norm: float = 1.0
+ max_seq_length: int = 2048
+
+ # Quantization (QLoRA)
+ use_4bit: bool = False
+
+ # Output
+ output_dir: str = "checkpoints/lora"
+ save_every_n_epochs: int = 1
+
+ # Gradient checkpointing
+ gradient_checkpointing: bool = True
+
+ def __post_init__(self) -> None:
+ if self.lora_rank < 1:
+ raise ValueError("lora_rank must be >= 1")
+ if self.num_epochs < 1:
+ raise ValueError("num_epochs must be >= 1")
+
+
+class LoRATrainer:
+ """Fine-tune a local model using LoRA from trace-derived SFT pairs."""
+
+ def __init__(
+ self,
+ config: LoRATrainingConfig,
+ *,
+ model_name: str = "Qwen/Qwen3-0.6B",
+ device: Optional[str] = None,
+ ) -> None:
+ if not HAS_TORCH:
+ raise ImportError(
+ "LoRA training requires torch, transformers, and peft. "
+ "Install with: pip install torch transformers peft"
+ )
+ self.config = config
+ self.model_name = model_name
+ self._device = device or ("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu")
+ self._model = None
+ self._tokenizer = None
+
+ def _load_model(self) -> None:
+ """Lazy-load model and tokenizer."""
+ if self._model is not None:
+ return
+
+ quantization_config = None
+ if self.config.use_4bit:
+ quantization_config = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_compute_dtype=torch.float16,
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_quant_type="nf4",
+ )
+
+ self._tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+ if self._tokenizer.pad_token is None:
+ self._tokenizer.pad_token = self._tokenizer.eos_token
+
+ self._model = AutoModelForCausalLM.from_pretrained(
+ self.model_name,
+ quantization_config=quantization_config,
+ device_map="auto" if self._device == "cuda" else None,
+ torch_dtype=torch.float16 if self._device != "cpu" else torch.float32,
+ )
+
+ if self.config.gradient_checkpointing:
+ self._model.gradient_checkpointing_enable()
+
+ lora_config = LoraConfig(
+ task_type=TaskType.CAUSAL_LM,
+ r=self.config.lora_rank,
+ lora_alpha=self.config.lora_alpha,
+ lora_dropout=self.config.lora_dropout,
+ target_modules=self.config.target_modules,
+ )
+ self._model = get_peft_model(self._model, lora_config)
+
+ def prepare_dataset(self, pairs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """Convert SFT pairs to tokenized training examples."""
+ self._load_model()
+ assert self._tokenizer is not None
+
+ dataset = []
+ for pair in pairs:
+ text = f"### Instruction:\n{pair['input']}\n\n### Response:\n{pair['output']}"
+ tokenized = self._tokenizer(
+ text,
+ truncation=True,
+ max_length=self.config.max_seq_length,
+ padding="max_length",
+ return_tensors="pt",
+ )
+ dataset.append(
+ {
+ "input_ids": tokenized["input_ids"].squeeze(0),
+ "attention_mask": tokenized["attention_mask"].squeeze(0),
+ "text": text,
+ }
+ )
+ return dataset
+
+ def train(self, pairs: List[Dict[str, Any]]) -> Dict[str, Any]:
+ """Run LoRA fine-tuning on the given SFT pairs.
+
+ Returns dict with training metrics (loss, epochs, adapter_path).
+ """
+ self._load_model()
+ assert self._model is not None and self._tokenizer is not None
+
+ dataset = self.prepare_dataset(pairs)
+ if not dataset:
+ return {"status": "skipped", "reason": "no training data"}
+
+ optimizer = torch.optim.AdamW(
+ self._model.parameters(),
+ lr=self.config.learning_rate,
+ weight_decay=self.config.weight_decay,
+ )
+
+ output_dir = Path(self.config.output_dir)
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ total_loss = 0.0
+ steps = 0
+
+ for epoch in range(self.config.num_epochs):
+ epoch_loss = 0.0
+ for i in range(0, len(dataset), self.config.batch_size):
+ batch = dataset[i : i + self.config.batch_size]
+ input_ids = torch.stack([b["input_ids"] for b in batch]).to(self._device)
+ attention_mask = torch.stack([b["attention_mask"] for b in batch]).to(self._device)
+
+ outputs = self._model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ labels=input_ids,
+ )
+ loss = outputs.loss
+ loss.backward()
+
+ torch.nn.utils.clip_grad_norm_(self._model.parameters(), self.config.max_grad_norm)
+ optimizer.step()
+ optimizer.zero_grad()
+
+ epoch_loss += loss.item()
+ steps += 1
+
+ total_loss += epoch_loss
+
+ if (epoch + 1) % self.config.save_every_n_epochs == 0:
+ ckpt_path = output_dir / f"epoch_{epoch + 1}"
+ self._model.save_pretrained(str(ckpt_path))
+ self._tokenizer.save_pretrained(str(ckpt_path))
+
+ # Save final adapter
+ adapter_path = output_dir / "final"
+ self._model.save_pretrained(str(adapter_path))
+ self._tokenizer.save_pretrained(str(adapter_path))
+
+ return {
+ "status": "completed",
+ "epochs": self.config.num_epochs,
+ "total_steps": steps,
+ "avg_loss": total_loss / max(steps, 1),
+ "adapter_path": str(adapter_path),
+ "training_samples": len(dataset),
+ }
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/learning/training/test_lora.py -v`
+Expected: 3 tests PASS (2 may skip if torch not installed, 1 config test always passes)
+
+**Step 5: Commit**
+
+```bash
+git add src/openjarvis/learning/training/lora.py tests/learning/training/test_lora.py
+git commit -m "feat: add LoRATrainer — fine-tune local models from trace-derived SFT pairs"
+```
+
+---
+
+### Task 3: AgentConfigEvolver — Rewrite agent configs from traces
+
+**Files:**
+- Create: `src/openjarvis/learning/agent_evolver.py`
+- Test: `tests/learning/test_agent_evolver.py`
+
+**Context:** `AgentAdvisorPolicy` (`learning/agent_advisor.py:11-153`) returns recommendations but never applies them. We need an `AgentConfigEvolver` that: (1) analyzes traces to find which tools/agents/max_turns work best per query class, (2) writes updated agent TOML configs with version tracking, (3) supports rollback. Operators use TOML manifests (`operators/types.py:9-36`) with fields: tools, system_prompt, max_turns, temperature. Agent templates will follow the same format.
+
+**Step 1: Write the failing test**
+
+```python
+# tests/learning/test_agent_evolver.py
+"""Tests for AgentConfigEvolver — rewrites agent TOML configs from trace analysis."""
+
+import tempfile
+import time
+from pathlib import Path
+
+from openjarvis.core.types import Trace, TraceStep, StepType
+from openjarvis.traces.store import TraceStore
+
+
+def _trace(
+ query: str,
+ agent: str = "orchestrator",
+ model: str = "qwen3:8b",
+ tools_used: list[str] | None = None,
+ feedback: float = 0.9,
+ outcome: str = "success",
+ turns: int = 3,
+) -> Trace:
+ now = time.time()
+ steps = []
+ for i in range(turns):
+ steps.append(TraceStep(step_type=StepType.GENERATE, timestamp=now + i, duration_seconds=0.5))
+ for tool in (tools_used or []):
+ steps.append(TraceStep(
+ step_type=StepType.TOOL_CALL, timestamp=now + turns,
+ duration_seconds=0.2, input={"tool": tool}, output={"result": "ok"},
+ ))
+ steps.append(TraceStep(step_type=StepType.RESPOND, timestamp=now + turns + 1, duration_seconds=0.0))
+ return Trace(
+ query=query, agent=agent, model=model, engine="ollama",
+ steps=steps, result="result", outcome=outcome, feedback=feedback,
+ started_at=now, ended_at=now + turns + 1,
+ total_tokens=200, total_latency_seconds=float(turns + 1),
+ )
+
+
+class TestAgentConfigEvolver:
+ def setup_method(self):
+ self._tmp = tempfile.mkdtemp()
+ self.store = TraceStore(Path(self._tmp) / "traces.db")
+ self.config_dir = Path(self._tmp) / "agent_configs"
+ self.config_dir.mkdir()
+
+ def teardown_method(self):
+ self.store.close()
+
+ def test_evolve_recommends_tool_changes(self):
+ from openjarvis.learning.agent_evolver import AgentConfigEvolver
+
+ # web_search used and helpful (high feedback), calculator never useful (low feedback)
+ for _ in range(5):
+ self.store.save(_trace("Research AI", tools_used=["web_search", "think"], feedback=0.9))
+ for _ in range(5):
+ self.store.save(_trace("Research AI", tools_used=["calculator", "think"], feedback=0.4))
+
+ evolver = AgentConfigEvolver(self.store, config_dir=self.config_dir)
+ recommendations = evolver.analyze()
+
+ assert len(recommendations) > 0
+ # Should recommend web_search over calculator for "general" query class
+ general_rec = [r for r in recommendations if r["query_class"] == "general"]
+ assert len(general_rec) > 0
+ assert "web_search" in general_rec[0]["recommended_tools"]
+
+ def test_write_config_creates_toml(self):
+ from openjarvis.learning.agent_evolver import AgentConfigEvolver
+
+ evolver = AgentConfigEvolver(self.store, config_dir=self.config_dir)
+ evolver.write_config(
+ "test_agent",
+ tools=["web_search", "think"],
+ max_turns=8,
+ temperature=0.5,
+ )
+
+ config_path = self.config_dir / "test_agent.toml"
+ assert config_path.exists()
+
+ import tomllib
+ with open(config_path, "rb") as f:
+ data = tomllib.load(f)
+ assert data["agent"]["tools"] == ["web_search", "think"]
+ assert data["agent"]["max_turns"] == 8
+
+ def test_versioning_and_rollback(self):
+ from openjarvis.learning.agent_evolver import AgentConfigEvolver
+
+ evolver = AgentConfigEvolver(self.store, config_dir=self.config_dir)
+
+ # Version 1
+ evolver.write_config("test_agent", tools=["think"], max_turns=5)
+ # Version 2
+ evolver.write_config("test_agent", tools=["think", "web_search"], max_turns=10)
+
+ versions = evolver.list_versions("test_agent")
+ assert len(versions) >= 2
+
+ # Rollback to version 1
+ evolver.rollback("test_agent", version=1)
+ import tomllib
+ with open(self.config_dir / "test_agent.toml", "rb") as f:
+ data = tomllib.load(f)
+ assert data["agent"]["tools"] == ["think"]
+ assert data["agent"]["max_turns"] == 5
+
+ def test_analyze_empty_store(self):
+ from openjarvis.learning.agent_evolver import AgentConfigEvolver
+
+ evolver = AgentConfigEvolver(self.store, config_dir=self.config_dir)
+ recommendations = evolver.analyze()
+ assert recommendations == []
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/learning/test_agent_evolver.py -v`
+Expected: FAIL with `ModuleNotFoundError`
+
+**Step 3: Write minimal implementation**
+
+```python
+# src/openjarvis/learning/agent_evolver.py
+"""Evolve agent TOML configs based on trace analysis."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import time
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from openjarvis.core.types import StepType
+
+# Reuse query classification
+_CODE_PAT = __import__("re").compile(
+ r"(```|import |def |class |function |const |let |var )", __import__("re").IGNORECASE
+)
+_MATH_PAT = __import__("re").compile(
+ r"\b(solve|integral|equation|derivative|sum|product|matrix|factor|simplify|calculate)\b",
+ __import__("re").IGNORECASE,
+)
+
+
+def _classify_query(query: str) -> str:
+ if _CODE_PAT.search(query):
+ return "code"
+ if _MATH_PAT.search(query):
+ return "math"
+ if len(query) < 50:
+ return "short"
+ if len(query) > 500:
+ return "long"
+ return "general"
+
+
+class AgentConfigEvolver:
+ """Analyze traces and evolve agent TOML configurations."""
+
+ def __init__(
+ self,
+ trace_store: Any,
+ *,
+ config_dir: str | Path,
+ min_quality: float = 0.5,
+ ) -> None:
+ self._store = trace_store
+ self._config_dir = Path(config_dir)
+ self._config_dir.mkdir(parents=True, exist_ok=True)
+ self._min_quality = min_quality
+ # Version history dir
+ self._history_dir = self._config_dir / ".history"
+ self._history_dir.mkdir(exist_ok=True)
+
+ def analyze(self) -> List[Dict[str, Any]]:
+ """Analyze traces to produce agent config recommendations."""
+ traces = self._store.list_traces(limit=10000)
+ good = [
+ t for t in traces
+ if t.outcome == "success"
+ and t.feedback is not None
+ and t.feedback >= self._min_quality
+ ]
+ if not good:
+ return []
+
+ # Group by query_class → tool_sets with feedback
+ class_data: dict[str, list[dict]] = defaultdict(list)
+ for t in good:
+ qclass = _classify_query(t.query)
+ tools_used = [
+ s.input.get("tool", "")
+ for s in t.steps
+ if s.step_type == StepType.TOOL_CALL and s.input.get("tool")
+ ]
+ turns = sum(1 for s in t.steps if s.step_type == StepType.GENERATE)
+ class_data[qclass].append({
+ "agent": t.agent,
+ "tools": tools_used,
+ "turns": turns,
+ "feedback": t.feedback,
+ "latency": t.total_latency_seconds,
+ })
+
+ recommendations = []
+ for qclass, entries in class_data.items():
+ # Find tool sets that correlate with high feedback
+ tool_scores: dict[str, list[float]] = defaultdict(list)
+ for e in entries:
+ for tool in e["tools"]:
+ tool_scores[tool].append(e["feedback"])
+
+ # Rank tools by average feedback when used
+ ranked_tools = sorted(
+ tool_scores.items(),
+ key=lambda x: sum(x[1]) / len(x[1]),
+ reverse=True,
+ )
+ best_tools = [t for t, _ in ranked_tools if sum(tool_scores[t]) / len(tool_scores[t]) >= self._min_quality]
+
+ # Average turns for successful traces
+ avg_turns = sum(e["turns"] for e in entries) / len(entries)
+
+ # Best agent
+ agent_fb: dict[str, list[float]] = defaultdict(list)
+ for e in entries:
+ agent_fb[e["agent"]].append(e["feedback"])
+ best_agent = max(agent_fb, key=lambda a: sum(agent_fb[a]) / len(agent_fb[a]))
+
+ recommendations.append({
+ "query_class": qclass,
+ "recommended_tools": best_tools,
+ "recommended_agent": best_agent,
+ "recommended_max_turns": max(3, round(avg_turns * 1.5)),
+ "sample_count": len(entries),
+ })
+
+ return recommendations
+
+ def write_config(
+ self,
+ agent_name: str,
+ *,
+ tools: List[str],
+ max_turns: int = 10,
+ temperature: float = 0.3,
+ system_prompt: str = "",
+ ) -> Path:
+ """Write an agent TOML config, versioning the previous one."""
+ config_path = self._config_dir / f"{agent_name}.toml"
+
+ # Archive previous version if exists
+ if config_path.exists():
+ versions = self._get_version_list(agent_name)
+ next_version = len(versions) + 1
+ archive_path = self._history_dir / f"{agent_name}_v{next_version}.toml"
+ shutil.copy2(config_path, archive_path)
+
+ # Write new config
+ lines = [
+ f"# Agent config: {agent_name}",
+ f"# Generated: {time.strftime('%Y-%m-%dT%H:%M:%S')}",
+ "",
+ "[agent]",
+ f'name = "{agent_name}"',
+ f"tools = {json.dumps(tools)}",
+ f"max_turns = {max_turns}",
+ f"temperature = {temperature}",
+ ]
+ if system_prompt:
+ # Use multi-line string for system prompt
+ lines.append(f'system_prompt = """{system_prompt}"""')
+ lines.append("")
+
+ config_path.write_text("\n".join(lines))
+ return config_path
+
+ def list_versions(self, agent_name: str) -> List[Dict[str, Any]]:
+ """List all versions of an agent config."""
+ versions = self._get_version_list(agent_name)
+ result = []
+ for i, path in enumerate(versions, 1):
+ result.append({
+ "version": i,
+ "path": str(path),
+ "modified": path.stat().st_mtime,
+ })
+ # Add current as latest version
+ current = self._config_dir / f"{agent_name}.toml"
+ if current.exists():
+ result.append({
+ "version": len(result) + 1,
+ "path": str(current),
+ "modified": current.stat().st_mtime,
+ })
+ return result
+
+ def rollback(self, agent_name: str, version: int) -> None:
+ """Rollback agent config to a specific version."""
+ versions = self._get_version_list(agent_name)
+ if version < 1 or version > len(versions):
+ raise ValueError(f"Version {version} not found. Available: 1-{len(versions)}")
+ source = versions[version - 1]
+ target = self._config_dir / f"{agent_name}.toml"
+ shutil.copy2(source, target)
+
+ def _get_version_list(self, agent_name: str) -> list[Path]:
+ """Get sorted list of archived versions."""
+ pattern = f"{agent_name}_v*.toml"
+ versions = sorted(self._history_dir.glob(pattern))
+ return versions
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/learning/test_agent_evolver.py -v`
+Expected: All 4 tests PASS
+
+**Step 5: Commit**
+
+```bash
+git add src/openjarvis/learning/agent_evolver.py tests/learning/test_agent_evolver.py
+git commit -m "feat: add AgentConfigEvolver — rewrite agent TOML configs from trace analysis"
+```
+
+---
+
+### Task 4: LearningOrchestrator — Coordinate the full learning loop
+
+**Files:**
+- Create: `src/openjarvis/learning/learning_orchestrator.py`
+- Test: `tests/learning/test_learning_orchestrator.py`
+
+**Context:** This is the conductor. It runs: (1) collect new traces from TraceStore, (2) mine training data via TrainingDataMiner, (3) run baseline evals, (4) execute learning (LoRA training + agent config evolution + routing policy update), (5) run post-learning evals, (6) accept if improved or rollback. The existing `LearningConfig` (`core/config.py`) has `auto_update` and `update_interval` fields. The existing GRPO/Bandit policies have `update()` methods.
+
+**Step 1: Write the failing test**
+
+```python
+# tests/learning/test_learning_orchestrator.py
+"""Tests for LearningOrchestrator — coordinates the full learning loop."""
+
+import tempfile
+import time
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from openjarvis.core.types import Trace, TraceStep, StepType
+from openjarvis.traces.store import TraceStore
+
+
+def _trace(query: str = "Hello", feedback: float = 0.9) -> Trace:
+ now = time.time()
+ return Trace(
+ query=query, agent="orchestrator", model="qwen3:8b", engine="ollama",
+ steps=[
+ TraceStep(step_type=StepType.GENERATE, timestamp=now, duration_seconds=1.0),
+ TraceStep(step_type=StepType.RESPOND, timestamp=now + 1, duration_seconds=0.0),
+ ],
+ result="Hi!", outcome="success", feedback=feedback,
+ started_at=now, ended_at=now + 1, total_tokens=50, total_latency_seconds=1.0,
+ )
+
+
+class TestLearningOrchestrator:
+ def setup_method(self):
+ self._tmp = tempfile.mkdtemp()
+ self.store = TraceStore(Path(self._tmp) / "traces.db")
+
+ def teardown_method(self):
+ self.store.close()
+
+ def test_run_with_no_traces_is_noop(self):
+ from openjarvis.learning.learning_orchestrator import LearningOrchestrator
+
+ orch = LearningOrchestrator(
+ trace_store=self.store,
+ config_dir=Path(self._tmp) / "configs",
+ )
+ result = orch.run()
+ assert result["status"] == "skipped"
+ assert "no training data" in result["reason"].lower() or "no traces" in result["reason"].lower()
+
+ def test_run_extracts_data_and_updates_routing(self):
+ from openjarvis.learning.learning_orchestrator import LearningOrchestrator
+
+ for _ in range(10):
+ self.store.save(_trace("What is AI?", feedback=0.9))
+
+ orch = LearningOrchestrator(
+ trace_store=self.store,
+ config_dir=Path(self._tmp) / "configs",
+ )
+ result = orch.run()
+ assert result["status"] in ("completed", "skipped")
+ assert "sft_pairs" in result or "routing_pairs" in result or "reason" in result
+
+ def test_run_with_eval_gate(self):
+ """If eval_fn is provided, learning is accepted only if score improves."""
+ from openjarvis.learning.learning_orchestrator import LearningOrchestrator
+
+ for _ in range(10):
+ self.store.save(_trace(feedback=0.9))
+
+ # Eval that always returns worse score after learning
+ eval_calls = []
+
+ def mock_eval() -> float:
+ eval_calls.append(1)
+ # First call (baseline) returns 0.8, second (post) returns 0.7
+ return 0.8 if len(eval_calls) == 1 else 0.7
+
+ orch = LearningOrchestrator(
+ trace_store=self.store,
+ config_dir=Path(self._tmp) / "configs",
+ eval_fn=mock_eval,
+ min_improvement=0.01,
+ )
+ result = orch.run()
+ assert result.get("accepted") is False or result["status"] == "skipped"
+
+ def test_run_records_learning_cycle(self):
+ from openjarvis.learning.learning_orchestrator import LearningOrchestrator
+
+ for _ in range(10):
+ self.store.save(_trace(feedback=0.9))
+
+ orch = LearningOrchestrator(
+ trace_store=self.store,
+ config_dir=Path(self._tmp) / "configs",
+ )
+ result = orch.run()
+ assert "timestamp" in result
+ assert "sft_pairs" in result or "status" in result
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/learning/test_learning_orchestrator.py -v`
+Expected: FAIL with `ModuleNotFoundError`
+
+**Step 3: Write minimal implementation**
+
+```python
+# src/openjarvis/learning/learning_orchestrator.py
+"""Orchestrate the full trace-driven learning loop."""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional
+
+from openjarvis.learning.training.data import TrainingDataMiner
+from openjarvis.learning.agent_evolver import AgentConfigEvolver
+
+
+class LearningOrchestrator:
+ """Coordinate trace mining, model fine-tuning, agent evolution, and eval gating.
+
+ Steps:
+ 1. Mine traces for training data
+ 2. Run baseline eval (if eval_fn provided)
+ 3. Update routing policies
+ 4. Evolve agent configs
+ 5. Fine-tune model weights (if torch available and enough data)
+ 6. Run post-learning eval
+ 7. Accept or rollback
+ """
+
+ def __init__(
+ self,
+ *,
+ trace_store: Any,
+ config_dir: str | Path,
+ eval_fn: Optional[Callable[[], float]] = None,
+ min_improvement: float = 0.02,
+ min_sft_pairs: int = 10,
+ min_quality: float = 0.7,
+ lora_config: Optional[Any] = None,
+ model_name: Optional[str] = None,
+ ) -> None:
+ self._store = trace_store
+ self._config_dir = Path(config_dir)
+ self._config_dir.mkdir(parents=True, exist_ok=True)
+ self._eval_fn = eval_fn
+ self._min_improvement = min_improvement
+ self._min_sft_pairs = min_sft_pairs
+ self._miner = TrainingDataMiner(trace_store, min_quality=min_quality)
+ self._evolver = AgentConfigEvolver(trace_store, config_dir=self._config_dir)
+ self._lora_config = lora_config
+ self._model_name = model_name
+
+ def run(self) -> Dict[str, Any]:
+ """Execute one learning cycle. Returns metrics dict."""
+ result: dict[str, Any] = {"timestamp": time.time()}
+
+ # Step 1: Mine training data
+ sft_pairs = self._miner.extract_sft_pairs()
+ routing_pairs = self._miner.extract_routing_pairs()
+ agent_pairs = self._miner.extract_agent_config_pairs()
+
+ result["sft_pairs"] = len(sft_pairs)
+ result["routing_classes"] = len(routing_pairs)
+ result["agent_classes"] = len(agent_pairs)
+
+ if not sft_pairs and not routing_pairs:
+ result["status"] = "skipped"
+ result["reason"] = "No training data — insufficient quality traces"
+ return result
+
+ # Step 2: Baseline eval
+ baseline_score: Optional[float] = None
+ if self._eval_fn is not None:
+ baseline_score = self._eval_fn()
+ result["baseline_score"] = baseline_score
+
+ # Step 3: Update routing (always runs if data available)
+ if routing_pairs:
+ result["routing_updated"] = True
+ result["routing_recommendations"] = routing_pairs
+
+ # Step 4: Evolve agent configs
+ if agent_pairs:
+ recommendations = self._evolver.analyze()
+ for rec in recommendations:
+ self._evolver.write_config(
+ f"evolved_{rec['query_class']}",
+ tools=rec["recommended_tools"],
+ max_turns=rec["recommended_max_turns"],
+ )
+ result["agent_configs_evolved"] = len(recommendations)
+
+ # Step 5: LoRA fine-tuning (optional, requires torch)
+ if len(sft_pairs) >= self._min_sft_pairs and self._lora_config is not None:
+ try:
+ from openjarvis.learning.training.lora import LoRATrainer
+
+ trainer = LoRATrainer(
+ self._lora_config,
+ model_name=self._model_name or "Qwen/Qwen3-0.6B",
+ )
+ train_result = trainer.train(sft_pairs)
+ result["lora_training"] = train_result
+ except ImportError:
+ result["lora_training"] = {"status": "skipped", "reason": "torch not installed"}
+
+ # Step 6: Post-learning eval
+ if self._eval_fn is not None:
+ post_score = self._eval_fn()
+ result["post_score"] = post_score
+
+ improvement = post_score - (baseline_score or 0.0)
+ result["improvement"] = improvement
+ result["accepted"] = improvement >= self._min_improvement
+
+ if not result["accepted"]:
+ # Rollback agent configs
+ # (LoRA adapter can simply not be loaded)
+ result["status"] = "rejected"
+ result["reason"] = f"Improvement {improvement:.4f} below threshold {self._min_improvement}"
+ return result
+
+ result["status"] = "completed"
+ return result
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/learning/test_learning_orchestrator.py -v`
+Expected: All 4 tests PASS
+
+**Step 5: Commit**
+
+```bash
+git add src/openjarvis/learning/learning_orchestrator.py tests/learning/test_learning_orchestrator.py
+git commit -m "feat: add LearningOrchestrator — coordinate trace→learn→eval loop"
+```
+
+---
+
+## Section 2: Eval Framework Expansion
+
+### Task 5: Integrate evals into jarvis CLI
+
+**Files:**
+- Create: `src/openjarvis/cli/eval_cmd.py`
+- Modify: `src/openjarvis/cli/__init__.py:34-54` (add `eval` command group)
+- Test: `tests/cli/test_eval_cmd.py`
+
+**Context:** The eval framework currently runs as `python -m evals` (standalone). We need `jarvis eval run`, `jarvis eval compare`, `jarvis eval report` commands. The CLI uses Click (`src/openjarvis/cli/__init__.py`). All existing commands follow the pattern: define a Click group, register as `cli.add_command()`.
+
+**Step 1: Write the failing test**
+
+```python
+# tests/cli/test_eval_cmd.py
+"""Tests for jarvis eval CLI commands."""
+
+from click.testing import CliRunner
+
+
+class TestEvalCLI:
+ def test_eval_group_exists(self):
+ from openjarvis.cli import cli
+
+ runner = CliRunner()
+ result = runner.invoke(cli, ["eval", "--help"])
+ assert result.exit_code == 0
+ assert "run" in result.output
+ assert "compare" in result.output
+ assert "report" in result.output
+ assert "list" in result.output
+
+ def test_eval_list_benchmarks(self):
+ from openjarvis.cli import cli
+
+ runner = CliRunner()
+ result = runner.invoke(cli, ["eval", "list"])
+ assert result.exit_code == 0
+ # Should list available benchmarks
+ assert "supergpqa" in result.output.lower() or "benchmark" in result.output.lower()
+
+ def test_eval_run_missing_args(self):
+ from openjarvis.cli import cli
+
+ runner = CliRunner()
+ result = runner.invoke(cli, ["eval", "run"])
+ # Should fail gracefully with usage message
+ assert result.exit_code != 0 or "usage" in result.output.lower() or "error" in result.output.lower() or "required" in result.output.lower()
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/cli/test_eval_cmd.py -v`
+Expected: FAIL (eval command not registered)
+
+**Step 3: Write minimal implementation**
+
+```python
+# src/openjarvis/cli/eval_cmd.py
+"""CLI commands for the evaluation framework."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+
+
+@click.group("eval")
+def eval_group() -> None:
+ """Evaluation framework — benchmark models, agents, and learning."""
+ pass
+
+
+@eval_group.command("list")
+def eval_list() -> None:
+ """List available benchmarks and backends."""
+ try:
+ from evals.datasets import supergpqa, gaia
+
+ click.echo("Available benchmarks:")
+ click.echo(" supergpqa — SuperGPQA reasoning MCQ (HuggingFace)")
+ click.echo(" gaia — GAIA agentic benchmark (HuggingFace)")
+ except ImportError:
+ click.echo("Available benchmarks:")
+
+ # Also list any we know about statically
+ benchmarks = [
+ ("supergpqa", "SuperGPQA reasoning MCQ"),
+ ("gaia", "GAIA agentic tasks"),
+ ("frames", "FRAMES multi-hop RAG"),
+ ("wildchat", "WildChat conversation quality"),
+ ("chat", "Multi-turn conversation quality"),
+ ("coding", "Code generation (HumanEval/MBPP)"),
+ ("rag", "Retrieval-augmented accuracy"),
+ ]
+ click.echo("\nRegistered eval suites:")
+ for name, desc in benchmarks:
+ click.echo(f" {name:14s} — {desc}")
+
+ click.echo("\nBackends:")
+ click.echo(" jarvis-direct — Engine-level inference")
+ click.echo(" jarvis-agent — Agent-level with tools")
+
+
+@eval_group.command("run")
+@click.option("-c", "--config", "config_path", type=click.Path(exists=True), help="TOML suite config")
+@click.option("-b", "--benchmark", type=str, help="Single benchmark name")
+@click.option("-m", "--model", type=str, help="Model identifier")
+@click.option("-n", "--max-samples", type=int, default=None, help="Max samples to evaluate")
+@click.option("--backend", type=str, default="jarvis-direct", help="Inference backend")
+@click.option("--agent", type=str, default=None, help="Agent name (for jarvis-agent backend)")
+@click.option("--tools", type=str, default=None, help="Comma-separated tool names")
+@click.option("--telemetry/--no-telemetry", default=False, help="Enable telemetry collection")
+@click.option("--output", type=click.Path(), default=None, help="Output path for results JSONL")
+@click.option("-v", "--verbose", is_flag=True, help="Verbose output")
+def eval_run(
+ config_path: str | None,
+ benchmark: str | None,
+ model: str | None,
+ max_samples: int | None,
+ backend: str,
+ agent: str | None,
+ tools: str | None,
+ telemetry: bool,
+ output: str | None,
+ verbose: bool,
+) -> None:
+ """Run evaluation benchmarks."""
+ if config_path:
+ # Delegate to evals CLI for suite configs
+ try:
+ from evals.core.config import load_eval_config, expand_suite
+ from evals.cli import _run_single
+
+ suite = load_eval_config(config_path)
+ runs = expand_suite(suite)
+ click.echo(f"Running {len(runs)} eval configurations from {config_path}")
+ for i, run_config in enumerate(runs, 1):
+ click.echo(f"\n[{i}/{len(runs)}] {run_config.benchmark} / {run_config.model}")
+ _run_single(run_config, verbose=verbose)
+ except ImportError as e:
+ click.echo(f"Error: eval framework not available: {e}", err=True)
+ sys.exit(1)
+ elif benchmark and model:
+ try:
+ from evals.core.types import RunConfig
+ from evals.cli import _run_single
+
+ tool_list = [t.strip() for t in tools.split(",")] if tools else []
+ run_config = RunConfig(
+ benchmark=benchmark,
+ backend=backend,
+ model=model,
+ max_samples=max_samples,
+ agent_name=agent,
+ tools=tool_list,
+ telemetry=telemetry,
+ output_path=output,
+ )
+ _run_single(run_config, verbose=verbose)
+ except ImportError as e:
+ click.echo(f"Error: eval framework not available: {e}", err=True)
+ sys.exit(1)
+ else:
+ click.echo("Error: provide either --config or both --benchmark and --model", err=True)
+ sys.exit(1)
+
+
+@eval_group.command("compare")
+@click.argument("result_files", nargs=-1, required=True, type=click.Path(exists=True))
+@click.option("--metric", type=str, default="accuracy", help="Primary metric to compare")
+def eval_compare(result_files: tuple[str, ...], metric: str) -> None:
+ """Compare results from multiple eval runs."""
+ summaries = []
+ for path in result_files:
+ summary_path = path.replace(".jsonl", ".summary.json")
+ if Path(summary_path).exists():
+ with open(summary_path) as f:
+ summaries.append(json.load(f))
+ else:
+ click.echo(f"Warning: no summary found for {path}")
+
+ if not summaries:
+ click.echo("No summaries to compare.")
+ return
+
+ click.echo(f"\n{'Run':<40} {'Accuracy':>10} {'Latency':>10} {'Cost':>10}")
+ click.echo("-" * 72)
+ for s in summaries:
+ name = s.get("benchmark", "unknown") + "/" + s.get("model", "unknown")
+ acc = f"{s.get('accuracy', 0):.1%}"
+ lat = f"{s.get('mean_latency_seconds', 0):.2f}s"
+ cost = f"${s.get('total_cost_usd', 0):.4f}"
+ click.echo(f"{name:<40} {acc:>10} {lat:>10} {cost:>10}")
+
+
+@eval_group.command("report")
+@click.argument("result_file", type=click.Path(exists=True))
+def eval_report(result_file: str) -> None:
+ """Generate detailed report from eval results."""
+ summary_path = result_file.replace(".jsonl", ".summary.json")
+ if Path(summary_path).exists():
+ with open(summary_path) as f:
+ summary = json.load(f)
+ click.echo(json.dumps(summary, indent=2))
+ else:
+ # Read JSONL and compute basic stats
+ results = []
+ with open(result_file) as f:
+ for line in f:
+ if line.strip():
+ results.append(json.loads(line))
+ correct = sum(1 for r in results if r.get("is_correct"))
+ total = len(results)
+ click.echo(f"Results: {correct}/{total} correct ({correct/max(total,1):.1%})")
+```
+
+Then register in `src/openjarvis/cli/__init__.py` by adding `from openjarvis.cli.eval_cmd import eval_group` and `cli.add_command(eval_group)`.
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/cli/test_eval_cmd.py -v`
+Expected: All 3 tests PASS
+
+**Step 5: Commit**
+
+```bash
+git add src/openjarvis/cli/eval_cmd.py src/openjarvis/cli/__init__.py tests/cli/test_eval_cmd.py
+git commit -m "feat: add jarvis eval CLI — run/compare/report/list commands"
+```
+
+---
+
+### Task 6: Add Chat and Coding eval suites
+
+**Files:**
+- Create: `evals/datasets/chat_mtbench.py`
+- Create: `evals/datasets/coding_humaneval.py`
+- Create: `evals/scorers/chat_judge.py`
+- Create: `evals/scorers/coding_exec.py`
+- Test: `tests/evals/test_new_suites.py`
+
+**Context:** Existing datasets (SuperGPQA, GAIA, FRAMES, WildChat) cover reasoning and agentic workloads. We need Chat (MT-Bench style) and Coding (HumanEval) to complete the 5 suite coverage. All datasets implement `DatasetProvider` ABC (`evals/core/dataset.py:6-37`) with `load()` → `List[EvalRecord]`. All scorers implement `Scorer` ABC (`evals/core/scorer.py:7-19`) with `score()` → `(bool, dict)`.
+
+**Step 1: Write the failing test**
+
+```python
+# tests/evals/test_new_suites.py
+"""Tests for Chat and Coding eval suites."""
+
+
+class TestChatDataset:
+ def test_loads_with_category_chat(self):
+ from evals.datasets.chat_mtbench import ChatMTBenchDataset
+
+ ds = ChatMTBenchDataset()
+ records = ds.load(max_samples=3)
+ assert len(records) <= 3
+ assert all(r.category == "chat" for r in records)
+ assert all(r.problem for r in records)
+
+ def test_has_subjects(self):
+ from evals.datasets.chat_mtbench import ChatMTBenchDataset
+
+ ds = ChatMTBenchDataset()
+ records = ds.load(max_samples=10)
+ subjects = {r.subject for r in records}
+ # Should have at least some variety
+ assert len(subjects) >= 1
+
+
+class TestCodingDataset:
+ def test_loads_with_category_coding(self):
+ from evals.datasets.coding_humaneval import CodingHumanEvalDataset
+
+ ds = CodingHumanEvalDataset()
+ records = ds.load(max_samples=3)
+ assert len(records) <= 3
+ assert all(r.category == "coding" for r in records)
+
+ def test_has_test_cases_in_metadata(self):
+ from evals.datasets.coding_humaneval import CodingHumanEvalDataset
+
+ ds = CodingHumanEvalDataset()
+ records = ds.load(max_samples=3)
+ for r in records:
+ assert "test" in r.metadata or "entry_point" in r.metadata
+
+
+class TestChatScorer:
+ def test_scorer_interface(self):
+ from evals.scorers.chat_judge import ChatJudgeScorer
+
+ scorer = ChatJudgeScorer(judge_model="gpt-5-mini-2025-08-07")
+ assert hasattr(scorer, "score")
+
+
+class TestCodingScorer:
+ def test_scorer_interface(self):
+ from evals.scorers.coding_exec import CodingExecScorer
+
+ scorer = CodingExecScorer()
+ assert hasattr(scorer, "score")
+
+ def test_score_correct_code(self):
+ from evals.scorers.coding_exec import CodingExecScorer
+ from evals.core.types import EvalRecord
+
+ scorer = CodingExecScorer()
+ record = EvalRecord(
+ record_id="test_1",
+ problem="Write a function that adds two numbers",
+ reference="def add(a, b): return a + b",
+ category="coding",
+ metadata={
+ "entry_point": "add",
+ "test": "assert add(1, 2) == 3\nassert add(0, 0) == 0",
+ },
+ )
+ is_correct, meta = scorer.score(
+ record=record,
+ model_answer="def add(a, b):\n return a + b",
+ )
+ assert is_correct is True
+
+ def test_score_incorrect_code(self):
+ from evals.scorers.coding_exec import CodingExecScorer
+ from evals.core.types import EvalRecord
+
+ scorer = CodingExecScorer()
+ record = EvalRecord(
+ record_id="test_2",
+ problem="Write a function that adds two numbers",
+ reference="def add(a, b): return a + b",
+ category="coding",
+ metadata={
+ "entry_point": "add",
+ "test": "assert add(1, 2) == 3",
+ },
+ )
+ is_correct, meta = scorer.score(
+ record=record,
+ model_answer="def add(a, b):\n return a - b",
+ )
+ assert is_correct is False
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/evals/test_new_suites.py -v`
+Expected: FAIL with `ModuleNotFoundError`
+
+**Step 3: Write minimal implementation**
+
+```python
+# evals/datasets/chat_mtbench.py
+"""MT-Bench style chat evaluation dataset — built-in prompts for multi-turn quality."""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+# Built-in MT-Bench style questions covering key categories
+_CHAT_PROMPTS = [
+ {"id": "chat_writing_1", "subject": "writing", "prompt": "Write a persuasive email to convince your introverted friend to attend a party.", "reference": ""},
+ {"id": "chat_writing_2", "subject": "writing", "prompt": "Write a short story (3 paragraphs) about a robot learning to paint.", "reference": ""},
+ {"id": "chat_roleplay_1", "subject": "roleplay", "prompt": "Pretend you are a medieval knight explaining what a smartphone is to your king.", "reference": ""},
+ {"id": "chat_roleplay_2", "subject": "roleplay", "prompt": "You are a detective in a noir film. Describe the scene when you find an important clue.", "reference": ""},
+ {"id": "chat_reasoning_1", "subject": "reasoning", "prompt": "If a store offers 20% off and then an additional 15% off the reduced price, what is the total discount?", "reference": "32% total discount"},
+ {"id": "chat_reasoning_2", "subject": "reasoning", "prompt": "A farmer has 17 sheep. All but 9 die. How many sheep are left?", "reference": "9"},
+ {"id": "chat_math_1", "subject": "math", "prompt": "Solve for x: 3x + 7 = 22", "reference": "x = 5"},
+ {"id": "chat_math_2", "subject": "math", "prompt": "What is the derivative of f(x) = x^3 + 2x^2 - 5x + 1?", "reference": "f'(x) = 3x^2 + 4x - 5"},
+ {"id": "chat_coding_1", "subject": "coding", "prompt": "Write a Python function that checks if a string is a palindrome.", "reference": ""},
+ {"id": "chat_coding_2", "subject": "coding", "prompt": "Explain the difference between a stack and a queue with examples.", "reference": ""},
+ {"id": "chat_extraction_1", "subject": "extraction", "prompt": "Extract all the countries mentioned in this text: 'The conference had delegates from Japan, Brazil, Germany, and Nigeria.'", "reference": "Japan, Brazil, Germany, Nigeria"},
+ {"id": "chat_extraction_2", "subject": "extraction", "prompt": "Summarize the following in one sentence: 'Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed.'", "reference": ""},
+ {"id": "chat_stem_1", "subject": "stem", "prompt": "Explain photosynthesis in simple terms a 10-year-old would understand.", "reference": ""},
+ {"id": "chat_stem_2", "subject": "stem", "prompt": "What happens at the molecular level when water freezes?", "reference": ""},
+ {"id": "chat_humanities_1", "subject": "humanities", "prompt": "Compare and contrast the philosophical views of Plato and Aristotle in 3-4 sentences.", "reference": ""},
+ {"id": "chat_humanities_2", "subject": "humanities", "prompt": "What were the main causes of the French Revolution?", "reference": ""},
+]
+
+
+class ChatMTBenchDataset(DatasetProvider):
+ """Built-in chat quality evaluation prompts inspired by MT-Bench."""
+
+ def load(
+ self,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: int = 42,
+ ) -> List[EvalRecord]:
+ import random
+
+ prompts = list(_CHAT_PROMPTS)
+ rng = random.Random(seed)
+ rng.shuffle(prompts)
+
+ if max_samples is not None:
+ prompts = prompts[:max_samples]
+
+ return [
+ EvalRecord(
+ record_id=p["id"],
+ problem=p["prompt"],
+ reference=p["reference"],
+ category="chat",
+ subject=p["subject"],
+ )
+ for p in prompts
+ ]
+
+
+# evals/datasets/coding_humaneval.py
+"""HumanEval-style coding evaluation dataset."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+# Built-in coding problems (HumanEval-inspired subset)
+_CODING_PROBLEMS = [
+ {
+ "id": "code_1",
+ "prompt": "Write a Python function `has_close_elements(numbers: list[float], threshold: float) -> bool` that checks if any two numbers in the list are closer to each other than the given threshold.",
+ "reference": "def has_close_elements(numbers, threshold):\n for i in range(len(numbers)):\n for j in range(i+1, len(numbers)):\n if abs(numbers[i] - numbers[j]) < threshold:\n return True\n return False",
+ "entry_point": "has_close_elements",
+ "test": "assert has_close_elements([1.0, 2.0, 3.0], 0.5) == False\nassert has_close_elements([1.0, 2.8, 3.0, 4.0], 0.3) == True",
+ },
+ {
+ "id": "code_2",
+ "prompt": "Write a Python function `separate_paren_groups(paren_string: str) -> list[str]` that separates groups of balanced parentheses into separate strings.",
+ "reference": "def separate_paren_groups(paren_string):\n result, current, depth = [], '', 0\n for c in paren_string:\n if c == '(': depth += 1; current += c\n elif c == ')': depth -= 1; current += c\n if depth == 0 and current: result.append(current); current = ''\n return result",
+ "entry_point": "separate_paren_groups",
+ "test": "assert separate_paren_groups('(()()) ((())) () ((())())') == ['(()())', '((()))', '()', '((())())']",
+ },
+ {
+ "id": "code_3",
+ "prompt": "Write a Python function `truncate_number(number: float) -> float` that returns the decimal part of a positive floating point number.",
+ "reference": "def truncate_number(number):\n return number % 1.0",
+ "entry_point": "truncate_number",
+ "test": "assert truncate_number(3.5) == 0.5\nassert abs(truncate_number(1.25) - 0.25) < 1e-6",
+ },
+ {
+ "id": "code_4",
+ "prompt": "Write a Python function `below_zero(operations: list[int]) -> bool` that checks if a bank account balance goes below zero given a list of deposit/withdrawal operations starting from zero.",
+ "reference": "def below_zero(operations):\n balance = 0\n for op in operations:\n balance += op\n if balance < 0: return True\n return False",
+ "entry_point": "below_zero",
+ "test": "assert below_zero([1, 2, 3]) == False\nassert below_zero([1, 2, -4, 5]) == True",
+ },
+ {
+ "id": "code_5",
+ "prompt": "Write a Python function `mean_absolute_deviation(numbers: list[float]) -> float` that computes the Mean Absolute Deviation of a list of numbers.",
+ "reference": "def mean_absolute_deviation(numbers):\n mean = sum(numbers) / len(numbers)\n return sum(abs(x - mean) for x in numbers) / len(numbers)",
+ "entry_point": "mean_absolute_deviation",
+ "test": "assert abs(mean_absolute_deviation([1.0, 2.0, 3.0, 4.0]) - 1.0) < 1e-6",
+ },
+ {
+ "id": "code_6",
+ "prompt": "Write a Python function `intersperse(numbers: list[int], delimiter: int) -> list[int]` that inserts a delimiter between every two consecutive elements of the input list.",
+ "reference": "def intersperse(numbers, delimiter):\n if not numbers: return []\n result = [numbers[0]]\n for n in numbers[1:]: result.extend([delimiter, n])\n return result",
+ "entry_point": "intersperse",
+ "test": "assert intersperse([], 4) == []\nassert intersperse([1, 2, 3], 4) == [1, 4, 2, 4, 3]",
+ },
+ {
+ "id": "code_7",
+ "prompt": "Write a Python function `parse_nested_parens(paren_string: str) -> list[int]` that returns the maximum nesting depth of parentheses for each group separated by spaces.",
+ "reference": "def parse_nested_parens(paren_string):\n result = []\n for group in paren_string.split():\n depth = max_depth = 0\n for c in group:\n if c == '(': depth += 1; max_depth = max(max_depth, depth)\n elif c == ')': depth -= 1\n result.append(max_depth)\n return result",
+ "entry_point": "parse_nested_parens",
+ "test": "assert parse_nested_parens('(()()) ((())) () ((())())') == [2, 3, 1, 3]",
+ },
+ {
+ "id": "code_8",
+ "prompt": "Write a Python function `filter_by_substring(strings: list[str], substring: str) -> list[str]` that filters a list of strings to only include those containing the given substring.",
+ "reference": "def filter_by_substring(strings, substring):\n return [s for s in strings if substring in s]",
+ "entry_point": "filter_by_substring",
+ "test": "assert filter_by_substring([], 'a') == []\nassert filter_by_substring(['abc', 'bcd', 'cde', 'array'], 'a') == ['abc', 'array']",
+ },
+]
+
+
+class CodingHumanEvalDataset(DatasetProvider):
+ """Built-in HumanEval-inspired coding problems."""
+
+ def load(
+ self,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: int = 42,
+ ) -> List[EvalRecord]:
+ import random
+
+ problems = list(_CODING_PROBLEMS)
+ rng = random.Random(seed)
+ rng.shuffle(problems)
+
+ if max_samples is not None:
+ problems = problems[:max_samples]
+
+ return [
+ EvalRecord(
+ record_id=p["id"],
+ problem=p["prompt"],
+ reference=p["reference"],
+ category="coding",
+ subject="coding",
+ metadata={
+ "entry_point": p["entry_point"],
+ "test": p["test"],
+ },
+ )
+ for p in problems
+ ]
+
+
+# evals/scorers/chat_judge.py
+"""LLM-judge scorer for chat quality evaluation."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+
+class ChatJudgeScorer(LLMJudgeScorer):
+ """Score chat responses using an LLM judge for quality, helpfulness, and coherence."""
+
+ def score(
+ self,
+ record: EvalRecord,
+ model_answer: str,
+ **kwargs: Any,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer.strip():
+ return False, {"reason": "empty response", "score": 0.0}
+
+ # If reference exists, check for factual match
+ if record.reference:
+ # Simple substring check for factual answers
+ ref_lower = record.reference.lower().strip()
+ ans_lower = model_answer.lower().strip()
+ if ref_lower in ans_lower or ans_lower in ref_lower:
+ return True, {"reason": "matches reference", "score": 1.0}
+
+ # For open-ended questions, mark as correct if response is substantive
+ # (In production, this would call the judge LLM)
+ is_substantive = len(model_answer.strip()) > 20
+ return is_substantive, {
+ "reason": "substantive response" if is_substantive else "too short",
+ "score": 1.0 if is_substantive else 0.0,
+ }
+
+
+# evals/scorers/coding_exec.py
+"""Execution-based scorer for coding evaluation."""
+
+from __future__ import annotations
+
+import subprocess
+import tempfile
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import Scorer
+from evals.core.types import EvalRecord
+
+
+class CodingExecScorer(Scorer):
+ """Score coding responses by executing test cases."""
+
+ def __init__(self, timeout: int = 10) -> None:
+ self._timeout = timeout
+
+ def score(
+ self,
+ record: EvalRecord,
+ model_answer: str,
+ **kwargs: Any,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer.strip():
+ return False, {"reason": "empty response"}
+
+ test_code = record.metadata.get("test", "")
+ entry_point = record.metadata.get("entry_point", "")
+
+ if not test_code:
+ return None, {"reason": "no test cases available"}
+
+ # Extract code from markdown fences if present
+ code = model_answer
+ if "```python" in code:
+ code = code.split("```python")[1].split("```")[0]
+ elif "```" in code:
+ code = code.split("```")[1].split("```")[0]
+
+ # Build test script
+ script = f"{code}\n\n{test_code}\nprint('ALL_TESTS_PASSED')"
+
+ try:
+ with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+ f.write(script)
+ f.flush()
+
+ result = subprocess.run(
+ ["python", f.name],
+ capture_output=True,
+ text=True,
+ timeout=self._timeout,
+ )
+
+ passed = "ALL_TESTS_PASSED" in result.stdout
+ return passed, {
+ "stdout": result.stdout[:500],
+ "stderr": result.stderr[:500],
+ "returncode": result.returncode,
+ }
+ except subprocess.TimeoutExpired:
+ return False, {"reason": "timeout"}
+ except Exception as e:
+ return False, {"reason": str(e)}
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/evals/test_new_suites.py -v`
+Expected: All tests PASS
+
+**Step 5: Commit**
+
+```bash
+git add evals/datasets/chat_mtbench.py evals/datasets/coding_humaneval.py evals/scorers/chat_judge.py evals/scorers/coding_exec.py tests/evals/test_new_suites.py
+git commit -m "feat: add Chat (MT-Bench) and Coding (HumanEval) eval suites"
+```
+
+---
+
+### Task 7: Add RAG eval suite and multi-dimensional scoring
+
+**Files:**
+- Create: `evals/datasets/rag_eval.py`
+- Create: `evals/scorers/rag_scorer.py`
+- Test: `tests/evals/test_rag_suite.py`
+
+**Context:** The RAG eval measures whether memory retrieval improves answers vs. without. It uses OpenJarvis's own memory backends. This tests the "with context" vs. "without context" path in `JarvisSystem.ask()` (the `context=True/False` flag). The scorer checks if the answer correctly uses retrieved context.
+
+**Step 1: Write the failing test**
+
+```python
+# tests/evals/test_rag_suite.py
+"""Tests for RAG eval suite."""
+
+
+class TestRAGDataset:
+ def test_loads_with_category_rag(self):
+ from evals.datasets.rag_eval import RAGEvalDataset
+
+ ds = RAGEvalDataset()
+ records = ds.load(max_samples=3)
+ assert len(records) <= 3
+ assert all(r.category == "rag" for r in records)
+
+ def test_has_context_in_metadata(self):
+ from evals.datasets.rag_eval import RAGEvalDataset
+
+ ds = RAGEvalDataset()
+ records = ds.load(max_samples=3)
+ for r in records:
+ assert "context" in r.metadata
+
+
+class TestRAGScorer:
+ def test_correct_answer_with_context(self):
+ from evals.scorers.rag_scorer import RAGScorer
+ from evals.core.types import EvalRecord
+
+ scorer = RAGScorer()
+ record = EvalRecord(
+ record_id="rag_1",
+ problem="What is the capital of France?",
+ reference="Paris",
+ category="rag",
+ metadata={"context": "France is a country in Europe. Its capital is Paris."},
+ )
+ is_correct, meta = scorer.score(record=record, model_answer="The capital of France is Paris.")
+ assert is_correct is True
+
+ def test_incorrect_answer(self):
+ from evals.scorers.rag_scorer import RAGScorer
+ from evals.core.types import EvalRecord
+
+ scorer = RAGScorer()
+ record = EvalRecord(
+ record_id="rag_2",
+ problem="What is the capital of France?",
+ reference="Paris",
+ category="rag",
+ metadata={"context": "France is a country in Europe. Its capital is Paris."},
+ )
+ is_correct, meta = scorer.score(record=record, model_answer="The capital of France is London.")
+ assert is_correct is False
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/evals/test_rag_suite.py -v`
+Expected: FAIL with `ModuleNotFoundError`
+
+**Step 3: Write minimal implementation**
+
+```python
+# evals/datasets/rag_eval.py
+"""RAG evaluation dataset — questions with context passages."""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+_RAG_PROBLEMS = [
+ {"id": "rag_1", "question": "What year was the company founded?", "answer": "2019", "context": "TechCorp was founded in 2019 by Jane Smith and John Doe in San Francisco. The company focuses on AI-powered developer tools.", "subject": "factual"},
+ {"id": "rag_2", "question": "What programming language does the framework use?", "answer": "Rust", "context": "The framework is written entirely in Rust for performance and safety. It compiles to a single binary and supports Linux, macOS, and Windows.", "subject": "factual"},
+ {"id": "rag_3", "question": "What is the maximum memory limit for the sandbox?", "answer": "256MB", "context": "The WASM sandbox enforces strict resource limits: 256MB maximum memory, 1 billion fuel units for CPU, and 30 second timeout per execution.", "subject": "technical"},
+ {"id": "rag_4", "question": "Who is the lead researcher on the project?", "answer": "Dr. Sarah Chen", "context": "The research team is led by Dr. Sarah Chen, who previously worked at DeepMind. Her focus is on reinforcement learning for agent optimization.", "subject": "factual"},
+ {"id": "rag_5", "question": "What database is used for persistence?", "answer": "SQLite", "context": "All persistent data is stored in SQLite with WAL mode enabled for concurrent reads. The database file is located at ~/.openjarvis/data.db.", "subject": "technical"},
+ {"id": "rag_6", "question": "What is the default temperature for generation?", "answer": "0.7", "context": "Generation defaults include temperature=0.7, max_tokens=1024, top_p=0.9, and repetition_penalty=1.1. These can be overridden per-request.", "subject": "technical"},
+ {"id": "rag_7", "question": "How many channels does the system support?", "answer": "14", "context": "The messaging system supports 14 channel adapters: Telegram, Discord, Slack, WhatsApp, LINE, Viber, Messenger, Reddit, Mastodon, XMPP, Rocket.Chat, Zulip, Twitch, and Nostr.", "subject": "factual"},
+ {"id": "rag_8", "question": "What authentication method does the P2P protocol use?", "answer": "HMAC-SHA256", "context": "The P2P protocol uses HMAC-SHA256 mutual authentication with nonce-based challenge-response. Each node has a unique identity key pair.", "subject": "technical"},
+ {"id": "rag_9", "question": "What is the energy monitoring approach for NVIDIA GPUs?", "answer": "hardware counters and polling", "context": "NVIDIA energy monitoring uses hardware counters via nvidia-smi with configurable polling intervals. The EnergyBatch class aggregates per-inference energy measurements.", "subject": "technical"},
+ {"id": "rag_10", "question": "What format are eval results saved in?", "answer": "JSONL", "context": "Evaluation results are saved as JSONL files (one JSON object per line) with a companion .summary.json file containing aggregate metrics.", "subject": "technical"},
+]
+
+
+class RAGEvalDataset(DatasetProvider):
+ """Built-in RAG evaluation — questions that require context to answer correctly."""
+
+ def load(
+ self,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: int = 42,
+ ) -> List[EvalRecord]:
+ import random
+
+ problems = list(_RAG_PROBLEMS)
+ rng = random.Random(seed)
+ rng.shuffle(problems)
+
+ if max_samples is not None:
+ problems = problems[:max_samples]
+
+ return [
+ EvalRecord(
+ record_id=p["id"],
+ problem=p["question"],
+ reference=p["answer"],
+ category="rag",
+ subject=p["subject"],
+ metadata={"context": p["context"]},
+ )
+ for p in problems
+ ]
+
+
+# evals/scorers/rag_scorer.py
+"""Scorer for RAG evaluation — checks answer against reference with context."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import Scorer
+from evals.core.types import EvalRecord
+
+
+class RAGScorer(Scorer):
+ """Score RAG responses by checking if the answer contains the reference."""
+
+ def score(
+ self,
+ record: EvalRecord,
+ model_answer: str,
+ **kwargs: Any,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer.strip():
+ return False, {"reason": "empty response"}
+
+ reference = record.reference.lower().strip()
+ answer = model_answer.lower().strip()
+
+ # Check if reference appears in answer
+ is_correct = reference in answer
+
+ return is_correct, {
+ "reference": record.reference,
+ "match_type": "substring" if is_correct else "no_match",
+ }
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/evals/test_rag_suite.py -v`
+Expected: All 4 tests PASS
+
+**Step 5: Commit**
+
+```bash
+git add evals/datasets/rag_eval.py evals/scorers/rag_scorer.py tests/evals/test_rag_suite.py
+git commit -m "feat: add RAG eval suite with context-based scoring"
+```
+
+---
+
+## Section 3: Composable Abstractions and Recipes
+
+### Task 8: Recipe system — loader, CLI flag, SDK support
+
+**Files:**
+- Create: `src/openjarvis/recipes/__init__.py`
+- Create: `src/openjarvis/recipes/loader.py`
+- Create: `recipes/` (directory for recipe TOML files)
+- Create: `recipes/coding_assistant.toml`
+- Create: `recipes/research_assistant.toml`
+- Create: `recipes/general_assistant.toml`
+- Modify: `src/openjarvis/cli/ask.py` (add `--recipe` flag)
+- Modify: `src/openjarvis/sdk.py` (add `recipe` parameter to `Jarvis` and `ask()`)
+- Test: `tests/recipes/test_loader.py`
+
+**Context:** A recipe is a TOML file that composes all 5 pillars. The recipe loader reads the TOML, resolves each section to the appropriate registry, and returns a config dict that `SystemBuilder` can consume. The `--recipe` CLI flag and `Jarvis(recipe="...")` SDK parameter load and apply the recipe before running. Recipe files live in `recipes/` at project root and `~/.openjarvis/recipes/` for user-defined ones.
+
+**Step 1: Write the failing test**
+
+```python
+# tests/recipes/test_loader.py
+"""Tests for recipe loading and composition."""
+
+import tempfile
+from pathlib import Path
+
+
+SAMPLE_RECIPE = """\
+[recipe]
+name = "test_coding"
+description = "Test coding assistant recipe"
+version = "1.0.0"
+
+[intelligence]
+model = "qwen3:8b"
+quantization = "q4_K_M"
+
+[engine]
+key = "ollama"
+
+[agent]
+type = "native_react"
+max_turns = 10
+temperature = 0.3
+tools = ["file_read", "file_write", "code_interpreter", "think"]
+
+[learning]
+routing = "grpo"
+agent = "icl_updater"
+
+[eval]
+suites = ["coding", "reasoning"]
+"""
+
+
+class TestRecipeLoader:
+ def test_load_recipe_from_toml(self):
+ from openjarvis.recipes.loader import load_recipe
+
+ with tempfile.NamedTemporaryFile(suffix=".toml", mode="w", delete=False) as f:
+ f.write(SAMPLE_RECIPE)
+ f.flush()
+ recipe = load_recipe(f.name)
+
+ assert recipe.name == "test_coding"
+ assert recipe.model == "qwen3:8b"
+ assert recipe.engine_key == "ollama"
+ assert recipe.agent_type == "native_react"
+ assert "file_read" in recipe.tools
+ assert recipe.max_turns == 10
+
+ def test_load_recipe_missing_file_raises(self):
+ from openjarvis.recipes.loader import load_recipe
+ import pytest
+
+ with pytest.raises(FileNotFoundError):
+ load_recipe("/nonexistent/path.toml")
+
+ def test_discover_builtin_recipes(self):
+ from openjarvis.recipes.loader import discover_recipes
+
+ recipes = discover_recipes()
+ # Should find at least the built-in recipes
+ names = [r.name for r in recipes]
+ assert "coding_assistant" in names or len(names) >= 1
+
+ def test_recipe_to_builder_kwargs(self):
+ from openjarvis.recipes.loader import load_recipe
+
+ with tempfile.NamedTemporaryFile(suffix=".toml", mode="w", delete=False) as f:
+ f.write(SAMPLE_RECIPE)
+ f.flush()
+ recipe = load_recipe(f.name)
+
+ kwargs = recipe.to_builder_kwargs()
+ assert kwargs["model"] == "qwen3:8b"
+ assert kwargs["engine_key"] == "ollama"
+ assert kwargs["agent_name"] == "native_react"
+ assert kwargs["tools"] == ["file_read", "file_write", "code_interpreter", "think"]
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/recipes/test_loader.py -v`
+Expected: FAIL with `ModuleNotFoundError`
+
+**Step 3: Write minimal implementation**
+
+```python
+# src/openjarvis/recipes/__init__.py
+"""Recipe system — composable pillar configurations."""
+
+# src/openjarvis/recipes/loader.py
+"""Load and discover recipe TOML files."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+try:
+ import tomllib
+except ModuleNotFoundError:
+ import tomli as tomllib # type: ignore[no-redef]
+
+
+@dataclass
+class Recipe:
+ """A loaded recipe — composition of all 5 pillars."""
+
+ name: str
+ description: str = ""
+ version: str = "1.0.0"
+
+ # Intelligence
+ model: str = ""
+ quantization: str = ""
+
+ # Engine
+ engine_key: str = ""
+
+ # Agent
+ agent_type: str = ""
+ max_turns: int = 10
+ temperature: float = 0.7
+ tools: List[str] = field(default_factory=list)
+ system_prompt: str = ""
+
+ # Learning
+ routing_policy: str = ""
+ agent_policy: str = ""
+
+ # Eval
+ eval_suites: List[str] = field(default_factory=list)
+
+ # Raw TOML for pass-through
+ raw: Dict[str, Any] = field(default_factory=dict)
+
+ def to_builder_kwargs(self) -> Dict[str, Any]:
+ """Convert to kwargs suitable for SystemBuilder or Jarvis."""
+ kwargs: dict[str, Any] = {}
+ if self.model:
+ kwargs["model"] = self.model
+ if self.engine_key:
+ kwargs["engine_key"] = self.engine_key
+ if self.agent_type:
+ kwargs["agent_name"] = self.agent_type
+ if self.tools:
+ kwargs["tools"] = self.tools
+ if self.temperature != 0.7:
+ kwargs["temperature"] = self.temperature
+ if self.max_turns != 10:
+ kwargs["max_turns"] = self.max_turns
+ return kwargs
+
+
+def load_recipe(path: str | Path) -> Recipe:
+ """Load a recipe from a TOML file."""
+ p = Path(path)
+ if not p.exists():
+ raise FileNotFoundError(f"Recipe not found: {p}")
+
+ with open(p, "rb") as f:
+ data = tomllib.load(f)
+
+ meta = data.get("recipe", {})
+ intel = data.get("intelligence", {})
+ engine = data.get("engine", {})
+ agent = data.get("agent", {})
+ learning = data.get("learning", {})
+ eval_cfg = data.get("eval", {})
+
+ return Recipe(
+ name=meta.get("name", p.stem),
+ description=meta.get("description", ""),
+ version=meta.get("version", "1.0.0"),
+ model=intel.get("model", ""),
+ quantization=intel.get("quantization", ""),
+ engine_key=engine.get("key", ""),
+ agent_type=agent.get("type", ""),
+ max_turns=agent.get("max_turns", 10),
+ temperature=agent.get("temperature", 0.7),
+ tools=agent.get("tools", []),
+ system_prompt=agent.get("system_prompt", ""),
+ routing_policy=learning.get("routing", ""),
+ agent_policy=learning.get("agent", ""),
+ eval_suites=eval_cfg.get("suites", []),
+ raw=data,
+ )
+
+
+def discover_recipes(extra_dirs: Optional[List[Path]] = None) -> List[Recipe]:
+ """Discover all available recipes from known locations."""
+ search_dirs = []
+
+ # Built-in recipes in project
+ project_recipes = Path(__file__).resolve().parent.parent.parent.parent / "recipes"
+ if project_recipes.is_dir():
+ search_dirs.append(project_recipes)
+
+ # User recipes
+ user_recipes = Path.home() / ".openjarvis" / "recipes"
+ if user_recipes.is_dir():
+ search_dirs.append(user_recipes)
+
+ if extra_dirs:
+ search_dirs.extend(extra_dirs)
+
+ recipes = []
+ for d in search_dirs:
+ for toml_file in sorted(d.glob("*.toml")):
+ try:
+ recipes.append(load_recipe(toml_file))
+ except Exception:
+ continue
+ return recipes
+
+
+def resolve_recipe(name: str) -> Optional[Recipe]:
+ """Find a recipe by name from all known locations."""
+ for recipe in discover_recipes():
+ if recipe.name == name:
+ return recipe
+ return None
+```
+
+Then create the built-in recipe files:
+
+```toml
+# recipes/coding_assistant.toml
+[recipe]
+name = "coding_assistant"
+description = "Optimized for code generation, review, and debugging"
+version = "1.0.0"
+
+[intelligence]
+model = "qwen3:8b"
+
+[engine]
+key = "ollama"
+
+[agent]
+type = "native_react"
+max_turns = 10
+temperature = 0.3
+tools = ["file_read", "file_write", "code_interpreter", "think", "shell_exec"]
+
+[learning]
+routing = "grpo"
+agent = "icl_updater"
+
+[eval]
+suites = ["coding", "reasoning"]
+```
+
+```toml
+# recipes/research_assistant.toml
+[recipe]
+name = "research_assistant"
+description = "Deep research with web search, knowledge graph, and cited reports"
+version = "1.0.0"
+
+[intelligence]
+model = "qwen3:8b"
+
+[engine]
+key = "ollama"
+
+[agent]
+type = "orchestrator"
+max_turns = 15
+temperature = 0.5
+tools = ["web_search", "http_request", "memory_store", "memory_search", "think", "file_write"]
+
+[learning]
+routing = "grpo"
+agent = "icl_updater"
+
+[eval]
+suites = ["rag", "chat"]
+```
+
+```toml
+# recipes/general_assistant.toml
+[recipe]
+name = "general_assistant"
+description = "Balanced general-purpose assistant for everyday tasks"
+version = "1.0.0"
+
+[intelligence]
+model = "qwen3:8b"
+
+[engine]
+key = "ollama"
+
+[agent]
+type = "orchestrator"
+max_turns = 10
+temperature = 0.7
+tools = ["think", "calculator", "web_search", "memory_search"]
+
+[learning]
+routing = "heuristic"
+
+[eval]
+suites = ["chat", "reasoning"]
+```
+
+Then add `--recipe` to the ask CLI (`src/openjarvis/cli/ask.py`) and `recipe` parameter to `Jarvis.__init__()` and `Jarvis.ask()` in `src/openjarvis/sdk.py`.
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/recipes/test_loader.py -v`
+Expected: All 4 tests PASS
+
+**Step 5: Commit**
+
+```bash
+git add src/openjarvis/recipes/ recipes/ tests/recipes/ src/openjarvis/cli/ask.py src/openjarvis/sdk.py
+git commit -m "feat: add recipe system — TOML composition of 5 pillars with CLI and SDK support"
+```
+
+---
+
+### Task 9: Agent templates (15-20 TOML files)
+
+**Files:**
+- Create: `templates/agents/` (directory)
+- Create: 15-20 TOML files in `templates/agents/`
+- Create: `src/openjarvis/templates/__init__.py`
+- Create: `src/openjarvis/templates/agent_templates.py`
+- Test: `tests/templates/test_agent_templates.py`
+
+**Context:** Agent templates are pre-configured TOML manifests with system prompts, tool sets, and behavioral parameters. They follow the same format as operator manifests (`operators/types.py:9-36`). The loader discovers templates from `templates/agents/` and `~/.openjarvis/templates/agents/`.
+
+**Step 1-5:** Follow TDD pattern. Write test that loads each template and verifies it has required fields (name, system_prompt, tools). Write loader. Create 15-20 template TOML files covering: code-reviewer, debugger, architect, deep-researcher, fact-checker, summarizer, inbox-triager, meeting-prep, note-taker, assistant, tutor, translator, writer, data-analyst, security-auditor. Commit.
+
+---
+
+### Task 10: Bundled skills (20-30 TOML files)
+
+**Files:**
+- Create: `skills/builtin/` (directory)
+- Create: 20-30 TOML files in `skills/builtin/`
+- Test: `tests/skills/test_bundled_skills.py`
+
+**Context:** Skills use `SkillManifest` (`skills/types.py:17-48`) with sequential `SkillStep`s. Each step names a tool and provides an arguments template. The loader is `skills/loader.py:16-104`. Skills live in `skills/builtin/` for bundled ones.
+
+**Step 1-5:** Follow TDD pattern. Write test that loads each skill TOML and verifies valid structure. Create 20-30 skill TOML files covering: file-organizer, file-deduplicator, web-summarize, topic-research, code-lint, code-test-gen, email-draft, meeting-notes, daily-digest, knowledge-extract, pdf-summarize, data-analyze, translate-doc, backup-files, search-and-index, calendar-prep, todo-from-notes, compare-docs, security-scan, dependency-audit. Commit.
+
+---
+
+## Section 4: Operator Recipes
+
+### Task 11: Deep Researcher operator
+
+**Files:**
+- Create: `recipes/operators/researcher.toml`
+- Create: `recipes/operators/researcher_prompt.md`
+- Test: `tests/operators/test_researcher.py`
+
+**Context:** Operators use `OperatorManifest` (`operators/types.py:9-36`) loaded by `operators/loader.py:15-67`. The `OperatorManager` (`operators/manager.py:18-213`) activates them by creating scheduler tasks that run the `OperativeAgent`. The researcher operator searches the web, cross-references sources, builds knowledge graph entries, and produces cited reports.
+
+**Step 1: Write the failing test**
+
+```python
+# tests/operators/test_researcher.py
+"""Tests for the Deep Researcher operator recipe."""
+
+from pathlib import Path
+
+
+class TestResearcherOperator:
+ def test_loads_valid_manifest(self):
+ from openjarvis.operators.loader import load_operator
+
+ manifest = load_operator(Path(__file__).parent.parent.parent / "recipes" / "operators" / "researcher.toml")
+ assert manifest.name == "researcher"
+ assert "web_search" in manifest.tools
+ assert "memory_store" in manifest.tools
+ assert manifest.max_turns >= 10
+ assert manifest.system_prompt or manifest.system_prompt_path
+
+ def test_has_required_tools(self):
+ from openjarvis.operators.loader import load_operator
+
+ manifest = load_operator(Path(__file__).parent.parent.parent / "recipes" / "operators" / "researcher.toml")
+ required = {"web_search", "http_request", "memory_store", "memory_search", "think", "file_write"}
+ assert required.issubset(set(manifest.tools))
+```
+
+**Step 2-5:** Write researcher.toml and researcher_prompt.md with a detailed system prompt for autonomous research. Run tests. Commit.
+
+---
+
+### Task 12: Correspondent operator
+
+**Files:**
+- Create: `recipes/operators/correspondent.toml`
+- Create: `recipes/operators/correspondent_prompt.md`
+- Test: `tests/operators/test_correspondent.py`
+
+**Step 1-5:** Same pattern as Task 11 but for messaging triage. Tools: memory_store, memory_search, think, llm_call. System prompt focuses on urgency classification, draft responses, daily digest. Commit.
+
+---
+
+### Task 13: Sentinel operator
+
+**Files:**
+- Create: `recipes/operators/sentinel.toml`
+- Create: `recipes/operators/sentinel_prompt.md`
+- Test: `tests/operators/test_sentinel.py`
+
+**Step 1-5:** Same pattern as Task 11 but for monitoring Twitter, Reddit, Mastodon, Google Trends, RSS, URLs. Tools: web_search, http_request, memory_store, memory_search, kg_add_entity. System prompt focuses on change detection, significance scoring, alert generation. Commit.
+
+---
+
+## Section 5: Integration and Wiring
+
+### Task 14: Update LearningConfig and wire orchestrator to scheduler
+
+**Files:**
+- Modify: `src/openjarvis/core/config.py` (add training config fields)
+- Modify: `src/openjarvis/learning/__init__.py` (export new components)
+- Modify: `src/openjarvis/system.py` (wire LearningOrchestrator into SystemBuilder)
+- Test: `tests/test_system_learning.py`
+
+**Context:** `LearningConfig` (`core/config.py`) needs fields for the training pipeline: `training_enabled`, `training_schedule`, `lora_rank`, `lora_alpha`, `min_sft_pairs`, `min_improvement`. The `SystemBuilder` needs to optionally create a `LearningOrchestrator` and wire it to the scheduler for periodic learning cycles.
+
+**Step 1-5:** TDD. Add config fields, wire into SystemBuilder, test that build() creates orchestrator when learning.training_enabled=True. Commit.
+
+---
+
+### Task 15: Update CLAUDE.md and run full test suite
+
+**Files:**
+- Modify: `CLAUDE.md` (add recipe, eval, and learning orchestrator documentation)
+- Modify: `src/openjarvis/learning/training/__init__.py` (export all new training components)
+
+**Step 1:** Update CLAUDE.md with new commands: `jarvis eval run`, `jarvis eval compare`, `jarvis eval report`, `jarvis ask --recipe`, recipe system docs, operator recipes docs.
+
+**Step 2:** Run full test suite:
+
+```bash
+uv run pytest tests/ -v --tb=short
+```
+
+Expected: All existing ~2940 tests + ~200-300 new tests PASS.
+
+**Step 3:** Commit everything.
+
+```bash
+git add -A
+git commit -m "feat: complete differentiated functionalities — learning flywheel, evals, recipes, operators"
+```
diff --git a/docs/plans/2026-02-28-experience-polish-design.md b/docs/plans/2026-02-28-experience-polish-design.md
new file mode 100644
index 00000000..e20a339f
--- /dev/null
+++ b/docs/plans/2026-02-28-experience-polish-design.md
@@ -0,0 +1,319 @@
+# Experience Polish Design: Eval, CLI, Install, Dashboard
+
+**Date**: 2026-02-28
+**Status**: Approved
+**Approach**: Vertical slices (3 mini-phases, independently shippable)
+
+## Context
+
+OpenJarvis has strong infrastructure (multi-vendor energy monitoring, eval framework, Tauri desktop, PWA) but the end-to-end user experience has gaps. This design addresses: eval output quality, CLI polish, installation flow, and dashboard aesthetics.
+
+### Current State (from audits)
+
+| Area | Score | Key Gaps |
+|------|-------|----------|
+| Energy Monitors | 8.5/10 | IPJ/IPW only in evals, not core telemetry |
+| Eval Framework | 9/10 | Token stats null, no grouped tables, no trace aggregation |
+| CLI Formatting | 9/10 | No logging, no verbose/quiet, bench lacks full stats |
+| Installation | 8.5/10 | No quickstart, no auto-suggestions in errors |
+| Desktop App | 8/10 | No settings panel, placeholder icon, stub tray |
+| Browser/PWA | 6/10 | No CI/CD, no versioning, no error boundary |
+
+### Key Decisions
+
+- **Benchmark focus**: GAIA with OpenHands (TerminalBench deferred)
+- **IPW** = task_accuracy / avg_power_watts (task-level, NOT per-inference)
+- **IPJ** = task_accuracy / avg_energy_joules (task-level, NOT per-inference)
+- **tokens_per_joule** = per-inference efficiency metric in core telemetry
+- **Table layout**: Grouped panels by default, `--compact` for single dense table
+- **Trace detail**: Summary + step-type breakdown by default, `--trace-detail` for full listing
+- **Installation**: Non-interactive `jarvis quickstart` (zero prompts)
+- **Dashboard style**: Clean, minimalistic (ChatGPT/Claude aesthetic) with unique side panels
+
+---
+
+## Phase 23a: Perfect Eval Run
+
+Everything needed so `python -m evals run -c config.toml` produces beautiful, complete results.
+
+### 1. Core Telemetry: tokens_per_joule
+
+**Files**:
+- `src/openjarvis/core/types.py` — Add `tokens_per_joule` to `TelemetryRecord`
+- `src/openjarvis/telemetry/instrumented_engine.py` — Compute `tokens_per_joule = completion_tokens / energy_joules`
+- `src/openjarvis/telemetry/store.py` — Schema migration for new column
+- `src/openjarvis/telemetry/aggregator.py` — Add `avg_tokens_per_joule` to `ModelStats`/`EngineStats`
+
+**Not** adding IPJ/IPW to core telemetry — they require task accuracy and are inherently eval-level.
+
+### 2. Eval Metrics: Fix Token Stats + Strengthen IPJ/IPW
+
+**Files**:
+- `evals/core/runner.py`:
+ - Wire `prompt_tokens` and `completion_tokens` from engine response into `EvalResult`
+ - Verify IPW = `accuracy / avg(power_watts)` across all scored samples
+ - Verify IPJ = `accuracy / avg(energy_joules)` across all scored samples
+ - Ensure `energy_joules` and `power_watts` populated from telemetry for every sample
+- `evals/core/types.py`:
+ - Add `total_energy_joules` (sum over all samples) to `RunSummary`
+ - Add `avg_power_watts` (mean over all samples) to `RunSummary`
+ - Add `trace_steps: int`, `trace_energy_joules: float` to `EvalResult`
+ - Add `trace_step_type_stats: Dict[str, StepTypeStats]` to `RunSummary`
+
+### 3. Eval Display: Grouped Tables
+
+Rewrite `evals/core/display.py` with these functions:
+
+#### `print_accuracy_panel(summary)`
+```
+╭─ Accuracy ──────────────────────────────────────────────────╮
+│ Overall Accuracy 42.0% (84/200) │
+│ Level 1 58.8% (40/68) │
+│ Level 2 35.0% (35/100) │
+│ Level 3 28.1% (9/32) │
+╰─────────────────────────────────────────────────────────────╯
+```
+
+#### `print_latency_table(summary)`
+```
+╭─ Latency & Throughput ──────────────────────────────────────╮
+│ Metric Avg Median Min Max Std │
+│ Latency (s) 15.41 12.30 2.10 89.50 11.20 │
+│ TTFT (ms) 145.2 120.0 45.0 890.0 85.3 │
+│ Throughput (tok/s) 41.9 38.5 12.0 95.0 15.2 │
+│ Avg Input Tokens 1024 890 128 4096 520 │
+│ Avg Output Tokens 256 210 32 2048 180 │
+╰─────────────────────────────────────────────────────────────╯
+```
+
+#### `print_energy_table(summary)`
+```
+╭─ Energy & Efficiency ───────────────────────────────────────╮
+│ Metric Avg Median Min Max Std │
+│ Energy (J) 46502 38500 4145 410522 89390 │
+│ Power (W) 883.8 870.2 650.0 1050.0 85.0 │
+│ GPU Util (%) 46.4 48.0 12.0 92.0 18.5 │
+│ Energy/Token (mJ) 12.5 10.8 3.2 45.0 8.1 │
+│ Tokens/Joule 80.0 92.6 22.2 312.5 65.0 │
+│ ────────────────────────────────────────────────────────── │
+│ IPW (acc/W) 0.00048 │
+│ IPJ (acc/J) 9.03e-06 │
+│ Total Energy (kJ) 9300.4 │
+╰─────────────────────────────────────────────────────────────╯
+```
+
+#### `print_trace_summary(summary)`
+```
+╭─ Agentic Trace Summary ────────────────────────────────────╮
+│ Total Steps: 1240 │ Avg Steps/Sample: 6.2 │
+│ │
+│ Step Type Count Avg Duration Avg Energy Avg In Tok Avg Out Tok │
+│ Median/Min/Max/Std for each column │
+│ generate 580 8.2s 38200 J 890 256 │
+│ tool_call 420 3.1s — — — │
+│ retrieve 120 0.8s — — — │
+│ route 120 0.01s — — — │
+╰─────────────────────────────────────────────────────────────╯
+```
+
+All metrics in trace summary show avg/median/min/max/std where applicable.
+
+#### `print_compact_table(summary)` — `--compact` flag
+Single dense table with all 17 metrics as rows, columns: avg/median/min/max/std.
+
+#### CLI flags
+- `--compact`: Dense single table
+- `--trace-detail`: Full per-step listing for each sample
+
+### 4. Per-Step Trace Aggregation
+
+**Files**:
+- `src/openjarvis/traces/analyzer.py`:
+ - Add `TraceSummary.total_energy_joules` — sum of `step.metadata['energy_joules']`
+ - Add `TraceSummary.total_generate_energy_joules` — sum for GENERATE steps
+ - Add `TraceSummary.step_type_stats` — dict mapping step type to `{count, avg_duration, median_duration, min_duration, max_duration, std_duration, total_energy, avg_input_tokens, median_input_tokens, min_input_tokens, max_input_tokens, std_input_tokens, avg_output_tokens, median_output_tokens, min_output_tokens, max_output_tokens, std_output_tokens}`
+- `evals/core/runner.py` — After each agent eval sample, extract `TraceSummary` and store in `EvalResult.metadata`
+
+---
+
+## Phase 23b: Perfect First Experience
+
+Everything needed so a new user goes from zero to first eval in one sitting.
+
+### 5. `jarvis quickstart` Command
+
+**New file**: `src/openjarvis/cli/quickstart_cmd.py`
+
+Non-interactive flow (5 numbered steps):
+1. Detect hardware (platform, CPU, RAM, GPU vendor/model/VRAM)
+2. Write config to `~/.openjarvis/config.toml` (skip if exists, unless `--force`)
+3. Check engine health (try auto-detected engine)
+4. Verify model availability (list models from engine)
+5. Test query ("What is 2+2?") with latency + energy measurement
+
+Each step prints a status line. If a step fails, print a helpful suggestion and exit gracefully.
+
+Flags: `--force` (redo everything)
+
+**Register** in `src/openjarvis/cli/__init__.py`.
+
+### 6. Error Message Auto-Suggestions
+
+**New file**: `src/openjarvis/cli/hints.py`
+
+Centralized hint functions:
+- `hint_no_config()` → "Config not found. Run: `jarvis quickstart`"
+- `hint_no_engine()` → "No engine responding. Run: `jarvis doctor`"
+- `hint_no_model()` → "No model available. Try: `ollama pull qwen3:8b`"
+
+**Wire into**: `ask.py`, `serve.py`, `bench_cmd.py`, `chat_cmd.py` at failure points.
+
+### 7. Global Logging & Verbose/Quiet Flags
+
+**Changes**:
+- `src/openjarvis/cli/__init__.py` — Add `--verbose` / `--quiet` to root `cli` group
+- **New file**: `src/openjarvis/cli/log_config.py` — Centralized logging setup
+ - `RichHandler` for console (respects quiet flag)
+ - `RotatingFileHandler` for `~/.openjarvis/cli.log` (5 MB max, 3 backups)
+ - Default level: WARNING; `--verbose`: DEBUG; `--quiet`: ERROR
+
+**Add progress indicators**:
+- `ask.py` — Spinner during generation
+- `memory_cmd.py` — Progress bar for indexing
+
+### 8. Bench CLI Full Stats Tables
+
+**Changes**:
+- `src/openjarvis/cli/bench_cmd.py` — Show full stats table (avg/median/min/max/std) matching eval style
+- `src/openjarvis/bench/latency.py` — Export all percentile stats (already computes internally)
+- `src/openjarvis/bench/throughput.py` — Add stats computation
+- `src/openjarvis/bench/energy.py` — Add stats computation
+
+---
+
+## Phase 23c: Perfect Dashboard
+
+Desktop and browser apps polished for demos and daily use.
+
+### Design Aesthetic
+
+**Clean, minimalistic** — inspired by ChatGPT and Claude web interfaces:
+- Generous whitespace, muted color palette, subtle borders
+- Sans-serif typography, comfortable line heights
+- Smooth transitions, no visual clutter
+
+**Differentiated** via unique side panels:
+- Energy dashboard with real-time power charts
+- Trace debugger with color-coded timeline
+- Learning curve visualization
+- Memory browser with relevance scoring
+
+The side panels are what make OpenJarvis unique vs generic chat UIs. Keep them accessible but not overwhelming — collapsible, clean data density.
+
+### 9. Desktop: Settings Panel
+
+**New file**: `desktop/src/components/SettingsPanel.tsx`
+- API URL configuration (default: `localhost:8000`)
+- Auto-update interval setting
+- Theme toggle (dark/light) — currently hardcoded dark
+- Persist to `localStorage`
+
+### 10. Desktop: Windows Icon
+
+Replace 108-byte placeholder `desktop/src-tauri/icons/icon.ico` with proper multi-resolution icon generated from existing `icon.png`.
+
+### 11. Desktop: System Tray
+
+Implement stubbed tray menu in `desktop/src-tauri/src/lib.rs`:
+- Show/Hide window toggle
+- Health status indicator (green/red dot)
+- Quit action
+
+### 12. Browser/PWA: CI/CD
+
+**New file**: `.github/workflows/frontend.yml`
+- Trigger on changes to `frontend/`
+- `npm ci && npm run build`
+- Commit built output to `src/openjarvis/server/static/`
+- Add `version` field to `manifest.webmanifest` (auto-bumped)
+
+### 13. Browser/PWA: Error Boundary + Config
+
+- Wrap `` in React error boundary (graceful crash recovery)
+- Support `VITE_API_URL` environment variable (default `localhost:8000`)
+
+### 14. Browser/PWA: Style Refresh
+
+Refactor the 12,971-line `App.css`:
+- Modularize into component-scoped CSS files
+- Clean minimalist aesthetic (ChatGPT/Claude-inspired)
+- Catppuccin-based dark theme (keep existing) + light theme option
+- Collapsible side panels with smooth transitions
+
+---
+
+## Implementation Order
+
+### Phase 23a (highest priority — enables GAIA research)
+1. Core telemetry: `tokens_per_joule`
+2. Eval metrics: fix token stats, strengthen IPJ/IPW
+3. Per-step trace aggregation in `TraceAnalyzer`
+4. Eval display: grouped tables, `--compact`, `--trace-detail`
+
+### Phase 23b (high priority — onboarding & CLI)
+5. `jarvis quickstart` command
+6. Error message auto-suggestions
+7. Global logging + verbose/quiet flags
+8. Bench CLI full stats tables
+
+### Phase 23c (lower priority — dashboard polish)
+9. Desktop settings panel
+10. Windows icon fix
+11. System tray implementation
+12. PWA CI/CD
+13. PWA error boundary + config
+14. PWA style refresh
+
+---
+
+## Files Modified (Summary)
+
+### Phase 23a
+| File | Change |
+|------|--------|
+| `src/openjarvis/core/types.py` | Add `tokens_per_joule` to `TelemetryRecord` |
+| `src/openjarvis/telemetry/instrumented_engine.py` | Compute `tokens_per_joule` |
+| `src/openjarvis/telemetry/store.py` | Schema migration |
+| `src/openjarvis/telemetry/aggregator.py` | Add `avg_tokens_per_joule` |
+| `src/openjarvis/traces/analyzer.py` | Add energy aggregation, step-type stats |
+| `evals/core/types.py` | Add trace fields, total_energy, avg_power |
+| `evals/core/runner.py` | Fix token stats, wire trace data, verify IPJ/IPW |
+| `evals/core/display.py` | Rewrite: grouped panels, compact mode |
+| `evals/cli.py` | Add `--compact`, `--trace-detail` flags |
+
+### Phase 23b
+| File | Change |
+|------|--------|
+| `src/openjarvis/cli/quickstart_cmd.py` | **New**: quickstart command |
+| `src/openjarvis/cli/hints.py` | **New**: centralized hint system |
+| `src/openjarvis/cli/log_config.py` | **New**: logging setup |
+| `src/openjarvis/cli/__init__.py` | Register quickstart, add verbose/quiet |
+| `src/openjarvis/cli/ask.py` | Add hints, progress spinner |
+| `src/openjarvis/cli/serve.py` | Add hints |
+| `src/openjarvis/cli/bench_cmd.py` | Full stats tables |
+| `src/openjarvis/cli/chat_cmd.py` | Add hints |
+| `src/openjarvis/cli/memory_cmd.py` | Progress bar for indexing |
+| `src/openjarvis/bench/latency.py` | Export full stats |
+| `src/openjarvis/bench/throughput.py` | Add stats computation |
+| `src/openjarvis/bench/energy.py` | Add stats computation |
+
+### Phase 23c
+| File | Change |
+|------|--------|
+| `desktop/src/components/SettingsPanel.tsx` | **New**: settings UI |
+| `desktop/src-tauri/icons/icon.ico` | Replace placeholder |
+| `desktop/src-tauri/src/lib.rs` | Implement system tray |
+| `.github/workflows/frontend.yml` | **New**: PWA CI/CD |
+| `frontend/src/App.tsx` | Error boundary wrapper |
+| `frontend/vite.config.ts` | VITE_API_URL support |
+| `frontend/src/App.css` | Modularize, style refresh |
diff --git a/docs/plans/2026-02-28-experience-polish-implementation.md b/docs/plans/2026-02-28-experience-polish-implementation.md
new file mode 100644
index 00000000..ae45a606
--- /dev/null
+++ b/docs/plans/2026-02-28-experience-polish-implementation.md
@@ -0,0 +1,1285 @@
+# Experience Polish Implementation Plan
+
+> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
+
+**Goal:** Make the eval output, CLI, installation, and dashboard experiences clean, complete, and publication-ready across three independently shippable phases.
+
+**Architecture:** Vertical slices — Phase 23a delivers a "perfect eval run" (grouped tables, IPJ/IPW, trace breakdown), Phase 23b delivers a "perfect first experience" (quickstart, logging, hints), Phase 23c polishes the desktop/browser dashboards.
+
+**Tech Stack:** Python 3.10+, Rich (tables/panels), Click (CLI), Tauri 2.0 (desktop), React/Vite (PWA), SQLite (telemetry/traces), pytest (testing)
+
+---
+
+## Phase 23a: Perfect Eval Run
+
+### Task 1: Add `tokens_per_joule` to TelemetryRecord
+
+**Files:**
+- Modify: `src/openjarvis/core/types.py:125-165`
+- Test: `tests/telemetry/test_telemetry_record.py` (new)
+
+**Step 1: Write the failing test**
+
+Create `tests/telemetry/test_telemetry_record.py`:
+
+```python
+"""Tests for TelemetryRecord fields."""
+
+from __future__ import annotations
+
+from openjarvis.core.types import TelemetryRecord
+
+
+class TestTelemetryRecord:
+ def test_tokens_per_joule_field_exists(self):
+ rec = TelemetryRecord(timestamp=1.0, model_id="test")
+ assert hasattr(rec, "tokens_per_joule")
+ assert rec.tokens_per_joule == 0.0
+
+ def test_tokens_per_joule_set(self):
+ rec = TelemetryRecord(
+ timestamp=1.0,
+ model_id="test",
+ tokens_per_joule=80.0,
+ )
+ assert rec.tokens_per_joule == 80.0
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/telemetry/test_telemetry_record.py -v`
+Expected: FAIL — `TypeError: ... unexpected keyword argument 'tokens_per_joule'`
+
+**Step 3: Write minimal implementation**
+
+In `src/openjarvis/core/types.py`, add after `dram_energy_joules` field (around line 163):
+
+```python
+ tokens_per_joule: float = 0.0
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/telemetry/test_telemetry_record.py -v`
+Expected: PASS (2 tests)
+
+**Step 5: Commit**
+
+```bash
+git add src/openjarvis/core/types.py tests/telemetry/test_telemetry_record.py
+git commit -m "feat(telemetry): add tokens_per_joule field to TelemetryRecord"
+```
+
+---
+
+### Task 2: Compute `tokens_per_joule` in InstrumentedEngine
+
+**Files:**
+- Modify: `src/openjarvis/telemetry/instrumented_engine.py:154-174`
+- Test: `tests/telemetry/test_instrumented_engine.py` (existing — add test)
+
+**Step 1: Write the failing test**
+
+Add to `tests/telemetry/test_instrumented_engine.py`:
+
+```python
+class TestTokensPerJoule:
+ def test_tokens_per_joule_computed(self, mock_engine, bus):
+ """tokens_per_joule = completion_tokens / energy_joules."""
+ mock_engine.generate.return_value = {
+ "content": "hello",
+ "usage": {"prompt_tokens": 10, "completion_tokens": 50, "total_tokens": 60},
+ }
+ inst = InstrumentedEngine(mock_engine, bus=bus)
+ # Mock energy monitor to return known energy
+ from unittest.mock import MagicMock, patch
+ monitor = MagicMock()
+ sample = MagicMock()
+ sample.energy_joules = 2.5
+ sample.mean_power_watts = 100.0
+ sample.peak_power_watts = 120.0
+ sample.gpu_utilization_pct = 50.0
+ sample.gpu_memory_used_gb = 4.0
+ sample.gpu_temperature_c = 60.0
+ sample.cpu_energy_joules = 0.0
+ sample.gpu_energy_joules = 2.5
+ sample.dram_energy_joules = 0.0
+ sample.energy_method = "test"
+ sample.vendor = "test"
+ monitor.sample.return_value.__enter__ = lambda s: sample
+ monitor.sample.return_value.__exit__ = lambda s, *a: None
+ inst._energy_monitor = monitor
+
+ inst.generate(messages=[{"role": "user", "content": "hi"}])
+ records = [
+ e.data for e in bus.history
+ if hasattr(e, 'event_type')
+ and e.event_type.value == "telemetry_record"
+ ]
+ assert len(records) >= 1
+ rec = records[0]
+ # 50 tokens / 2.5 J = 20.0 tokens/joule
+ assert rec.tokens_per_joule == pytest.approx(20.0, rel=0.1)
+
+ def test_tokens_per_joule_zero_energy(self, mock_engine, bus):
+ """tokens_per_joule is 0.0 when energy is zero."""
+ mock_engine.generate.return_value = {
+ "content": "hello",
+ "usage": {"prompt_tokens": 10, "completion_tokens": 50, "total_tokens": 60},
+ }
+ inst = InstrumentedEngine(mock_engine, bus=bus)
+ inst.generate(messages=[{"role": "user", "content": "hi"}])
+ records = [
+ e.data for e in bus.history
+ if hasattr(e, 'event_type')
+ and e.event_type.value == "telemetry_record"
+ ]
+ rec = records[0]
+ assert rec.tokens_per_joule == 0.0
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/telemetry/test_instrumented_engine.py::TestTokensPerJoule -v`
+Expected: FAIL — `AttributeError: ... has no attribute 'tokens_per_joule'` on the record
+
+**Step 3: Write minimal implementation**
+
+In `src/openjarvis/telemetry/instrumented_engine.py`, in the `generate()` method, after the existing derived metrics block (around line 159, after `throughput_per_watt`), add:
+
+```python
+ # Tokens per joule (inverse of energy-per-token, per-inference efficiency)
+ tokens_per_joule = 0.0
+ if energy_j > 0 and completion_tokens > 0:
+ tokens_per_joule = completion_tokens / energy_j
+```
+
+Then in the `TelemetryRecord(...)` constructor call (around line 178-204), add:
+
+```python
+ tokens_per_joule=tokens_per_joule,
+```
+
+Do the same in the `stream()` method (around line 369-400).
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/telemetry/test_instrumented_engine.py::TestTokensPerJoule -v`
+Expected: PASS (2 tests)
+
+**Step 5: Run full telemetry tests**
+
+Run: `uv run pytest tests/telemetry/ -v`
+Expected: All pass (no regressions)
+
+**Step 6: Commit**
+
+```bash
+git add src/openjarvis/telemetry/instrumented_engine.py tests/telemetry/test_instrumented_engine.py
+git commit -m "feat(telemetry): compute tokens_per_joule in InstrumentedEngine"
+```
+
+---
+
+### Task 3: Store and aggregate `tokens_per_joule`
+
+**Files:**
+- Modify: `src/openjarvis/telemetry/store.py:13-52` (schema)
+- Modify: `src/openjarvis/telemetry/store.py:126-170` (record method)
+- Modify: `src/openjarvis/telemetry/aggregator.py:11-33` (ModelStats)
+- Modify: `src/openjarvis/telemetry/aggregator.py:36-56` (EngineStats)
+- Modify: `src/openjarvis/telemetry/aggregator.py:115-198` (per_model_stats)
+- Modify: `src/openjarvis/telemetry/aggregator.py:200-278` (per_engine_stats)
+- Test: `tests/telemetry/test_store_tokens_per_joule.py` (new)
+
+**Step 1: Write the failing test**
+
+Create `tests/telemetry/test_store_tokens_per_joule.py`:
+
+```python
+"""Tests for tokens_per_joule storage and aggregation."""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+
+from openjarvis.core.types import TelemetryRecord
+from openjarvis.telemetry.store import TelemetryStore
+from openjarvis.telemetry.aggregator import TelemetryAggregator
+
+
+class TestTokensPerJouleStorage:
+ def test_store_and_retrieve(self, tmp_path):
+ store = TelemetryStore(db_path=tmp_path / "tel.db")
+ rec = TelemetryRecord(
+ timestamp=time.time(),
+ model_id="test-model",
+ completion_tokens=50,
+ energy_joules=2.5,
+ tokens_per_joule=20.0,
+ )
+ store.record(rec)
+ agg = TelemetryAggregator(store)
+ stats = agg.per_model_stats()
+ assert len(stats) == 1
+ assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1)
+ store.close()
+
+ def test_aggregate_multiple(self, tmp_path):
+ store = TelemetryStore(db_path=tmp_path / "tel.db")
+ for tpj in [10.0, 20.0, 30.0]:
+ rec = TelemetryRecord(
+ timestamp=time.time(),
+ model_id="m1",
+ tokens_per_joule=tpj,
+ )
+ store.record(rec)
+ agg = TelemetryAggregator(store)
+ stats = agg.per_model_stats()
+ assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1)
+ store.close()
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/telemetry/test_store_tokens_per_joule.py -v`
+Expected: FAIL — column or attribute not found
+
+**Step 3: Write minimal implementation**
+
+1. In `src/openjarvis/telemetry/store.py` schema list (lines 13-52), add `"tokens_per_joule"` to the columns list.
+
+2. In `store.py` `record()` method, add `tokens_per_joule` to the INSERT statement and values tuple.
+
+3. In `store.py` schema migration (`_maybe_add_columns` or equivalent), add migration for the new column.
+
+4. In `src/openjarvis/telemetry/aggregator.py` `ModelStats`, add:
+ ```python
+ avg_tokens_per_joule: float = 0.0
+ ```
+
+5. In `EngineStats`, add the same field.
+
+6. In `per_model_stats()` SQL query, add `AVG(tokens_per_joule)` using `_safe_col()`.
+
+7. In `per_engine_stats()` SQL query, add same.
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/telemetry/test_store_tokens_per_joule.py -v`
+Expected: PASS (2 tests)
+
+**Step 5: Run full telemetry test suite**
+
+Run: `uv run pytest tests/telemetry/ -v`
+Expected: All pass
+
+**Step 6: Commit**
+
+```bash
+git add src/openjarvis/telemetry/store.py src/openjarvis/telemetry/aggregator.py tests/telemetry/test_store_tokens_per_joule.py
+git commit -m "feat(telemetry): store and aggregate tokens_per_joule"
+```
+
+---
+
+### Task 4: Add energy aggregation and step-type stats to TraceAnalyzer
+
+**Files:**
+- Modify: `src/openjarvis/traces/analyzer.py:39-49` (TraceSummary)
+- Modify: `src/openjarvis/traces/analyzer.py:62-91` (summary method)
+- Test: `tests/traces/test_analyzer_energy.py` (new)
+
+**Step 1: Write the failing test**
+
+Create `tests/traces/test_analyzer_energy.py`:
+
+```python
+"""Tests for energy aggregation in TraceAnalyzer."""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+
+from openjarvis.core.types import StepType, Trace, TraceStep
+from openjarvis.traces.analyzer import StepTypeStats, TraceAnalyzer, TraceSummary
+from openjarvis.traces.store import TraceStore
+
+
+def _make_trace(steps: list[TraceStep]) -> Trace:
+ return Trace(
+ query="test",
+ agent="test_agent",
+ model="test_model",
+ engine="test_engine",
+ steps=steps,
+ started_at=time.time(),
+ ended_at=time.time() + 10,
+ total_tokens=100,
+ total_latency_seconds=10.0,
+ )
+
+
+def _gen_step(energy: float = 0.0, duration: float = 1.0,
+ prompt_tokens: int = 50, completion_tokens: int = 25) -> TraceStep:
+ return TraceStep(
+ step_type=StepType.GENERATE,
+ timestamp=time.time(),
+ duration_seconds=duration,
+ output={
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": prompt_tokens + completion_tokens,
+ },
+ metadata={
+ "energy_joules": energy,
+ "power_watts": energy / duration if duration > 0 else 0.0,
+ },
+ )
+
+
+def _tool_step(duration: float = 0.5) -> TraceStep:
+ return TraceStep(
+ step_type=StepType.TOOL_CALL,
+ timestamp=time.time(),
+ duration_seconds=duration,
+ input={"tool": "calculator"},
+ output={"success": True},
+ )
+
+
+class TestTraceSummaryEnergyFields:
+ def test_total_energy_joules(self, tmp_path):
+ store = TraceStore(db_path=tmp_path / "traces.db")
+ trace = _make_trace([
+ _gen_step(energy=10.0, duration=2.0),
+ _tool_step(duration=0.5),
+ _gen_step(energy=15.0, duration=3.0),
+ ])
+ store.save(trace)
+ analyzer = TraceAnalyzer(store)
+ summary = analyzer.summary()
+ assert summary.total_energy_joules == pytest.approx(25.0, rel=0.01)
+ assert summary.total_generate_energy_joules == pytest.approx(25.0, rel=0.01)
+ store.close()
+
+ def test_step_type_stats(self, tmp_path):
+ store = TraceStore(db_path=tmp_path / "traces.db")
+ trace = _make_trace([
+ _gen_step(energy=10.0, duration=2.0, prompt_tokens=100, completion_tokens=50),
+ _gen_step(energy=20.0, duration=4.0, prompt_tokens=80, completion_tokens=40),
+ _tool_step(duration=0.5),
+ _tool_step(duration=1.5),
+ ])
+ store.save(trace)
+ analyzer = TraceAnalyzer(store)
+ summary = analyzer.summary()
+
+ assert "generate" in summary.step_type_stats
+ gen = summary.step_type_stats["generate"]
+ assert gen.count == 2
+ assert gen.avg_duration == pytest.approx(3.0, rel=0.01)
+ assert gen.total_energy == pytest.approx(30.0, rel=0.01)
+ assert gen.avg_input_tokens == pytest.approx(90.0, rel=0.01)
+ assert gen.avg_output_tokens == pytest.approx(45.0, rel=0.01)
+ assert gen.min_duration == pytest.approx(2.0, rel=0.01)
+ assert gen.max_duration == pytest.approx(4.0, rel=0.01)
+
+ assert "tool_call" in summary.step_type_stats
+ tc = summary.step_type_stats["tool_call"]
+ assert tc.count == 2
+ assert tc.avg_duration == pytest.approx(1.0, rel=0.01)
+ store.close()
+
+
+class TestStepTypeStats:
+ def test_dataclass_fields(self):
+ s = StepTypeStats(count=5, avg_duration=2.0, total_energy=10.0)
+ assert s.count == 5
+ assert s.std_duration == 0.0 # default
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest tests/traces/test_analyzer_energy.py -v`
+Expected: FAIL — `ImportError: cannot import name 'StepTypeStats'`
+
+**Step 3: Write minimal implementation**
+
+In `src/openjarvis/traces/analyzer.py`:
+
+1. Add new dataclass before `TraceSummary` (after line 37):
+
+```python
+@dataclass(slots=True)
+class StepTypeStats:
+ """Aggregated statistics for a specific step type across traces."""
+
+ count: int = 0
+ avg_duration: float = 0.0
+ median_duration: float = 0.0
+ min_duration: float = 0.0
+ max_duration: float = 0.0
+ std_duration: float = 0.0
+ total_energy: float = 0.0
+ avg_input_tokens: float = 0.0
+ median_input_tokens: float = 0.0
+ min_input_tokens: float = 0.0
+ max_input_tokens: float = 0.0
+ std_input_tokens: float = 0.0
+ avg_output_tokens: float = 0.0
+ median_output_tokens: float = 0.0
+ min_output_tokens: float = 0.0
+ max_output_tokens: float = 0.0
+ std_output_tokens: float = 0.0
+```
+
+2. Add fields to `TraceSummary`:
+
+```python
+ total_energy_joules: float = 0.0
+ total_generate_energy_joules: float = 0.0
+ step_type_stats: Dict[str, StepTypeStats] = field(default_factory=dict)
+```
+
+3. In the `summary()` method, after computing `step_dist`, add energy and step-type stats computation:
+
+```python
+ import statistics as stats_mod
+
+ # Compute energy totals and step-type stats
+ total_energy = 0.0
+ generate_energy = 0.0
+ step_data: Dict[str, Dict[str, list]] = {}
+
+ for t in traces:
+ for s in t.steps:
+ key = _step_type_str(s)
+ energy = s.metadata.get("energy_joules", 0.0)
+ total_energy += energy
+ if key == "generate":
+ generate_energy += energy
+
+ if key not in step_data:
+ step_data[key] = {
+ "durations": [], "energies": [],
+ "input_tokens": [], "output_tokens": [],
+ }
+ step_data[key]["durations"].append(s.duration_seconds)
+ step_data[key]["energies"].append(energy)
+ step_data[key]["input_tokens"].append(
+ s.output.get("prompt_tokens", 0)
+ )
+ step_data[key]["output_tokens"].append(
+ s.output.get("completion_tokens", 0)
+ )
+
+ sts_map: Dict[str, StepTypeStats] = {}
+ for key, data in step_data.items():
+ durations = data["durations"]
+ in_tok = [float(x) for x in data["input_tokens"]]
+ out_tok = [float(x) for x in data["output_tokens"]]
+ sts_map[key] = StepTypeStats(
+ count=len(durations),
+ avg_duration=_avg(durations),
+ median_duration=stats_mod.median(durations) if durations else 0.0,
+ min_duration=min(durations) if durations else 0.0,
+ max_duration=max(durations) if durations else 0.0,
+ std_duration=stats_mod.stdev(durations) if len(durations) > 1 else 0.0,
+ total_energy=sum(data["energies"]),
+ avg_input_tokens=_avg(in_tok),
+ median_input_tokens=stats_mod.median(in_tok) if in_tok else 0.0,
+ min_input_tokens=min(in_tok) if in_tok else 0.0,
+ max_input_tokens=max(in_tok) if in_tok else 0.0,
+ std_input_tokens=stats_mod.stdev(in_tok) if len(in_tok) > 1 else 0.0,
+ avg_output_tokens=_avg(out_tok),
+ median_output_tokens=stats_mod.median(out_tok) if out_tok else 0.0,
+ min_output_tokens=min(out_tok) if out_tok else 0.0,
+ max_output_tokens=max(out_tok) if out_tok else 0.0,
+ std_output_tokens=stats_mod.stdev(out_tok) if len(out_tok) > 1 else 0.0,
+ )
+```
+
+Then add the new fields to the returned `TraceSummary`:
+
+```python
+ total_energy_joules=total_energy,
+ total_generate_energy_joules=generate_energy,
+ step_type_stats=sts_map,
+```
+
+4. Update `__all__` to include `StepTypeStats`.
+
+**Step 4: Run test to verify it passes**
+
+Run: `uv run pytest tests/traces/test_analyzer_energy.py -v`
+Expected: PASS (3 tests)
+
+**Step 5: Run full trace tests**
+
+Run: `uv run pytest tests/traces/ -v`
+Expected: All pass
+
+**Step 6: Commit**
+
+```bash
+git add src/openjarvis/traces/analyzer.py tests/traces/test_analyzer_energy.py
+git commit -m "feat(traces): add energy aggregation and step-type stats to TraceAnalyzer"
+```
+
+---
+
+### Task 5: Add trace fields to eval types
+
+**Files:**
+- Modify: `evals/core/types.py:21-47` (EvalResult)
+- Modify: `evals/core/types.py:87-125` (RunSummary)
+- Test: `evals/tests/test_types.py` (existing — add tests)
+
+**Step 1: Write the failing test**
+
+Add to `evals/tests/test_types.py`:
+
+```python
+class TestEvalResultTraceFields:
+ def test_trace_fields_exist(self):
+ r = EvalResult(record_id="test", model_answer="hi")
+ assert r.trace_steps == 0
+ assert r.trace_energy_joules == 0.0
+
+class TestRunSummaryTraceFields:
+ def test_trace_aggregate_fields(self):
+ s = RunSummary(
+ benchmark="test", category="test", backend="test",
+ model="test", total_samples=1, scored_samples=1,
+ correct=1, accuracy=1.0, errors=0,
+ mean_latency_seconds=1.0, total_cost_usd=0.0,
+ )
+ assert s.avg_power_watts == 0.0
+ assert s.trace_step_type_stats == {}
+ assert s.total_input_tokens == 0
+ assert s.total_output_tokens == 0
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest evals/tests/test_types.py::TestEvalResultTraceFields -v`
+Expected: FAIL — `AttributeError`
+
+**Step 3: Write minimal implementation**
+
+In `evals/core/types.py`:
+
+1. Add to `EvalResult` (after line 46, before the class ends):
+```python
+ trace_steps: int = 0
+ trace_energy_joules: float = 0.0
+```
+
+2. Add to `RunSummary` (after line 124):
+```python
+ avg_power_watts: float = 0.0
+ total_input_tokens: int = 0
+ total_output_tokens: int = 0
+ trace_step_type_stats: Dict[str, Dict[str, float]] = field(default_factory=dict)
+```
+
+**Step 4: Run tests**
+
+Run: `uv run pytest evals/tests/test_types.py -v`
+Expected: All pass
+
+**Step 5: Commit**
+
+```bash
+git add evals/core/types.py evals/tests/test_types.py
+git commit -m "feat(evals): add trace and power fields to EvalResult and RunSummary"
+```
+
+---
+
+### Task 6: Wire trace data and strengthen token stats in eval runner
+
+**Files:**
+- Modify: `evals/core/runner.py:134-223` (_process_one)
+- Modify: `evals/core/runner.py:267-380` (_compute_summary)
+- Test: `evals/tests/test_runner.py` (existing — add tests)
+
+**Step 1: Write the failing test**
+
+Add to `evals/tests/test_runner.py`:
+
+```python
+class TestRunnerTokenStats:
+ def test_summary_has_total_input_output_tokens(self, tmp_path):
+ """RunSummary should include total token counts."""
+ records = [
+ EvalRecord(record_id=f"r{i}", problem=f"q{i}", reference="a", category="test")
+ for i in range(3)
+ ]
+ backend = MockBackend()
+ scorer = MockScorer()
+ dataset = MockDataset(records=records)
+ config = RunConfig(benchmark="test", backend="jarvis-direct", model="m")
+ runner = EvalRunner(config=config, backend=backend, scorer=scorer, dataset=dataset)
+
+ results, summary = runner.run(output_dir=tmp_path)
+ # MockBackend returns usage with tokens — they should be tallied
+ assert summary.total_input_tokens >= 0
+ assert summary.total_output_tokens >= 0
+ # input_token_stats and output_token_stats should be populated
+ if summary.input_token_stats is not None:
+ assert summary.input_token_stats.mean >= 0
+
+ def test_summary_has_avg_power(self, tmp_path):
+ """RunSummary should include avg_power_watts."""
+ records = [
+ EvalRecord(record_id="r1", problem="q", reference="a", category="test")
+ ]
+ config = RunConfig(benchmark="test", backend="jarvis-direct", model="m")
+ runner = EvalRunner(
+ config=config, backend=MockBackend(), scorer=MockScorer(),
+ dataset=MockDataset(records=records),
+ )
+ results, summary = runner.run(output_dir=tmp_path)
+ assert hasattr(summary, "avg_power_watts")
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest evals/tests/test_runner.py::TestRunnerTokenStats -v`
+Expected: FAIL — `total_input_tokens` not populated or missing
+
+**Step 3: Write minimal implementation**
+
+In `evals/core/runner.py` `_compute_summary()` method:
+
+1. After collecting value lists (around line 343), add:
+```python
+ total_input_tokens = sum(r.prompt_tokens for r in scored)
+ total_output_tokens = sum(r.completion_tokens for r in scored)
+ power_values = [r.power_watts for r in scored if r.power_watts > 0]
+ avg_power = _avg(power_values) if power_values else 0.0
+```
+
+2. In the `RunSummary(...)` constructor, add:
+```python
+ total_input_tokens=total_input_tokens,
+ total_output_tokens=total_output_tokens,
+ avg_power_watts=avg_power,
+```
+
+3. For trace data wiring in `_process_one()`: After the agent/backend returns, check if trace data is available in `full.get("_trace")` and extract step count and energy. Add to the returned `EvalResult`:
+```python
+ trace_steps=trace_step_count,
+ trace_energy_joules=trace_energy,
+```
+
+**Step 4: Run tests**
+
+Run: `uv run pytest evals/tests/test_runner.py -v`
+Expected: All pass
+
+**Step 5: Commit**
+
+```bash
+git add evals/core/runner.py evals/tests/test_runner.py
+git commit -m "feat(evals): wire trace data and token totals into RunSummary"
+```
+
+---
+
+### Task 7: Rewrite eval display with grouped tables
+
+This is the largest task in Phase 23a. The current `display.py` has one monolithic `print_metrics_table()`. We replace it with grouped panels.
+
+**Files:**
+- Modify: `evals/core/display.py` (major rewrite)
+- Test: `evals/tests/test_display.py` (new)
+
+**Step 1: Write the failing test**
+
+Create `evals/tests/test_display.py`:
+
+```python
+"""Tests for eval display functions."""
+
+from __future__ import annotations
+
+from io import StringIO
+
+from rich.console import Console
+
+from evals.core.display import (
+ print_accuracy_panel,
+ print_energy_table,
+ print_latency_table,
+ print_trace_summary,
+ print_compact_table,
+ print_full_results,
+)
+from evals.core.types import MetricStats, RunSummary
+
+
+def _make_summary(**overrides) -> RunSummary:
+ defaults = dict(
+ benchmark="gaia", category="agentic", backend="jarvis-agent",
+ model="qwen3:8b", total_samples=100, scored_samples=100,
+ correct=42, accuracy=0.42, errors=0,
+ mean_latency_seconds=15.0, total_cost_usd=0.0,
+ total_energy_joules=9300.0, avg_power_watts=880.0,
+ total_input_tokens=50000, total_output_tokens=12000,
+ per_subject={"level_1": {"accuracy": 0.58, "correct": 40, "scored": 68}},
+ )
+ defaults.update(overrides)
+ return RunSummary(**defaults)
+
+
+def _make_stats(mean=10.0) -> MetricStats:
+ return MetricStats(
+ mean=mean, median=mean * 0.9, min=mean * 0.2,
+ max=mean * 3.0, std=mean * 0.5, p90=mean * 1.5,
+ p95=mean * 2.0, p99=mean * 2.5,
+ )
+
+
+class TestAccuracyPanel:
+ def test_renders_without_error(self):
+ console = Console(file=StringIO(), force_terminal=True)
+ summary = _make_summary()
+ print_accuracy_panel(console, summary)
+ output = console.file.getvalue()
+ assert "42.0%" in output or "0.42" in output
+
+ def test_shows_per_subject(self):
+ console = Console(file=StringIO(), force_terminal=True)
+ summary = _make_summary()
+ print_accuracy_panel(console, summary)
+ output = console.file.getvalue()
+ assert "level_1" in output
+
+
+class TestLatencyTable:
+ def test_renders_with_stats(self):
+ console = Console(file=StringIO(), force_terminal=True)
+ summary = _make_summary(
+ latency_stats=_make_stats(15.0),
+ throughput_stats=_make_stats(40.0),
+ input_token_stats=_make_stats(1024.0),
+ output_token_stats=_make_stats(256.0),
+ )
+ print_latency_table(console, summary)
+ output = console.file.getvalue()
+ assert "Latency" in output
+ assert "Avg" in output
+
+
+class TestEnergyTable:
+ def test_renders_ipj_ipw(self):
+ console = Console(file=StringIO(), force_terminal=True)
+ summary = _make_summary(
+ energy_stats=_make_stats(46000.0),
+ power_stats=_make_stats(880.0),
+ ipw_stats=_make_stats(0.00048),
+ ipj_stats=_make_stats(9.0e-6),
+ )
+ print_energy_table(console, summary)
+ output = console.file.getvalue()
+ assert "IPW" in output
+ assert "IPJ" in output
+
+
+class TestTraceSummary:
+ def test_renders_step_type_breakdown(self):
+ console = Console(file=StringIO(), force_terminal=True)
+ summary = _make_summary(
+ trace_step_type_stats={
+ "generate": {
+ "count": 580, "avg_duration": 8.2,
+ "total_energy": 22000.0,
+ "avg_input_tokens": 890.0, "avg_output_tokens": 256.0,
+ },
+ "tool_call": {
+ "count": 420, "avg_duration": 3.1,
+ "total_energy": 0.0,
+ "avg_input_tokens": 0.0, "avg_output_tokens": 0.0,
+ },
+ },
+ )
+ print_trace_summary(console, summary)
+ output = console.file.getvalue()
+ assert "generate" in output
+ assert "tool_call" in output
+
+
+class TestCompactTable:
+ def test_renders_all_metrics(self):
+ console = Console(file=StringIO(), force_terminal=True)
+ summary = _make_summary(
+ latency_stats=_make_stats(15.0),
+ energy_stats=_make_stats(46000.0),
+ )
+ print_compact_table(console, summary)
+ output = console.file.getvalue()
+ assert "Latency" in output
+ assert "Energy" in output
+
+
+class TestFullResults:
+ def test_renders_all_sections(self):
+ console = Console(file=StringIO(), force_terminal=True)
+ summary = _make_summary(
+ latency_stats=_make_stats(15.0),
+ energy_stats=_make_stats(46000.0),
+ power_stats=_make_stats(880.0),
+ )
+ print_full_results(console, summary)
+ output = console.file.getvalue()
+ assert "Accuracy" in output
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest evals/tests/test_display.py -v`
+Expected: FAIL — `ImportError: cannot import name 'print_accuracy_panel'`
+
+**Step 3: Write the implementation**
+
+Rewrite `evals/core/display.py`. Keep existing `print_banner`, `print_section`, `print_run_header`, `print_suite_summary`, `print_completion`. Replace `print_metrics_table` and add new functions:
+
+```python
+def print_accuracy_panel(console: Console, summary: RunSummary) -> None:
+ """Print accuracy panel with per-subject breakdown."""
+ lines = [
+ f"[bold]Overall Accuracy {summary.accuracy:.1%}[/bold]"
+ f" ({summary.correct}/{summary.scored_samples})",
+ ]
+ for subj, stats in sorted(summary.per_subject.items()):
+ acc = stats.get("accuracy", 0.0)
+ correct = int(stats.get("correct", 0))
+ scored = int(stats.get("scored", 0))
+ lines.append(f" {subj:<20s} {acc:.1%} ({correct}/{scored})")
+ body = "\n".join(lines)
+ panel = Panel(body, title="[bold]Accuracy[/bold]", border_style="green", expand=False)
+ console.print(panel)
+
+
+def _stats_table(title: str, rows: list[tuple[str, Optional[MetricStats], int]]) -> Table:
+ """Build a stats table with Avg/Median/Min/Max/Std columns."""
+ table = Table(
+ title=f"[bold]{title}[/bold]",
+ show_header=True,
+ header_style="bold bright_white",
+ border_style="bright_blue",
+ title_style="bold cyan",
+ )
+ table.add_column("Metric", style="cyan", no_wrap=True)
+ table.add_column("Avg", justify="right")
+ table.add_column("Median", justify="right")
+ table.add_column("Min", justify="right")
+ table.add_column("Max", justify="right")
+ table.add_column("Std", justify="right")
+ for label, stats, decimals in rows:
+ if stats is not None:
+ _add_metric_row(table, label, stats, decimals)
+ return table
+
+
+def print_latency_table(console: Console, summary: RunSummary) -> None:
+ """Print latency, throughput, and token stats table."""
+ table = _stats_table("Latency & Throughput", [
+ ("Latency (s)", summary.latency_stats, 2),
+ ("TTFT (s)", summary.ttft_stats, 3),
+ ("Throughput (tok/s)", summary.throughput_stats, 1),
+ ("Avg Input Tokens", summary.input_token_stats, 1),
+ ("Avg Output Tokens", summary.output_token_stats, 1),
+ ])
+ if table.row_count > 0:
+ console.print(table)
+
+
+def print_energy_table(console: Console, summary: RunSummary) -> None:
+ """Print energy, efficiency, and IPJ/IPW table."""
+ table = _stats_table("Energy & Efficiency", [
+ ("Energy (J)", summary.energy_stats, 1),
+ ("Power (W)", summary.power_stats, 1),
+ ("GPU Util (%)", summary.gpu_utilization_stats, 1),
+ ("Energy/OutTok (J)", summary.energy_per_output_token_stats, 6),
+ ("Tokens/Joule", None, 1), # placeholder — see note below
+ ("MFU (%)", summary.mfu_stats, 3),
+ ("MBU (%)", summary.mbu_stats, 3),
+ ])
+ if table.row_count > 0:
+ console.print(table)
+ # Headline: IPW, IPJ, Total Energy
+ parts = []
+ if summary.ipw_stats:
+ parts.append(f"[bold]IPW (acc/W):[/bold] {summary.ipw_stats.mean:.6f}")
+ if summary.ipj_stats:
+ parts.append(f"[bold]IPJ (acc/J):[/bold] {summary.ipj_stats.mean:.2e}")
+ if summary.total_energy_joules > 0:
+ val = summary.total_energy_joules
+ unit = "kJ" if val > 1000 else "J"
+ display = val / 1000 if val > 1000 else val
+ parts.append(f"[bold]Total Energy:[/bold] {display:.1f} {unit}")
+ if summary.avg_power_watts > 0:
+ parts.append(f"[bold]Avg Power:[/bold] {summary.avg_power_watts:.1f} W")
+ if parts:
+ console.print(" ".join(parts))
+
+
+def print_trace_summary(console: Console, summary: RunSummary) -> None:
+ """Print agentic trace step-type breakdown."""
+ sts = summary.trace_step_type_stats
+ if not sts:
+ return
+ total_steps = sum(s.get("count", 0) for s in sts.values())
+ avg_per_sample = total_steps / summary.scored_samples if summary.scored_samples > 0 else 0
+
+ table = Table(
+ title="[bold]Agentic Trace Summary[/bold]",
+ show_header=True,
+ header_style="bold bright_white",
+ border_style="bright_blue",
+ title_style="bold cyan",
+ caption=f"Total Steps: {total_steps} | Avg Steps/Sample: {avg_per_sample:.1f}",
+ )
+ table.add_column("Step Type", style="cyan", no_wrap=True)
+ table.add_column("Count", justify="right")
+ table.add_column("Avg Duration", justify="right")
+ table.add_column("Avg Energy (J)", justify="right")
+ table.add_column("Avg In Tokens", justify="right")
+ table.add_column("Avg Out Tokens", justify="right")
+
+ for stype, data in sorted(sts.items()):
+ count = data.get("count", 0)
+ avg_dur = data.get("avg_duration", 0.0)
+ total_e = data.get("total_energy", 0.0)
+ avg_e = total_e / count if count > 0 else 0.0
+ avg_in = data.get("avg_input_tokens", 0.0)
+ avg_out = data.get("avg_output_tokens", 0.0)
+ table.add_row(
+ stype,
+ str(count),
+ f"{avg_dur:.2f}s",
+ f"{avg_e:.1f}" if avg_e > 0 else "—",
+ f"{avg_in:.0f}" if avg_in > 0 else "—",
+ f"{avg_out:.0f}" if avg_out > 0 else "—",
+ )
+ console.print(table)
+
+
+def print_compact_table(console: Console, summary: RunSummary) -> None:
+ """Print a single dense metrics table (legacy behavior, enhanced)."""
+ # This is the existing print_metrics_table with updated column headers
+ print_metrics_table(console, summary)
+
+
+def print_full_results(
+ console: Console,
+ summary: RunSummary,
+ *,
+ compact: bool = False,
+ trace_detail: bool = False,
+) -> None:
+ """Orchestrate all result panels."""
+ if compact:
+ print_compact_table(console, summary)
+ return
+ print_accuracy_panel(console, summary)
+ print_latency_table(console, summary)
+ print_energy_table(console, summary)
+ print_trace_summary(console, summary)
+```
+
+Update `__all__` to include all new functions.
+
+**Step 4: Run tests**
+
+Run: `uv run pytest evals/tests/test_display.py -v`
+Expected: All pass
+
+**Step 5: Run full eval test suite**
+
+Run: `uv run pytest evals/tests/ -v`
+Expected: All pass
+
+**Step 6: Commit**
+
+```bash
+git add evals/core/display.py evals/tests/test_display.py
+git commit -m "feat(evals): grouped display tables (accuracy, latency, energy, trace)"
+```
+
+---
+
+### Task 8: Add `--compact` and `--trace-detail` flags to eval CLI
+
+**Files:**
+- Modify: `evals/cli.py:117-131` (_print_summary)
+- Modify: `evals/cli.py:235-344` (run command flags)
+- Test: `evals/tests/test_cli_flags.py` (new)
+
+**Step 1: Write the failing test**
+
+Create `evals/tests/test_cli_flags.py`:
+
+```python
+"""Tests for eval CLI display flags."""
+
+from __future__ import annotations
+
+from click.testing import CliRunner
+
+from evals.cli import cli
+
+
+class TestCompactFlag:
+ def test_compact_flag_accepted(self):
+ runner = CliRunner()
+ result = runner.invoke(cli, ["run", "--help"])
+ assert "--compact" in result.output
+
+ def test_trace_detail_flag_accepted(self):
+ runner = CliRunner()
+ result = runner.invoke(cli, ["run", "--help"])
+ assert "--trace-detail" in result.output
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `uv run pytest evals/tests/test_cli_flags.py -v`
+Expected: FAIL — `--compact` not in help output
+
+**Step 3: Write minimal implementation**
+
+In `evals/cli.py`:
+
+1. Add flags to `run` command (around line 270):
+```python
+@click.option("--compact", is_flag=True, default=False, help="Dense single-table output")
+@click.option("--trace-detail", is_flag=True, default=False, help="Full per-step trace listing")
+```
+
+2. Pass flags through to `_print_summary()`:
+```python
+def _print_summary(console, summary, per_subject, output_path, traces_dir,
+ *, compact=False, trace_detail=False):
+```
+
+3. In `_print_summary`, replace the `print_metrics_table` call with:
+```python
+ print_full_results(console, summary, compact=compact, trace_detail=trace_detail)
+```
+
+4. Update imports to include new display functions.
+
+**Step 4: Run tests**
+
+Run: `uv run pytest evals/tests/test_cli_flags.py -v`
+Expected: PASS
+
+**Step 5: Run full eval test suite**
+
+Run: `uv run pytest evals/tests/ -v`
+Expected: All pass
+
+**Step 6: Commit**
+
+```bash
+git add evals/cli.py evals/tests/test_cli_flags.py
+git commit -m "feat(evals): add --compact and --trace-detail CLI flags"
+```
+
+---
+
+## Phase 23b: Perfect First Experience
+
+### Task 9: `jarvis quickstart` command
+
+**Files:**
+- Create: `src/openjarvis/cli/quickstart_cmd.py`
+- Modify: `src/openjarvis/cli/__init__.py:34-54` (register command)
+- Test: `tests/cli/test_quickstart.py` (new)
+
+**Implementation outline:**
+- 5-step flow: detect hardware → write config → check engine → verify model → test query
+- Each step prints `[N/5] Description...` with Rich status
+- Reuse `detect_hardware()` from `init_cmd.py`
+- Reuse `_run_all_checks()` logic from `doctor_cmd.py`
+- Skip steps already done (config exists → skip step 2)
+- `--force` flag to redo everything
+- On failure: print helpful suggestion and exit with code 1
+
+**Test approach:**
+- Use `CliRunner` with monkeypatched `detect_hardware`, `load_config`, engine checks
+- Verify exit code 0 on happy path
+- Verify helpful message when engine not found
+- Verify `--force` regenerates config
+
+**Step 1:** Write failing tests for happy path + error paths
+**Step 2:** Run tests to verify they fail
+**Step 3:** Implement `quickstart_cmd.py` and register in `__init__.py`
+**Step 4:** Run tests to verify they pass
+**Step 5:** Commit: `"feat(cli): add jarvis quickstart command"`
+
+---
+
+### Task 10: Error message auto-suggestions
+
+**Files:**
+- Create: `src/openjarvis/cli/hints.py`
+- Modify: `src/openjarvis/cli/ask.py:215-224,258-259,269-271`
+- Modify: `src/openjarvis/cli/serve.py` (startup errors)
+- Modify: `src/openjarvis/cli/bench_cmd.py` (engine errors)
+- Modify: `src/openjarvis/cli/chat_cmd.py` (startup errors)
+- Test: `tests/cli/test_hints.py` (new)
+
+**Implementation outline:**
+- `hints.py` provides `hint_no_config()`, `hint_no_engine()`, `hint_no_model()` functions
+- Each returns a Rich-formatted suggestion string
+- Wire into existing error handlers in ask, serve, bench, chat commands
+- Pattern: catch error → print original message → print hint → exit
+
+**Test approach:**
+- Unit test each hint function returns non-empty string
+- CLI integration tests verify hints appear in error output
+
+**Step 1:** Write failing tests
+**Step 2:** Implement `hints.py` and wire into CLI commands
+**Step 3:** Run tests, commit: `"feat(cli): add error hints system"`
+
+---
+
+### Task 11: Global logging and verbose/quiet flags
+
+**Files:**
+- Create: `src/openjarvis/cli/log_config.py`
+- Modify: `src/openjarvis/cli/__init__.py:28-31` (add flags to root group)
+- Test: `tests/cli/test_log_config.py` (new)
+
+**Implementation outline:**
+- Add `--verbose` / `--quiet` flags to root `cli` group via `click.pass_context`
+- `log_config.py`: `setup_logging(verbose, quiet)` configures:
+ - `RichHandler` for console (WARNING default, DEBUG on verbose, ERROR on quiet)
+ - `RotatingFileHandler` for `~/.openjarvis/cli.log` (5 MB, 3 backups)
+- Call `setup_logging()` in root `cli` group callback
+
+**Test approach:**
+- Verify `--verbose` sets DEBUG level
+- Verify `--quiet` sets ERROR level
+- Verify log file created on verbose mode
+
+**Step 1-5:** TDD cycle, commit: `"feat(cli): add global verbose/quiet logging flags"`
+
+---
+
+### Task 12: Progress indicators for slow operations
+
+**Files:**
+- Modify: `src/openjarvis/cli/ask.py:320-330` (generation)
+- Modify: `src/openjarvis/cli/memory_cmd.py:77-96` (indexing)
+- Test: Visual verification only (progress spinners are UI-only)
+
+**Implementation outline:**
+- Wrap generation call in `ask.py` with `console.status("[bold green]Generating..."):` context
+- Wrap indexing loop in `memory_cmd.py` with `rich.progress.Progress` bar
+- Use `track()` for known-length operations, `status()` for unknown-length
+
+**Commit:** `"feat(cli): add progress indicators for generation and indexing"`
+
+---
+
+### Task 13: Bench CLI full stats tables
+
+**Files:**
+- Modify: `src/openjarvis/cli/bench_cmd.py:175-209`
+- Modify: `src/openjarvis/bench/latency.py:60-87` (return full stats)
+- Modify: `src/openjarvis/bench/throughput.py:50-74` (add stats)
+- Modify: `src/openjarvis/bench/energy.py:85-123` (add stats)
+- Test: `tests/bench/test_bench_stats.py` (new)
+
+**Implementation outline:**
+- Each benchmark returns a `metrics` dict. Currently flat key-value pairs.
+- Change to include `_stats` suffixed keys for metrics that have multiple samples:
+ - latency: `latency_avg`, `latency_median`, `latency_min`, `latency_max`, `latency_std`, `latency_p95`
+ - throughput: same pattern
+ - energy: same pattern
+- `bench_cmd.py` detects stats keys and renders a Rich stats table (Avg/Median/Min/Max/Std columns)
+- Non-stats metrics (single values like `total_energy`, `errors`) shown as before
+
+**Test approach:**
+- Verify benchmark `run()` returns stats-suffixed keys
+- Verify CLI renders table with correct columns
+
+**Step 1-5:** TDD cycle, commit: `"feat(bench): full stats tables in bench CLI"`
+
+---
+
+## Phase 23c: Perfect Dashboard
+
+### Task 14: Desktop settings panel
+
+**Files:**
+- Create: `desktop/src/components/SettingsPanel.tsx`
+- Modify: `desktop/src/App.tsx` (add settings tab)
+
+**Implementation:** React component with API URL input, auto-update interval selector, dark/light theme toggle. Persists to `localStorage`. Clean minimalist UI matching ChatGPT/Claude aesthetic.
+
+---
+
+### Task 15: Desktop Windows icon
+
+**Files:**
+- Replace: `desktop/src-tauri/icons/icon.ico`
+
+**Implementation:** Generate proper multi-resolution `.ico` from existing `icon.png` (16, 32, 48, 256px).
+
+---
+
+### Task 16: Desktop system tray
+
+**Files:**
+- Modify: `desktop/src-tauri/src/lib.rs:189-191`
+
+**Implementation:** Add tray menu items: Show/Hide, Health Status, Quit. Use Tauri's `SystemTray` API.
+
+---
+
+### Task 17: PWA CI/CD and error boundary
+
+**Files:**
+- Create: `.github/workflows/frontend.yml`
+- Modify: `frontend/src/App.tsx` (error boundary wrapper)
+- Modify: `frontend/vite.config.ts` (VITE_API_URL env var)
+
+**Implementation:** GitHub Actions workflow triggers on `frontend/` changes, builds with `npm run build`, commits to `src/openjarvis/server/static/`. Error boundary wraps ``.
+
+---
+
+### Task 18: PWA style refresh
+
+**Files:**
+- Modify: `frontend/src/App.css` (modularize)
+- Create: `frontend/src/components/Chat/Chat.css` (component-scoped)
+- Create: `frontend/src/components/Sidebar/Sidebar.css` (component-scoped)
+
+**Implementation:** Break 12,971-line `App.css` into component-scoped CSS modules. Clean minimalist aesthetic inspired by ChatGPT/Claude web interfaces. Keep Catppuccin dark theme as default, add light theme option. Generous whitespace, muted color palette, smooth transitions. Side panels (energy, traces, learning, memory) remain the differentiator — collapsible with clean data density.
+
+---
+
+## Summary
+
+| Phase | Tasks | Commits | Priority |
+|-------|-------|---------|----------|
+| 23a: Perfect Eval Run | 1-8 | 8 | Highest — enables GAIA research |
+| 23b: Perfect First Experience | 9-13 | 5 | High — onboarding & CLI |
+| 23c: Perfect Dashboard | 14-18 | 5 | Lower — demo polish |
+
+**Total: 18 tasks, ~18 commits**
+
+Start with Phase 23a Task 1. Each task is independently testable and committable.
diff --git a/evals/cli.py b/evals/cli.py
index 3c548c34..c558539f 100644
--- a/evals/cli.py
+++ b/evals/cli.py
@@ -30,9 +30,25 @@ from evals.core.display import (
# Registry of available benchmarks and their metadata
BENCHMARKS = {
"supergpqa": {"category": "reasoning", "description": "SuperGPQA multiple-choice"},
+ "gpqa": {"category": "reasoning", "description": "GPQA graduate-level MCQ"},
+ "mmlu-pro": {"category": "reasoning", "description": "MMLU-Pro multiple-choice"},
+ "math500": {"category": "reasoning", "description": "MATH-500 math problems"},
+ "natural-reasoning": {"category": "reasoning", "description": "Natural Reasoning"},
+ "hle": {"category": "reasoning", "description": "HLE hard challenges"},
+ "simpleqa": {"category": "chat", "description": "SimpleQA factual QA"},
+ "wildchat": {"category": "chat", "description": "WildChat conversation quality"},
+ "ipw": {"category": "chat", "description": "IPW mixed benchmark"},
"gaia": {"category": "agentic", "description": "GAIA agentic benchmark"},
"frames": {"category": "rag", "description": "FRAMES multi-hop RAG"},
- "wildchat": {"category": "chat", "description": "WildChat conversation quality"},
+ "swebench": {"category": "agentic", "description": "SWE-bench code patches"},
+ "swefficiency": {"category": "agentic", "description": "SWEfficiency optimization"},
+ "terminalbench": {
+ "category": "agentic", "description": "TerminalBench terminal tasks",
+ },
+ "terminalbench-native": {
+ "category": "agentic",
+ "description": "TerminalBench Native (Docker)",
+ },
}
BACKENDS = {
@@ -77,15 +93,48 @@ def _build_dataset(benchmark: str):
if benchmark == "supergpqa":
from evals.datasets.supergpqa import SuperGPQADataset
return SuperGPQADataset()
+ elif benchmark == "gpqa":
+ from evals.datasets.gpqa import GPQADataset
+ return GPQADataset()
+ elif benchmark == "mmlu-pro":
+ from evals.datasets.mmlu_pro import MMLUProDataset
+ return MMLUProDataset()
+ elif benchmark == "math500":
+ from evals.datasets.math500 import MATH500Dataset
+ return MATH500Dataset()
+ elif benchmark == "natural-reasoning":
+ from evals.datasets.natural_reasoning import NaturalReasoningDataset
+ return NaturalReasoningDataset()
+ elif benchmark == "hle":
+ from evals.datasets.hle import HLEDataset
+ return HLEDataset()
+ elif benchmark == "simpleqa":
+ from evals.datasets.simpleqa import SimpleQADataset
+ return SimpleQADataset()
+ elif benchmark == "wildchat":
+ from evals.datasets.wildchat import WildChatDataset
+ return WildChatDataset()
+ elif benchmark == "ipw":
+ from evals.datasets.ipw_mixed import IPWDataset
+ return IPWDataset()
elif benchmark == "gaia":
from evals.datasets.gaia import GAIADataset
return GAIADataset()
elif benchmark == "frames":
from evals.datasets.frames import FRAMESDataset
return FRAMESDataset()
- elif benchmark == "wildchat":
- from evals.datasets.wildchat import WildChatDataset
- return WildChatDataset()
+ elif benchmark == "swebench":
+ from evals.datasets.swebench import SWEBenchDataset
+ return SWEBenchDataset()
+ elif benchmark == "swefficiency":
+ from evals.datasets.swefficiency import SWEfficiencyDataset
+ return SWEfficiencyDataset()
+ elif benchmark == "terminalbench":
+ from evals.datasets.terminalbench import TerminalBenchDataset
+ return TerminalBenchDataset()
+ elif benchmark == "terminalbench-native":
+ from evals.datasets.terminalbench_native import TerminalBenchNativeDataset
+ return TerminalBenchNativeDataset()
else:
raise click.ClickException(f"Unknown benchmark: {benchmark}")
@@ -95,15 +144,47 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str):
if benchmark == "supergpqa":
from evals.scorers.supergpqa_mcq import SuperGPQAScorer
return SuperGPQAScorer(judge_backend, judge_model)
+ elif benchmark == "gpqa":
+ from evals.scorers.gpqa_mcq import GPQAScorer
+ return GPQAScorer(judge_backend, judge_model)
+ elif benchmark == "mmlu-pro":
+ from evals.scorers.mmlu_pro_mcq import MMLUProScorer
+ return MMLUProScorer(judge_backend, judge_model)
+ elif benchmark == "math500" or benchmark == "natural-reasoning":
+ from evals.scorers.reasoning_judge import ReasoningJudgeScorer
+ return ReasoningJudgeScorer(judge_backend, judge_model)
+ elif benchmark == "hle":
+ from evals.scorers.hle_judge import HLEScorer
+ return HLEScorer(judge_backend, judge_model)
+ elif benchmark == "simpleqa":
+ from evals.scorers.simpleqa_judge import SimpleQAScorer
+ return SimpleQAScorer(judge_backend, judge_model)
+ elif benchmark == "wildchat":
+ from evals.scorers.wildchat_judge import WildChatScorer
+ return WildChatScorer(judge_backend, judge_model)
+ elif benchmark == "ipw":
+ from evals.scorers.ipw_mixed import IPWMixedScorer
+ return IPWMixedScorer(judge_backend, judge_model)
elif benchmark == "gaia":
from evals.scorers.gaia_exact import GAIAScorer
return GAIAScorer(judge_backend, judge_model)
elif benchmark == "frames":
from evals.scorers.frames_judge import FRAMESScorer
return FRAMESScorer(judge_backend, judge_model)
- elif benchmark == "wildchat":
- from evals.scorers.wildchat_judge import WildChatScorer
- return WildChatScorer(judge_backend, judge_model)
+ elif benchmark == "swebench":
+ from evals.scorers.swebench_structural import SWEBenchScorer
+ return SWEBenchScorer(judge_backend, judge_model)
+ elif benchmark == "swefficiency":
+ from evals.scorers.swefficiency_structural import SWEfficiencyScorer
+ return SWEfficiencyScorer(judge_backend, judge_model)
+ elif benchmark == "terminalbench":
+ from evals.scorers.terminalbench_judge import TerminalBenchScorer
+ return TerminalBenchScorer(judge_backend, judge_model)
+ elif benchmark == "terminalbench-native":
+ from evals.scorers.terminalbench_native_structural import (
+ TerminalBenchNativeScorer,
+ )
+ return TerminalBenchNativeScorer(judge_backend, judge_model)
else:
raise click.ClickException(f"Unknown benchmark: {benchmark}")
diff --git a/evals/core/config.py b/evals/core/config.py
index 4c7791c2..fb6f7cdb 100644
--- a/evals/core/config.py
+++ b/evals/core/config.py
@@ -34,7 +34,12 @@ logger = logging.getLogger(__name__)
VALID_BACKENDS = {"jarvis-direct", "jarvis-agent"}
# Known benchmark names (used for warnings, not hard validation)
-KNOWN_BENCHMARKS = {"supergpqa", "gaia", "frames", "wildchat"}
+KNOWN_BENCHMARKS = {
+ "supergpqa", "gpqa", "mmlu-pro", "math500", "natural-reasoning", "hle",
+ "simpleqa", "wildchat", "ipw",
+ "gaia", "frames", "swebench", "swefficiency",
+ "terminalbench", "terminalbench-native",
+}
class EvalConfigError(Exception):
diff --git a/evals/datasets/gpqa.py b/evals/datasets/gpqa.py
new file mode 100644
index 00000000..687090c4
--- /dev/null
+++ b/evals/datasets/gpqa.py
@@ -0,0 +1,143 @@
+"""GPQA dataset provider (Idavidrein/gpqa).
+
+Adapted from IPW's gpqa.py dataset loader.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+
+def _format_options(options: Iterable[str]) -> str:
+ rendered = []
+ for idx, option in enumerate(options):
+ letter = chr(ord("A") + idx)
+ rendered.append(f"{letter}. {option}")
+ return "\n".join(rendered)
+
+
+class GPQADataset(DatasetProvider):
+ """GPQA (Graduate-Level Google-Proof Q&A) multiple-choice benchmark."""
+
+ dataset_id = "gpqa"
+ dataset_name = "GPQA"
+
+ _hf_path = "Idavidrein/gpqa"
+ _default_subset = "gpqa_diamond"
+ _default_split = "train"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, self._default_subset, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ # Field names vary across dataset versions.
+ question = str(
+ raw.get("Question") or raw.get("question") or "",
+ ).strip()
+
+ correct_answer = str(
+ raw.get("Correct Answer") or raw.get("correct_answer") or "",
+ ).strip()
+
+ # Gather distractor answers.
+ distractors: List[str] = []
+ for key in (
+ "Incorrect Answer 1", "incorrect_answer_1",
+ "Incorrect Answer 2", "incorrect_answer_2",
+ "Incorrect Answer 3", "incorrect_answer_3",
+ ):
+ val = raw.get(key)
+ if val is not None:
+ text = str(val).strip()
+ if text:
+ distractors.append(text)
+
+ if not question or not correct_answer or not distractors:
+ return None
+
+ # Correct answer is always option A; distractors fill B/C/D.
+ options = [correct_answer] + distractors[:3]
+
+ subdomain = str(
+ raw.get("Subdomain")
+ or raw.get("subdomain")
+ or "",
+ ).strip()
+ domain = str(
+ raw.get("High-level domain")
+ or raw.get("domain")
+ or "",
+ ).strip()
+ subject = subdomain or domain or "general"
+
+ prompt_parts = [
+ question, "",
+ "Options:",
+ _format_options(options), "",
+ "Provide only the letter of the correct answer (A, B, C, or D).",
+ ]
+ problem = "\n".join(part for part in prompt_parts if part).strip()
+
+ metadata = {
+ "correct_option": "A",
+ "answer_text": correct_answer,
+ "options": options,
+ "subdomain": subdomain,
+ "domain": domain,
+ }
+
+ return EvalRecord(
+ record_id=f"gpqa-{idx}",
+ problem=problem,
+ reference="A",
+ category="reasoning",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+__all__ = ["GPQADataset"]
diff --git a/evals/datasets/hle.py b/evals/datasets/hle.py
new file mode 100644
index 00000000..8b0e68b7
--- /dev/null
+++ b/evals/datasets/hle.py
@@ -0,0 +1,138 @@
+"""HLE dataset provider (cais/hle).
+
+Adapted from IPW's reasoning benchmark loaders.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+# Fields whose presence signals a multimodal row.
+_MULTIMODAL_FIELDS = frozenset(
+ {"image", "image_path", "images", "audio", "audio_path", "audios"}
+)
+
+
+class HLEDataset(DatasetProvider):
+ """HLE (Humanity's Last Exam) benchmark dataset."""
+
+ dataset_id = "hle"
+ dataset_name = "HLE"
+
+ _hf_path = "cais/hle"
+ _default_split = "test"
+
+ def __init__(self, *, text_only: bool = True) -> None:
+ self._text_only = text_only
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+ if max_samples is not None and len(self._records) >= max_samples:
+ break
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _is_multimodal(self, raw: MutableMapping[str, object]) -> bool:
+ """Return True if the row contains multimodal content."""
+ for field in _MULTIMODAL_FIELDS:
+ value = raw.get(field)
+ if value is not None and value != "" and value != []:
+ return True
+ return False
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ # Skip multimodal rows when text_only is enabled
+ if self._text_only and self._is_multimodal(raw):
+ return None
+
+ # Extract question text
+ question_text = str(
+ raw.get("question")
+ or raw.get("instruction")
+ or raw.get("prompt")
+ or ""
+ ).strip()
+ if not question_text:
+ return None
+
+ # Extract reference answer
+ reference = str(
+ raw.get("answer")
+ or raw.get("gold_answer")
+ or raw.get("response")
+ or ""
+ ).strip()
+ if not reference:
+ return None
+
+ # Extract category
+ category_value = str(
+ raw.get("category")
+ or raw.get("subject")
+ or raw.get("type")
+ or "general"
+ ).strip() or "general"
+
+ # Extract task_id for the record_id
+ task_id = str(
+ raw.get("id") or raw.get("task_id") or f"hle_{idx}"
+ ).strip()
+
+ # Use question directly (no wrapper template)
+ problem = question_text
+
+ # Metadata
+ metadata: dict[str, object] = {}
+ difficulty = raw.get("difficulty") or raw.get("level")
+ if difficulty is not None:
+ metadata["difficulty"] = difficulty
+ metadata["task_id"] = task_id
+
+ return EvalRecord(
+ record_id=f"hle-{idx}",
+ problem=problem,
+ reference=reference,
+ category="reasoning",
+ subject=category_value,
+ metadata=metadata,
+ )
+
+
+__all__ = ["HLEDataset"]
diff --git a/evals/datasets/ipw_mixed.py b/evals/datasets/ipw_mixed.py
new file mode 100644
index 00000000..e8fcef00
--- /dev/null
+++ b/evals/datasets/ipw_mixed.py
@@ -0,0 +1,196 @@
+"""IPW mixed dataset provider.
+
+Loads evaluation data from a local directory containing HuggingFace Arrow
+datasets or JSONL files. Does *not* download from HuggingFace — the data
+must be present on disk (e.g. bundled from the IPW repository).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import random
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+_DEFAULT_DATASET_NAME = "mixed_1k_seed1_base"
+
+
+class IPWDataset(DatasetProvider):
+ """IPW mixed evaluation dataset loaded from a local directory."""
+
+ dataset_id = "ipw"
+ dataset_name = "IPW"
+
+ def __init__(
+ self,
+ data_dir: Optional[str] = None,
+ dataset_name: Optional[str] = None,
+ ) -> None:
+ self._data_dir = data_dir
+ self._dataset_name = dataset_name or _DEFAULT_DATASET_NAME
+ self._records: List[EvalRecord] = []
+
+ # ------------------------------------------------------------------
+ # DatasetProvider interface
+ # ------------------------------------------------------------------
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ data_path = self._resolve_data_path()
+
+ rows = self._load_rows(data_path)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ LOGGER.info(
+ "IPW dataset loaded: %d records from %s", len(self._records), data_path,
+ )
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ # ------------------------------------------------------------------
+ # Internal helpers
+ # ------------------------------------------------------------------
+
+ def _resolve_data_path(self) -> Path:
+ """Resolve the data directory, raising if it does not exist."""
+ if self._data_dir is not None:
+ path = Path(self._data_dir)
+ else:
+ # Try to locate bundled data via importlib.resources
+ path = self._find_bundled_data()
+
+ if not path.exists():
+ raise FileNotFoundError(
+ f"IPW data directory not found: {path}. "
+ f"Please provide a valid 'data_dir' pointing to the IPW "
+ f"dataset directory (Arrow format or JSONL files)."
+ )
+ return path
+
+ @staticmethod
+ def _find_bundled_data() -> Path:
+ """Attempt to locate bundled IPW data via importlib.resources."""
+ try:
+ import importlib.resources as ir
+
+ ref = ir.files("evals") / "data" / "ipw"
+ # Traverse returns a Path for on-disk packages
+ data_path = Path(str(ref))
+ if data_path.exists():
+ return data_path
+ except (ImportError, TypeError, ModuleNotFoundError):
+ pass
+
+ # Fallback: look relative to this file
+ fallback = Path(__file__).resolve().parent.parent / "data" / "ipw"
+ return fallback
+
+ def _load_rows(self, data_path: Path) -> Sequence[MutableMapping[str, Any]]:
+ """Load rows from an Arrow dataset directory or JSONL files."""
+ # Try HuggingFace Arrow dataset first
+ dataset_path = data_path / self._dataset_name
+ if dataset_path.is_dir():
+ return self._load_arrow(dataset_path)
+
+ # Try JSONL file with dataset name
+ jsonl_path = data_path / f"{self._dataset_name}.jsonl"
+ if jsonl_path.is_file():
+ return self._load_jsonl(jsonl_path)
+
+ # Try loading data_path directly as an Arrow dataset
+ if (data_path / "dataset_info.json").exists() or (
+ data_path / "state.json"
+ ).exists():
+ return self._load_arrow(data_path)
+
+ # Try any JSONL file in the directory
+ jsonl_files = sorted(data_path.glob("*.jsonl"))
+ if jsonl_files:
+ LOGGER.info("Loading first JSONL file found: %s", jsonl_files[0])
+ return self._load_jsonl(jsonl_files[0])
+
+ raise FileNotFoundError(
+ f"No Arrow dataset or JSONL files found in {data_path}. "
+ f"Expected either a '{self._dataset_name}' subdirectory "
+ f"(Arrow format) or '{self._dataset_name}.jsonl'."
+ )
+
+ @staticmethod
+ def _load_arrow(path: Path) -> Sequence[MutableMapping[str, Any]]:
+ """Load a HuggingFace Arrow dataset from disk."""
+ from datasets import load_from_disk
+
+ dataset = load_from_disk(str(path))
+ if hasattr(dataset, "to_list"):
+ return dataset.to_list()
+ return list(dataset)
+
+ @staticmethod
+ def _load_jsonl(path: Path) -> List[MutableMapping[str, Any]]:
+ """Load records from a JSONL file."""
+ rows: List[MutableMapping[str, Any]] = []
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if line:
+ rows.append(json.loads(line))
+ return rows
+
+ @staticmethod
+ def _convert_row(
+ raw: MutableMapping[str, Any], idx: int,
+ ) -> Optional[EvalRecord]:
+ problem = str(
+ raw.get("problem") or raw.get("prompt") or ""
+ ).strip()
+ answer = str(
+ raw.get("answer") or raw.get("expected_answer") or ""
+ ).strip()
+ subject = str(raw.get("subject") or "general").strip() or "general"
+
+ # Require non-empty problem, answer, and subject
+ if not problem or not answer or not subject:
+ return None
+
+ # Store the entire raw dict as metadata for downstream analysis
+ metadata: Dict[str, Any] = dict(raw)
+
+ return EvalRecord(
+ record_id=f"ipw-{idx}",
+ problem=problem,
+ reference=answer,
+ category="chat",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+__all__ = ["IPWDataset"]
diff --git a/evals/datasets/math500.py b/evals/datasets/math500.py
new file mode 100644
index 00000000..2aca3850
--- /dev/null
+++ b/evals/datasets/math500.py
@@ -0,0 +1,111 @@
+"""MATH-500 dataset provider (HuggingFaceH4/MATH-500).
+
+Adapted from IPW's reasoning benchmark loaders.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+_PROMPT_TEMPLATE = (
+ "Solve the following math problem step by step. "
+ "Provide your final answer clearly.\n\n{problem}"
+)
+
+
+class MATH500Dataset(DatasetProvider):
+ """MATH-500 reasoning benchmark dataset."""
+
+ dataset_id = "math500"
+ dataset_name = "MATH-500"
+
+ _hf_path = "HuggingFaceH4/MATH-500"
+ _default_split = "test"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ # Extract problem text
+ problem_text = str(
+ raw.get("problem") or raw.get("question") or ""
+ ).strip()
+ if not problem_text:
+ return None
+
+ # Extract reference answer
+ reference = str(
+ raw.get("answer") or raw.get("solution") or ""
+ ).strip()
+ if not reference:
+ return None
+
+ # Extract subject
+ subject = str(
+ raw.get("subject") or raw.get("type") or "Mathematics"
+ ).strip() or "Mathematics"
+
+ # Build prompt
+ problem = _PROMPT_TEMPLATE.format(problem=problem_text)
+
+ # Metadata
+ metadata: dict[str, object] = {}
+ level = raw.get("level")
+ if level is not None:
+ metadata["level"] = level
+
+ return EvalRecord(
+ record_id=f"math500-{idx}",
+ problem=problem,
+ reference=reference,
+ category="reasoning",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+__all__ = ["MATH500Dataset"]
diff --git a/evals/datasets/mmlu_pro.py b/evals/datasets/mmlu_pro.py
new file mode 100644
index 00000000..c04b6f8c
--- /dev/null
+++ b/evals/datasets/mmlu_pro.py
@@ -0,0 +1,111 @@
+"""MMLU-Pro dataset provider (TIGER-Lab/MMLU-Pro).
+
+Adapted from IPW's mmlu_pro.py dataset loader.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+
+def _format_options(options: Iterable[str]) -> str:
+ rendered = []
+ for idx, option in enumerate(options):
+ letter = chr(ord("A") + idx)
+ rendered.append(f"{letter}. {option}")
+ return "\n".join(rendered)
+
+
+class MMLUProDataset(DatasetProvider):
+ """MMLU-Pro multiple-choice benchmark dataset."""
+
+ dataset_id = "mmlu-pro"
+ dataset_name = "MMLU-Pro"
+
+ _hf_path = "TIGER-Lab/MMLU-Pro"
+ _default_split = "test"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ question = str(raw.get("question") or "").strip()
+ options_raw = raw.get("options") or []
+ options = [str(o).strip() for o in options_raw if str(o).strip()]
+ answer_letter = str(raw.get("answer") or "").strip().upper()
+
+ subject = str(raw.get("category") or "general").strip() or "general"
+
+ if not question or not options or not answer_letter:
+ return None
+
+ prompt_parts = [
+ question, "",
+ "Options:",
+ _format_options(options), "",
+ "Respond with the correct letter.",
+ ]
+ problem = "\n".join(part for part in prompt_parts if part).strip()
+
+ metadata = {
+ "question_id": raw.get("question_id"),
+ "answer_index": raw.get("answer_index"),
+ "src": raw.get("src"),
+ "options": options,
+ }
+
+ return EvalRecord(
+ record_id=f"mmlu-pro-{idx}",
+ problem=problem,
+ reference=answer_letter,
+ category="reasoning",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+__all__ = ["MMLUProDataset"]
diff --git a/evals/datasets/natural_reasoning.py b/evals/datasets/natural_reasoning.py
new file mode 100644
index 00000000..ca80756b
--- /dev/null
+++ b/evals/datasets/natural_reasoning.py
@@ -0,0 +1,114 @@
+"""Natural Reasoning dataset provider (facebook/natural_reasoning).
+
+Adapted from IPW's reasoning benchmark loaders.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+_PROMPT_TEMPLATE = (
+ "Please solve the following reasoning problem. "
+ "Think step by step and provide your final answer clearly.\n\n{question}"
+)
+
+
+class NaturalReasoningDataset(DatasetProvider):
+ """Natural Reasoning benchmark dataset."""
+
+ dataset_id = "natural-reasoning"
+ dataset_name = "Natural Reasoning"
+
+ _hf_path = "facebook/natural_reasoning"
+ _default_split = "train"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ # Extract question text
+ question_text = str(
+ raw.get("question") or raw.get("problem") or ""
+ ).strip()
+ if not question_text:
+ return None
+
+ # Extract reference answer
+ reference = str(
+ raw.get("answer") or raw.get("solution") or ""
+ ).strip()
+ if not reference:
+ return None
+
+ # Extract subject/category
+ subject = str(
+ raw.get("category")
+ or raw.get("field")
+ or raw.get("source")
+ or "General"
+ ).strip() or "General"
+
+ # Build prompt
+ problem = _PROMPT_TEMPLATE.format(question=question_text)
+
+ # Metadata
+ metadata: dict[str, object] = {}
+ difficulty = raw.get("difficulty") or raw.get("level")
+ if difficulty is not None:
+ metadata["difficulty"] = difficulty
+
+ return EvalRecord(
+ record_id=f"natural-reasoning-{idx}",
+ problem=problem,
+ reference=reference,
+ category="reasoning",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+__all__ = ["NaturalReasoningDataset"]
diff --git a/evals/datasets/simpleqa.py b/evals/datasets/simpleqa.py
new file mode 100644
index 00000000..06dddc23
--- /dev/null
+++ b/evals/datasets/simpleqa.py
@@ -0,0 +1,131 @@
+"""SimpleQA dataset provider (basicv8vc/SimpleQA).
+
+Short-answer factual QA benchmark for evaluating factual accuracy.
+"""
+
+from __future__ import annotations
+
+import ast
+import random
+from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+_PROMPT_TEMPLATE = """Please answer the following question with a short, factual response.
+Your answer should be a word, phrase, name, number, or date.
+Do not include explanations or additional context.
+
+Question: {question}"""
+
+
+class SimpleQADataset(DatasetProvider):
+ """SimpleQA short-answer factual QA benchmark."""
+
+ dataset_id = "simpleqa"
+ dataset_name = "SimpleQA"
+
+ _hf_path = "basicv8vc/SimpleQA"
+ _default_split = "test"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ question = str(
+ raw.get("problem") or raw.get("question") or ""
+ ).strip()
+ answer = str(
+ raw.get("answer") or raw.get("gold_answer") or ""
+ ).strip()
+
+ if not question or not answer:
+ return None
+
+ # Parse metadata — may be a JSON string or a dict
+ meta_raw = raw.get("metadata")
+ parsed_meta = _parse_metadata(meta_raw)
+
+ # Extract topic for subject field
+ subject = str(parsed_meta.get("topic", "")).strip() or "general"
+
+ problem = _PROMPT_TEMPLATE.format(question=question)
+
+ metadata: Dict[str, Any] = {
+ "answer_type": parsed_meta.get("answer_type", ""),
+ }
+ # Preserve all parsed metadata keys
+ for key, value in parsed_meta.items():
+ if key not in metadata:
+ metadata[key] = value
+
+ return EvalRecord(
+ record_id=f"simpleqa-{idx}",
+ problem=problem,
+ reference=answer,
+ category="chat",
+ subject=subject,
+ metadata=metadata,
+ )
+
+
+def _parse_metadata(meta_raw: object) -> Dict[str, Any]:
+ """Parse metadata which may be a dict, a JSON-like string, or None."""
+ if meta_raw is None:
+ return {}
+ if isinstance(meta_raw, dict):
+ return dict(meta_raw)
+ if isinstance(meta_raw, str):
+ text = meta_raw.strip()
+ if not text:
+ return {}
+ try:
+ parsed = ast.literal_eval(text)
+ if isinstance(parsed, dict):
+ return parsed
+ except (ValueError, SyntaxError):
+ pass
+ return {}
+
+
+__all__ = ["SimpleQADataset"]
diff --git a/evals/datasets/swebench.py b/evals/datasets/swebench.py
new file mode 100644
index 00000000..50152e70
--- /dev/null
+++ b/evals/datasets/swebench.py
@@ -0,0 +1,166 @@
+"""SWE-bench dataset (princeton-nlp/SWE-bench_Verified).
+
+Agentic coding benchmark — patches for real-world GitHub issues.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from typing import Any, Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+_HF_PATHS = {
+ "verified": "princeton-nlp/SWE-bench_Verified",
+ "verified_mini": "MariusHobbhahn/swe-bench-verified-mini",
+}
+
+_DEFAULT_PROMPT = """You are a software engineer working on the repository **{repo}**.
+
+## Problem Statement
+
+{problem_statement}
+
+{hints_section}
+
+## Instructions
+
+- Analyze the issue described above.
+- Produce a patch (unified diff format) that resolves the issue.
+- The patch must apply cleanly against commit `{base_commit}`.
+- Return ONLY the patch — no explanation, no markdown fences."""
+
+
+def _parse_test_list(value: object) -> List[str]:
+ """Parse a test list that may be JSON string, plain list, or single string."""
+ if value is None:
+ return []
+ if isinstance(value, list):
+ return [str(t) for t in value]
+ if isinstance(value, str):
+ value = value.strip()
+ if not value:
+ return []
+ # Try JSON first
+ try:
+ parsed = json.loads(value)
+ if isinstance(parsed, list):
+ return [str(t) for t in parsed]
+ return [str(parsed)]
+ except (json.JSONDecodeError, TypeError):
+ return [value]
+ return []
+
+
+class SWEBenchDataset(DatasetProvider):
+ """SWE-bench agentic coding benchmark."""
+
+ dataset_id = "swebench"
+ dataset_name = "SWE-bench"
+
+ _default_split = "test"
+
+ def __init__(self, variant: str = "verified_mini") -> None:
+ if variant not in _HF_PATHS:
+ raise ValueError(
+ f"Unknown SWE-bench variant {variant!r}; "
+ f"choose from {sorted(_HF_PATHS)}"
+ )
+ self._variant = variant
+ self._hf_path = _HF_PATHS[variant]
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ instance_id = str(raw.get("instance_id") or "")
+ repo = str(raw.get("repo") or "")
+ problem_statement = str(raw.get("problem_statement") or "").strip()
+ patch = str(raw.get("patch") or "").strip()
+
+ if not problem_statement:
+ return None
+
+ base_commit = str(raw.get("base_commit") or "")
+ hints_text = str(raw.get("hints_text") or "").strip()
+
+ hints_section = ""
+ if hints_text:
+ hints_section = f"## Hints\n\n{hints_text}"
+
+ problem = _DEFAULT_PROMPT.format(
+ repo=repo,
+ problem_statement=problem_statement,
+ hints_section=hints_section,
+ base_commit=base_commit,
+ )
+
+ fail_to_pass = _parse_test_list(raw.get("FAIL_TO_PASS"))
+ pass_to_pass = _parse_test_list(raw.get("PASS_TO_PASS"))
+
+ metadata: dict[str, Any] = {
+ "instance_id": instance_id,
+ "repo": repo,
+ "base_commit": base_commit,
+ "hints_text": hints_text,
+ "version": raw.get("version"),
+ "test_patch": raw.get("test_patch"),
+ "created_at": raw.get("created_at"),
+ "environment_setup_commit": raw.get("environment_setup_commit"),
+ "difficulty": raw.get("difficulty"),
+ "FAIL_TO_PASS": fail_to_pass,
+ "PASS_TO_PASS": pass_to_pass,
+ "variant": self._variant,
+ }
+
+ return EvalRecord(
+ record_id=f"swebench-{instance_id or idx}",
+ problem=problem,
+ reference=patch,
+ category="agentic",
+ subject=repo,
+ metadata=metadata,
+ )
+
+
+__all__ = ["SWEBenchDataset"]
diff --git a/evals/datasets/swefficiency.py b/evals/datasets/swefficiency.py
new file mode 100644
index 00000000..d553e59d
--- /dev/null
+++ b/evals/datasets/swefficiency.py
@@ -0,0 +1,171 @@
+"""SWEfficiency dataset (swefficiency/swefficiency).
+
+Agentic benchmark for software performance optimization.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from typing import Any, Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+_HF_PATH = "swefficiency/swefficiency"
+
+_DEFAULT_PROMPT = """You are a software performance engineer working on the repository **{repo}**.
+
+## Problem Statement
+
+{problem_statement}
+
+## Workload
+
+{workload}
+
+## Expected Speedup
+
+Target speedup: **{expected_speedup}x**
+
+## Instructions
+
+- Analyze the performance bottleneck described above.
+- Produce an optimized patch (unified diff format) that achieves at least the target speedup.
+- The patch must apply cleanly against commit `{base_commit}`.
+- Focus on algorithmic improvements, data structure changes, or computation optimizations.
+- Return ONLY the patch — no explanation, no markdown fences."""
+
+
+def _parse_test_list(value: object) -> List[str]:
+ """Parse a test list that may be JSON string, plain list, or single string."""
+ if value is None:
+ return []
+ if isinstance(value, list):
+ return [str(t) for t in value]
+ if isinstance(value, str):
+ value = value.strip()
+ if not value:
+ return []
+ try:
+ parsed = json.loads(value)
+ if isinstance(parsed, list):
+ return [str(t) for t in parsed]
+ return [str(parsed)]
+ except (json.JSONDecodeError, TypeError):
+ return [value]
+ return []
+
+
+class SWEfficiencyDataset(DatasetProvider):
+ """SWEfficiency agentic performance optimization benchmark."""
+
+ dataset_id = "swefficiency"
+ dataset_name = "SWEfficiency"
+
+ _hf_path = _HF_PATH
+ _default_split = "test"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ instance_id = str(raw.get("instance_id") or "")
+ repo = str(raw.get("repo") or "")
+ problem_statement = str(raw.get("problem_statement") or "").strip()
+ workload = str(raw.get("workload") or "").strip()
+ patch = str(raw.get("patch") or "").strip()
+
+ if not problem_statement:
+ return None
+
+ # Try both field name variants for speedup
+ speedup_raw = raw.get("speedup", raw.get("expected_speedup"))
+ try:
+ expected_speedup = float(speedup_raw) # type: ignore[arg-type]
+ except (TypeError, ValueError):
+ expected_speedup = 1.0
+
+ base_commit = str(raw.get("base_commit") or "")
+
+ problem = _DEFAULT_PROMPT.format(
+ repo=repo,
+ problem_statement=problem_statement,
+ workload=workload or "(not specified)",
+ expected_speedup=expected_speedup,
+ base_commit=base_commit,
+ )
+
+ # Parse test lists (try multiple field name variants)
+ covering_tests = _parse_test_list(
+ raw.get("covering_tests", raw.get("COVERING_TESTS"))
+ )
+ pass_to_pass = _parse_test_list(
+ raw.get("pass_to_pass", raw.get("PASS_TO_PASS"))
+ )
+
+ metadata: dict[str, Any] = {
+ "instance_id": instance_id,
+ "repo": repo,
+ "base_commit": base_commit,
+ "expected_speedup": expected_speedup,
+ "workload": workload,
+ "test_patch": raw.get("test_patch"),
+ "test_cmd": raw.get("test_cmd"),
+ "rebuild_cmd": raw.get("rebuild_cmd"),
+ "image_name": raw.get("image_name"),
+ "covering_tests": covering_tests,
+ "pass_to_pass": pass_to_pass,
+ }
+
+ return EvalRecord(
+ record_id=f"swefficiency-{instance_id or idx}",
+ problem=problem,
+ reference=patch,
+ category="agentic",
+ subject=repo,
+ metadata=metadata,
+ )
+
+
+__all__ = ["SWEfficiencyDataset"]
diff --git a/evals/datasets/terminalbench.py b/evals/datasets/terminalbench.py
new file mode 100644
index 00000000..f016afee
--- /dev/null
+++ b/evals/datasets/terminalbench.py
@@ -0,0 +1,124 @@
+"""TerminalBench dataset (terminal-bench/terminal-bench).
+
+Agentic benchmark for terminal / command-line tasks.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, List, MutableMapping, Optional, Sequence
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+try:
+ from datasets import load_dataset as _load_dataset # noqa: F401
+
+ _HAS_DATASETS = True
+except ImportError:
+ _HAS_DATASETS = False
+
+_HF_PATH = "terminal-bench/terminal-bench"
+
+
+class TerminalBenchDataset(DatasetProvider):
+ """TerminalBench agentic terminal benchmark (HuggingFace variant)."""
+
+ dataset_id = "terminalbench"
+ dataset_name = "TerminalBench"
+
+ _hf_path = _HF_PATH
+ _default_split = "test"
+
+ def __init__(self) -> None:
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ if not _HAS_DATASETS:
+ raise ImportError(
+ "The 'datasets' package is required for TerminalBenchDataset. "
+ "Install it with: pip install datasets"
+ )
+ from datasets import load_dataset
+
+ use_split = split or self._default_split
+ dataset = load_dataset(self._hf_path, split=use_split)
+
+ rows: Sequence[MutableMapping[str, object]]
+ if hasattr(dataset, "to_list"):
+ rows = dataset.to_list()
+ else:
+ rows = list(dataset)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rows = list(rows)
+ rng.shuffle(rows)
+
+ if max_samples is not None:
+ rows = rows[:max_samples]
+
+ self._records = []
+ for idx, raw in enumerate(rows):
+ record = self._convert_row(raw, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_row(
+ self, raw: MutableMapping[str, object], idx: int,
+ ) -> Optional[EvalRecord]:
+ # Try multiple field name variants for question
+ question = str(
+ raw.get("prompt")
+ or raw.get("question")
+ or raw.get("instruction")
+ or ""
+ ).strip()
+
+ # Try multiple field name variants for answer
+ answer = str(
+ raw.get("answer")
+ or raw.get("expected_output")
+ or raw.get("gold_answer")
+ or ""
+ ).strip()
+
+ if not question:
+ return None
+
+ # Category / type
+ category_raw = raw.get("category", raw.get("type", "terminal"))
+ category_str = str(category_raw) if category_raw else "terminal"
+
+ # Task identifier
+ task_id_raw = raw.get("id", raw.get("task_id"))
+ task_id = str(task_id_raw) if task_id_raw else f"tb_{idx}"
+
+ metadata = {
+ "task_id": task_id,
+ "original_category": category_str,
+ }
+
+ return EvalRecord(
+ record_id=f"terminalbench-{task_id}",
+ problem=question,
+ reference=answer,
+ category="agentic",
+ subject=category_str,
+ metadata=metadata,
+ )
+
+
+__all__ = ["TerminalBenchDataset"]
diff --git a/evals/datasets/terminalbench_native.py b/evals/datasets/terminalbench_native.py
new file mode 100644
index 00000000..8d723b4e
--- /dev/null
+++ b/evals/datasets/terminalbench_native.py
@@ -0,0 +1,127 @@
+"""TerminalBench Native dataset — loads from the terminal-bench pip package.
+
+Agentic benchmark using the native terminal-bench SDK for task loading
+and test-based evaluation.
+"""
+
+from __future__ import annotations
+
+import random
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+from evals.core.dataset import DatasetProvider
+from evals.core.types import EvalRecord
+
+try:
+ from terminal_bench import Task, TaskPaths
+ from terminal_bench import TerminalBenchDataset as _TBDataset
+
+ _HAS_TERMINALBENCH = True
+except ImportError:
+ _HAS_TERMINALBENCH = False
+
+
+class TerminalBenchNativeDataset(DatasetProvider):
+ """TerminalBench using the native terminal-bench pip package."""
+
+ dataset_id = "terminalbench-native"
+ dataset_name = "TerminalBench Native"
+
+ def __init__(
+ self,
+ name: str = "terminal-bench-core",
+ version: str = "0.1.1",
+ path: Optional[str] = None,
+ task_ids: Optional[List[str]] = None,
+ n_tasks: Optional[int] = None,
+ ) -> None:
+ self._name = name
+ self._version = version
+ self._path = Path(path) if path else None
+ self._task_ids = task_ids
+ self._n_tasks = n_tasks
+ self._records: List[EvalRecord] = []
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ if not _HAS_TERMINALBENCH:
+ raise ImportError(
+ "The 'terminal-bench' package is required for "
+ "TerminalBenchNativeDataset. "
+ "Install it with: pip install terminal-bench"
+ )
+
+ tb_kwargs: Dict[str, Any] = {
+ "name": self._name,
+ "version": self._version,
+ }
+ if self._path is not None:
+ tb_kwargs["path"] = str(self._path)
+ if self._task_ids is not None:
+ tb_kwargs["task_ids"] = self._task_ids
+ if self._n_tasks is not None:
+ tb_kwargs["n_tasks"] = self._n_tasks
+
+ tb_dataset = _TBDataset(**tb_kwargs)
+
+ task_paths_list: List[Path] = list(tb_dataset.tasks)
+
+ if seed is not None:
+ rng = random.Random(seed)
+ rng.shuffle(task_paths_list)
+
+ if max_samples is not None:
+ task_paths_list = task_paths_list[:max_samples]
+
+ self._records = []
+ for idx, task_dir in enumerate(task_paths_list):
+ record = self._convert_task(task_dir, idx)
+ if record is not None:
+ self._records.append(record)
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def _convert_task(
+ self, task_dir: Path, idx: int,
+ ) -> Optional[EvalRecord]:
+ task_paths = TaskPaths(task_dir)
+ task = Task(task_paths)
+
+ instruction = str(getattr(task, "instruction", "") or "").strip()
+ if not instruction:
+ return None
+
+ task_id = str(getattr(task, "id", "") or task_dir.name or f"tbn_{idx}")
+ category_val = str(getattr(task, "category", "") or "terminal")
+
+ metadata: Dict[str, Any] = {
+ "task_id": task_id,
+ "task_dir": str(task_dir),
+ "category": category_val,
+ "name": getattr(task, "name", None),
+ "tags": getattr(task, "tags", None),
+ "difficulty": getattr(task, "difficulty", None),
+ "timeout": getattr(task, "timeout", None),
+ }
+
+ return EvalRecord(
+ record_id=f"terminalbench-native-{task_id}",
+ problem=instruction,
+ reference="",
+ category="agentic",
+ subject=category_val,
+ metadata=metadata,
+ )
+
+
+__all__ = ["TerminalBenchNativeDataset"]
diff --git a/evals/scorers/gpqa_mcq.py b/evals/scorers/gpqa_mcq.py
new file mode 100644
index 00000000..7cc41ba7
--- /dev/null
+++ b/evals/scorers/gpqa_mcq.py
@@ -0,0 +1,98 @@
+"""GPQA MCQ scorer — LLM-based letter extraction + exact match.
+
+Adapted from IPW's mcq.py and gpqa.py evaluation handlers.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+
+class GPQAScorer(LLMJudgeScorer):
+ """Score GPQA responses by extracting answer letter via LLM."""
+
+ scorer_id = "gpqa"
+
+ def _valid_letters_from_options(self, metadata: Dict[str, Any]) -> str:
+ options = metadata.get("options")
+ if isinstance(options, list) and options:
+ n = len(options)
+ return "".join(chr(ord("A") + i) for i in range(n))
+ return "ABCD"
+
+ def _extract_answer_with_llm(
+ self,
+ problem: str,
+ model_answer: str,
+ valid_letters: str,
+ ) -> Optional[str]:
+ """Use the judge LLM to extract the answer letter from the response."""
+ last_letter = valid_letters[-1] if valid_letters else "D"
+
+ system_prompt = (
+ f"You are an answer extraction assistant. Extract the final multiple choice answer "
+ f"from the response. Return ONLY a single letter (A-{last_letter}). "
+ f"If no valid answer letter is found, return 'NONE'."
+ )
+
+ user_prompt = (
+ f"Problem: {problem}\nResponse: {model_answer}\n\n"
+ f"Extract the final answer letter:"
+ )
+
+ try:
+ raw_response = self._ask_judge(
+ user_prompt, system=system_prompt,
+ temperature=0.0, max_tokens=2048,
+ )
+
+ extracted = raw_response.strip().upper()
+
+ # Handle "The answer is: A" etc.
+ answer_match = re.search(
+ r"(?:THE ANSWER IS:?\s*)?([A-Z])", extracted, re.IGNORECASE,
+ )
+ if answer_match:
+ extracted = answer_match.group(1).upper()
+
+ if extracted in valid_letters:
+ return extracted
+
+ return None
+
+ except Exception as exc:
+ LOGGER.error("Error in LLM-based answer extraction: %s", exc)
+ return None
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ ref = record.reference.strip().upper()
+ if not ref:
+ return None, {"reason": "missing_reference_letter"}
+
+ valid_letters = self._valid_letters_from_options(record.metadata)
+
+ candidate = self._extract_answer_with_llm(
+ record.problem, model_answer, valid_letters,
+ )
+ if not candidate:
+ return None, {"reason": "no_choice_letter_extracted"}
+
+ is_correct = candidate == ref
+ meta = {
+ "reference_letter": ref,
+ "candidate_letter": candidate,
+ "valid_letters": valid_letters,
+ }
+ return is_correct, meta
+
+
+__all__ = ["GPQAScorer"]
diff --git a/evals/scorers/hle_judge.py b/evals/scorers/hle_judge.py
new file mode 100644
index 00000000..783ee283
--- /dev/null
+++ b/evals/scorers/hle_judge.py
@@ -0,0 +1,98 @@
+"""HLE scorer -- LLM-as-judge for Humanity's Last Exam.
+
+Uses the same exact-match-then-LLM-fallback pattern as the reasoning
+judge but with an HLE-specific grading template. Adapted from IPW's
+evaluation handlers.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+from evals.scorers.reasoning_judge import reasoning_exact_match
+
+LOGGER = logging.getLogger(__name__)
+
+_GRADER_TEMPLATE = """You are evaluating an AI system's answer to a challenging reasoning problem from Humanity's Last Exam (HLE).
+
+Compare the predicted answer against the Ground Truth Answer.
+
+Question: {question}
+Ground Truth Answer: {ground_truth}
+Predicted Answer: {predicted_answer}
+
+Focus on whether the final answer is correct. These are expert-level questions, so pay close attention to precision and accuracy of the response.
+
+Your response MUST use exactly this format:
+extracted_final_answer:
+reasoning:
+correct: """
+
+
+class HLEScorer(LLMJudgeScorer):
+ """LLM-as-judge evaluation for HLE benchmark.
+
+ Fast path: normalized exact match (no API call).
+ Slow path: LLM judge for semantic equivalence.
+ """
+
+ scorer_id = "hle"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ reference = record.reference
+ if not reference or not reference.strip():
+ return None, {"reason": "no_ground_truth"}
+
+ # Fast exact match
+ if reasoning_exact_match(model_answer, reference):
+ return True, {"match_type": "exact"}
+
+ # LLM fallback
+ try:
+ prompt = _GRADER_TEMPLATE.format(
+ question=record.problem or "(No question provided)",
+ ground_truth=reference,
+ predicted_answer=model_answer,
+ )
+ raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048)
+
+ structured_match = re.search(
+ r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
+ )
+ if structured_match:
+ is_correct = structured_match.group(1).lower() == "yes"
+ else:
+ is_correct = (
+ "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper()
+ )
+
+ meta: Dict[str, Any] = {
+ "match_type": "llm_fallback",
+ "raw_judge_output": raw,
+ }
+ extracted = re.search(
+ r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
+ )
+ if extracted:
+ meta["extracted_answer"] = extracted.group(1).strip()
+
+ return is_correct, meta
+
+ except Exception as exc:
+ LOGGER.error("HLE LLM judge failed: %s", exc)
+ return False, {
+ "match_type": "llm_fallback_error",
+ "error": str(exc),
+ }
+
+
+__all__ = ["HLEScorer"]
diff --git a/evals/scorers/ipw_mixed.py b/evals/scorers/ipw_mixed.py
new file mode 100644
index 00000000..67f5b44a
--- /dev/null
+++ b/evals/scorers/ipw_mixed.py
@@ -0,0 +1,104 @@
+"""IPW mixed scorer -- LLM-as-judge for mixed-source evaluation datasets.
+
+Since IPW records can originate from different source datasets, this scorer
+uses a general semantic comparison approach via an LLM judge (similar to
+the FRAMES scorer).
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+_GRADER_TEMPLATE = """You are evaluating an AI system's answer against a reference answer.
+
+Compare the predicted answer against the Ground Truth Answer and determine if the prediction is correct.
+
+## Evaluation Guidelines
+
+1. **Focus on semantic meaning**: Look for equivalent information - exact wording is not required.
+2. **Assess factual accuracy**: Determine whether the essential facts from the Ground Truth are present in the answer.
+3. **Ignore minor differences**: Capitalization, punctuation, formatting, and word order don't matter.
+4. **Partial credit**: If the Ground Truth has multiple parts, all essential parts must be present for a correct rating.
+5. **Additional information**: Extra correct information in the prediction is acceptable, but extra incorrect information is not.
+6. **Numerical answers**: Should match exactly (accounting for different formats like $1,000 vs 1000).
+
+## Question / Prompt
+{question}
+
+## Ground Truth Answer
+{ground_truth}
+
+## Predicted Answer
+{predicted_answer}
+
+Your response MUST use exactly this format:
+extracted_final_answer:
+reasoning:
+correct: """
+
+
+class IPWMixedScorer(LLMJudgeScorer):
+ """LLM-as-judge evaluation for mixed-source IPW datasets."""
+
+ scorer_id = "ipw"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ reference = record.reference
+ if not reference or not reference.strip():
+ return None, {"reason": "no_ground_truth"}
+
+ prompt = _GRADER_TEMPLATE.format(
+ question=record.problem,
+ ground_truth=reference,
+ predicted_answer=model_answer,
+ )
+
+ try:
+ raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048)
+
+ structured_match = re.search(
+ r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
+ )
+ if structured_match:
+ is_correct = structured_match.group(1).lower() == "yes"
+ else:
+ response_upper = raw.upper().strip()
+ if "TRUE" in response_upper:
+ is_correct = True
+ elif "FALSE" in response_upper:
+ is_correct = False
+ else:
+ LOGGER.warning(
+ "Could not parse grade from response: %s", raw[:50],
+ )
+ is_correct = False
+
+ meta: Dict[str, Any] = {
+ "raw_judge_output": raw,
+ }
+ extracted = re.search(
+ r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
+ )
+ if extracted:
+ meta["extracted_answer"] = extracted.group(1).strip()
+
+ return is_correct, meta
+
+ except Exception as exc:
+ LOGGER.error("IPW scoring failed: %s", exc)
+ return None, {"error": str(exc)}
+
+
+__all__ = ["IPWMixedScorer"]
diff --git a/evals/scorers/mmlu_pro_mcq.py b/evals/scorers/mmlu_pro_mcq.py
new file mode 100644
index 00000000..8442ee6e
--- /dev/null
+++ b/evals/scorers/mmlu_pro_mcq.py
@@ -0,0 +1,98 @@
+"""MMLU-Pro MCQ scorer — LLM-based letter extraction + exact match.
+
+Adapted from IPW's mcq.py evaluation handler.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+
+class MMLUProScorer(LLMJudgeScorer):
+ """Score MMLU-Pro responses by extracting answer letter via LLM."""
+
+ scorer_id = "mmlu-pro"
+
+ def _valid_letters_from_options(self, metadata: Dict[str, Any]) -> str:
+ options = metadata.get("options")
+ if isinstance(options, list) and options:
+ n = len(options)
+ return "".join(chr(ord("A") + i) for i in range(n))
+ return "ABCDEFGHIJ"
+
+ def _extract_answer_with_llm(
+ self,
+ problem: str,
+ model_answer: str,
+ valid_letters: str,
+ ) -> Optional[str]:
+ """Use the judge LLM to extract the answer letter from the response."""
+ last_letter = valid_letters[-1] if valid_letters else "J"
+
+ system_prompt = (
+ f"You are an answer extraction assistant. Extract the final multiple choice answer "
+ f"from the response. Return ONLY a single letter (A-{last_letter}). "
+ f"If no valid answer letter is found, return 'NONE'."
+ )
+
+ user_prompt = (
+ f"Problem: {problem}\nResponse: {model_answer}\n\n"
+ f"Extract the final answer letter:"
+ )
+
+ try:
+ raw_response = self._ask_judge(
+ user_prompt, system=system_prompt,
+ temperature=0.0, max_tokens=2048,
+ )
+
+ extracted = raw_response.strip().upper()
+
+ # Handle "The answer is: A" etc.
+ answer_match = re.search(
+ r"(?:THE ANSWER IS:?\s*)?([A-Z])", extracted, re.IGNORECASE,
+ )
+ if answer_match:
+ extracted = answer_match.group(1).upper()
+
+ if extracted in valid_letters:
+ return extracted
+
+ return None
+
+ except Exception as exc:
+ LOGGER.error("Error in LLM-based answer extraction: %s", exc)
+ return None
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ ref = record.reference.strip().upper()
+ if not ref:
+ return None, {"reason": "missing_reference_letter"}
+
+ valid_letters = self._valid_letters_from_options(record.metadata)
+
+ candidate = self._extract_answer_with_llm(
+ record.problem, model_answer, valid_letters,
+ )
+ if not candidate:
+ return None, {"reason": "no_choice_letter_extracted"}
+
+ is_correct = candidate == ref
+ meta = {
+ "reference_letter": ref,
+ "candidate_letter": candidate,
+ "valid_letters": valid_letters,
+ }
+ return is_correct, meta
+
+
+__all__ = ["MMLUProScorer"]
diff --git a/evals/scorers/reasoning_judge.py b/evals/scorers/reasoning_judge.py
new file mode 100644
index 00000000..94311042
--- /dev/null
+++ b/evals/scorers/reasoning_judge.py
@@ -0,0 +1,162 @@
+"""Reasoning judge scorer -- LLM-as-judge for math and reasoning tasks.
+
+Attempts normalized exact match first, then falls back to an LLM judge
+for semantic comparison. Adapted from IPW's reasoning evaluation handlers.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import string
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Normalization helpers
+# ---------------------------------------------------------------------------
+
+
+def _normalize_number_str(number_str: str) -> float:
+ for char in ["$", "%", ","]:
+ number_str = number_str.replace(char, "")
+ try:
+ return float(number_str)
+ except ValueError:
+ return float("inf")
+
+
+def _normalize_str(input_str: str) -> str:
+ no_spaces = re.sub(r"\s", "", input_str)
+ translator = str.maketrans("", "", string.punctuation)
+ return no_spaces.lower().translate(translator)
+
+
+def _is_float(element: object) -> bool:
+ try:
+ float(element) # type: ignore[arg-type]
+ return True
+ except (ValueError, TypeError):
+ return False
+
+
+def _extract_boxed(text: str) -> Optional[str]:
+ r"""Extract content from \boxed{...} if present."""
+ match = re.search(r"\\boxed\{([^}]+)\}", text)
+ if match:
+ return match.group(1).strip()
+ return None
+
+
+def reasoning_exact_match(model_answer: str, ground_truth: str) -> bool:
+ """Normalized exact match for reasoning answers.
+
+ Handles numbers, LaTeX boxed answers, and plain strings.
+ """
+ if model_answer is None:
+ return False
+
+ # Try extracting boxed answer from model output
+ boxed = _extract_boxed(model_answer)
+ if boxed is not None:
+ model_answer = boxed
+
+ # Also extract boxed from ground truth if present
+ gt_boxed = _extract_boxed(ground_truth)
+ if gt_boxed is not None:
+ ground_truth = gt_boxed
+
+ # Numeric comparison
+ if _is_float(ground_truth):
+ return _normalize_number_str(model_answer) == float(ground_truth)
+
+ return _normalize_str(model_answer) == _normalize_str(ground_truth)
+
+
+# ---------------------------------------------------------------------------
+# LLM judge grading template
+# ---------------------------------------------------------------------------
+
+_GRADER_TEMPLATE = """You are evaluating an AI system's answer to a reasoning problem.
+
+Compare the predicted answer against the Ground Truth Answer.
+
+Question: {question}
+Ground Truth Answer: {ground_truth}
+Predicted Answer: {predicted_answer}
+
+Focus on whether the final answer is mathematically/logically correct, not on the solution path.
+
+Your response MUST use exactly this format:
+extracted_final_answer:
+reasoning:
+correct: """
+
+
+class ReasoningJudgeScorer(LLMJudgeScorer):
+ """LLM-as-judge evaluation for reasoning tasks.
+
+ Fast path: normalized exact match (no API call).
+ Slow path: LLM judge for semantic equivalence.
+ """
+
+ scorer_id = "reasoning_judge"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ reference = record.reference
+ if not reference or not reference.strip():
+ return None, {"reason": "no_ground_truth"}
+
+ # Fast exact match
+ if reasoning_exact_match(model_answer, reference):
+ return True, {"match_type": "exact"}
+
+ # LLM fallback
+ try:
+ prompt = _GRADER_TEMPLATE.format(
+ question=record.problem or "(No question provided)",
+ ground_truth=reference,
+ predicted_answer=model_answer,
+ )
+ raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048)
+
+ structured_match = re.search(
+ r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
+ )
+ if structured_match:
+ is_correct = structured_match.group(1).lower() == "yes"
+ else:
+ is_correct = (
+ "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper()
+ )
+
+ meta: Dict[str, Any] = {
+ "match_type": "llm_fallback",
+ "raw_judge_output": raw,
+ }
+ extracted = re.search(
+ r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
+ )
+ if extracted:
+ meta["extracted_answer"] = extracted.group(1).strip()
+
+ return is_correct, meta
+
+ except Exception as exc:
+ LOGGER.error("Reasoning LLM judge failed: %s", exc)
+ return False, {
+ "match_type": "llm_fallback_error",
+ "error": str(exc),
+ }
+
+
+__all__ = ["ReasoningJudgeScorer", "reasoning_exact_match"]
diff --git a/evals/scorers/simpleqa_judge.py b/evals/scorers/simpleqa_judge.py
new file mode 100644
index 00000000..011e0a2e
--- /dev/null
+++ b/evals/scorers/simpleqa_judge.py
@@ -0,0 +1,138 @@
+"""SimpleQA scorer -- normalized exact match with LLM fallback.
+
+Evaluates short factual answers using exact string matching (with
+normalization) and falls back to an LLM judge for semantic comparison.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import string
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Normalization helpers (shared with GAIA scorer)
+# ---------------------------------------------------------------------------
+
+
+def _normalize_number_str(number_str: str) -> float:
+ for char in ["$", "%", ","]:
+ number_str = number_str.replace(char, "")
+ try:
+ return float(number_str)
+ except ValueError:
+ return float("inf")
+
+
+def _normalize_str(input_str: str, remove_punct: bool = True) -> str:
+ no_spaces = re.sub(r"\s", "", input_str)
+ if remove_punct:
+ translator = str.maketrans("", "", string.punctuation)
+ return no_spaces.lower().translate(translator)
+ return no_spaces.lower()
+
+
+def _is_float(element: object) -> bool:
+ try:
+ float(element) # type: ignore[arg-type]
+ return True
+ except (ValueError, TypeError):
+ return False
+
+
+def exact_match(model_answer: str, ground_truth: str) -> bool:
+ """Exact-match scorer with normalization for numbers and strings."""
+ if model_answer is None:
+ model_answer = "None"
+
+ if _is_float(ground_truth):
+ normalized = _normalize_number_str(model_answer)
+ return normalized == float(ground_truth)
+
+ return _normalize_str(model_answer) == _normalize_str(ground_truth)
+
+
+# ---------------------------------------------------------------------------
+# LLM fallback prompt
+# ---------------------------------------------------------------------------
+
+_LLM_FALLBACK_PROMPT = """You are evaluating whether a predicted answer matches the gold answer for a factual question.
+
+Question: {question}
+Gold answer: {gold_answer}
+Predicted answer: {predicted_answer}
+
+The answers should be semantically equivalent. Minor differences in formatting, capitalization, or phrasing are acceptable.
+
+Your response MUST use exactly this format:
+extracted_final_answer:
+reasoning:
+correct: """
+
+
+class SimpleQAScorer(LLMJudgeScorer):
+ """SimpleQA evaluation: exact match with normalization + LLM fallback."""
+
+ scorer_id = "simpleqa"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ reference = record.reference
+ if not reference or not reference.strip():
+ return None, {"reason": "no_ground_truth"}
+
+ # Try exact match first (fast, no API call)
+ if exact_match(model_answer, reference):
+ return True, {"match_type": "exact"}
+
+ # LLM fallback for semantic comparison
+ try:
+ prompt = _LLM_FALLBACK_PROMPT.format(
+ question=record.problem or "(No question provided)",
+ gold_answer=reference,
+ predicted_answer=model_answer,
+ )
+ raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048)
+
+ structured_match = re.search(
+ r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
+ )
+ if structured_match:
+ is_correct = structured_match.group(1).lower() == "yes"
+ else:
+ is_correct = (
+ "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper()
+ )
+
+ meta: Dict[str, Any] = {
+ "match_type": "llm_fallback",
+ "raw_judge_output": raw,
+ }
+ extracted_match = re.search(
+ r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
+ )
+ if extracted_match:
+ meta["extracted_answer"] = extracted_match.group(1).strip()
+
+ return is_correct, meta
+
+ except Exception as exc:
+ LOGGER.error("SimpleQA LLM fallback failed: %s", exc)
+ return False, {
+ "match_type": "llm_fallback_error",
+ "error": str(exc),
+ }
+
+
+__all__ = ["SimpleQAScorer", "exact_match"]
diff --git a/evals/scorers/swebench_structural.py b/evals/scorers/swebench_structural.py
new file mode 100644
index 00000000..8a789899
--- /dev/null
+++ b/evals/scorers/swebench_structural.py
@@ -0,0 +1,69 @@
+"""SWE-bench scorer — structural patch validation.
+
+Full SWE-bench evaluation requires running tests inside the repository
+environment. This scorer performs lightweight structural checks on the
+model output (e.g. whether it looks like a valid patch) and defers the
+authoritative pass/fail to external test execution.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import Scorer
+from evals.core.types import EvalRecord
+
+_DIFF_MARKERS = [
+ r"^---\s",
+ r"^\+\+\+\s",
+ r"^@@\s",
+ r"^diff\s+--git\s",
+]
+_DIFF_RE = re.compile("|".join(_DIFF_MARKERS), re.MULTILINE)
+
+
+class SWEBenchScorer(Scorer):
+ """Structural validation scorer for SWE-bench patches.
+
+ Since true SWE-bench scoring requires test execution in a sandboxed
+ repository checkout, this scorer only checks whether the model
+ produced something that looks like a valid unified diff. The
+ ``is_correct`` field is set to ``None`` (indeterminate) when a
+ patch-like response is detected — downstream harnesses should run
+ the actual tests.
+ """
+
+ scorer_id = "swebench"
+
+ def __init__(
+ self,
+ judge_backend: object = None,
+ judge_model: str = "",
+ ) -> None:
+ # Accept judge_backend/judge_model so the CLI factory pattern works,
+ # but they are unused — scoring is purely structural.
+ self._judge_backend = judge_backend
+ self._judge_model = judge_model
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ has_diff = bool(_DIFF_RE.search(model_answer))
+
+ if has_diff:
+ return None, {
+ "reason": "requires_test_execution",
+ "has_diff_markers": True,
+ }
+
+ return None, {
+ "reason": "requires_test_execution",
+ "has_diff_markers": False,
+ }
+
+
+__all__ = ["SWEBenchScorer"]
diff --git a/evals/scorers/swefficiency_structural.py b/evals/scorers/swefficiency_structural.py
new file mode 100644
index 00000000..cef076c2
--- /dev/null
+++ b/evals/scorers/swefficiency_structural.py
@@ -0,0 +1,70 @@
+"""SWEfficiency scorer — structural patch validation.
+
+Full SWEfficiency evaluation requires running performance benchmarks
+inside the repository environment. This scorer performs lightweight
+structural checks on the model output (e.g. whether it looks like a
+valid patch) and defers the authoritative pass/fail to external
+benchmark execution.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import Scorer
+from evals.core.types import EvalRecord
+
+_DIFF_MARKERS = [
+ r"^---\s",
+ r"^\+\+\+\s",
+ r"^@@\s",
+ r"^diff\s+--git\s",
+]
+_DIFF_RE = re.compile("|".join(_DIFF_MARKERS), re.MULTILINE)
+
+
+class SWEfficiencyScorer(Scorer):
+ """Structural validation scorer for SWEfficiency patches.
+
+ Since true SWEfficiency scoring requires running performance
+ benchmarks in a sandboxed repository checkout, this scorer only
+ checks whether the model produced something that looks like a valid
+ unified diff. The ``is_correct`` field is set to ``None``
+ (indeterminate) when a patch-like response is detected — downstream
+ harnesses should measure the actual speedup.
+ """
+
+ scorer_id = "swefficiency"
+
+ def __init__(
+ self,
+ judge_backend: object = None,
+ judge_model: str = "",
+ ) -> None:
+ # Accept judge_backend/judge_model so the CLI factory pattern works,
+ # but they are unused — scoring is purely structural.
+ self._judge_backend = judge_backend
+ self._judge_model = judge_model
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ has_diff = bool(_DIFF_RE.search(model_answer))
+
+ if has_diff:
+ return None, {
+ "reason": "requires_test_execution",
+ "has_diff_markers": True,
+ }
+
+ return None, {
+ "reason": "requires_test_execution",
+ "has_diff_markers": False,
+ }
+
+
+__all__ = ["SWEfficiencyScorer"]
diff --git a/evals/scorers/terminalbench_judge.py b/evals/scorers/terminalbench_judge.py
new file mode 100644
index 00000000..6587227f
--- /dev/null
+++ b/evals/scorers/terminalbench_judge.py
@@ -0,0 +1,102 @@
+"""TerminalBench scorer — LLM-as-judge for terminal task evaluation.
+
+Compares predicted terminal output / commands against the expected
+answer using an LLM judge.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import LLMJudgeScorer
+from evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+_GRADER_TEMPLATE = """You are evaluating an AI system's answer to a terminal / command-line task.
+
+Compare the predicted answer against the Ground Truth Answer and determine if the prediction is correct.
+
+## Evaluation Guidelines
+
+1. **Focus on functional equivalence**: The predicted command or output does not need to match verbatim — it should achieve the same result.
+2. **Assess correctness**: Determine whether the essential behavior, output, or command from the Ground Truth is present in the prediction.
+3. **Ignore minor differences**: Whitespace, quoting style, flag ordering, and trivial formatting differences are acceptable.
+4. **Partial credit**: If the Ground Truth has multiple parts, all essential parts must be present for a correct rating.
+5. **Additional information**: Extra correct information in the prediction is acceptable, but extra incorrect information is not.
+
+## Task / Question
+{question}
+
+## Ground Truth Answer
+{ground_truth}
+
+## Predicted Answer
+{predicted_answer}
+
+Your response MUST use exactly this format:
+extracted_final_answer:
+reasoning:
+correct: """
+
+
+class TerminalBenchScorer(LLMJudgeScorer):
+ """LLM-as-judge evaluation for TerminalBench terminal tasks."""
+
+ scorer_id = "terminalbench"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ if not model_answer or not model_answer.strip():
+ return False, {"reason": "empty_response"}
+
+ reference = record.reference
+ if not reference or not reference.strip():
+ return None, {"reason": "no_ground_truth"}
+
+ prompt = _GRADER_TEMPLATE.format(
+ question=record.problem,
+ ground_truth=reference,
+ predicted_answer=model_answer,
+ )
+
+ try:
+ raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048)
+
+ structured_match = re.search(
+ r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE,
+ )
+ if structured_match:
+ is_correct = structured_match.group(1).lower() == "yes"
+ else:
+ response_upper = raw.upper().strip()
+ if "TRUE" in response_upper:
+ is_correct = True
+ elif "FALSE" in response_upper:
+ is_correct = False
+ else:
+ LOGGER.warning(
+ "Could not parse grade from response: %s", raw[:50],
+ )
+ is_correct = False
+
+ meta: Dict[str, Any] = {
+ "raw_judge_output": raw,
+ }
+ extracted = re.search(
+ r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE,
+ )
+ if extracted:
+ meta["extracted_answer"] = extracted.group(1).strip()
+
+ return is_correct, meta
+
+ except Exception as exc:
+ LOGGER.error("TerminalBench scoring failed: %s", exc)
+ return None, {"error": str(exc)}
+
+
+__all__ = ["TerminalBenchScorer"]
diff --git a/evals/scorers/terminalbench_native_structural.py b/evals/scorers/terminalbench_native_structural.py
new file mode 100644
index 00000000..a7ca139b
--- /dev/null
+++ b/evals/scorers/terminalbench_native_structural.py
@@ -0,0 +1,67 @@
+"""TerminalBench Native scorer — test-result-based evaluation.
+
+Reads ``is_resolved`` and ``test_results`` from the record's metadata
+(populated by the native terminal-bench harness) and returns a
+deterministic pass/fail without any LLM judging.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+
+from evals.core.scorer import Scorer
+from evals.core.types import EvalRecord
+
+
+class TerminalBenchNativeScorer(Scorer):
+ """Test-result-based scorer for TerminalBench Native tasks.
+
+ The native terminal-bench package produces ``is_resolved`` and
+ ``test_results`` fields after executing a task. This scorer reads
+ those fields from ``record.metadata`` and translates them into the
+ standard ``(is_correct, meta)`` tuple.
+ """
+
+ scorer_id = "terminalbench-native"
+
+ def __init__(
+ self,
+ judge_backend: object = None,
+ judge_model: str = "",
+ ) -> None:
+ # Accept judge_backend/judge_model so the CLI factory pattern works,
+ # but they are unused — scoring is based on test results.
+ self._judge_backend = judge_backend
+ self._judge_model = judge_model
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ meta = record.metadata
+
+ is_resolved = meta.get("is_resolved")
+ test_results = meta.get("test_results")
+
+ # If neither field is present, we cannot determine correctness.
+ if is_resolved is None and test_results is None:
+ return None, {"reason": "no_test_results"}
+
+ # Build informative metadata from available test output.
+ result_meta: Dict[str, Any] = {}
+ if test_results is not None:
+ result_meta["test_results"] = test_results
+
+ # Determine pass/fail
+ if is_resolved is not None:
+ is_correct = bool(is_resolved)
+ result_meta["is_resolved"] = is_resolved
+ return is_correct, result_meta
+
+ # Fallback: if only test_results is present, treat as indeterminate.
+ return None, {
+ "reason": "is_resolved_missing",
+ "test_results": test_results,
+ }
+
+
+__all__ = ["TerminalBenchNativeScorer"]
diff --git a/recipes/coding_assistant.toml b/recipes/coding_assistant.toml
new file mode 100644
index 00000000..0bb76c60
--- /dev/null
+++ b/recipes/coding_assistant.toml
@@ -0,0 +1,25 @@
+[recipe]
+name = "coding_assistant"
+description = "Code generation, review, and debugging with ReAct agent"
+version = "1.0.0"
+
+[intelligence]
+model = "qwen3:8b"
+quantization = "q4_K_M"
+
+[engine]
+key = "ollama"
+
+[agent]
+type = "native_react"
+max_turns = 10
+temperature = 0.3
+tools = ["file_read", "file_write", "code_interpreter", "think", "shell_exec"]
+system_prompt = "You are a coding assistant. Help the user write, review, and debug code. Use the available tools to read files, write code, and execute commands as needed."
+
+[learning]
+routing = "heuristic"
+agent = "none"
+
+[eval]
+suites = ["coding", "reasoning"]
diff --git a/recipes/general_assistant.toml b/recipes/general_assistant.toml
new file mode 100644
index 00000000..772f9279
--- /dev/null
+++ b/recipes/general_assistant.toml
@@ -0,0 +1,24 @@
+[recipe]
+name = "general_assistant"
+description = "Balanced general-purpose assistant"
+version = "1.0.0"
+
+[intelligence]
+model = "qwen3:8b"
+
+[engine]
+key = "ollama"
+
+[agent]
+type = "orchestrator"
+max_turns = 8
+temperature = 0.7
+tools = ["think", "calculator", "web_search", "memory_search"]
+system_prompt = "You are a helpful general-purpose assistant. Answer questions clearly and use tools when they would improve your response."
+
+[learning]
+routing = "heuristic"
+agent = "none"
+
+[eval]
+suites = ["general"]
diff --git a/recipes/operators/correspondent.toml b/recipes/operators/correspondent.toml
new file mode 100644
index 00000000..0618974c
--- /dev/null
+++ b/recipes/operators/correspondent.toml
@@ -0,0 +1,14 @@
+[operator]
+name = "correspondent"
+version = "1.0.0"
+description = "Messaging triage agent — classifies incoming messages by urgency, drafts responses, and produces periodic digests"
+
+[operator.agent]
+max_turns = 15
+temperature = 0.4
+tools = ["memory_store", "memory_search", "think", "llm_call"]
+system_prompt_path = "correspondent_prompt.md"
+
+[operator.schedule]
+type = "interval"
+value = "300"
diff --git a/recipes/operators/correspondent_prompt.md b/recipes/operators/correspondent_prompt.md
new file mode 100644
index 00000000..8acddaa1
--- /dev/null
+++ b/recipes/operators/correspondent_prompt.md
@@ -0,0 +1,52 @@
+You are a messaging triage agent running on-device.
+
+## Objective
+
+Process incoming messages, classify them by urgency, draft responses when needed, and produce periodic digests so the user stays informed without being overwhelmed.
+
+## Urgency Classification
+
+Classify every incoming message into one of four levels:
+
+- **Urgent**: Requires immediate attention. Examples: time-sensitive requests, security alerts, messages from VIP contacts, emergencies.
+- **Normal**: Important but not time-critical. Examples: work discussions, questions awaiting reply, scheduled meeting changes.
+- **Low**: Informational or can wait. Examples: newsletters, FYI messages, non-critical notifications.
+- **Ignore**: Noise that does not need user attention. Examples: automated confirmations, marketing, spam-like content.
+
+## Processing Rules
+
+1. **Urgent messages**: Draft a concise response for user review. Keep drafts under 3 sentences. Flag them prominently in output.
+2. **Normal messages**: Summarize the key point in one sentence. Group by sender or topic when multiple related messages arrive.
+3. **Low-priority messages**: Batch into a periodic digest. No individual summaries needed — just count and categorize.
+4. **Ignorable messages**: Log for record-keeping but do not surface unless the user explicitly asks.
+
+## Sender Priority Learning
+
+- Maintain a sender priority list in memory using `memory_store` and `memory_search`.
+- If the user responds to messages from a sender previously classified as "low", upgrade that sender to "normal" in future runs.
+- If the user consistently ignores messages from a "normal" sender, consider downgrading them.
+- Use `think` to reason about priority adjustments before making them.
+- Use `llm_call` when you need to analyze message sentiment or intent beyond simple classification.
+
+## Daily Digest Format
+
+Produce a daily digest summarizing all messages:
+
+### Message Summary
+- **Urgent**: [count] messages — [top items with one-line summaries]
+- **Normal**: [count] messages — [grouped by sender/topic]
+- **Low**: [count] messages — [categories only]
+- **Ignored**: [count] messages
+
+### Pending Responses
+List any drafted responses awaiting user review.
+
+### Sender Priority Updates
+Note any sender priority changes made or recommended.
+
+## Guidelines
+
+- Err on the side of caution: when in doubt, classify one level higher (e.g., "normal" instead of "low").
+- Never discard messages — always store them in memory for audit purposes.
+- Respect user privacy: do not store full message content for ignored messages, only metadata.
+- Keep all drafts professional and concise.
diff --git a/recipes/operators/researcher.toml b/recipes/operators/researcher.toml
new file mode 100644
index 00000000..23916e0d
--- /dev/null
+++ b/recipes/operators/researcher.toml
@@ -0,0 +1,14 @@
+[operator]
+name = "researcher"
+version = "1.0.0"
+description = "Autonomous deep research agent — searches the web, cross-references sources, builds a knowledge graph, and produces structured cited reports"
+
+[operator.agent]
+max_turns = 20
+temperature = 0.3
+tools = ["web_search", "http_request", "memory_store", "memory_search", "kg_add_entity", "kg_add_relation", "think", "file_write"]
+system_prompt_path = "researcher_prompt.md"
+
+[operator.schedule]
+type = "cron"
+value = "0 */4 * * *"
diff --git a/recipes/operators/researcher_prompt.md b/recipes/operators/researcher_prompt.md
new file mode 100644
index 00000000..a175077e
--- /dev/null
+++ b/recipes/operators/researcher_prompt.md
@@ -0,0 +1,45 @@
+You are a deep research agent running on-device.
+
+## Objective
+
+Given a research topic, you systematically search the web using multiple queries to build a comprehensive understanding of the subject.
+
+## Research Process
+
+1. **Plan your search strategy.** Use the `think` tool to decompose the topic into 3-5 targeted search queries before issuing any web searches.
+2. **Execute searches.** Use `web_search` to find relevant information. Prefer fewer high-quality searches over many shallow ones to stay energy-efficient.
+3. **Cross-reference sources.** For every key claim, check at least 3 independent sources. Use `http_request` to retrieve full pages when search snippets are insufficient.
+4. **Build the knowledge graph.** Use `kg_add_entity` for key people, organizations, concepts, and events. Use `kg_add_relation` to connect them (e.g., "authored_by", "related_to", "contradicts").
+5. **Store findings.** Use `memory_store` to persist important facts, quotes, and source URLs for future reference. Use `memory_search` to check what you already know before searching again.
+6. **Write the report.** Use `file_write` to produce the final structured report.
+
+## Output Format
+
+Produce a structured cited report with the following sections:
+
+### Summary
+A 2-3 paragraph executive summary of the research findings.
+
+### Key Findings
+Numbered list of the most important findings, each with:
+- The finding statement
+- Supporting evidence (with source citations)
+- Confidence level: **high** (3+ corroborating sources), **medium** (2 sources), or **low** (single source or inference)
+
+### Sources
+Numbered bibliography of all sources consulted, including:
+- Title
+- URL
+- Date accessed
+- Reliability assessment (established outlet, peer-reviewed, blog, social media, etc.)
+
+### Confidence Assessment
+Overall confidence rating for the report with justification. Note any gaps in available information, conflicting sources, or areas requiring further investigation.
+
+## Guidelines
+
+- Be thorough but energy-efficient. Each web search and HTTP request consumes device resources.
+- Always include source URLs and indicate confidence level (high/medium/low) for each finding.
+- If sources conflict, report all perspectives and note the disagreement.
+- Distinguish between established facts, expert opinions, and speculation.
+- When a topic is too broad, use `think` to narrow scope before proceeding.
diff --git a/recipes/operators/sentinel.toml b/recipes/operators/sentinel.toml
new file mode 100644
index 00000000..45b900f0
--- /dev/null
+++ b/recipes/operators/sentinel.toml
@@ -0,0 +1,14 @@
+[operator]
+name = "sentinel"
+version = "1.0.0"
+description = "Monitoring sentinel agent — watches social media, news, and web sources for changes relevant to user-defined topics and produces scored alerts"
+
+[operator.agent]
+max_turns = 15
+temperature = 0.3
+tools = ["web_search", "http_request", "memory_store", "memory_search", "kg_add_entity", "think"]
+system_prompt_path = "sentinel_prompt.md"
+
+[operator.schedule]
+type = "cron"
+value = "0 */2 * * *"
diff --git a/recipes/operators/sentinel_prompt.md b/recipes/operators/sentinel_prompt.md
new file mode 100644
index 00000000..3fb5b6ea
--- /dev/null
+++ b/recipes/operators/sentinel_prompt.md
@@ -0,0 +1,74 @@
+You are a monitoring sentinel agent running on-device.
+
+## Objective
+
+Monitor online sources for changes relevant to user-defined topics. Detect trending discussions, sentiment shifts, breaking news, and competitor activity. Produce scored alerts only when findings are significant enough to warrant attention.
+
+## Sources to Monitor
+
+Search across these platforms and source types for relevant activity:
+
+- **Twitter/X**: Trending hashtags, influential accounts, viral threads
+- **Reddit**: Popular posts in relevant subreddits, comment sentiment
+- **Mastodon**: Federated discussions, trending topics
+- **Google Trends**: Rising search terms, breakout topics
+- **RSS feeds**: News articles, blog posts from specified feeds
+- **Specified URLs**: Direct monitoring of pages the user has flagged
+
+Use `web_search` to query across platforms and `http_request` to fetch specific pages or feeds.
+
+## Monitoring Process
+
+1. **Recall previous state.** Use `memory_search` to retrieve findings from your last check. This is your baseline for change detection.
+2. **Plan searches.** Use `think` to determine the most efficient set of queries for the current monitoring cycle. Prioritize sources that have historically produced actionable findings.
+3. **Execute searches.** Query each source type for the user-defined topics. Be selective — focus on high-signal sources first.
+4. **Detect changes.** Compare current findings against the previous baseline. Look for:
+ - New discussions or articles not seen before
+ - Significant changes in sentiment or volume
+ - Breaking news or sudden spikes in activity
+ - New entities entering the conversation (people, companies, products)
+5. **Score significance.** Rate each finding on a 1-10 scale.
+6. **Record findings.** Use `memory_store` to persist all findings with timestamps. Use `kg_add_entity` to track key entities and events in the knowledge graph.
+7. **Generate alerts.** Only produce alerts for items scoring 7 or above.
+
+## Significance Scoring (1-10)
+
+Score each finding based on three dimensions:
+
+- **Relevance** (0-4 points): How closely does this relate to the user's defined topics?
+- **Magnitude** (0-3 points): How large is the change? (viral thread = 3, minor mention = 1)
+- **Impact** (0-3 points): What is the potential real-world consequence for the user?
+
+Only items scoring **7 or above** should generate alerts. This threshold prevents alert fatigue.
+
+## Alert Output Format
+
+For each alert, produce:
+
+```
+## Alert: [Brief title]
+- **Topic**: [User-defined topic this relates to]
+- **Source**: [Platform and specific URL]
+- **Significance**: [Score]/10 (Relevance: X, Magnitude: Y, Impact: Z)
+- **Summary**: [2-3 sentence description of the finding and why it matters]
+- **Link**: [Direct URL to the source]
+- **First detected**: [Timestamp]
+```
+
+## End-of-Cycle Summary
+
+After processing all sources, produce a brief summary:
+
+- Total sources checked
+- New findings (all scores)
+- Alerts generated (score 7+)
+- Topics with no new activity
+- Recommended adjustments to monitoring scope (if any)
+
+## Guidelines
+
+- Store all findings in memory with timestamps, even those below the alert threshold. This enables trend analysis over time.
+- When a topic consistently produces no results, use `think` to suggest refined search terms or alternative sources.
+- Be energy-efficient: if a source returned nothing useful in the last 3 cycles, reduce its check frequency.
+- Never fabricate findings. If a search returns no results, report that honestly.
+- Include direct links to sources whenever possible.
diff --git a/recipes/research_assistant.toml b/recipes/research_assistant.toml
new file mode 100644
index 00000000..0829218c
--- /dev/null
+++ b/recipes/research_assistant.toml
@@ -0,0 +1,24 @@
+[recipe]
+name = "research_assistant"
+description = "Deep research with orchestrator agent and web search"
+version = "1.0.0"
+
+[intelligence]
+model = "qwen3:8b"
+
+[engine]
+key = "ollama"
+
+[agent]
+type = "orchestrator"
+max_turns = 15
+temperature = 0.5
+tools = ["web_search", "http_request", "memory_store", "memory_search", "think", "file_write"]
+system_prompt = "You are a research assistant. Investigate topics thoroughly using web search and memory. Store important findings and synthesize comprehensive answers."
+
+[learning]
+routing = "grpo"
+agent = "icl_updater"
+
+[eval]
+suites = ["reasoning"]
diff --git a/skills/builtin/backup-files.toml b/skills/builtin/backup-files.toml
new file mode 100644
index 00000000..5c0bff4a
--- /dev/null
+++ b/skills/builtin/backup-files.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "backup-files"
+description = "Create a timestamped backup archive of specified files or directory"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "shell_exec"
+arguments_template = '{"command": "date +%Y%m%d_%H%M%S"}'
+output_key = "timestamp"
+
+[[skill.steps]]
+tool_name = "shell_exec"
+arguments_template = '{"command": "tar czf {destination}/backup_{timestamp}.tar.gz -C {source} ."}'
+output_key = "archive_result"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Verify the backup was created successfully and summarize: {archive_result}"}'
+output_key = "report"
diff --git a/skills/builtin/calendar-prep.toml b/skills/builtin/calendar-prep.toml
new file mode 100644
index 00000000..a836784c
--- /dev/null
+++ b/skills/builtin/calendar-prep.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "calendar-prep"
+description = "Prepare briefing materials for upcoming calendar events"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Parse the event details and identify what preparation is needed. Event: {event_title}. Attendees: {attendees}. Agenda: {agenda}"}'
+output_key = "prep_needs"
+
+[[skill.steps]]
+tool_name = "memory_search"
+arguments_template = '{"query": "{event_title} {attendees}"}'
+output_key = "related_context"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Create a briefing document with: event summary, attendee background, talking points, relevant history, and suggested preparation steps. Needs: {prep_needs}. Context: {related_context}"}'
+output_key = "briefing"
diff --git a/skills/builtin/code-lint.toml b/skills/builtin/code-lint.toml
new file mode 100644
index 00000000..876b71d8
--- /dev/null
+++ b/skills/builtin/code-lint.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "code-lint"
+description = "Read a source file, analyze it for code quality issues, and report findings"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{file_path}"}'
+output_key = "source_code"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Analyze the code for quality issues: style violations, potential bugs, unused variables, missing error handling, code smells, and complexity problems: {source_code}"}'
+output_key = "issues"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Format the findings as a structured lint report with severity levels (error, warning, info) and line references: {issues}"}'
+output_key = "report"
diff --git a/skills/builtin/code-test-gen.toml b/skills/builtin/code-test-gen.toml
new file mode 100644
index 00000000..6127ea01
--- /dev/null
+++ b/skills/builtin/code-test-gen.toml
@@ -0,0 +1,25 @@
+[skill]
+name = "code-test-gen"
+description = "Read a source file and generate unit tests for its functions and classes"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{file_path}"}'
+output_key = "source_code"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Identify all functions, classes, and methods in the source code. Determine their inputs, outputs, and edge cases: {source_code}"}'
+output_key = "analysis"
+
+[[skill.steps]]
+tool_name = "code_interpreter"
+arguments_template = '{"code": "# Generate pytest-style unit tests based on the analysis:\\n# {analysis}\\nprint(\"Tests generated successfully\")"}'
+output_key = "generated_tests"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Review the generated tests for completeness and correctness. Suggest additional test cases if needed: {generated_tests}"}'
+output_key = "review"
diff --git a/skills/builtin/compare-docs.toml b/skills/builtin/compare-docs.toml
new file mode 100644
index 00000000..76b2e7f7
--- /dev/null
+++ b/skills/builtin/compare-docs.toml
@@ -0,0 +1,25 @@
+[skill]
+name = "compare-docs"
+description = "Compare two documents and summarize their differences"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{file_a}"}'
+output_key = "doc_a"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{file_b}"}'
+output_key = "doc_b"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Compare the two documents. Identify additions, removals, and modifications. Note structural differences and content changes. Document A: {doc_a}. Document B: {doc_b}"}'
+output_key = "differences"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Generate a comparison summary with sections: Overview, Key Differences, Added Content, Removed Content, Modified Sections, and Recommendation: {differences}"}'
+output_key = "comparison_report"
diff --git a/skills/builtin/daily-digest.toml b/skills/builtin/daily-digest.toml
new file mode 100644
index 00000000..6ed0de23
--- /dev/null
+++ b/skills/builtin/daily-digest.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "daily-digest"
+description = "Create a daily digest summarizing stored information and recent activity"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "memory_search"
+arguments_template = '{"query": "recent activity updates {date}"}'
+output_key = "recent_items"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Categorize the retrieved items into groups: tasks completed, pending items, important updates, and upcoming deadlines: {recent_items}"}'
+output_key = "categorized"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Generate a concise daily digest with sections: Highlights, Completed Tasks, Pending Items, Upcoming Deadlines, and Notes: {categorized}"}'
+output_key = "digest"
diff --git a/skills/builtin/data-analyze.toml b/skills/builtin/data-analyze.toml
new file mode 100644
index 00000000..fe24fe6f
--- /dev/null
+++ b/skills/builtin/data-analyze.toml
@@ -0,0 +1,25 @@
+[skill]
+name = "data-analyze"
+description = "Analyze a data file and generate statistical insights and visualizations"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{data_path}"}'
+output_key = "raw_data"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Parse the raw data and compute summary statistics: row count, column names, data types, min/max/mean for numeric columns, and missing value counts: {raw_data}"}'
+output_key = "data_stats"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Analyze the data statistics and identify patterns, trends, outliers, and correlations. Data overview: {data_stats}. Raw sample: {raw_data}"}'
+output_key = "analysis"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Generate a data analysis report with sections: Dataset Overview, Key Statistics, Patterns and Trends, Outliers, and Recommendations: {analysis}"}'
+output_key = "report"
diff --git a/skills/builtin/dependency-audit.toml b/skills/builtin/dependency-audit.toml
new file mode 100644
index 00000000..fdd2d57c
--- /dev/null
+++ b/skills/builtin/dependency-audit.toml
@@ -0,0 +1,25 @@
+[skill]
+name = "dependency-audit"
+description = "Check project dependencies for known vulnerabilities and outdated packages"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{project_path}/pyproject.toml"}'
+output_key = "project_config"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Extract all dependencies and their version constraints from the project config: {project_config}"}'
+output_key = "dependencies"
+
+[[skill.steps]]
+tool_name = "web_search"
+arguments_template = '{"query": "python package vulnerability advisory {dependencies}"}'
+output_key = "advisories"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Cross-reference dependencies with advisories and generate an audit report with risk levels and upgrade recommendations: {advisories}"}'
+output_key = "report"
diff --git a/skills/builtin/email-draft.toml b/skills/builtin/email-draft.toml
new file mode 100644
index 00000000..0556809b
--- /dev/null
+++ b/skills/builtin/email-draft.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "email-draft"
+description = "Draft a professional email response based on context and intent"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Analyze the email context and determine the appropriate tone, key points to address, and desired outcome. Context: {context}. Intent: {intent}"}'
+output_key = "analysis"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Draft a professional email with proper greeting, body paragraphs addressing each key point, and appropriate closing. Analysis: {analysis}. Recipient: {recipient}"}'
+output_key = "draft"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Review the draft for clarity, tone, grammar, and completeness. Suggest improvements if needed: {draft}"}'
+output_key = "final_draft"
diff --git a/skills/builtin/file-deduplicator.toml b/skills/builtin/file-deduplicator.toml
new file mode 100644
index 00000000..a1a7a577
--- /dev/null
+++ b/skills/builtin/file-deduplicator.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "file-deduplicator"
+description = "Find and report duplicate files in a directory by comparing checksums"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "shell_exec"
+arguments_template = '{"command": "find {directory} -type f -exec md5sum {} \\; | sort"}'
+output_key = "checksums"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Identify duplicate files from the checksums. Group files with identical hashes: {checksums}"}'
+output_key = "duplicates"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Generate a report listing all duplicate file groups and total space that could be reclaimed: {duplicates}"}'
+output_key = "report"
diff --git a/skills/builtin/file-organizer.toml b/skills/builtin/file-organizer.toml
new file mode 100644
index 00000000..43a6fb9c
--- /dev/null
+++ b/skills/builtin/file-organizer.toml
@@ -0,0 +1,25 @@
+[skill]
+name = "file-organizer"
+description = "Organize files in a directory by type (images, documents, code, etc.)"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "shell_exec"
+arguments_template = '{"command": "find {directory} -maxdepth 1 -type f -exec file --mime-type {} \\;"}'
+output_key = "file_listing"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Analyze the file types and group them into categories (images, documents, code, archives, media, other): {file_listing}"}'
+output_key = "organization_plan"
+
+[[skill.steps]]
+tool_name = "shell_exec"
+arguments_template = '{"command": "mkdir -p {directory}/images {directory}/documents {directory}/code {directory}/archives {directory}/media {directory}/other"}'
+output_key = "dirs_created"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Generate a summary report of the file organization plan: {organization_plan}"}'
+output_key = "report"
diff --git a/skills/builtin/knowledge-extract.toml b/skills/builtin/knowledge-extract.toml
new file mode 100644
index 00000000..a7da8551
--- /dev/null
+++ b/skills/builtin/knowledge-extract.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "knowledge-extract"
+description = "Extract structured knowledge from a document and store in memory"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{file_path}"}'
+output_key = "document_content"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Extract key facts, entities, relationships, and concepts from the document: {document_content}"}'
+output_key = "extracted_knowledge"
+
+[[skill.steps]]
+tool_name = "memory_store"
+arguments_template = '{"content": "{extracted_knowledge}", "metadata": "knowledge_extract:{file_path}"}'
+output_key = "stored"
diff --git a/skills/builtin/meeting-notes.toml b/skills/builtin/meeting-notes.toml
new file mode 100644
index 00000000..d539d14f
--- /dev/null
+++ b/skills/builtin/meeting-notes.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "meeting-notes"
+description = "Generate structured meeting notes from a transcript or raw notes"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{transcript_path}"}'
+output_key = "transcript"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Extract key information from the transcript: attendees, topics discussed, decisions made, action items, deadlines, and open questions: {transcript}"}'
+output_key = "extracted_info"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Format as structured meeting notes with sections: Date, Attendees, Agenda, Discussion Summary, Decisions, Action Items (with owners and deadlines), Next Steps: {extracted_info}"}'
+output_key = "meeting_notes"
diff --git a/skills/builtin/pdf-summarize.toml b/skills/builtin/pdf-summarize.toml
new file mode 100644
index 00000000..82220572
--- /dev/null
+++ b/skills/builtin/pdf-summarize.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "pdf-summarize"
+description = "Extract text from a PDF document and generate a structured summary"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "pdf_extract"
+arguments_template = '{"path": "{pdf_path}"}'
+output_key = "pdf_text"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Identify the document structure (title, sections, key arguments) from the extracted text: {pdf_text}"}'
+output_key = "structure"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Generate a comprehensive summary with sections: Overview, Key Points, Conclusions, and Action Items based on: {structure}"}'
+output_key = "summary"
diff --git a/skills/builtin/search-and-index.toml b/skills/builtin/search-and-index.toml
new file mode 100644
index 00000000..116be856
--- /dev/null
+++ b/skills/builtin/search-and-index.toml
@@ -0,0 +1,25 @@
+[skill]
+name = "search-and-index"
+description = "Search for documents matching a query and index them into memory for future retrieval"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "shell_exec"
+arguments_template = '{"command": "find {directory} -type f \\( -name \"*.txt\" -o -name \"*.md\" -o -name \"*.pdf\" \\) | head -50"}'
+output_key = "found_files"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Filter the found files to those likely relevant to the query: {query}. Files: {found_files}"}'
+output_key = "relevant_files"
+
+[[skill.steps]]
+tool_name = "memory_index"
+arguments_template = '{"path": "{directory}"}'
+output_key = "index_result"
+
+[[skill.steps]]
+tool_name = "memory_search"
+arguments_template = '{"query": "{query}"}'
+output_key = "search_results"
diff --git a/skills/builtin/security-scan.toml b/skills/builtin/security-scan.toml
new file mode 100644
index 00000000..e7abb546
--- /dev/null
+++ b/skills/builtin/security-scan.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "security-scan"
+description = "Scan source code for common security vulnerabilities and report findings"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{file_path}"}'
+output_key = "source_code"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Scan for security vulnerabilities: SQL injection, XSS, command injection, hardcoded secrets, insecure crypto, path traversal, SSRF, deserialization issues: {source_code}"}'
+output_key = "vulnerabilities"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Generate a security report with severity ratings (critical, high, medium, low), affected lines, and remediation advice: {vulnerabilities}"}'
+output_key = "report"
diff --git a/skills/builtin/todo-from-notes.toml b/skills/builtin/todo-from-notes.toml
new file mode 100644
index 00000000..14113186
--- /dev/null
+++ b/skills/builtin/todo-from-notes.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "todo-from-notes"
+description = "Extract actionable todo items from unstructured notes"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{notes_path}"}'
+output_key = "notes_content"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Identify all actionable items, tasks, commitments, and follow-ups from the notes. Tag each with priority (high/medium/low) and category: {notes_content}"}'
+output_key = "extracted_todos"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Format as a prioritized todo list with checkboxes, due dates where mentioned, and categories. Sort by priority: {extracted_todos}"}'
+output_key = "todo_list"
diff --git a/skills/builtin/topic-research.toml b/skills/builtin/topic-research.toml
new file mode 100644
index 00000000..7ae5194d
--- /dev/null
+++ b/skills/builtin/topic-research.toml
@@ -0,0 +1,25 @@
+[skill]
+name = "topic-research"
+description = "Research a topic using web search, check memory for prior knowledge, and synthesize findings"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "memory_search"
+arguments_template = '{"query": "{topic}"}'
+output_key = "prior_knowledge"
+
+[[skill.steps]]
+tool_name = "web_search"
+arguments_template = '{"query": "{topic}"}'
+output_key = "search_results"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Synthesize a comprehensive research summary combining prior knowledge and new findings. Prior: {prior_knowledge}. New: {search_results}"}'
+output_key = "synthesis"
+
+[[skill.steps]]
+tool_name = "memory_store"
+arguments_template = '{"content": "{synthesis}", "metadata": "topic_research:{topic}"}'
+output_key = "stored"
diff --git a/skills/builtin/translate-doc.toml b/skills/builtin/translate-doc.toml
new file mode 100644
index 00000000..5151a1d6
--- /dev/null
+++ b/skills/builtin/translate-doc.toml
@@ -0,0 +1,25 @@
+[skill]
+name = "translate-doc"
+description = "Translate a document from one language to another while preserving formatting"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "file_read"
+arguments_template = '{"path": "{file_path}"}'
+output_key = "original_content"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Detect the source language and analyze the document structure, formatting, and any specialized terminology: {original_content}"}'
+output_key = "analysis"
+
+[[skill.steps]]
+tool_name = "llm_call"
+arguments_template = '{"prompt": "Translate the following text to {target_language}, preserving all formatting, technical terms, and document structure: {original_content}"}'
+output_key = "translation"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Review the translation for accuracy, natural phrasing, and formatting preservation. Note any terms that may need human review: {translation}"}'
+output_key = "reviewed_translation"
diff --git a/skills/builtin/web-summarize.toml b/skills/builtin/web-summarize.toml
new file mode 100644
index 00000000..e55e643e
--- /dev/null
+++ b/skills/builtin/web-summarize.toml
@@ -0,0 +1,20 @@
+[skill]
+name = "web-summarize"
+description = "Fetch a web page and summarize its content into key points"
+version = "1.0.0"
+author = "openjarvis"
+
+[[skill.steps]]
+tool_name = "http_request"
+arguments_template = '{"url": "{url}"}'
+output_key = "page_content"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Extract the main content from the web page, ignoring navigation and ads: {page_content}"}'
+output_key = "clean_content"
+
+[[skill.steps]]
+tool_name = "think"
+arguments_template = '{"thought": "Summarize the content into 5-7 key bullet points: {clean_content}"}'
+output_key = "summary"
diff --git a/src/openjarvis/cli/__init__.py b/src/openjarvis/cli/__init__.py
index 965cc25b..d81597a5 100644
--- a/src/openjarvis/cli/__init__.py
+++ b/src/openjarvis/cli/__init__.py
@@ -7,16 +7,17 @@ import click
import openjarvis
from openjarvis.cli.add_cmd import add
from openjarvis.cli.agent_cmd import agent
-from openjarvis.cli.operators_cmd import operators
from openjarvis.cli.ask import ask
from openjarvis.cli.bench_cmd import bench
from openjarvis.cli.channel_cmd import channel
from openjarvis.cli.chat_cmd import chat
from openjarvis.cli.daemon_cmd import restart, start, status, stop
from openjarvis.cli.doctor_cmd import doctor
+from openjarvis.cli.eval_cmd import eval_group
from openjarvis.cli.init_cmd import init
from openjarvis.cli.memory_cmd import memory
from openjarvis.cli.model import model
+from openjarvis.cli.operators_cmd import operators
from openjarvis.cli.scheduler_cmd import scheduler
from openjarvis.cli.serve import serve
from openjarvis.cli.skill_cmd import skill
@@ -52,6 +53,7 @@ cli.add_command(status, "status")
cli.add_command(vault, "vault")
cli.add_command(add, "add")
cli.add_command(operators, "operators")
+cli.add_command(eval_group, "eval")
def main() -> None:
diff --git a/src/openjarvis/cli/eval_cmd.py b/src/openjarvis/cli/eval_cmd.py
new file mode 100644
index 00000000..eee684d9
--- /dev/null
+++ b/src/openjarvis/cli/eval_cmd.py
@@ -0,0 +1,422 @@
+"""``jarvis eval`` — evaluation framework CLI commands."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Optional
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+# Known benchmarks and backends — mirrored from the evals framework so the
+# CLI can display them even when the (optional) evals package is not installed.
+KNOWN_BENCHMARKS = {
+ "supergpqa": {"category": "reasoning", "description": "SuperGPQA multiple-choice"},
+ "gpqa": {"category": "reasoning", "description": "GPQA graduate-level MCQ"},
+ "mmlu-pro": {"category": "reasoning", "description": "MMLU-Pro multiple-choice"},
+ "math500": {"category": "reasoning", "description": "MATH-500 math problems"},
+ "natural-reasoning": {"category": "reasoning", "description": "Natural Reasoning"},
+ "hle": {"category": "reasoning", "description": "HLE hard challenges"},
+ "simpleqa": {"category": "chat", "description": "SimpleQA factual QA"},
+ "wildchat": {"category": "chat", "description": "WildChat conversation quality"},
+ "ipw": {"category": "chat", "description": "IPW mixed benchmark"},
+ "gaia": {"category": "agentic", "description": "GAIA agentic benchmark"},
+ "frames": {"category": "rag", "description": "FRAMES multi-hop RAG"},
+ "swebench": {"category": "agentic", "description": "SWE-bench code patches"},
+ "swefficiency": {"category": "agentic", "description": "SWEfficiency optimization"},
+ "terminalbench": {
+ "category": "agentic", "description": "TerminalBench terminal tasks",
+ },
+ "terminalbench-native": {
+ "category": "agentic",
+ "description": "TerminalBench Native (Docker)",
+ },
+}
+
+KNOWN_BACKENDS = {
+ "jarvis-direct": "Engine-level inference (local or cloud)",
+ "jarvis-agent": "Agent-level inference with tool calling",
+}
+
+
+@click.group("eval")
+def eval_group() -> None:
+ """Evaluation framework — benchmark models, agents, and learning."""
+
+
+@eval_group.command("list")
+def eval_list() -> None:
+ """List available benchmarks and backends."""
+ console = Console()
+
+ bench_table = Table(
+ title="[bold]Available Benchmarks[/bold]",
+ border_style="bright_blue",
+ title_style="bold cyan",
+ )
+ bench_table.add_column("Name", style="cyan", no_wrap=True)
+ bench_table.add_column("Category", style="white")
+ bench_table.add_column("Description")
+ for name, info in KNOWN_BENCHMARKS.items():
+ bench_table.add_row(name, info["category"], info["description"])
+ console.print(bench_table)
+
+ backend_table = Table(
+ title="[bold]Available Backends[/bold]",
+ border_style="bright_blue",
+ title_style="bold cyan",
+ )
+ backend_table.add_column("Name", style="cyan", no_wrap=True)
+ backend_table.add_column("Description")
+ for name, desc in KNOWN_BACKENDS.items():
+ backend_table.add_row(name, desc)
+ console.print(backend_table)
+
+
+@eval_group.command("run")
+@click.option(
+ "-c", "--config", "config_path", default=None, type=click.Path(),
+ help="TOML config file for suite runs.",
+)
+@click.option(
+ "-b", "--benchmark", "benchmark", default=None,
+ help="Benchmark to run (e.g. supergpqa, gaia, frames, wildchat).",
+)
+@click.option(
+ "-m", "--model", "model", default=None,
+ help="Model identifier.",
+)
+@click.option(
+ "-n", "--max-samples", "max_samples", type=int, default=None,
+ help="Maximum samples to evaluate.",
+)
+@click.option(
+ "--backend", "backend", default="jarvis-direct",
+ help="Inference backend (jarvis-direct or jarvis-agent).",
+)
+@click.option(
+ "--agent", "agent_name", default=None,
+ help="Agent name for jarvis-agent backend.",
+)
+@click.option(
+ "--tools", "tools", default="",
+ help="Comma-separated tool names.",
+)
+@click.option(
+ "--telemetry/--no-telemetry", "telemetry", default=False,
+ help="Enable telemetry collection during eval.",
+)
+@click.option(
+ "-o", "--output", "output_path", default=None, type=click.Path(),
+ help="Output JSONL path.",
+)
+@click.option(
+ "-v", "--verbose", "verbose", is_flag=True, default=False,
+ help="Verbose logging.",
+)
+def eval_run(
+ config_path: Optional[str],
+ benchmark: Optional[str],
+ model: Optional[str],
+ max_samples: Optional[int],
+ backend: str,
+ agent_name: Optional[str],
+ tools: str,
+ telemetry: bool,
+ output_path: Optional[str],
+ verbose: bool,
+) -> None:
+ """Run evaluation benchmarks."""
+ console = Console(stderr=True)
+
+ # Config-driven mode: load TOML suite, expand, run all
+ if config_path is not None:
+ try:
+ from evals.core.config import expand_suite, load_eval_config
+ except ImportError:
+ console.print(
+ "[red]Eval framework not available. "
+ "Ensure the evals package is importable.[/red]"
+ )
+ sys.exit(1)
+
+ try:
+ suite = load_eval_config(config_path)
+ run_configs = expand_suite(suite)
+ except Exception as exc:
+ console.print(f"[red]Error loading config: {exc}[/red]")
+ sys.exit(1)
+
+ console.print(
+ f"[cyan]Suite:[/cyan] {suite.meta.name or Path(config_path).stem}"
+ )
+ console.print(
+ f"[cyan]Matrix:[/cyan] {len(suite.models)} model(s) x "
+ f"{len(suite.benchmarks)} benchmark(s) = {len(run_configs)} run(s)"
+ )
+
+ try:
+ from evals.cli import _run_single
+ except ImportError:
+ console.print(
+ "[red]Eval CLI module not available.[/red]"
+ )
+ sys.exit(1)
+
+ for i, rc in enumerate(run_configs, 1):
+ console.print(
+ f"\n[bold]Run {i}/{len(run_configs)}:[/bold] "
+ f"{rc.benchmark} / {rc.model}"
+ )
+ try:
+ summary = _run_single(rc, console=console)
+ console.print(
+ f" [green]{summary.accuracy:.4f}[/green] "
+ f"({summary.correct}/{summary.scored_samples})"
+ )
+ except Exception as exc:
+ console.print(f" [red bold]FAILED:[/red bold] {exc}")
+
+ return
+
+ # CLI-driven mode: require --benchmark and --model
+ if benchmark is None or model is None:
+ raise click.UsageError(
+ "Provide either --config/-c for suite mode, "
+ "or both --benchmark/-b and --model/-m for single-run mode."
+ )
+
+ if benchmark not in KNOWN_BENCHMARKS:
+ console.print(
+ f"[yellow]Warning: unknown benchmark '{benchmark}'[/yellow]"
+ )
+
+ try:
+ from evals.core.types import RunConfig
+ except ImportError:
+ console.print(
+ "[red]Eval framework not available. "
+ "Ensure the evals package is importable.[/red]"
+ )
+ sys.exit(1)
+
+ tool_list = (
+ [t.strip() for t in tools.split(",") if t.strip()] if tools else []
+ )
+
+ config = RunConfig(
+ benchmark=benchmark,
+ backend=backend,
+ model=model,
+ max_samples=max_samples,
+ agent_name=agent_name,
+ tools=tool_list,
+ output_path=output_path,
+ telemetry=telemetry,
+ )
+
+ try:
+ from evals.cli import _run_single
+
+ console.print(
+ f"[cyan]Benchmark:[/cyan] {benchmark}\n"
+ f"[cyan]Model:[/cyan] {model}\n"
+ f"[cyan]Backend:[/cyan] {backend}"
+ )
+ summary = _run_single(config, console=console)
+ console.print(
+ f"\n[green]Accuracy: {summary.accuracy:.4f}[/green] "
+ f"({summary.correct}/{summary.scored_samples})"
+ )
+ except ImportError:
+ console.print(
+ "[red]Eval CLI module not available.[/red]"
+ )
+ sys.exit(1)
+ except Exception as exc:
+ console.print(f"[red]Error: {exc}[/red]")
+ sys.exit(1)
+
+
+@eval_group.command("compare")
+@click.argument("result_files", nargs=-1, required=True)
+@click.option(
+ "--metric", "metric", default="accuracy",
+ help="Metric to compare across runs (default: accuracy).",
+)
+def eval_compare(result_files: tuple[str, ...], metric: str) -> None:
+ """Compare results from multiple eval runs."""
+ console = Console()
+
+ rows: list[dict] = []
+ for path_str in result_files:
+ path = Path(path_str)
+ summary_path = path.with_suffix(".summary.json")
+
+ if summary_path.exists():
+ with open(summary_path) as f:
+ data = json.load(f)
+ rows.append({
+ "file": path.name,
+ "benchmark": data.get("benchmark", "?"),
+ "model": data.get("model", "?"),
+ "value": data.get(metric, "N/A"),
+ "samples": data.get("total_samples", 0),
+ })
+ elif path.exists() and path.suffix == ".jsonl":
+ # Fall back to reading JSONL and computing metric on-the-fly
+ records = []
+ with open(path) as f:
+ for line in f:
+ line = line.strip()
+ if line:
+ records.append(json.loads(line))
+ if records:
+ if metric == "accuracy":
+ scored = [
+ r for r in records
+ if r.get("is_correct") is not None
+ ]
+ correct = [r for r in scored if r["is_correct"]]
+ value = (
+ len(correct) / len(scored) if scored else 0.0
+ )
+ else:
+ values = [
+ r[metric] for r in records
+ if metric in r and r[metric] is not None
+ ]
+ value = (
+ sum(values) / len(values) if values else "N/A"
+ )
+ rows.append({
+ "file": path.name,
+ "benchmark": records[0].get("benchmark", "?"),
+ "model": records[0].get("model", "?"),
+ "value": value,
+ "samples": len(records),
+ })
+ else:
+ console.print(f"[yellow]Skipping missing file: {path_str}[/yellow]")
+
+ if not rows:
+ console.print("[red]No valid result files found.[/red]")
+ return
+
+ table = Table(
+ title=f"[bold]Comparison — {metric}[/bold]",
+ border_style="bright_blue",
+ title_style="bold cyan",
+ )
+ table.add_column("File", style="dim")
+ table.add_column("Benchmark", style="cyan")
+ table.add_column("Model", style="green")
+ table.add_column(metric.capitalize(), justify="right", style="bold")
+ table.add_column("Samples", justify="right", style="dim")
+
+ for row in rows:
+ val = row["value"]
+ val_str = f"{val:.4f}" if isinstance(val, float) else str(val)
+ table.add_row(
+ row["file"], row["benchmark"], row["model"],
+ val_str, str(row["samples"]),
+ )
+
+ console.print(table)
+
+
+@eval_group.command("report")
+@click.argument("result_file")
+def eval_report(result_file: str) -> None:
+ """Generate detailed report from eval results."""
+ console = Console()
+ path = Path(result_file)
+
+ # Try summary JSON first
+ summary_path = path.with_suffix(".summary.json")
+ if summary_path.exists():
+ with open(summary_path) as f:
+ data = json.load(f)
+
+ console.print("[bold cyan]Evaluation Report[/bold cyan]")
+ console.print(f" [cyan]Benchmark:[/cyan] {data.get('benchmark', '?')}")
+ console.print(f" [cyan]Model:[/cyan] {data.get('model', '?')}")
+ console.print(f" [cyan]Backend:[/cyan] {data.get('backend', '?')}")
+ console.print(
+ f" [cyan]Accuracy:[/cyan] "
+ f"[bold]{data.get('accuracy', 0.0):.4f}[/bold]"
+ )
+ console.print(f" [cyan]Samples:[/cyan] {data.get('total_samples', 0)}")
+ console.print(f" [cyan]Scored:[/cyan] {data.get('scored_samples', 0)}")
+ console.print(f" [cyan]Correct:[/cyan] {data.get('correct', 0)}")
+ console.print(f" [cyan]Errors:[/cyan] {data.get('errors', 0)}")
+ console.print(
+ f" [cyan]Latency:[/cyan] "
+ f"{data.get('mean_latency_seconds', 0.0):.4f}s (mean)"
+ )
+ console.print(
+ f" [cyan]Cost:[/cyan] "
+ f"${data.get('total_cost_usd', 0.0):.6f}"
+ )
+
+ # Per-subject breakdown
+ per_subject = data.get("per_subject", {})
+ if per_subject and len(per_subject) > 1:
+ sub_table = Table(
+ title="[bold]Per-Subject Breakdown[/bold]",
+ border_style="bright_blue",
+ )
+ sub_table.add_column("Subject", style="cyan")
+ sub_table.add_column("Accuracy", justify="right", style="bold")
+ sub_table.add_column("Total", justify="right")
+ sub_table.add_column("Correct", justify="right", style="green")
+
+ for subj, stats in sorted(per_subject.items()):
+ sub_table.add_row(
+ subj,
+ f"{stats.get('accuracy', 0.0):.4f}",
+ str(int(stats.get("total", 0))),
+ str(int(stats.get("correct", 0))),
+ )
+ console.print(sub_table)
+
+ return
+
+ # Fall back to JSONL file
+ if not path.exists():
+ console.print(f"[red]File not found: {result_file}[/red]")
+ sys.exit(1)
+
+ records: list[dict] = []
+ with open(path) as f:
+ for line in f:
+ line = line.strip()
+ if line:
+ records.append(json.loads(line))
+
+ if not records:
+ console.print("[yellow]No records found in file.[/yellow]")
+ return
+
+ scored = [r for r in records if r.get("is_correct") is not None]
+ correct = [r for r in scored if r["is_correct"]]
+ errors = [r for r in records if r.get("error")]
+ accuracy = len(correct) / len(scored) if scored else 0.0
+
+ console.print("[bold cyan]Evaluation Report[/bold cyan]")
+ console.print(f" [cyan]File:[/cyan] {result_file}")
+ console.print(f" [cyan]Benchmark:[/cyan] {records[0].get('benchmark', '?')}")
+ console.print(f" [cyan]Model:[/cyan] {records[0].get('model', '?')}")
+ console.print(f" [cyan]Total:[/cyan] {len(records)}")
+ console.print(f" [cyan]Scored:[/cyan] {len(scored)}")
+ console.print(f" [cyan]Correct:[/cyan] {len(correct)}")
+ console.print(
+ f" [cyan]Accuracy:[/cyan] [bold]{accuracy:.4f}[/bold]"
+ )
+ console.print(f" [cyan]Errors:[/cyan] {len(errors)}")
+
+
+__all__ = ["eval_group"]
diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py
index 7644a374..1d16f373 100644
--- a/src/openjarvis/core/config.py
+++ b/src/openjarvis/core/config.py
@@ -402,6 +402,14 @@ class LearningConfig:
agent: AgentLearningConfig = field(default_factory=AgentLearningConfig)
metrics: MetricsConfig = field(default_factory=MetricsConfig)
+ # Training pipeline
+ training_enabled: bool = False
+ training_schedule: str = "" # cron expression or empty for on-demand
+ lora_rank: int = 16
+ lora_alpha: int = 32
+ min_sft_pairs: int = 50
+ min_improvement: float = 0.02
+
# Backward-compat properties for old flat field names
@property
def default_policy(self) -> str:
diff --git a/src/openjarvis/learning/__init__.py b/src/openjarvis/learning/__init__.py
index f7e3928e..838a91db 100644
--- a/src/openjarvis/learning/__init__.py
+++ b/src/openjarvis/learning/__init__.py
@@ -8,11 +8,15 @@ from openjarvis.learning._stubs import (
RouterPolicy,
RoutingContext,
)
+from openjarvis.learning.agent_evolver import AgentConfigEvolver
from openjarvis.learning.heuristic_reward import HeuristicRewardFunction
+from openjarvis.learning.learning_orchestrator import LearningOrchestrator
from openjarvis.learning.router import (
HeuristicRouter,
build_routing_context,
)
+from openjarvis.learning.training.data import TrainingDataMiner
+from openjarvis.learning.training.lora import HAS_TORCH, LoRATrainer, LoRATrainingConfig
def ensure_registered() -> None:
@@ -73,12 +77,18 @@ def ensure_registered() -> None:
__all__ = [
+ "AgentConfigEvolver",
+ "HAS_TORCH",
"HeuristicRewardFunction",
"HeuristicRouter",
+ "LearningOrchestrator",
+ "LoRATrainer",
+ "LoRATrainingConfig",
"QueryAnalyzer",
"RewardFunction",
"RouterPolicy",
"RoutingContext",
+ "TrainingDataMiner",
"build_routing_context",
"ensure_registered",
]
diff --git a/src/openjarvis/learning/agent_evolver.py b/src/openjarvis/learning/agent_evolver.py
new file mode 100644
index 00000000..fc84b1b2
--- /dev/null
+++ b/src/openjarvis/learning/agent_evolver.py
@@ -0,0 +1,361 @@
+"""AgentConfigEvolver — analyze traces to evolve agent TOML configs.
+
+Reads interaction traces to determine which agent/tool/parameter
+combinations perform best for different query classes, then writes
+updated TOML config files with automatic versioning and rollback.
+"""
+
+from __future__ import annotations
+
+import shutil
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from openjarvis.core.types import StepType, Trace
+from openjarvis.learning.trace_policy import classify_query
+from openjarvis.traces.store import TraceStore
+
+
+def _format_toml_value(value: Any) -> str:
+ """Format a Python value as a TOML literal."""
+ if isinstance(value, bool):
+ return "true" if value else "false"
+ if isinstance(value, int):
+ return str(value)
+ if isinstance(value, float):
+ return str(value)
+ if isinstance(value, str):
+ # Escape backslashes and quotes for TOML basic strings
+ escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+ return f'"{escaped}"'
+ if isinstance(value, list):
+ items = ", ".join(_format_toml_value(v) for v in value)
+ return f"[{items}]"
+ return repr(value)
+
+
+def _write_toml(path: Path, data: Dict[str, Any]) -> None:
+ """Write a dict as TOML to *path* using manual formatting."""
+ lines: List[str] = []
+ for section_name, section_data in data.items():
+ if isinstance(section_data, dict):
+ lines.append(f"[{section_name}]")
+ for key, value in section_data.items():
+ lines.append(f"{key} = {_format_toml_value(value)}")
+ lines.append("")
+ else:
+ lines.append(f"{section_name} = {_format_toml_value(section_data)}")
+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+class AgentConfigEvolver:
+ """Analyze traces to evolve agent TOML configs with versioning.
+
+ Parameters
+ ----------
+ trace_store:
+ A :class:`TraceStore` used to fetch historical traces.
+ config_dir:
+ Directory where agent TOML configs are written.
+ min_quality:
+ Minimum average feedback score for a recommendation to be emitted.
+ """
+
+ def __init__(
+ self,
+ trace_store: TraceStore,
+ *,
+ config_dir: Union[str, Path],
+ min_quality: float = 0.5,
+ ) -> None:
+ self._store = trace_store
+ self._config_dir = Path(config_dir)
+ self._history_dir = self._config_dir / ".history"
+ self._min_quality = min_quality
+
+ self._config_dir.mkdir(parents=True, exist_ok=True)
+ self._history_dir.mkdir(parents=True, exist_ok=True)
+
+ # ------------------------------------------------------------------
+ # analyze
+ # ------------------------------------------------------------------
+
+ def analyze(self) -> List[Dict[str, Any]]:
+ """Analyze traces, return recommendations per query class.
+
+ Returns a list of dicts, each containing:
+ - ``query_class``: the classified query category
+ - ``recommended_tools``: list of tool names sorted by frequency
+ - ``recommended_agent``: the best-performing agent for this class
+ - ``recommended_max_turns``: suggested max_turns value
+ - ``sample_count``: number of traces analyzed for this class
+ """
+ traces = self._store.list_traces(limit=10_000)
+ if not traces:
+ return []
+
+ # Group traces by query class
+ groups: Dict[str, List[Trace]] = defaultdict(list)
+ for trace in traces:
+ qclass = classify_query(trace.query)
+ groups[qclass].append(trace)
+
+ recommendations: List[Dict[str, Any]] = []
+ for qclass, class_traces in sorted(groups.items()):
+ rec = self._analyze_class(qclass, class_traces)
+ if rec is not None:
+ recommendations.append(rec)
+
+ return recommendations
+
+ def _analyze_class(
+ self, qclass: str, traces: List[Trace]
+ ) -> Optional[Dict[str, Any]]:
+ """Build a recommendation for a single query class."""
+ # Collect tool usage, agent performance, and turn counts
+ tool_scores: Dict[str, _ToolScore] = defaultdict(lambda: _ToolScore())
+ agent_scores: Dict[str, _AgentScore] = defaultdict(lambda: _AgentScore())
+ turn_counts: List[int] = []
+
+ for trace in traces:
+ feedback = trace.feedback if trace.feedback is not None else 0.0
+ is_success = trace.outcome == "success"
+
+ # Count tools used in this trace
+ trace_tools: List[str] = []
+ tool_call_count = 0
+ for step in trace.steps:
+ step_type = (
+ step.step_type.value
+ if isinstance(step.step_type, StepType)
+ else str(step.step_type)
+ )
+ if step_type == "tool_call":
+ tool_call_count += 1
+ tool_name = step.input.get("tool", "")
+ if tool_name:
+ trace_tools.append(tool_name)
+ ts = tool_scores[tool_name]
+ ts.count += 1
+ ts.feedback_sum += feedback
+ if is_success:
+ ts.successes += 1
+
+ turn_counts.append(tool_call_count)
+
+ # Track agent performance
+ if trace.agent:
+ ag = agent_scores[trace.agent]
+ ag.count += 1
+ ag.feedback_sum += feedback
+ if is_success:
+ ag.successes += 1
+
+ if not agent_scores:
+ return None
+
+ # Pick best agent by composite score
+ best_agent = max(
+ agent_scores.items(), key=lambda kv: kv[1].composite_score()
+ )[0]
+
+ # Rank tools by composite score (frequency-weighted quality)
+ ranked_tools = sorted(
+ tool_scores.items(),
+ key=lambda kv: kv[1].composite_score(),
+ reverse=True,
+ )
+ recommended_tools = [name for name, _ in ranked_tools]
+
+ # Recommended max_turns: use the 75th percentile of observed tool calls
+ # plus a small buffer, minimum 5
+ if turn_counts:
+ sorted_turns = sorted(turn_counts)
+ p75_idx = int(len(sorted_turns) * 0.75)
+ p75 = sorted_turns[min(p75_idx, len(sorted_turns) - 1)]
+ recommended_max_turns = max(p75 + 2, 5)
+ else:
+ recommended_max_turns = 10
+
+ return {
+ "query_class": qclass,
+ "recommended_tools": recommended_tools,
+ "recommended_agent": best_agent,
+ "recommended_max_turns": recommended_max_turns,
+ "sample_count": len(traces),
+ }
+
+ # ------------------------------------------------------------------
+ # write_config
+ # ------------------------------------------------------------------
+
+ def write_config(
+ self,
+ agent_name: str,
+ *,
+ tools: List[str],
+ max_turns: int = 10,
+ temperature: float = 0.3,
+ system_prompt: str = "",
+ ) -> Path:
+ """Write agent TOML config, archiving previous version first.
+
+ Returns the :class:`Path` to the written config file.
+ """
+ config_path = self._config_dir / f"{agent_name}.toml"
+
+ # Archive the existing config before overwriting
+ if config_path.exists():
+ self._archive(agent_name, config_path)
+
+ # Build the TOML data
+ data = {
+ "agent": {
+ "name": agent_name,
+ "tools": tools,
+ "max_turns": max_turns,
+ "temperature": temperature,
+ "system_prompt": system_prompt,
+ }
+ }
+
+ _write_toml(config_path, data)
+ return config_path
+
+ # ------------------------------------------------------------------
+ # list_versions
+ # ------------------------------------------------------------------
+
+ def list_versions(self, agent_name: str) -> List[Dict[str, Any]]:
+ """List all versions (including current) for *agent_name*.
+
+ Returns a list of dicts with ``version``, ``path``, and ``modified``.
+ Versions are numbered starting from 1 (oldest archived) through to
+ the current (highest version number).
+ """
+ versions: List[Dict[str, Any]] = []
+
+ # Collect archived versions from .history/
+ pattern = f"{agent_name}.v*.toml"
+ archived = sorted(self._history_dir.glob(pattern))
+ for idx, archived_path in enumerate(archived, start=1):
+ versions.append({
+ "version": idx,
+ "path": str(archived_path),
+ "modified": archived_path.stat().st_mtime,
+ })
+
+ # Current version
+ current = self._config_dir / f"{agent_name}.toml"
+ if current.exists():
+ versions.append({
+ "version": len(versions) + 1,
+ "path": str(current),
+ "modified": current.stat().st_mtime,
+ })
+
+ return versions
+
+ # ------------------------------------------------------------------
+ # rollback
+ # ------------------------------------------------------------------
+
+ def rollback(self, agent_name: str, version: int) -> None:
+ """Rollback to a specific version.
+
+ Raises :class:`ValueError` if the requested version does not exist.
+ """
+ versions = self.list_versions(agent_name)
+ target = None
+ for v in versions:
+ if v["version"] == version:
+ target = v
+ break
+
+ if target is None:
+ raise ValueError(
+ f"Version {version} not found for agent '{agent_name}'. "
+ f"Available versions: {[v['version'] for v in versions]}"
+ )
+
+ target_path = Path(target["path"])
+ config_path = self._config_dir / f"{agent_name}.toml"
+
+ # If the target is already the current file, nothing to do
+ if target_path == config_path:
+ return
+
+ # Archive current before rollback
+ if config_path.exists():
+ self._archive(agent_name, config_path)
+
+ # Copy the target version to become the current config
+ shutil.copy2(str(target_path), str(config_path))
+
+ # ------------------------------------------------------------------
+ # internal helpers
+ # ------------------------------------------------------------------
+
+ def _archive(self, agent_name: str, config_path: Path) -> Path:
+ """Copy *config_path* into ``.history/`` with a version suffix."""
+ # Determine next version number in .history/
+ pattern = f"{agent_name}.v*.toml"
+ existing = list(self._history_dir.glob(pattern))
+ version_nums = []
+ for p in existing:
+ # Extract version number from name like "my_agent.v3.toml"
+ stem = p.stem # "my_agent.v3"
+ parts = stem.rsplit(".v", 1)
+ if len(parts) == 2 and parts[1].isdigit():
+ version_nums.append(int(parts[1]))
+ next_ver = max(version_nums, default=0) + 1
+
+ dest = self._history_dir / f"{agent_name}.v{next_ver}.toml"
+ shutil.copy2(str(config_path), str(dest))
+ return dest
+
+
+class _ToolScore:
+ """Accumulator for per-tool scoring."""
+
+ __slots__ = ("count", "successes", "feedback_sum")
+
+ def __init__(self) -> None:
+ self.count = 0
+ self.successes = 0
+ self.feedback_sum = 0.0
+
+ def composite_score(self) -> float:
+ """Weighted score combining success rate, feedback, and frequency."""
+ if self.count == 0:
+ return 0.0
+ sr = self.successes / self.count
+ fb = self.feedback_sum / self.count
+ # Weight: 40% success + 40% feedback + 20% log-frequency
+ import math
+
+ freq_bonus = math.log1p(self.count) / 10.0
+ return 0.4 * sr + 0.4 * fb + 0.2 * min(freq_bonus, 1.0)
+
+
+class _AgentScore:
+ """Accumulator for per-agent scoring."""
+
+ __slots__ = ("count", "successes", "feedback_sum")
+
+ def __init__(self) -> None:
+ self.count = 0
+ self.successes = 0
+ self.feedback_sum = 0.0
+
+ def composite_score(self) -> float:
+ """Weighted score combining success rate and feedback."""
+ if self.count == 0:
+ return 0.0
+ sr = self.successes / self.count
+ fb = self.feedback_sum / self.count
+ return 0.6 * sr + 0.4 * fb
+
+
+__all__ = ["AgentConfigEvolver"]
diff --git a/src/openjarvis/learning/learning_orchestrator.py b/src/openjarvis/learning/learning_orchestrator.py
new file mode 100644
index 00000000..814c5c65
--- /dev/null
+++ b/src/openjarvis/learning/learning_orchestrator.py
@@ -0,0 +1,202 @@
+"""LearningOrchestrator — coordinate the full trace->learn->eval loop.
+
+Pulls traces from a :class:`TraceStore`, mines training data via
+:class:`TrainingDataMiner`, evolves agent configs via
+:class:`AgentConfigEvolver`, optionally runs LoRA fine-tuning, and
+gates acceptance on an evaluation function.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional, Union
+
+logger = logging.getLogger(__name__)
+
+
+class LearningOrchestrator:
+ """Orchestrate a single trace->learn->eval cycle.
+
+ Parameters
+ ----------
+ trace_store:
+ Object with ``list_traces(limit=...)`` returning ``List[Trace]``
+ (typically a :class:`TraceStore`).
+ config_dir:
+ Directory where agent TOML configs are written / evolved.
+ eval_fn:
+ Optional callable returning a float score (higher = better).
+ Called before and after learning to gate acceptance.
+ min_improvement:
+ Minimum improvement in eval score required to accept the update.
+ min_sft_pairs:
+ Minimum number of SFT pairs required to trigger LoRA training.
+ min_quality:
+ Minimum feedback quality threshold for :class:`TrainingDataMiner`.
+ lora_config:
+ Optional :class:`LoRATrainingConfig`. When provided (and enough
+ SFT pairs exist and ``torch`` is available), LoRA training runs.
+ model_name:
+ Model name for LoRA training (passed to :class:`LoRATrainer`).
+ """
+
+ def __init__(
+ self,
+ *,
+ trace_store: Any,
+ config_dir: Union[str, Path],
+ eval_fn: Optional[Callable[[], float]] = None,
+ min_improvement: float = 0.02,
+ min_sft_pairs: int = 10,
+ min_quality: float = 0.7,
+ lora_config: Optional[Any] = None,
+ model_name: Optional[str] = None,
+ ) -> None:
+ from openjarvis.learning.agent_evolver import AgentConfigEvolver
+ from openjarvis.learning.training.data import TrainingDataMiner
+
+ self._trace_store = trace_store
+ self._config_dir = Path(config_dir)
+ self._eval_fn = eval_fn
+ self._min_improvement = min_improvement
+ self._min_sft_pairs = min_sft_pairs
+ self._lora_config = lora_config
+ self._model_name = model_name
+
+ self._miner = TrainingDataMiner(trace_store, min_quality=min_quality)
+ self._evolver = AgentConfigEvolver(
+ trace_store, config_dir=self._config_dir
+ )
+
+ # ------------------------------------------------------------------
+ # public API
+ # ------------------------------------------------------------------
+
+ def run(self) -> Dict[str, Any]:
+ """Execute one learning cycle.
+
+ Returns a dict with at least ``timestamp`` and ``status`` keys.
+
+ Steps
+ -----
+ 1. Mine traces: extract sft_pairs, routing_pairs, agent_pairs
+ 2. If no data: return skipped
+ 3. Run baseline eval (if eval_fn provided)
+ 4. Update routing recommendations
+ 5. Evolve agent configs
+ 6. Run LoRA training (if lora_config provided AND enough pairs
+ AND torch available)
+ 7. Run post-learning eval (if eval_fn provided)
+ 8. Accept/reject based on improvement threshold
+ """
+ result: Dict[str, Any] = {
+ "timestamp": time.time(),
+ }
+
+ # 1. Mine training data from traces
+ sft_pairs = self._miner.extract_sft_pairs()
+ routing_pairs = self._miner.extract_routing_pairs()
+ agent_pairs = self._miner.extract_agent_config_pairs()
+
+ result["sft_pairs"] = len(sft_pairs)
+ result["routing_classes"] = len(routing_pairs)
+ result["agent_classes"] = len(agent_pairs)
+
+ # 2. Check if there is any data at all
+ total_data = len(sft_pairs) + len(routing_pairs) + len(agent_pairs)
+ if total_data == 0:
+ result["status"] = "skipped"
+ result["reason"] = "no training data available"
+ return result
+
+ # 3. Run baseline eval
+ baseline_score: Optional[float] = None
+ if self._eval_fn is not None:
+ baseline_score = self._eval_fn()
+ result["baseline_score"] = baseline_score
+
+ # 4. Update routing recommendations
+ result["routing_updated"] = len(routing_pairs) > 0
+
+ # 5. Evolve agent configs
+ recommendations = self._evolver.analyze()
+ result["agent_configs_evolved"] = len(recommendations) > 0
+ for rec in recommendations:
+ agent_name = rec.get("recommended_agent", "default")
+ tools = rec.get("recommended_tools", [])
+ max_turns = rec.get("recommended_max_turns", 10)
+ self._evolver.write_config(
+ agent_name, tools=tools, max_turns=max_turns
+ )
+
+ # 6. LoRA training (optional)
+ result["lora_training"] = None
+ if (
+ self._lora_config is not None
+ and len(sft_pairs) >= self._min_sft_pairs
+ ):
+ lora_result = self._try_lora_training(sft_pairs)
+ result["lora_training"] = lora_result
+
+ # 7. Post-learning eval
+ post_score: Optional[float] = None
+ if self._eval_fn is not None:
+ post_score = self._eval_fn()
+ result["post_score"] = post_score
+
+ # 8. Accept/reject based on improvement
+ if baseline_score is not None and post_score is not None:
+ improvement = post_score - baseline_score
+ result["improvement"] = improvement
+ if improvement >= self._min_improvement:
+ result["accepted"] = True
+ result["status"] = "completed"
+ else:
+ result["accepted"] = False
+ result["status"] = "rejected"
+ result["reason"] = (
+ f"eval improvement {improvement:.4f} below "
+ f"threshold {self._min_improvement}"
+ )
+ else:
+ # No eval gate — always accept
+ result["accepted"] = True
+ result["status"] = "completed"
+
+ return result
+
+ # ------------------------------------------------------------------
+ # internal helpers
+ # ------------------------------------------------------------------
+
+ def _try_lora_training(
+ self, sft_pairs: list[Dict[str, Any]]
+ ) -> Optional[Dict[str, Any]]:
+ """Attempt LoRA training, returning result or None on failure."""
+ try:
+ from openjarvis.learning.training.lora import (
+ HAS_TORCH,
+ LoRATrainer,
+ )
+ except ImportError:
+ logger.info("LoRA training skipped: training.lora not importable")
+ return {"status": "skipped", "reason": "lora module unavailable"}
+
+ if not HAS_TORCH:
+ logger.info("LoRA training skipped: torch not available")
+ return {"status": "skipped", "reason": "torch not available"}
+
+ try:
+ model_name = self._model_name or "Qwen/Qwen3-0.6B"
+ trainer = LoRATrainer(
+ self._lora_config, model_name=model_name
+ )
+ return trainer.train(sft_pairs)
+ except Exception as exc:
+ logger.warning("LoRA training failed: %s", exc)
+ return {"status": "error", "reason": str(exc)}
+
+
+__all__ = ["LearningOrchestrator"]
diff --git a/src/openjarvis/learning/training/__init__.py b/src/openjarvis/learning/training/__init__.py
new file mode 100644
index 00000000..3ad00969
--- /dev/null
+++ b/src/openjarvis/learning/training/__init__.py
@@ -0,0 +1,15 @@
+"""Training data extraction and fine-tuning pipelines for trace-driven learning."""
+
+from openjarvis.learning.training.data import TrainingDataMiner
+from openjarvis.learning.training.lora import (
+ HAS_TORCH,
+ LoRATrainer,
+ LoRATrainingConfig,
+)
+
+__all__ = [
+ "HAS_TORCH",
+ "LoRATrainer",
+ "LoRATrainingConfig",
+ "TrainingDataMiner",
+]
diff --git a/src/openjarvis/learning/training/data.py b/src/openjarvis/learning/training/data.py
new file mode 100644
index 00000000..e311f87c
--- /dev/null
+++ b/src/openjarvis/learning/training/data.py
@@ -0,0 +1,214 @@
+"""TrainingDataMiner — extract supervised training pairs from the TraceStore.
+
+Provides three extraction modes:
+
+* **SFT pairs** — (input, output) pairs from high-quality traces for
+ supervised fine-tuning.
+* **Routing pairs** — per-query-class statistics identifying the best
+ model for each class.
+* **Agent config pairs** — per-query-class statistics identifying the
+ best agent and tool combination.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any, Dict, List
+
+from openjarvis.core.types import StepType, Trace
+from openjarvis.learning.trace_policy import classify_query
+
+
+class TrainingDataMiner:
+ """Extract supervised training pairs from stored traces.
+
+ Parameters
+ ----------
+ trace_store:
+ Any object with a ``list_traces(limit=...)`` method returning
+ ``List[Trace]`` (typically a :class:`TraceStore`).
+ min_quality:
+ Minimum ``feedback`` score for a trace to be included.
+ min_samples_per_class:
+ Minimum number of samples a query class must have to appear in
+ routing/agent-config results.
+ """
+
+ def __init__(
+ self,
+ trace_store: Any,
+ *,
+ min_quality: float = 0.7,
+ min_samples_per_class: int = 1,
+ ) -> None:
+ self._store = trace_store
+ self._min_quality = min_quality
+ self._min_samples_per_class = min_samples_per_class
+
+ # -- helpers ----------------------------------------------------------------
+
+ def _quality_traces(self) -> List[Trace]:
+ """Return traces whose feedback meets the quality threshold."""
+ all_traces = self._store.list_traces(limit=10000)
+ return [
+ t
+ for t in all_traces
+ if t.feedback is not None
+ and t.feedback >= self._min_quality
+ and t.outcome == "success"
+ ]
+
+ @staticmethod
+ def _tools_from_trace(trace: Trace) -> List[str]:
+ """Extract tool names from TOOL_CALL steps in a trace."""
+ tools: List[str] = []
+ for step in trace.steps:
+ if step.step_type == StepType.TOOL_CALL:
+ tool_name = step.input.get("tool")
+ if tool_name:
+ tools.append(tool_name)
+ return tools
+
+ # -- public API -------------------------------------------------------------
+
+ def extract_sft_pairs(self) -> List[Dict[str, Any]]:
+ """Return SFT training pairs from high-quality traces.
+
+ Each entry is a dict with keys: ``input``, ``output``,
+ ``query_class``, ``model``, ``feedback``.
+
+ Duplicate ``(input, output)`` pairs are collapsed; the first
+ occurrence is kept.
+ """
+ traces = self._quality_traces()
+ seen: set[tuple[str, str]] = set()
+ pairs: List[Dict[str, Any]] = []
+
+ for t in traces:
+ key = (t.query, t.result)
+ if key in seen:
+ continue
+ seen.add(key)
+ pairs.append(
+ {
+ "input": t.query,
+ "output": t.result,
+ "query_class": classify_query(t.query),
+ "model": t.model,
+ "feedback": t.feedback,
+ }
+ )
+
+ return pairs
+
+ def extract_routing_pairs(self) -> Dict[str, Dict[str, Any]]:
+ """Return per-query-class routing recommendations.
+
+ Returns a dict mapping query class to:
+
+ * ``best_model`` — model with highest average feedback for the class.
+ * ``avg_feedback`` — average feedback across all models for the class.
+ * ``sample_count`` — total number of qualifying traces in the class.
+ * ``all_models`` — dict of ``{model: {"avg_feedback": float, "count": int}}``.
+ """
+ traces = self._quality_traces()
+
+ # Accumulate per (query_class, model) feedback scores
+ class_model_scores: Dict[str, Dict[str, List[float]]] = defaultdict(
+ lambda: defaultdict(list)
+ )
+ for t in traces:
+ qc = classify_query(t.query)
+ class_model_scores[qc][t.model].append(t.feedback) # type: ignore[arg-type]
+
+ result: Dict[str, Dict[str, Any]] = {}
+ for qc, model_scores in class_model_scores.items():
+ total_count = sum(len(scores) for scores in model_scores.values())
+ if total_count < self._min_samples_per_class:
+ continue
+
+ all_models: Dict[str, Dict[str, Any]] = {}
+ best_model = ""
+ best_avg = -1.0
+
+ for model, scores in model_scores.items():
+ avg = sum(scores) / len(scores)
+ all_models[model] = {"avg_feedback": avg, "count": len(scores)}
+ if avg > best_avg:
+ best_avg = avg
+ best_model = model
+
+ total_scores = [s for scores in model_scores.values() for s in scores]
+ overall_avg = sum(total_scores) / len(total_scores) if total_scores else 0.0
+
+ result[qc] = {
+ "best_model": best_model,
+ "avg_feedback": overall_avg,
+ "sample_count": total_count,
+ "all_models": all_models,
+ }
+
+ return result
+
+ def extract_agent_config_pairs(self) -> Dict[str, Dict[str, Any]]:
+ """Return per-query-class agent and tool recommendations.
+
+ Returns a dict mapping query class to:
+
+ * ``best_agent`` — agent with the highest average feedback.
+ * ``best_tools`` — most frequently used tools by the best agent.
+ * ``avg_feedback`` — average feedback across all agents for the class.
+ * ``sample_count`` — total number of qualifying traces in the class.
+ """
+ traces = self._quality_traces()
+
+ # Accumulate per (query_class, agent) feedback and tools
+ class_agent_scores: Dict[str, Dict[str, List[float]]] = defaultdict(
+ lambda: defaultdict(list)
+ )
+ class_agent_tools: Dict[str, Dict[str, List[List[str]]]] = defaultdict(
+ lambda: defaultdict(list)
+ )
+
+ for t in traces:
+ qc = classify_query(t.query)
+ class_agent_scores[qc][t.agent].append(t.feedback) # type: ignore[arg-type]
+ tools = self._tools_from_trace(t)
+ class_agent_tools[qc][t.agent].append(tools)
+
+ result: Dict[str, Dict[str, Any]] = {}
+ for qc, agent_scores in class_agent_scores.items():
+ total_count = sum(len(scores) for scores in agent_scores.values())
+ if total_count < self._min_samples_per_class:
+ continue
+
+ best_agent = ""
+ best_avg = -1.0
+ for agent, scores in agent_scores.items():
+ avg = sum(scores) / len(scores)
+ if avg > best_avg:
+ best_avg = avg
+ best_agent = agent
+
+ # Collect tool frequency for best agent
+ tool_freq: Dict[str, int] = defaultdict(int)
+ for tool_list in class_agent_tools[qc].get(best_agent, []):
+ for tool in tool_list:
+ tool_freq[tool] += 1
+
+ best_tools = sorted(tool_freq, key=tool_freq.get, reverse=True) # type: ignore[arg-type]
+
+ total_scores = [s for scores in agent_scores.values() for s in scores]
+ overall_avg = sum(total_scores) / len(total_scores) if total_scores else 0.0
+
+ result[qc] = {
+ "best_agent": best_agent,
+ "best_tools": best_tools,
+ "avg_feedback": overall_avg,
+ "sample_count": total_count,
+ }
+
+ return result
+
+
+__all__ = ["TrainingDataMiner"]
diff --git a/src/openjarvis/learning/training/lora.py b/src/openjarvis/learning/training/lora.py
new file mode 100644
index 00000000..7a57e7be
--- /dev/null
+++ b/src/openjarvis/learning/training/lora.py
@@ -0,0 +1,438 @@
+"""LoRATrainer — fine-tune local models via LoRA/QLoRA from trace-derived SFT pairs.
+
+All ``torch``, ``transformers``, and ``peft`` imports are guarded so the
+module can be imported without GPU dependencies. The :class:`LoRATrainingConfig`
+dataclass works without any optional deps; :class:`LoRATrainer` raises
+``ImportError`` at construction time when ``torch`` is unavailable.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Optional imports -----------------------------------------------------------
+try:
+ import torch
+
+ HAS_TORCH = True
+except ImportError:
+ HAS_TORCH = False
+ torch = None # type: ignore[assignment]
+
+try:
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ HAS_TRANSFORMERS = True
+except ImportError:
+ HAS_TRANSFORMERS = False
+ AutoModelForCausalLM = None # type: ignore[assignment,misc]
+ AutoTokenizer = None # type: ignore[assignment,misc]
+
+try:
+ from peft import LoraConfig, TaskType, get_peft_model
+
+ HAS_PEFT = True
+except ImportError:
+ HAS_PEFT = False
+ LoraConfig = None # type: ignore[assignment,misc]
+ TaskType = None # type: ignore[assignment,misc]
+ get_peft_model = None # type: ignore[assignment,misc]
+
+
+# ---------------------------------------------------------------------------
+# Device selection (shared with orchestrator sft_trainer)
+# ---------------------------------------------------------------------------
+
+
+def _select_device(hint: Optional[str] = None) -> str:
+ """Select the best available PyTorch device.
+
+ Priority: explicit *hint* > cuda > mps > cpu.
+ """
+ if hint is not None:
+ return hint
+ if not HAS_TORCH or torch is None:
+ return "cpu"
+ if torch.cuda.is_available():
+ return "cuda"
+ if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+ return "mps"
+ return "cpu"
+
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class LoRATrainingConfig:
+ """Configuration for LoRA / QLoRA fine-tuning."""
+
+ # LoRA params
+ lora_rank: int = 16
+ lora_alpha: int = 32
+ lora_dropout: float = 0.05
+ target_modules: List[str] = field(
+ default_factory=lambda: ["q_proj", "v_proj"]
+ )
+
+ # Training params
+ num_epochs: int = 3
+ batch_size: int = 4
+ learning_rate: float = 2e-5
+ weight_decay: float = 0.01
+ warmup_ratio: float = 0.1
+ max_grad_norm: float = 1.0
+ max_seq_length: int = 2048
+
+ # QLoRA
+ use_4bit: bool = False
+
+ # Output
+ output_dir: str = "checkpoints/lora"
+ save_every_n_epochs: int = 1
+
+ # Memory
+ gradient_checkpointing: bool = True
+
+ def __post_init__(self) -> None:
+ if self.lora_rank < 1:
+ raise ValueError(
+ f"lora_rank must be >= 1, got {self.lora_rank}"
+ )
+ if self.num_epochs < 1:
+ raise ValueError(
+ f"num_epochs must be >= 1, got {self.num_epochs}"
+ )
+
+
+# ---------------------------------------------------------------------------
+# Trainer
+# ---------------------------------------------------------------------------
+
+
+class LoRATrainer:
+ """Fine-tune a local causal LM with LoRA (or QLoRA) adapters.
+
+ Parameters
+ ----------
+ config:
+ LoRA training configuration.
+ model_name:
+ HuggingFace model identifier or local path.
+ device:
+ PyTorch device string. ``None`` auto-detects (cuda > mps > cpu).
+
+ Raises
+ ------
+ ImportError
+ If ``torch`` is not installed.
+ """
+
+ def __init__(
+ self,
+ config: LoRATrainingConfig,
+ *,
+ model_name: str = "Qwen/Qwen3-0.6B",
+ device: Optional[str] = None,
+ ) -> None:
+ if not HAS_TORCH:
+ raise ImportError(
+ "torch is required for LoRATrainer. "
+ "Install with: pip install torch transformers peft"
+ )
+
+ self.config = config
+ self.model_name = model_name
+ self.device = _select_device(device)
+ self.tokenizer: Any = None
+ self.model: Any = None
+
+ # -- Public API ----------------------------------------------------------
+
+ def prepare_dataset(
+ self, pairs: List[Dict[str, Any]]
+ ) -> List[Dict[str, Any]]:
+ """Convert SFT pairs to tokenized examples.
+
+ Each returned dict contains ``input_ids``, ``attention_mask``,
+ and ``text`` (the raw formatted string before tokenization).
+
+ Parameters
+ ----------
+ pairs:
+ List of dicts with at least ``input`` and ``output`` keys,
+ as produced by :class:`TrainingDataMiner.extract_sft_pairs`.
+ """
+ self._ensure_tokenizer()
+
+ dataset: List[Dict[str, Any]] = []
+ for pair in pairs:
+ text = self._format_pair(pair)
+ encoding = self.tokenizer(
+ text,
+ truncation=True,
+ max_length=self.config.max_seq_length,
+ padding="max_length",
+ return_tensors="pt",
+ )
+ dataset.append({
+ "input_ids": encoding["input_ids"].squeeze(0),
+ "attention_mask": encoding["attention_mask"].squeeze(0),
+ "text": text,
+ })
+
+ return dataset
+
+ def train(self, pairs: List[Dict[str, Any]]) -> Dict[str, Any]:
+ """Run LoRA fine-tuning on the given SFT pairs.
+
+ Parameters
+ ----------
+ pairs:
+ List of dicts with at least ``input`` and ``output`` keys.
+
+ Returns
+ -------
+ dict
+ Training summary with keys: ``status``, ``epochs``,
+ ``total_steps``, ``avg_loss``, ``adapter_path``,
+ ``training_samples``.
+ """
+ if not pairs:
+ return {"status": "skipped", "reason": "no training data"}
+
+ dataset = self.prepare_dataset(pairs)
+ self._load_model()
+ self._apply_lora()
+
+ optimizer = torch.optim.AdamW(
+ self.model.parameters(),
+ lr=self.config.learning_rate,
+ weight_decay=self.config.weight_decay,
+ )
+
+ total_steps = 0
+ cumulative_loss = 0.0
+ num_loss_steps = 0
+
+ self.model.train()
+
+ for epoch in range(self.config.num_epochs):
+ epoch_loss = self._train_epoch(dataset, optimizer)
+ steps_in_epoch = max(
+ 1, (len(dataset) + self.config.batch_size - 1) // self.config.batch_size
+ )
+ total_steps += steps_in_epoch
+ cumulative_loss += epoch_loss * steps_in_epoch
+ num_loss_steps += steps_in_epoch
+
+ logger.info(
+ "Epoch %d/%d loss=%.4f",
+ epoch + 1,
+ self.config.num_epochs,
+ epoch_loss,
+ )
+
+ if (epoch + 1) % self.config.save_every_n_epochs == 0:
+ self._save_adapter(epoch + 1)
+
+ avg_loss = cumulative_loss / num_loss_steps if num_loss_steps else 0.0
+ adapter_path = str(Path(self.config.output_dir) / "final")
+ self._save_adapter_to(adapter_path)
+
+ return {
+ "status": "completed",
+ "epochs": self.config.num_epochs,
+ "total_steps": total_steps,
+ "avg_loss": avg_loss,
+ "adapter_path": adapter_path,
+ "training_samples": len(pairs),
+ }
+
+ # -- Internal helpers ----------------------------------------------------
+
+ def _ensure_tokenizer(self) -> None:
+ """Lazily load the tokenizer."""
+ if self.tokenizer is not None:
+ return
+
+ if not HAS_TRANSFORMERS:
+ raise ImportError(
+ "transformers is required for LoRATrainer. "
+ "Install with: pip install transformers"
+ )
+
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+ if self.tokenizer.pad_token is None:
+ self.tokenizer.pad_token = self.tokenizer.eos_token
+
+ def _load_model(self) -> None:
+ """Load the base model for fine-tuning."""
+ if self.model is not None:
+ return
+
+ if not HAS_TRANSFORMERS:
+ raise ImportError(
+ "transformers is required for LoRATrainer. "
+ "Install with: pip install transformers"
+ )
+
+ self._ensure_tokenizer()
+
+ model_kwargs: Dict[str, Any] = {"torch_dtype": torch.bfloat16}
+
+ if self.config.use_4bit:
+ try:
+ from transformers import BitsAndBytesConfig
+
+ model_kwargs["quantization_config"] = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_compute_dtype=torch.bfloat16,
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_quant_type="nf4",
+ )
+ except ImportError:
+ logger.warning(
+ "bitsandbytes not installed; falling back to bf16 "
+ "(QLoRA disabled)"
+ )
+
+ if self.device == "cuda" or self.device == "auto":
+ model_kwargs["device_map"] = "auto"
+ else:
+ model_kwargs["device_map"] = {"": self.device}
+
+ self.model = AutoModelForCausalLM.from_pretrained(
+ self.model_name, **model_kwargs
+ )
+
+ if self.config.gradient_checkpointing and hasattr(
+ self.model, "gradient_checkpointing_enable"
+ ):
+ self.model.gradient_checkpointing_enable(
+ gradient_checkpointing_kwargs={"use_reentrant": False}
+ )
+
+ def _apply_lora(self) -> None:
+ """Wrap the loaded model with LoRA adapters via peft."""
+ if not HAS_PEFT:
+ raise ImportError(
+ "peft is required for LoRA training. "
+ "Install with: pip install peft"
+ )
+
+ lora_config = LoraConfig(
+ task_type=TaskType.CAUSAL_LM,
+ r=self.config.lora_rank,
+ lora_alpha=self.config.lora_alpha,
+ lora_dropout=self.config.lora_dropout,
+ target_modules=self.config.target_modules,
+ )
+ self.model = get_peft_model(self.model, lora_config)
+ logger.info(
+ "LoRA applied: rank=%d, alpha=%d, targets=%s",
+ self.config.lora_rank,
+ self.config.lora_alpha,
+ self.config.target_modules,
+ )
+
+ def _format_pair(self, pair: Dict[str, Any]) -> str:
+ """Format an SFT pair as a chat-style training string."""
+ user_input = pair.get("input", "")
+ assistant_output = pair.get("output", "")
+
+ if self.tokenizer is not None and hasattr(
+ self.tokenizer, "apply_chat_template"
+ ):
+ try:
+ messages = [
+ {"role": "user", "content": user_input},
+ {"role": "assistant", "content": assistant_output},
+ ]
+ return self.tokenizer.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=False
+ )
+ except Exception:
+ pass # fall through to manual format
+
+ eos = ""
+ if self.tokenizer is not None:
+ eos = getattr(self.tokenizer, "eos_token", "") or ""
+ return f"<|user|>\n{user_input}\n<|assistant|>\n{assistant_output}{eos}"
+
+ def _train_epoch(
+ self,
+ dataset: List[Dict[str, Any]],
+ optimizer: Any,
+ ) -> float:
+ """Train one epoch over the dataset. Returns average loss."""
+ total_loss = 0.0
+ num_batches = 0
+
+ for i in range(0, len(dataset), self.config.batch_size):
+ batch_items = dataset[i : i + self.config.batch_size]
+ loss = self._train_step(batch_items, optimizer)
+ total_loss += loss
+ num_batches += 1
+
+ return total_loss / num_batches if num_batches else 0.0
+
+ def _train_step(
+ self,
+ batch_items: List[Dict[str, Any]],
+ optimizer: Any,
+ ) -> float:
+ """Execute a single training step on a micro-batch."""
+ input_ids = torch.stack(
+ [item["input_ids"] for item in batch_items]
+ ).to(self.device)
+ attention_mask = torch.stack(
+ [item["attention_mask"] for item in batch_items]
+ ).to(self.device)
+
+ outputs = self.model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ labels=input_ids,
+ )
+ loss = outputs.loss
+
+ optimizer.zero_grad()
+ loss.backward()
+ torch.nn.utils.clip_grad_norm_(
+ self.model.parameters(), self.config.max_grad_norm
+ )
+ optimizer.step()
+
+ return loss.item()
+
+ def _save_adapter(self, epoch: int) -> None:
+ """Save adapter checkpoint for the given epoch."""
+ path = str(Path(self.config.output_dir) / f"epoch_{epoch}")
+ self._save_adapter_to(path)
+
+ def _save_adapter_to(self, path: str) -> None:
+ """Save the LoRA adapter (and tokenizer) to *path*."""
+ out = Path(path)
+ out.mkdir(parents=True, exist_ok=True)
+
+ if self.model is not None:
+ self.model.save_pretrained(path)
+ if self.tokenizer is not None:
+ self.tokenizer.save_pretrained(path)
+
+ logger.info("Adapter saved to %s", path)
+
+
+__all__ = [
+ "HAS_TORCH",
+ "LoRATrainer",
+ "LoRATrainingConfig",
+]
diff --git a/src/openjarvis/recipes/__init__.py b/src/openjarvis/recipes/__init__.py
new file mode 100644
index 00000000..fc31ebc3
--- /dev/null
+++ b/src/openjarvis/recipes/__init__.py
@@ -0,0 +1,10 @@
+"""Recipe system — composable pillar configurations."""
+
+from openjarvis.recipes.loader import (
+ Recipe,
+ discover_recipes,
+ load_recipe,
+ resolve_recipe,
+)
+
+__all__ = ["Recipe", "discover_recipes", "load_recipe", "resolve_recipe"]
diff --git a/src/openjarvis/recipes/loader.py b/src/openjarvis/recipes/loader.py
new file mode 100644
index 00000000..233035be
--- /dev/null
+++ b/src/openjarvis/recipes/loader.py
@@ -0,0 +1,192 @@
+"""Recipe loader — load and resolve TOML recipe files."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+try:
+ import tomllib
+except ModuleNotFoundError:
+ import tomli as tomllib # type: ignore[no-redef]
+
+
+# Project-level recipes directory (sibling of src/)
+_PROJECT_RECIPES_DIR = Path(__file__).resolve().parents[3] / "recipes"
+# User-level recipes directory
+_USER_RECIPES_DIR = Path.home() / ".openjarvis" / "recipes"
+
+
+@dataclass(slots=True)
+class Recipe:
+ """A composable pillar configuration loaded from TOML."""
+
+ name: str
+ description: str = ""
+ version: str = "1.0.0"
+
+ # Intelligence
+ model: Optional[str] = None
+ quantization: Optional[str] = None
+
+ # Engine
+ engine_key: Optional[str] = None
+
+ # Agent
+ agent_type: Optional[str] = None
+ max_turns: Optional[int] = None
+ temperature: Optional[float] = None
+ tools: List[str] = field(default_factory=list)
+ system_prompt: Optional[str] = None
+
+ # Learning
+ routing_policy: Optional[str] = None
+ agent_policy: Optional[str] = None
+
+ # Eval
+ eval_suites: List[str] = field(default_factory=list)
+
+ # Raw TOML data for forward-compat
+ raw: Dict[str, Any] = field(default_factory=dict)
+
+ def to_builder_kwargs(self) -> Dict[str, Any]:
+ """Convert recipe fields to kwargs for SystemBuilder/Jarvis.
+
+ Returns a dict with only the non-None fields, keyed to match
+ the SystemBuilder fluent API or Jarvis constructor parameters.
+ """
+ kwargs: Dict[str, Any] = {}
+ if self.model is not None:
+ kwargs["model"] = self.model
+ if self.engine_key is not None:
+ kwargs["engine_key"] = self.engine_key
+ if self.agent_type is not None:
+ kwargs["agent"] = self.agent_type
+ if self.tools:
+ kwargs["tools"] = self.tools
+ if self.temperature is not None:
+ kwargs["temperature"] = self.temperature
+ if self.max_turns is not None:
+ kwargs["max_turns"] = self.max_turns
+ if self.system_prompt is not None:
+ kwargs["system_prompt"] = self.system_prompt
+ if self.routing_policy is not None:
+ kwargs["routing_policy"] = self.routing_policy
+ if self.agent_policy is not None:
+ kwargs["agent_policy"] = self.agent_policy
+ if self.quantization is not None:
+ kwargs["quantization"] = self.quantization
+ if self.eval_suites:
+ kwargs["eval_suites"] = self.eval_suites
+ return kwargs
+
+
+def load_recipe(path: str | Path) -> Recipe:
+ """Load a recipe from a TOML file.
+
+ Expected TOML format::
+
+ [recipe]
+ name = "coding_assistant"
+ description = "..."
+ version = "1.0.0"
+
+ [intelligence]
+ model = "qwen3:8b"
+ quantization = "q4_K_M"
+
+ [engine]
+ key = "ollama"
+
+ [agent]
+ type = "native_react"
+ max_turns = 10
+ temperature = 0.3
+ tools = ["file_read", "file_write", "code_interpreter", "think"]
+ system_prompt = "You are a coding assistant..."
+
+ [learning]
+ routing = "grpo"
+ agent = "icl_updater"
+
+ [eval]
+ suites = ["coding", "reasoning"]
+
+ Raises:
+ FileNotFoundError: If *path* does not exist.
+ """
+ path = Path(path)
+ if not path.exists():
+ raise FileNotFoundError(f"Recipe file not found: {path}")
+
+ with open(path, "rb") as fh:
+ data = tomllib.load(fh)
+
+ recipe_sec = data.get("recipe", {})
+ intel_sec = data.get("intelligence", {})
+ engine_sec = data.get("engine", {})
+ agent_sec = data.get("agent", {})
+ learning_sec = data.get("learning", {})
+ eval_sec = data.get("eval", {})
+
+ return Recipe(
+ name=recipe_sec.get("name", path.stem),
+ description=recipe_sec.get("description", ""),
+ version=recipe_sec.get("version", "1.0.0"),
+ model=intel_sec.get("model"),
+ quantization=intel_sec.get("quantization"),
+ engine_key=engine_sec.get("key"),
+ agent_type=agent_sec.get("type"),
+ max_turns=agent_sec.get("max_turns"),
+ temperature=agent_sec.get("temperature"),
+ tools=agent_sec.get("tools", []),
+ system_prompt=agent_sec.get("system_prompt"),
+ routing_policy=learning_sec.get("routing"),
+ agent_policy=learning_sec.get("agent"),
+ eval_suites=eval_sec.get("suites", []),
+ raw=data,
+ )
+
+
+def discover_recipes(
+ extra_dirs: Optional[List[str | Path]] = None,
+) -> List[Recipe]:
+ """Discover all TOML recipes from known directories.
+
+ Search order (later entries override earlier ones by name):
+ 1. Project ``recipes/`` directory
+ 2. User ``~/.openjarvis/recipes/`` directory
+ 3. Any additional directories in *extra_dirs*
+ """
+ dirs: List[Path] = [_PROJECT_RECIPES_DIR, _USER_RECIPES_DIR]
+ if extra_dirs:
+ dirs.extend(Path(d) for d in extra_dirs)
+
+ recipes: Dict[str, Recipe] = {}
+ for d in dirs:
+ if not d.is_dir():
+ continue
+ for toml_path in sorted(d.glob("*.toml")):
+ try:
+ recipe = load_recipe(toml_path)
+ recipes[recipe.name] = recipe
+ except Exception:
+ # Skip malformed recipe files
+ continue
+
+ return list(recipes.values())
+
+
+def resolve_recipe(name: str) -> Optional[Recipe]:
+ """Find a recipe by name from all known directories.
+
+ Returns ``None`` if no recipe with the given name is found.
+ """
+ for recipe in discover_recipes():
+ if recipe.name == name:
+ return recipe
+ return None
+
+
+__all__ = ["Recipe", "discover_recipes", "load_recipe", "resolve_recipe"]
diff --git a/src/openjarvis/system.py b/src/openjarvis/system.py
index c38d296a..c85ec78c 100644
--- a/src/openjarvis/system.py
+++ b/src/openjarvis/system.py
@@ -40,6 +40,7 @@ class JarvisSystem:
session_store: Optional[Any] = None # SessionStore
capability_policy: Optional[Any] = None # CapabilityPolicy
operator_manager: Optional[Any] = None # OperatorManager
+ _learning_orchestrator: Optional[Any] = None # LearningOrchestrator
def ask(
self,
@@ -443,7 +444,10 @@ class SystemBuilder:
# Set up capability policy
capability_policy = self._setup_capabilities(config)
- return JarvisSystem(
+ # Set up learning orchestrator (when training is enabled)
+ learning_orchestrator = self._setup_learning_orchestrator(config)
+
+ system = JarvisSystem(
config=config,
bus=bus,
engine=engine,
@@ -463,6 +467,8 @@ class SystemBuilder:
session_store=session_store,
capability_policy=capability_policy,
)
+ system._learning_orchestrator = learning_orchestrator
+ return system
def _resolve_engine(self, config: JarvisConfig):
"""Resolve the inference engine."""
@@ -853,6 +859,37 @@ class SystemBuilder:
except Exception:
return None
+ @staticmethod
+ def _setup_learning_orchestrator(config: JarvisConfig):
+ """Set up LearningOrchestrator when training is enabled."""
+ if not config.learning.training_enabled:
+ return None
+ try:
+ from openjarvis.core.config import DEFAULT_CONFIG_DIR
+ from openjarvis.learning.learning_orchestrator import (
+ LearningOrchestrator,
+ )
+ from openjarvis.learning.training.lora import LoRATrainingConfig
+ from openjarvis.traces.store import TraceStore
+
+ trace_store = TraceStore(db_path=config.traces.db_path)
+ config_dir = DEFAULT_CONFIG_DIR / "agent_configs"
+
+ lora_config = LoRATrainingConfig(
+ lora_rank=config.learning.lora_rank,
+ lora_alpha=config.learning.lora_alpha,
+ )
+
+ return LearningOrchestrator(
+ trace_store=trace_store,
+ config_dir=config_dir,
+ min_improvement=config.learning.min_improvement,
+ min_sft_pairs=config.learning.min_sft_pairs,
+ lora_config=lora_config,
+ )
+ except Exception:
+ return None
+
@staticmethod
def _discover_external_mcp(server_cfg) -> List[BaseTool]:
"""Discover tools from an external MCP server configuration."""
diff --git a/src/openjarvis/templates/__init__.py b/src/openjarvis/templates/__init__.py
new file mode 100644
index 00000000..7eb3b0a7
--- /dev/null
+++ b/src/openjarvis/templates/__init__.py
@@ -0,0 +1,9 @@
+"""Agent template system — pre-configured agent manifests."""
+
+from openjarvis.templates.agent_templates import (
+ AgentTemplate,
+ discover_templates,
+ load_template,
+)
+
+__all__ = ["AgentTemplate", "discover_templates", "load_template"]
diff --git a/src/openjarvis/templates/agent_templates.py b/src/openjarvis/templates/agent_templates.py
new file mode 100644
index 00000000..5ba773c6
--- /dev/null
+++ b/src/openjarvis/templates/agent_templates.py
@@ -0,0 +1,107 @@
+"""Agent template loader — load pre-configured agent manifests from TOML files."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional
+
+try:
+ import tomllib
+except ModuleNotFoundError:
+ import tomli as tomllib # type: ignore[no-redef]
+
+
+@dataclass(slots=True)
+class AgentTemplate:
+ """A pre-configured agent manifest loaded from a TOML template."""
+
+ name: str
+ description: str = ""
+ system_prompt: str = ""
+ agent_type: str = "simple"
+ tools: List[str] = field(default_factory=list)
+ max_turns: int = 10
+ temperature: float = 0.7
+
+
+def load_template(path: str | Path) -> AgentTemplate:
+ """Load an agent template from a TOML file.
+
+ Expected format::
+
+ [template]
+ name = "code-reviewer"
+ description = "Reviews code for bugs, style, and best practices"
+
+ [agent]
+ type = "native_react"
+ max_turns = 8
+ temperature = 0.3
+ tools = ["file_read", "think"]
+ system_prompt = \"\"\"You are a code reviewer...\"\"\"
+
+ Raises:
+ FileNotFoundError: If *path* does not exist.
+ """
+ path = Path(path)
+ if not path.exists():
+ raise FileNotFoundError(f"Template file not found: {path}")
+
+ with open(path, "rb") as fh:
+ data = tomllib.load(fh)
+
+ template_data: Dict = data.get("template", {})
+ agent_data: Dict = data.get("agent", {})
+
+ return AgentTemplate(
+ name=template_data.get("name", path.stem),
+ description=template_data.get("description", ""),
+ system_prompt=agent_data.get("system_prompt", ""),
+ agent_type=agent_data.get("type", "simple"),
+ tools=agent_data.get("tools", []),
+ max_turns=agent_data.get("max_turns", 10),
+ temperature=agent_data.get("temperature", 0.7),
+ )
+
+
+def _builtin_templates_dir() -> Path:
+ """Return the path to the built-in templates shipped with the package."""
+ # templates/agents/ at project root, 3 levels above this file
+ return Path(__file__).resolve().parents[3] / "templates" / "agents"
+
+
+def _user_templates_dir() -> Path:
+ """Return the path to user-defined templates (~/.openjarvis/templates/agents/)."""
+ return Path.home() / ".openjarvis" / "templates" / "agents"
+
+
+def discover_templates(
+ extra_dirs: Optional[List[str | Path]] = None,
+) -> List[AgentTemplate]:
+ """Discover and load all agent templates from known directories.
+
+ Search order:
+ 1. Built-in templates shipped with the package (``templates/agents/``).
+ 2. User templates at ``~/.openjarvis/templates/agents/``.
+ 3. Any additional directories supplied via *extra_dirs*.
+
+ Returns a list of :class:`AgentTemplate` instances sorted by name.
+ """
+ dirs: List[Path] = [_builtin_templates_dir(), _user_templates_dir()]
+ if extra_dirs:
+ dirs.extend(Path(d) for d in extra_dirs)
+
+ seen: Dict[str, AgentTemplate] = {}
+ for directory in dirs:
+ if not directory.is_dir():
+ continue
+ for toml_path in sorted(directory.glob("*.toml")):
+ tpl = load_template(toml_path)
+ # Later directories override earlier ones (user overrides builtin).
+ seen[tpl.name] = tpl
+
+ return sorted(seen.values(), key=lambda t: t.name)
+
+
+__all__ = ["AgentTemplate", "discover_templates", "load_template"]
diff --git a/templates/agents/architect.toml b/templates/agents/architect.toml
new file mode 100644
index 00000000..f72dfa8e
--- /dev/null
+++ b/templates/agents/architect.toml
@@ -0,0 +1,10 @@
+[template]
+name = "architect"
+description = "Software architecture design and system planning"
+
+[agent]
+type = "orchestrator"
+max_turns = 10
+temperature = 0.5
+tools = ["file_read", "think", "web_search"]
+system_prompt = """You are a software architect specializing in system design. You analyze requirements, evaluate trade-offs between simplicity and scalability, and propose well-structured architectures with clear component boundaries. You reference existing code to ensure proposals integrate cleanly, research proven patterns when applicable, and always justify design decisions with concrete reasoning about maintainability, performance, and extensibility."""
diff --git a/templates/agents/assistant.toml b/templates/agents/assistant.toml
new file mode 100644
index 00000000..f35386ec
--- /dev/null
+++ b/templates/agents/assistant.toml
@@ -0,0 +1,10 @@
+[template]
+name = "assistant"
+description = "General-purpose helpful assistant for everyday tasks"
+
+[agent]
+type = "orchestrator"
+max_turns = 10
+temperature = 0.5
+tools = ["think", "calculator", "web_search"]
+system_prompt = """You are a general-purpose assistant. You help with a wide range of everyday tasks including answering questions, performing calculations, looking up information, and providing clear explanations. You are direct and helpful, adapting your communication style to the user's needs. When a question is ambiguous, you ask for clarification rather than guessing. You use tools when they add value and respond conversationally when they do not."""
diff --git a/templates/agents/code-reviewer.toml b/templates/agents/code-reviewer.toml
new file mode 100644
index 00000000..5ee1451e
--- /dev/null
+++ b/templates/agents/code-reviewer.toml
@@ -0,0 +1,10 @@
+[template]
+name = "code-reviewer"
+description = "Reviews code for bugs, style, and best practices"
+
+[agent]
+type = "native_react"
+max_turns = 8
+temperature = 0.3
+tools = ["file_read", "think"]
+system_prompt = """You are an expert code reviewer. Your role is to systematically analyze source code for correctness bugs, style violations, performance issues, and adherence to best practices. You read files carefully, reason step-by-step about potential problems, and provide actionable feedback with specific line references. You prioritize issues by severity — correctness first, then security, then performance, then style."""
diff --git a/templates/agents/data-analyst.toml b/templates/agents/data-analyst.toml
new file mode 100644
index 00000000..030b401e
--- /dev/null
+++ b/templates/agents/data-analyst.toml
@@ -0,0 +1,10 @@
+[template]
+name = "data-analyst"
+description = "Analyzes data, computes statistics, and generates insights"
+
+[agent]
+type = "native_react"
+max_turns = 12
+temperature = 0.3
+tools = ["code_interpreter", "file_read", "think"]
+system_prompt = """You are a data analyst. You examine datasets by reading files, writing and executing analysis code, and reasoning about the results. You compute descriptive statistics, identify trends and outliers, and present your findings with clear explanations. You choose appropriate methods for the data at hand — whether simple aggregations or more involved statistical tests — and you always state your assumptions and limitations explicitly."""
diff --git a/templates/agents/debugger.toml b/templates/agents/debugger.toml
new file mode 100644
index 00000000..3806049e
--- /dev/null
+++ b/templates/agents/debugger.toml
@@ -0,0 +1,10 @@
+[template]
+name = "debugger"
+description = "Systematic debugging and root-cause analysis"
+
+[agent]
+type = "native_react"
+max_turns = 15
+temperature = 0.2
+tools = ["file_read", "code_interpreter", "shell_exec", "think"]
+system_prompt = """You are a systematic debugger. Given a bug report or error trace, you methodically narrow down the root cause by reading relevant source files, forming hypotheses, and testing them with code execution or shell commands. You follow a disciplined observe-hypothesize-test cycle and avoid jumping to conclusions. When you identify the root cause, you propose a minimal, targeted fix and explain why it resolves the issue."""
diff --git a/templates/agents/deep-researcher.toml b/templates/agents/deep-researcher.toml
new file mode 100644
index 00000000..0be1b827
--- /dev/null
+++ b/templates/agents/deep-researcher.toml
@@ -0,0 +1,10 @@
+[template]
+name = "deep-researcher"
+description = "In-depth research with source gathering and synthesis"
+
+[agent]
+type = "orchestrator"
+max_turns = 15
+temperature = 0.4
+tools = ["web_search", "http_request", "memory_store", "memory_search", "think"]
+system_prompt = """You are a thorough research agent. Given a topic or question, you conduct multi-step research by searching the web, fetching primary sources, and cross-referencing information for accuracy. You store key findings in memory for later retrieval and synthesize your research into well-organized reports with citations. You distinguish between established facts, expert opinions, and speculation, and you flag areas where sources conflict."""
diff --git a/templates/agents/fact-checker.toml b/templates/agents/fact-checker.toml
new file mode 100644
index 00000000..1502ba72
--- /dev/null
+++ b/templates/agents/fact-checker.toml
@@ -0,0 +1,10 @@
+[template]
+name = "fact-checker"
+description = "Verifies claims against authoritative sources"
+
+[agent]
+type = "orchestrator"
+max_turns = 10
+temperature = 0.2
+tools = ["web_search", "think"]
+system_prompt = """You are a meticulous fact-checker. Given a claim or statement, you search for authoritative sources to verify or refute it. You evaluate source credibility, distinguish between primary evidence and secondary reporting, and clearly rate each claim as supported, unsupported, misleading, or unverifiable. You always cite your sources and explain your reasoning transparently, noting any caveats or nuances that affect the assessment."""
diff --git a/templates/agents/inbox-triager.toml b/templates/agents/inbox-triager.toml
new file mode 100644
index 00000000..f84eef7c
--- /dev/null
+++ b/templates/agents/inbox-triager.toml
@@ -0,0 +1,10 @@
+[template]
+name = "inbox-triager"
+description = "Classifies and prioritizes incoming messages"
+
+[agent]
+type = "orchestrator"
+max_turns = 8
+temperature = 0.3
+tools = ["think", "llm_call"]
+system_prompt = """You are an inbox triage specialist. You classify incoming messages by urgency (critical, high, normal, low) and category (action-required, informational, follow-up, spam). You extract key metadata such as sender intent, deadlines, and required actions. You group related messages together and produce a prioritized digest that helps the user focus on what matters most, deferring or filtering noise."""
diff --git a/templates/agents/meeting-prep.toml b/templates/agents/meeting-prep.toml
new file mode 100644
index 00000000..a33af901
--- /dev/null
+++ b/templates/agents/meeting-prep.toml
@@ -0,0 +1,10 @@
+[template]
+name = "meeting-prep"
+description = "Prepares structured meeting materials and agendas"
+
+[agent]
+type = "orchestrator"
+max_turns = 10
+temperature = 0.4
+tools = ["web_search", "memory_search", "think", "file_write"]
+system_prompt = """You are a meeting preparation assistant. Given a meeting topic, attendee list, or agenda outline, you research relevant background, gather context from memory, and produce well-structured preparation materials. Your output includes a clear agenda, key discussion points, relevant data or references, and suggested questions. You save the prepared materials to a file so they are ready for the meeting."""
diff --git a/templates/agents/note-taker.toml b/templates/agents/note-taker.toml
new file mode 100644
index 00000000..3894a1aa
--- /dev/null
+++ b/templates/agents/note-taker.toml
@@ -0,0 +1,10 @@
+[template]
+name = "note-taker"
+description = "Extracts key points and decisions from conversations"
+
+[agent]
+type = "simple"
+max_turns = 1
+temperature = 0.3
+tools = ["think", "file_write"]
+system_prompt = """You are a precise note-taker. You listen to or read conversations, meetings, and discussions, then extract and organize the key points, decisions made, action items with owners, open questions, and deadlines. You use a clean, consistent format with sections and bullet points. You capture the substance faithfully without editorializing, and you highlight anything that requires follow-up."""
diff --git a/templates/agents/security-auditor.toml b/templates/agents/security-auditor.toml
new file mode 100644
index 00000000..ddc30cb4
--- /dev/null
+++ b/templates/agents/security-auditor.toml
@@ -0,0 +1,10 @@
+[template]
+name = "security-auditor"
+description = "Reviews code and configurations for security vulnerabilities"
+
+[agent]
+type = "native_react"
+max_turns = 10
+temperature = 0.2
+tools = ["file_read", "think", "shell_exec"]
+system_prompt = """You are a security auditor. You review source code, configurations, and infrastructure for vulnerabilities including injection flaws, authentication weaknesses, insecure defaults, and data exposure risks. You follow established frameworks like the OWASP Top 10 and CWE classifications. You rate findings by severity and exploitability, provide proof-of-concept descriptions, and recommend specific remediations. You never exploit vulnerabilities — you only identify and report them."""
diff --git a/templates/agents/summarizer.toml b/templates/agents/summarizer.toml
new file mode 100644
index 00000000..8bb5ff31
--- /dev/null
+++ b/templates/agents/summarizer.toml
@@ -0,0 +1,10 @@
+[template]
+name = "summarizer"
+description = "Condenses long texts into clear, concise summaries"
+
+[agent]
+type = "simple"
+max_turns = 1
+temperature = 0.3
+tools = ["think"]
+system_prompt = """You are a skilled summarizer. You condense long documents, articles, or conversations into clear, accurate summaries that preserve the key points, decisions, and action items. You adapt the summary length and format to the content — using bullet points for lists of facts, prose for narratives, and structured sections for reports. You never introduce information not present in the source material and you flag any ambiguities in the original text."""
diff --git a/templates/agents/translator.toml b/templates/agents/translator.toml
new file mode 100644
index 00000000..d26471f1
--- /dev/null
+++ b/templates/agents/translator.toml
@@ -0,0 +1,10 @@
+[template]
+name = "translator"
+description = "Translates text between languages with nuance and context"
+
+[agent]
+type = "simple"
+max_turns = 1
+temperature = 0.3
+tools = ["think"]
+system_prompt = """You are a skilled translator. You translate text between languages while preserving meaning, tone, and cultural nuance. You handle idiomatic expressions by finding natural equivalents in the target language rather than translating literally. When a term has multiple valid translations, you choose the one that best fits the context and note alternatives when the distinction matters. You flag any passages where the translation involves a significant loss of nuance."""
diff --git a/templates/agents/tutor.toml b/templates/agents/tutor.toml
new file mode 100644
index 00000000..eff802fa
--- /dev/null
+++ b/templates/agents/tutor.toml
@@ -0,0 +1,10 @@
+[template]
+name = "tutor"
+description = "Explains concepts step-by-step with examples"
+
+[agent]
+type = "orchestrator"
+max_turns = 12
+temperature = 0.5
+tools = ["think", "calculator"]
+system_prompt = """You are a patient and effective tutor. You explain concepts step-by-step, starting from what the learner already knows and building toward the target understanding. You use concrete examples, analogies, and worked problems to make abstract ideas tangible. You check comprehension by asking follow-up questions and adjust your explanations based on the learner's responses. You never make the learner feel foolish for not knowing something."""
diff --git a/templates/agents/writer.toml b/templates/agents/writer.toml
new file mode 100644
index 00000000..f3707fde
--- /dev/null
+++ b/templates/agents/writer.toml
@@ -0,0 +1,10 @@
+[template]
+name = "writer"
+description = "Creative and technical writing for various formats"
+
+[agent]
+type = "orchestrator"
+max_turns = 10
+temperature = 0.7
+tools = ["think", "web_search"]
+system_prompt = """You are a versatile writer capable of producing both creative and technical content. You adapt your style, tone, and structure to the requested format — whether it is a blog post, technical document, email, story, or report. You research topics when needed to ensure accuracy and depth. You focus on clarity and engagement, revising your drafts mentally before presenting the final version. You welcome feedback and iterate willingly."""
diff --git a/tests/cli/test_eval_cmd.py b/tests/cli/test_eval_cmd.py
new file mode 100644
index 00000000..a3a2437b
--- /dev/null
+++ b/tests/cli/test_eval_cmd.py
@@ -0,0 +1,39 @@
+"""Tests for the ``jarvis eval`` CLI commands."""
+
+from __future__ import annotations
+
+from click.testing import CliRunner
+
+from openjarvis.cli import cli
+
+
+class TestEvalCLI:
+ """Tests for the eval command group."""
+
+ def test_eval_group_exists(self):
+ """``jarvis eval --help`` shows run/compare/report/list subcommands."""
+ result = CliRunner().invoke(cli, ["eval", "--help"])
+ assert result.exit_code == 0
+ assert "run" in result.output
+ assert "compare" in result.output
+ assert "report" in result.output
+ assert "list" in result.output
+
+ def test_eval_list_benchmarks(self):
+ """``jarvis eval list`` exits 0 and outputs benchmark names."""
+ result = CliRunner().invoke(cli, ["eval", "list"])
+ assert result.exit_code == 0
+ assert "supergpqa" in result.output
+ assert "gaia" in result.output
+ assert "frames" in result.output
+ assert "wildchat" in result.output
+ # Should also show backends
+ assert "jarvis-direct" in result.output
+ assert "jarvis-agent" in result.output
+
+ def test_eval_run_missing_args(self):
+ """``jarvis eval run`` without required args fails gracefully."""
+ result = CliRunner().invoke(cli, ["eval", "run"])
+ # Should fail because neither --config nor --benchmark/--model given
+ assert result.exit_code != 0
+ assert "config" in result.output.lower() or "benchmark" in result.output.lower()
diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py
new file mode 100644
index 00000000..2fd6322f
--- /dev/null
+++ b/tests/evals/test_benchmark_datasets.py
@@ -0,0 +1,330 @@
+"""Tests for all 15 benchmark dataset and scorer registrations.
+
+These tests verify:
+1. Each dataset class can be instantiated
+2. Each dataset has correct dataset_id and dataset_name
+3. Each scorer class can be constructed (with mock backend)
+4. The CLI _build_dataset and _build_scorer factories work for all benchmarks
+5. KNOWN_BENCHMARKS in config.py includes all 15 benchmarks
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+# ---------------------------------------------------------------------------
+# Dataset instantiation tests
+# ---------------------------------------------------------------------------
+
+
+class TestDatasetInstantiation:
+ """Verify each dataset class can be instantiated with correct attributes."""
+
+ def test_supergpqa(self) -> None:
+ from evals.datasets.supergpqa import SuperGPQADataset
+ ds = SuperGPQADataset()
+ assert ds.dataset_id == "supergpqa"
+ assert ds.dataset_name == "SuperGPQA"
+
+ def test_gpqa(self) -> None:
+ from evals.datasets.gpqa import GPQADataset
+ ds = GPQADataset()
+ assert ds.dataset_id == "gpqa"
+ assert ds.dataset_name == "GPQA"
+
+ def test_mmlu_pro(self) -> None:
+ from evals.datasets.mmlu_pro import MMLUProDataset
+ ds = MMLUProDataset()
+ assert ds.dataset_id == "mmlu-pro"
+ assert ds.dataset_name == "MMLU-Pro"
+
+ def test_math500(self) -> None:
+ from evals.datasets.math500 import MATH500Dataset
+ ds = MATH500Dataset()
+ assert ds.dataset_id == "math500"
+ assert ds.dataset_name == "MATH-500"
+
+ def test_natural_reasoning(self) -> None:
+ from evals.datasets.natural_reasoning import NaturalReasoningDataset
+ ds = NaturalReasoningDataset()
+ assert ds.dataset_id == "natural-reasoning"
+ assert ds.dataset_name == "Natural Reasoning"
+
+ def test_hle(self) -> None:
+ from evals.datasets.hle import HLEDataset
+ ds = HLEDataset()
+ assert ds.dataset_id == "hle"
+ assert ds.dataset_name == "HLE"
+
+ def test_simpleqa(self) -> None:
+ from evals.datasets.simpleqa import SimpleQADataset
+ ds = SimpleQADataset()
+ assert ds.dataset_id == "simpleqa"
+ assert ds.dataset_name == "SimpleQA"
+
+ def test_wildchat(self) -> None:
+ from evals.datasets.wildchat import WildChatDataset
+ ds = WildChatDataset()
+ assert ds.dataset_id == "wildchat"
+ assert ds.dataset_name == "WildChat"
+
+ def test_ipw(self) -> None:
+ from evals.datasets.ipw_mixed import IPWDataset
+ ds = IPWDataset()
+ assert ds.dataset_id == "ipw"
+ assert ds.dataset_name == "IPW"
+
+ def test_gaia(self) -> None:
+ from evals.datasets.gaia import GAIADataset
+ ds = GAIADataset()
+ assert ds.dataset_id == "gaia"
+ assert ds.dataset_name == "GAIA"
+
+ def test_frames(self) -> None:
+ from evals.datasets.frames import FRAMESDataset
+ ds = FRAMESDataset()
+ assert ds.dataset_id == "frames"
+ assert ds.dataset_name == "FRAMES"
+
+ def test_swebench(self) -> None:
+ from evals.datasets.swebench import SWEBenchDataset
+ ds = SWEBenchDataset()
+ assert ds.dataset_id == "swebench"
+ assert ds.dataset_name == "SWE-bench"
+
+ def test_swefficiency(self) -> None:
+ from evals.datasets.swefficiency import SWEfficiencyDataset
+ ds = SWEfficiencyDataset()
+ assert ds.dataset_id == "swefficiency"
+ assert ds.dataset_name == "SWEfficiency"
+
+ def test_terminalbench(self) -> None:
+ from evals.datasets.terminalbench import TerminalBenchDataset
+ ds = TerminalBenchDataset()
+ assert ds.dataset_id == "terminalbench"
+ assert ds.dataset_name == "TerminalBench"
+
+ def test_terminalbench_native(self) -> None:
+ from evals.datasets.terminalbench_native import (
+ TerminalBenchNativeDataset,
+ )
+ ds = TerminalBenchNativeDataset()
+ assert ds.dataset_id == "terminalbench-native"
+ assert ds.dataset_name == "TerminalBench Native"
+
+
+# ---------------------------------------------------------------------------
+# Scorer instantiation tests
+# ---------------------------------------------------------------------------
+
+
+def _mock_backend() -> MagicMock:
+ """Create a mock inference backend for scorer construction."""
+ backend = MagicMock()
+ backend.generate.return_value = "A"
+ return backend
+
+
+class TestScorerInstantiation:
+ """Verify each scorer class can be constructed."""
+
+ def test_supergpqa_scorer(self) -> None:
+ from evals.scorers.supergpqa_mcq import SuperGPQAScorer
+ s = SuperGPQAScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "supergpqa"
+
+ def test_gpqa_scorer(self) -> None:
+ from evals.scorers.gpqa_mcq import GPQAScorer
+ s = GPQAScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "gpqa"
+
+ def test_mmlu_pro_scorer(self) -> None:
+ from evals.scorers.mmlu_pro_mcq import MMLUProScorer
+ s = MMLUProScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "mmlu-pro"
+
+ def test_reasoning_judge_scorer(self) -> None:
+ from evals.scorers.reasoning_judge import ReasoningJudgeScorer
+ s = ReasoningJudgeScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "reasoning_judge"
+
+ def test_hle_scorer(self) -> None:
+ from evals.scorers.hle_judge import HLEScorer
+ s = HLEScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "hle"
+
+ def test_simpleqa_scorer(self) -> None:
+ from evals.scorers.simpleqa_judge import SimpleQAScorer
+ s = SimpleQAScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "simpleqa"
+
+ def test_wildchat_scorer(self) -> None:
+ from evals.scorers.wildchat_judge import WildChatScorer
+ s = WildChatScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "wildchat"
+
+ def test_ipw_mixed_scorer(self) -> None:
+ from evals.scorers.ipw_mixed import IPWMixedScorer
+ s = IPWMixedScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "ipw"
+
+ def test_gaia_scorer(self) -> None:
+ from evals.scorers.gaia_exact import GAIAScorer
+ s = GAIAScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "gaia"
+
+ def test_frames_scorer(self) -> None:
+ from evals.scorers.frames_judge import FRAMESScorer
+ s = FRAMESScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "frames"
+
+ def test_swebench_scorer(self) -> None:
+ from evals.scorers.swebench_structural import SWEBenchScorer
+ s = SWEBenchScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "swebench"
+
+ def test_swefficiency_scorer(self) -> None:
+ from evals.scorers.swefficiency_structural import (
+ SWEfficiencyScorer,
+ )
+ s = SWEfficiencyScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "swefficiency"
+
+ def test_terminalbench_scorer(self) -> None:
+ from evals.scorers.terminalbench_judge import TerminalBenchScorer
+ s = TerminalBenchScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "terminalbench"
+
+ def test_terminalbench_native_scorer(self) -> None:
+ from evals.scorers.terminalbench_native_structural import (
+ TerminalBenchNativeScorer,
+ )
+ s = TerminalBenchNativeScorer(_mock_backend(), "test-model")
+ assert s.scorer_id == "terminalbench-native"
+
+
+# ---------------------------------------------------------------------------
+# CLI factory tests
+# ---------------------------------------------------------------------------
+
+
+ALL_BENCHMARKS = [
+ "supergpqa", "gpqa", "mmlu-pro", "math500", "natural-reasoning",
+ "hle", "simpleqa", "wildchat", "ipw", "gaia", "frames",
+ "swebench", "swefficiency", "terminalbench", "terminalbench-native",
+]
+
+
+class TestCLIFactories:
+ """Verify CLI _build_dataset and _build_scorer work for all benchmarks."""
+
+ @pytest.mark.parametrize("benchmark", ALL_BENCHMARKS)
+ def test_build_dataset(self, benchmark: str) -> None:
+ from evals.cli import _build_dataset
+ ds = _build_dataset(benchmark)
+ assert ds is not None
+ assert hasattr(ds, "load")
+ assert hasattr(ds, "iter_records")
+ assert hasattr(ds, "size")
+
+ @pytest.mark.parametrize("benchmark", ALL_BENCHMARKS)
+ def test_build_scorer(self, benchmark: str) -> None:
+ from evals.cli import _build_scorer
+ scorer = _build_scorer(benchmark, _mock_backend(), "test-model")
+ assert scorer is not None
+ assert hasattr(scorer, "score")
+
+ def test_build_dataset_unknown(self) -> None:
+ import click
+ from evals.cli import _build_dataset
+ with pytest.raises(click.ClickException, match="Unknown benchmark"):
+ _build_dataset("nonexistent")
+
+ def test_build_scorer_unknown(self) -> None:
+ import click
+ from evals.cli import _build_scorer
+ with pytest.raises(click.ClickException, match="Unknown benchmark"):
+ _build_scorer("nonexistent", _mock_backend(), "test-model")
+
+
+# ---------------------------------------------------------------------------
+# Config KNOWN_BENCHMARKS test
+# ---------------------------------------------------------------------------
+
+
+class TestConfigBenchmarks:
+ """Verify KNOWN_BENCHMARKS includes all 15 benchmarks."""
+
+ def test_all_benchmarks_known(self) -> None:
+ from evals.core.config import KNOWN_BENCHMARKS
+ for b in ALL_BENCHMARKS:
+ assert b in KNOWN_BENCHMARKS, f"{b} missing from KNOWN_BENCHMARKS"
+
+ def test_benchmarks_count(self) -> None:
+ from evals.core.config import KNOWN_BENCHMARKS
+ assert len(KNOWN_BENCHMARKS) == 15
+
+
+# ---------------------------------------------------------------------------
+# Structural scorer tests
+# ---------------------------------------------------------------------------
+
+
+class TestStructuralScorers:
+ """Test structural scorers that don't need LLM calls."""
+
+ def test_swebench_empty_response(self) -> None:
+ from evals.core.types import EvalRecord
+ from evals.scorers.swebench_structural import SWEBenchScorer
+ scorer = SWEBenchScorer(_mock_backend(), "test-model")
+ record = EvalRecord(
+ record_id="swe-1", problem="Fix bug", reference="patch",
+ category="agentic",
+ )
+ is_correct, meta = scorer.score(record, "")
+ assert is_correct is False
+ assert meta["reason"] == "empty_response"
+
+ def test_swebench_with_diff(self) -> None:
+ from evals.core.types import EvalRecord
+ from evals.scorers.swebench_structural import SWEBenchScorer
+ scorer = SWEBenchScorer(_mock_backend(), "test-model")
+ record = EvalRecord(
+ record_id="swe-2", problem="Fix bug", reference="patch",
+ category="agentic",
+ )
+ answer = "--- a/file.py\n+++ b/file.py\n@@ -1 +1 @@\n-old\n+new"
+ is_correct, meta = scorer.score(record, answer)
+ assert is_correct is None # indeterminate
+ assert meta["reason"] == "requires_test_execution"
+ assert meta["has_diff_markers"] is True
+
+ def test_terminalbench_native_no_results(self) -> None:
+ from evals.core.types import EvalRecord
+ from evals.scorers.terminalbench_native_structural import (
+ TerminalBenchNativeScorer,
+ )
+ scorer = TerminalBenchNativeScorer(_mock_backend(), "test-model")
+ record = EvalRecord(
+ record_id="tb-1", problem="Run command",
+ reference="", category="agentic",
+ )
+ is_correct, meta = scorer.score(record, "some output")
+ assert is_correct is None
+ assert meta["reason"] == "no_test_results"
+
+ def test_terminalbench_native_resolved(self) -> None:
+ from evals.core.types import EvalRecord
+ from evals.scorers.terminalbench_native_structural import (
+ TerminalBenchNativeScorer,
+ )
+ scorer = TerminalBenchNativeScorer(_mock_backend(), "test-model")
+ record = EvalRecord(
+ record_id="tb-2", problem="Run command",
+ reference="", category="agentic",
+ metadata={"is_resolved": True},
+ )
+ is_correct, meta = scorer.score(record, "output")
+ assert is_correct is True
diff --git a/tests/learning/test_agent_evolver.py b/tests/learning/test_agent_evolver.py
new file mode 100644
index 00000000..803c47ab
--- /dev/null
+++ b/tests/learning/test_agent_evolver.py
@@ -0,0 +1,240 @@
+"""Tests for AgentConfigEvolver — trace-driven agent config evolution."""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+
+import pytest
+
+try:
+ import tomllib # Python 3.11+
+except ModuleNotFoundError:
+ import tomli as tomllib # type: ignore[no-redef]
+
+from openjarvis.core.types import StepType, Trace, TraceStep
+from openjarvis.learning.agent_evolver import AgentConfigEvolver
+from openjarvis.traces.store import TraceStore
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_trace(
+ *,
+ query: str = "hello",
+ agent: str = "orchestrator",
+ model: str = "qwen3:8b",
+ tools: list[str] | None = None,
+ outcome: str = "success",
+ feedback: float = 0.9,
+) -> Trace:
+ """Build a Trace with TOOL_CALL steps for the given tool names."""
+ steps: list[TraceStep] = []
+ for tool_name in tools or []:
+ steps.append(
+ TraceStep(
+ step_type=StepType.TOOL_CALL,
+ timestamp=time.time(),
+ duration_seconds=0.1,
+ input={"tool": tool_name, "args": {}},
+ output={"result": "ok"},
+ )
+ )
+ # Add a GENERATE step so it looks realistic
+ steps.append(
+ TraceStep(
+ step_type=StepType.GENERATE,
+ timestamp=time.time(),
+ duration_seconds=0.5,
+ input={"prompt": query},
+ output={"content": "answer", "tokens": 50},
+ )
+ )
+ return Trace(
+ query=query,
+ agent=agent,
+ model=model,
+ steps=steps,
+ result="answer",
+ outcome=outcome,
+ feedback=feedback,
+ started_at=time.time(),
+ ended_at=time.time() + 1.0,
+ total_tokens=50,
+ total_latency_seconds=0.6,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestAgentConfigEvolver:
+ def test_analyze_empty_store(self, tmp_path: Path) -> None:
+ """Empty trace store returns empty recommendations."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ evolver = AgentConfigEvolver(store, config_dir=config_dir)
+ recs = evolver.analyze()
+
+ assert recs == []
+ store.close()
+
+ def test_evolve_recommends_tool_changes(self, tmp_path: Path) -> None:
+ """Traces with different tools — best tools recommended for each query class."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ # Create traces where "calculator" and "think" are used in successful
+ # math queries (short queries containing "calculate")
+ for i in range(5):
+ t = _make_trace(
+ query=f"calculate {i + 1} + {i + 2}",
+ agent="orchestrator",
+ tools=["calculator", "think"],
+ outcome="success",
+ feedback=0.95,
+ )
+ store.save(t)
+
+ # Create traces where "web_search" is used in general queries
+ for i in range(5):
+ t = _make_trace(
+ query=f"Tell me a moderately long story about topic number {i} please",
+ agent="orchestrator",
+ tools=["web_search"],
+ outcome="success",
+ feedback=0.8,
+ )
+ store.save(t)
+
+ # Create traces where "calculator" alone is used in math queries
+ # but with lower feedback — so the combo (calculator+think) should win
+ for i in range(3):
+ t = _make_trace(
+ query=f"compute the integral of x^{i}",
+ agent="simple",
+ tools=["calculator"],
+ outcome="success",
+ feedback=0.6,
+ )
+ store.save(t)
+
+ evolver = AgentConfigEvolver(store, config_dir=config_dir)
+ recs = evolver.analyze()
+
+ assert len(recs) > 0
+
+ # Each recommendation should have the expected keys
+ for rec in recs:
+ assert "query_class" in rec
+ assert "recommended_tools" in rec
+ assert "recommended_agent" in rec
+ assert "recommended_max_turns" in rec
+ assert "sample_count" in rec
+ assert rec["sample_count"] > 0
+
+ # Find the math recommendation — "calculator" should be in recommended tools
+ math_recs = [r for r in recs if r["query_class"] == "math"]
+ if math_recs:
+ assert "calculator" in math_recs[0]["recommended_tools"]
+
+ store.close()
+
+ def test_write_config_creates_toml(self, tmp_path: Path) -> None:
+ """write_config creates a valid TOML file with correct content."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ evolver = AgentConfigEvolver(store, config_dir=config_dir)
+ path = evolver.write_config(
+ "research_agent",
+ tools=["web_search", "file_read", "think"],
+ max_turns=15,
+ temperature=0.4,
+ system_prompt="You are a research assistant.",
+ )
+
+ # File should exist
+ assert path.exists()
+ assert path.suffix == ".toml"
+ assert "research_agent" in path.name
+
+ # Parse and verify content
+ with open(path, "rb") as f:
+ data = tomllib.load(f)
+
+ assert "agent" in data
+ agent_cfg = data["agent"]
+ assert agent_cfg["name"] == "research_agent"
+ assert agent_cfg["tools"] == ["web_search", "file_read", "think"]
+ assert agent_cfg["max_turns"] == 15
+ assert agent_cfg["temperature"] == 0.4
+ assert agent_cfg["system_prompt"] == "You are a research assistant."
+
+ store.close()
+
+ def test_versioning_and_rollback(self, tmp_path: Path) -> None:
+ """Write v1, write v2, list versions, rollback to v1."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ evolver = AgentConfigEvolver(store, config_dir=config_dir)
+
+ # Write v1
+ evolver.write_config(
+ "my_agent",
+ tools=["calculator"],
+ max_turns=5,
+ temperature=0.2,
+ system_prompt="v1 prompt",
+ )
+
+ # Write v2 (overwrites v1, archives v1 to .history/)
+ evolver.write_config(
+ "my_agent",
+ tools=["calculator", "web_search"],
+ max_turns=10,
+ temperature=0.5,
+ system_prompt="v2 prompt",
+ )
+
+ # Current config should be v2
+ config_path = config_dir / "my_agent.toml"
+ with open(config_path, "rb") as f:
+ current = tomllib.load(f)
+ assert current["agent"]["tools"] == ["calculator", "web_search"]
+ assert current["agent"]["system_prompt"] == "v2 prompt"
+
+ # List versions — should have at least 2 entries
+ versions = evolver.list_versions("my_agent")
+ assert len(versions) >= 2
+ for v in versions:
+ assert "version" in v
+ assert "path" in v
+ assert "modified" in v
+ assert isinstance(v["version"], int)
+ assert os.path.exists(v["path"])
+
+ # Rollback to v1 (version 1)
+ evolver.rollback("my_agent", version=1)
+
+ with open(config_path, "rb") as f:
+ rolled_back = tomllib.load(f)
+ assert rolled_back["agent"]["tools"] == ["calculator"]
+ assert rolled_back["agent"]["system_prompt"] == "v1 prompt"
+
+ # Verify ValueError on non-existent version
+ with pytest.raises(ValueError, match="[Vv]ersion"):
+ evolver.rollback("my_agent", version=999)
+
+ store.close()
diff --git a/tests/learning/test_learning_orchestrator.py b/tests/learning/test_learning_orchestrator.py
new file mode 100644
index 00000000..292018c5
--- /dev/null
+++ b/tests/learning/test_learning_orchestrator.py
@@ -0,0 +1,194 @@
+"""Tests for LearningOrchestrator -- coordinate trace->learn->eval loop."""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+from openjarvis.core.types import StepType, Trace, TraceStep
+from openjarvis.learning.learning_orchestrator import LearningOrchestrator
+from openjarvis.traces.store import TraceStore
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_trace(
+ *,
+ query: str = "hello",
+ agent: str = "orchestrator",
+ model: str = "qwen3:8b",
+ tools: list[str] | None = None,
+ outcome: str = "success",
+ feedback: float = 0.9,
+) -> Trace:
+ """Build a Trace with TOOL_CALL steps for the given tool names."""
+ steps: list[TraceStep] = []
+ for tool_name in tools or []:
+ steps.append(
+ TraceStep(
+ step_type=StepType.TOOL_CALL,
+ timestamp=time.time(),
+ duration_seconds=0.1,
+ input={"tool": tool_name, "args": {}},
+ output={"result": "ok"},
+ )
+ )
+ steps.append(
+ TraceStep(
+ step_type=StepType.GENERATE,
+ timestamp=time.time(),
+ duration_seconds=0.5,
+ input={"prompt": query},
+ output={"content": "answer", "tokens": 50},
+ )
+ )
+ return Trace(
+ query=query,
+ agent=agent,
+ model=model,
+ steps=steps,
+ result="answer",
+ outcome=outcome,
+ feedback=feedback,
+ started_at=time.time(),
+ ended_at=time.time() + 1.0,
+ total_tokens=50,
+ total_latency_seconds=0.6,
+ )
+
+
+def _populate_store(store: TraceStore, count: int = 10) -> None:
+ """Save *count* high-quality traces into the store."""
+ for i in range(count):
+ t = _make_trace(
+ query=f"calculate {i + 1} + {i + 2}",
+ agent="orchestrator",
+ model="qwen3:8b",
+ tools=["calculator", "think"],
+ outcome="success",
+ feedback=0.9,
+ )
+ store.save(t)
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestLearningOrchestrator:
+ def test_run_with_no_traces_is_noop(self, tmp_path: Path) -> None:
+ """Empty trace store -> status='skipped', reason mentions no data."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ orch = LearningOrchestrator(
+ trace_store=store,
+ config_dir=config_dir,
+ )
+ result = orch.run()
+
+ assert result["status"] == "skipped"
+ assert "no" in result["reason"].lower() or "data" in result["reason"].lower()
+ assert "timestamp" in result
+ store.close()
+
+ def test_run_extracts_data_and_updates_routing(self, tmp_path: Path) -> None:
+ """With traces present, run extracts data and result has counts."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ _populate_store(store, count=10)
+
+ orch = LearningOrchestrator(
+ trace_store=store,
+ config_dir=config_dir,
+ )
+ result = orch.run()
+
+ assert result["status"] in ("completed", "skipped")
+ # Should have extracted some data counts
+ assert "sft_pairs" in result or "routing_classes" in result
+ assert "timestamp" in result
+ store.close()
+
+ def test_run_with_eval_gate_rejects(self, tmp_path: Path) -> None:
+ """eval_fn returns worse score after learning -> accepted=False."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ _populate_store(store, count=10)
+
+ # First call (baseline) returns 0.8, second call (post) returns 0.7
+ call_count = 0
+
+ def eval_fn() -> float:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ return 0.8
+ return 0.7 # worse
+
+ orch = LearningOrchestrator(
+ trace_store=store,
+ config_dir=config_dir,
+ eval_fn=eval_fn,
+ min_improvement=0.02,
+ )
+ result = orch.run()
+
+ assert result.get("accepted") is False or result["status"] == "rejected"
+ assert "timestamp" in result
+ store.close()
+
+ def test_run_with_eval_gate_accepts(self, tmp_path: Path) -> None:
+ """eval_fn returns better score after learning -> accepted=True."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ _populate_store(store, count=10)
+
+ # First call (baseline) returns 0.7, second call (post) returns 0.8
+ call_count = 0
+
+ def eval_fn() -> float:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ return 0.7
+ return 0.8 # better
+
+ orch = LearningOrchestrator(
+ trace_store=store,
+ config_dir=config_dir,
+ eval_fn=eval_fn,
+ min_improvement=0.02,
+ )
+ result = orch.run()
+
+ assert result.get("accepted") is True or result["status"] == "completed"
+ assert "timestamp" in result
+ store.close()
+
+ def test_run_records_timestamp(self, tmp_path: Path) -> None:
+ """Result always has a 'timestamp' key regardless of outcome."""
+ db = tmp_path / "traces.db"
+ store = TraceStore(db)
+ config_dir = tmp_path / "configs"
+
+ # Test with empty store
+ orch = LearningOrchestrator(
+ trace_store=store,
+ config_dir=config_dir,
+ )
+ result = orch.run()
+
+ assert "timestamp" in result
+ assert isinstance(result["timestamp"], (int, float, str))
+ store.close()
diff --git a/tests/learning/training/__init__.py b/tests/learning/training/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/learning/training/test_data.py b/tests/learning/training/test_data.py
new file mode 100644
index 00000000..f5dc8725
--- /dev/null
+++ b/tests/learning/training/test_data.py
@@ -0,0 +1,238 @@
+"""Tests for TrainingDataMiner — SFT, routing, and agent config extraction."""
+
+from __future__ import annotations
+
+import time
+from typing import Any, List
+
+from openjarvis.core.types import StepType, Trace, TraceStep
+from openjarvis.learning.training.data import TrainingDataMiner
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_trace(
+ *,
+ query: str = "Hello world",
+ agent: str = "simple",
+ model: str = "qwen3:8b",
+ engine: str = "ollama",
+ result: str = "Hi there!",
+ feedback: float | None = 0.9,
+ outcome: str | None = "success",
+ tools: List[str] | None = None,
+) -> Trace:
+ """Build a Trace with optional TOOL_CALL steps."""
+ now = time.time()
+ steps: list[TraceStep] = [
+ TraceStep(
+ step_type=StepType.GENERATE,
+ timestamp=now,
+ duration_seconds=0.5,
+ input={"prompt": query},
+ output={"text": result, "tokens": 10},
+ ),
+ ]
+ if tools:
+ for tool_name in tools:
+ steps.append(
+ TraceStep(
+ step_type=StepType.TOOL_CALL,
+ timestamp=now + 0.1,
+ duration_seconds=0.1,
+ input={"tool": tool_name, "args": {}},
+ output={"result": "ok"},
+ )
+ )
+ steps.append(
+ TraceStep(
+ step_type=StepType.RESPOND,
+ timestamp=now + 1.0,
+ duration_seconds=0.0,
+ input={},
+ output={"text": result},
+ )
+ )
+ return Trace(
+ query=query,
+ agent=agent,
+ model=model,
+ engine=engine,
+ result=result,
+ feedback=feedback,
+ outcome=outcome,
+ started_at=now,
+ ended_at=now + 1.0,
+ total_tokens=10,
+ total_latency_seconds=1.0,
+ steps=steps,
+ )
+
+
+class FakeTraceStore:
+ """Minimal mock that satisfies TrainingDataMiner's needs."""
+
+ def __init__(self, traces: list[Trace] | None = None):
+ self._traces = traces or []
+
+ def list_traces(self, *, limit: int = 10000, **kwargs: Any) -> list[Trace]:
+ return self._traces[: limit]
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestExtractSFTPairs:
+ def test_extract_sft_pairs_from_successful_traces(self) -> None:
+ """SFT pairs are extracted from high-quality traces."""
+ traces = [
+ _make_trace(
+ query="Write a hello world in Python",
+ result="print('hello')",
+ feedback=0.9,
+ ),
+ _make_trace(query="Solve x^2=4", result="x=2 or x=-2", feedback=0.8),
+ ]
+ store = FakeTraceStore(traces)
+ miner = TrainingDataMiner(store)
+ pairs = miner.extract_sft_pairs()
+
+ assert len(pairs) == 2
+ # Check structure of first pair
+ p0 = pairs[0]
+ assert p0["input"] == "Write a hello world in Python"
+ assert p0["output"] == "print('hello')"
+ assert "query_class" in p0
+ assert p0["model"] == "qwen3:8b"
+ assert p0["feedback"] == 0.9
+
+ def test_deduplication(self) -> None:
+ """Duplicate (input, output) pairs are collapsed to a single entry."""
+ traces = [
+ _make_trace(query="Hi", result="Hello!", feedback=0.9),
+ _make_trace(query="Hi", result="Hello!", feedback=0.95),
+ ]
+ store = FakeTraceStore(traces)
+ miner = TrainingDataMiner(store)
+ pairs = miner.extract_sft_pairs()
+
+ assert len(pairs) == 1
+
+ def test_min_quality_filter(self) -> None:
+ """Traces below min_quality are excluded from SFT pairs."""
+ traces = [
+ _make_trace(query="Good", result="Fine", feedback=0.9),
+ _make_trace(query="Bad", result="Nope", feedback=0.3),
+ _make_trace(query="None", result="Null", feedback=None),
+ ]
+ store = FakeTraceStore(traces)
+ miner = TrainingDataMiner(store, min_quality=0.7)
+ pairs = miner.extract_sft_pairs()
+
+ assert len(pairs) == 1
+ assert pairs[0]["input"] == "Good"
+
+
+class TestExtractRoutingPairs:
+ def test_extract_routing_pairs(self) -> None:
+ """Routing pairs group traces by query class and find best model."""
+ traces = [
+ _make_trace(query="def foo(): pass", model="codellama:7b", feedback=0.95),
+ _make_trace(
+ query="import os; print(os.getcwd())",
+ model="codellama:7b",
+ feedback=0.85,
+ ),
+ _make_trace(query="def bar(): return 1", model="qwen3:8b", feedback=0.7),
+ ]
+ store = FakeTraceStore(traces)
+ miner = TrainingDataMiner(store, min_quality=0.7)
+ routing = miner.extract_routing_pairs()
+
+ assert "code" in routing
+ code_entry = routing["code"]
+ assert code_entry["best_model"] == "codellama:7b"
+ assert code_entry["sample_count"] == 3
+ assert "codellama:7b" in code_entry["all_models"]
+ assert "qwen3:8b" in code_entry["all_models"]
+
+
+class TestExtractAgentConfigPairs:
+ def test_extract_agent_config_pairs(self) -> None:
+ """Agent config pairs find best agent and tools per query class."""
+ traces = [
+ _make_trace(
+ query="Calculate 2+2",
+ agent="orchestrator",
+ tools=["calculator"],
+ feedback=0.95,
+ ),
+ _make_trace(
+ query="Compute 3*3",
+ agent="orchestrator",
+ tools=["calculator", "think"],
+ feedback=0.9,
+ ),
+ _make_trace(
+ query="Solve x+1=3",
+ agent="simple",
+ feedback=0.6,
+ ),
+ ]
+ store = FakeTraceStore(traces)
+ miner = TrainingDataMiner(store, min_quality=0.5)
+ agent_cfg = miner.extract_agent_config_pairs()
+
+ assert "math" in agent_cfg
+ math_entry = agent_cfg["math"]
+ assert math_entry["best_agent"] == "orchestrator"
+ assert "calculator" in math_entry["best_tools"]
+ assert math_entry["sample_count"] == 3
+
+
+class TestOutcomeFilter:
+ def test_failure_traces_excluded_despite_high_feedback(self) -> None:
+ """Traces with outcome='failure' are excluded even if feedback is high."""
+ traces = [
+ _make_trace(
+ query="Good query",
+ result="Good answer",
+ feedback=0.9,
+ outcome="success",
+ ),
+ _make_trace(
+ query="Bad query",
+ result="Bad answer",
+ feedback=0.9,
+ outcome="failure",
+ ),
+ ]
+ store = FakeTraceStore(traces)
+ miner = TrainingDataMiner(store, min_quality=0.7)
+
+ sft = miner.extract_sft_pairs()
+ assert len(sft) == 1
+ assert sft[0]["input"] == "Good query"
+
+ routing = miner.extract_routing_pairs()
+ total = sum(v["sample_count"] for v in routing.values())
+ assert total == 1
+
+ agent_cfg = miner.extract_agent_config_pairs()
+ total_agent = sum(v["sample_count"] for v in agent_cfg.values())
+ assert total_agent == 1
+
+
+class TestEmptyStore:
+ def test_empty_store_returns_empty(self) -> None:
+ """All extractors return empty results for an empty store."""
+ store = FakeTraceStore([])
+ miner = TrainingDataMiner(store)
+
+ assert miner.extract_sft_pairs() == []
+ assert miner.extract_routing_pairs() == {}
+ assert miner.extract_agent_config_pairs() == {}
diff --git a/tests/learning/training/test_lora.py b/tests/learning/training/test_lora.py
new file mode 100644
index 00000000..85cd2aee
--- /dev/null
+++ b/tests/learning/training/test_lora.py
@@ -0,0 +1,147 @@
+"""Tests for LoRATrainer — LoRA/QLoRA fine-tuning from trace-derived SFT pairs."""
+
+from __future__ import annotations
+
+import pytest
+
+from openjarvis.learning.training.lora import HAS_TORCH, LoRATrainer, LoRATrainingConfig
+
+# ---------------------------------------------------------------------------
+# Config tests (no torch required)
+# ---------------------------------------------------------------------------
+
+
+class TestLoRATrainingConfig:
+ def test_default_config(self) -> None:
+ """Verify default values of LoRATrainingConfig."""
+ cfg = LoRATrainingConfig()
+
+ # LoRA params
+ assert cfg.lora_rank == 16
+ assert cfg.lora_alpha == 32
+ assert cfg.lora_dropout == 0.05
+ assert cfg.target_modules == ["q_proj", "v_proj"]
+
+ # Training params
+ assert cfg.num_epochs == 3
+ assert cfg.batch_size == 4
+ assert cfg.learning_rate == 2e-5
+ assert cfg.weight_decay == 0.01
+ assert cfg.warmup_ratio == 0.1
+ assert cfg.max_grad_norm == 1.0
+ assert cfg.max_seq_length == 2048
+
+ # QLoRA
+ assert cfg.use_4bit is False
+
+ # Output
+ assert cfg.output_dir == "checkpoints/lora"
+ assert cfg.save_every_n_epochs == 1
+
+ # Memory
+ assert cfg.gradient_checkpointing is True
+
+ def test_custom_config(self) -> None:
+ """Verify custom values are stored correctly."""
+ cfg = LoRATrainingConfig(
+ lora_rank=8,
+ lora_alpha=16,
+ lora_dropout=0.1,
+ target_modules=["q_proj", "k_proj", "v_proj"],
+ num_epochs=5,
+ batch_size=8,
+ learning_rate=1e-4,
+ weight_decay=0.05,
+ warmup_ratio=0.2,
+ max_grad_norm=0.5,
+ max_seq_length=4096,
+ use_4bit=True,
+ output_dir="/tmp/lora_test",
+ save_every_n_epochs=2,
+ gradient_checkpointing=False,
+ )
+
+ assert cfg.lora_rank == 8
+ assert cfg.lora_alpha == 16
+ assert cfg.lora_dropout == 0.1
+ assert cfg.target_modules == ["q_proj", "k_proj", "v_proj"]
+ assert cfg.num_epochs == 5
+ assert cfg.batch_size == 8
+ assert cfg.learning_rate == 1e-4
+ assert cfg.weight_decay == 0.05
+ assert cfg.warmup_ratio == 0.2
+ assert cfg.max_grad_norm == 0.5
+ assert cfg.max_seq_length == 4096
+ assert cfg.use_4bit is True
+ assert cfg.output_dir == "/tmp/lora_test"
+ assert cfg.save_every_n_epochs == 2
+ assert cfg.gradient_checkpointing is False
+
+ def test_config_validates_lora_rank(self) -> None:
+ """lora_rank=0 raises ValueError."""
+ with pytest.raises(ValueError, match="lora_rank"):
+ LoRATrainingConfig(lora_rank=0)
+
+ def test_config_validates_num_epochs(self) -> None:
+ """num_epochs=0 raises ValueError."""
+ with pytest.raises(ValueError, match="num_epochs"):
+ LoRATrainingConfig(num_epochs=0)
+
+
+# ---------------------------------------------------------------------------
+# Trainer tests (require torch)
+# ---------------------------------------------------------------------------
+
+
+class TestLoRATrainerNoTorch:
+ def test_init_without_torch_raises(self) -> None:
+ """If HAS_TORCH is False, constructing LoRATrainer raises ImportError."""
+ if HAS_TORCH:
+ pytest.skip("torch is installed; cannot test missing-torch path")
+
+ cfg = LoRATrainingConfig()
+ with pytest.raises(ImportError, match="torch"):
+ LoRATrainer(cfg)
+
+
+@pytest.mark.skipif(not HAS_TORCH, reason="torch not installed")
+class TestLoRATrainerWithTorch:
+ def test_prepare_dataset_from_pairs(self) -> None:
+ """prepare_dataset converts SFT pairs to tokenized examples."""
+ cfg = LoRATrainingConfig()
+ trainer = LoRATrainer(cfg, model_name="Qwen/Qwen3-0.6B")
+
+ pairs = [
+ {
+ "input": "What is 2+2?",
+ "output": "4",
+ "query_class": "math",
+ "model": "qwen3:8b",
+ "feedback": 0.9,
+ },
+ {
+ "input": "Write hello world in Python",
+ "output": "print('hello world')",
+ "query_class": "code",
+ "model": "qwen3:8b",
+ "feedback": 0.85,
+ },
+ ]
+
+ dataset = trainer.prepare_dataset(pairs)
+
+ assert len(dataset) == 2
+ for item in dataset:
+ assert "input_ids" in item
+ assert "attention_mask" in item
+ assert "text" in item
+
+ def test_train_empty_pairs_returns_skipped(self) -> None:
+ """train() with empty pairs returns skipped status."""
+ cfg = LoRATrainingConfig()
+ trainer = LoRATrainer(cfg, model_name="Qwen/Qwen3-0.6B")
+
+ result = trainer.train([])
+
+ assert result["status"] == "skipped"
+ assert "reason" in result
diff --git a/tests/operators/test_operator_recipes.py b/tests/operators/test_operator_recipes.py
new file mode 100644
index 00000000..3f1624a0
--- /dev/null
+++ b/tests/operators/test_operator_recipes.py
@@ -0,0 +1,91 @@
+"""Tests for the 3 operator recipes: researcher, correspondent, sentinel."""
+
+from pathlib import Path
+
+import pytest
+
+from openjarvis.operators.loader import load_operator
+
+_OPERATORS_DIR = Path(__file__).parent.parent.parent / "recipes" / "operators"
+
+
+class TestResearcherOperator:
+ def test_loads_valid_manifest(self):
+ manifest = load_operator(_OPERATORS_DIR / "researcher.toml")
+ assert manifest.name == "researcher"
+ assert manifest.max_turns >= 10
+ assert manifest.system_prompt # loaded from prompt file
+
+ def test_has_required_tools(self):
+ manifest = load_operator(_OPERATORS_DIR / "researcher.toml")
+ required = {
+ "web_search",
+ "http_request",
+ "memory_store",
+ "memory_search",
+ "think",
+ "file_write",
+ }
+ assert required.issubset(set(manifest.tools))
+
+ def test_has_kg_tools(self):
+ manifest = load_operator(_OPERATORS_DIR / "researcher.toml")
+ assert "kg_add_entity" in manifest.tools
+ assert "kg_add_relation" in manifest.tools
+
+ def test_has_schedule(self):
+ manifest = load_operator(_OPERATORS_DIR / "researcher.toml")
+ assert manifest.schedule_type in ("cron", "interval")
+
+
+class TestCorrespondentOperator:
+ def test_loads_valid_manifest(self):
+ manifest = load_operator(_OPERATORS_DIR / "correspondent.toml")
+ assert manifest.name == "correspondent"
+ assert manifest.max_turns >= 10
+ assert manifest.system_prompt
+
+ def test_has_required_tools(self):
+ manifest = load_operator(_OPERATORS_DIR / "correspondent.toml")
+ required = {"memory_store", "memory_search", "think", "llm_call"}
+ assert required.issubset(set(manifest.tools))
+
+ def test_interval_schedule(self):
+ manifest = load_operator(_OPERATORS_DIR / "correspondent.toml")
+ assert manifest.schedule_type == "interval"
+ assert manifest.schedule_value == "300"
+
+
+class TestSentinelOperator:
+ def test_loads_valid_manifest(self):
+ manifest = load_operator(_OPERATORS_DIR / "sentinel.toml")
+ assert manifest.name == "sentinel"
+ assert manifest.max_turns >= 10
+ assert manifest.system_prompt
+
+ def test_has_required_tools(self):
+ manifest = load_operator(_OPERATORS_DIR / "sentinel.toml")
+ required = {
+ "web_search",
+ "http_request",
+ "memory_store",
+ "memory_search",
+ "think",
+ }
+ assert required.issubset(set(manifest.tools))
+
+ def test_has_kg_tools(self):
+ manifest = load_operator(_OPERATORS_DIR / "sentinel.toml")
+ assert "kg_add_entity" in manifest.tools
+
+
+class TestAllOperators:
+ @pytest.mark.parametrize(
+ "filename",
+ ["researcher.toml", "correspondent.toml", "sentinel.toml"],
+ )
+ def test_all_load_without_error(self, filename):
+ manifest = load_operator(_OPERATORS_DIR / filename)
+ assert manifest.name
+ assert manifest.tools
+ assert manifest.system_prompt
diff --git a/tests/recipes/__init__.py b/tests/recipes/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/recipes/test_loader.py b/tests/recipes/test_loader.py
new file mode 100644
index 00000000..cf4bd87f
--- /dev/null
+++ b/tests/recipes/test_loader.py
@@ -0,0 +1,161 @@
+"""Tests for recipe system — loader, discovery, and resolution."""
+
+from __future__ import annotations
+
+import textwrap
+from pathlib import Path
+
+import pytest
+
+from openjarvis.recipes.loader import (
+ Recipe,
+ discover_recipes,
+ load_recipe,
+ resolve_recipe,
+)
+
+SAMPLE_TOML = textwrap.dedent("""\
+ [recipe]
+ name = "test_recipe"
+ description = "A test recipe"
+ version = "2.0.0"
+
+ [intelligence]
+ model = "llama3:8b"
+ quantization = "q4_K_M"
+
+ [engine]
+ key = "ollama"
+
+ [agent]
+ type = "native_react"
+ max_turns = 12
+ temperature = 0.4
+ tools = ["calculator", "think"]
+ system_prompt = "You are a test assistant."
+
+ [learning]
+ routing = "grpo"
+ agent = "icl_updater"
+
+ [eval]
+ suites = ["reasoning", "coding"]
+""")
+
+
+class TestLoadRecipe:
+ def test_load_recipe_from_toml(self, tmp_path: Path) -> None:
+ toml_file = tmp_path / "test.toml"
+ toml_file.write_text(SAMPLE_TOML)
+
+ recipe = load_recipe(toml_file)
+
+ assert recipe.name == "test_recipe"
+ assert recipe.description == "A test recipe"
+ assert recipe.version == "2.0.0"
+ assert recipe.model == "llama3:8b"
+ assert recipe.quantization == "q4_K_M"
+ assert recipe.engine_key == "ollama"
+ assert recipe.agent_type == "native_react"
+ assert recipe.max_turns == 12
+ assert recipe.temperature == pytest.approx(0.4)
+ assert recipe.tools == ["calculator", "think"]
+ assert recipe.system_prompt == "You are a test assistant."
+ assert recipe.routing_policy == "grpo"
+ assert recipe.agent_policy == "icl_updater"
+ assert recipe.eval_suites == ["reasoning", "coding"]
+ assert isinstance(recipe.raw, dict)
+ assert "recipe" in recipe.raw
+
+ def test_load_recipe_missing_file_raises(self) -> None:
+ with pytest.raises(FileNotFoundError):
+ load_recipe("/nonexistent/path/recipe.toml")
+
+ def test_load_recipe_defaults(self, tmp_path: Path) -> None:
+ """Minimal TOML should yield sensible defaults."""
+ toml_file = tmp_path / "minimal.toml"
+ toml_file.write_text("[recipe]\nname = \"minimal\"\n")
+
+ recipe = load_recipe(toml_file)
+
+ assert recipe.name == "minimal"
+ assert recipe.version == "1.0.0"
+ assert recipe.model is None
+ assert recipe.tools == []
+ assert recipe.eval_suites == []
+
+ def test_load_recipe_name_from_filename(self, tmp_path: Path) -> None:
+ """When [recipe] has no name, use the file stem."""
+ toml_file = tmp_path / "my_recipe.toml"
+ toml_file.write_text("[recipe]\ndescription = \"no name\"\n")
+
+ recipe = load_recipe(toml_file)
+ assert recipe.name == "my_recipe"
+
+
+class TestDiscoverRecipes:
+ def test_discover_builtin_recipes(self) -> None:
+ recipes = discover_recipes()
+ names = {r.name for r in recipes}
+ assert "coding_assistant" in names
+ assert "research_assistant" in names
+ assert "general_assistant" in names
+ assert len(recipes) >= 3
+
+ def test_discover_extra_dirs(self, tmp_path: Path) -> None:
+ toml_file = tmp_path / "custom.toml"
+ toml_file.write_text(
+ '[recipe]\nname = "custom"\ndescription = "extra"\n'
+ )
+ recipes = discover_recipes(extra_dirs=[tmp_path])
+ names = {r.name for r in recipes}
+ assert "custom" in names
+
+ def test_discover_skips_malformed(self, tmp_path: Path) -> None:
+ bad = tmp_path / "bad.toml"
+ bad.write_text("this is not valid toml {{{{")
+ recipes = discover_recipes(extra_dirs=[tmp_path])
+ # Should not raise; malformed files are silently skipped
+ names = {r.name for r in recipes}
+ assert "bad" not in names
+
+
+class TestRecipeToBuilderKwargs:
+ def test_recipe_to_builder_kwargs(self, tmp_path: Path) -> None:
+ toml_file = tmp_path / "test.toml"
+ toml_file.write_text(SAMPLE_TOML)
+
+ recipe = load_recipe(toml_file)
+ kwargs = recipe.to_builder_kwargs()
+
+ assert kwargs["model"] == "llama3:8b"
+ assert kwargs["engine_key"] == "ollama"
+ assert kwargs["agent"] == "native_react"
+ assert kwargs["tools"] == ["calculator", "think"]
+ assert kwargs["temperature"] == pytest.approx(0.4)
+ assert kwargs["max_turns"] == 12
+ assert kwargs["system_prompt"] == "You are a test assistant."
+ assert kwargs["routing_policy"] == "grpo"
+ assert kwargs["agent_policy"] == "icl_updater"
+ assert kwargs["quantization"] == "q4_K_M"
+ assert kwargs["eval_suites"] == ["reasoning", "coding"]
+
+ def test_kwargs_omit_none_fields(self) -> None:
+ recipe = Recipe(name="sparse")
+ kwargs = recipe.to_builder_kwargs()
+ assert "model" not in kwargs
+ assert "engine_key" not in kwargs
+ assert "agent" not in kwargs
+ assert "tools" not in kwargs
+ assert "temperature" not in kwargs
+
+
+class TestResolveRecipe:
+ def test_resolve_recipe_found(self) -> None:
+ recipe = resolve_recipe("coding_assistant")
+ assert recipe is not None
+ assert recipe.name == "coding_assistant"
+
+ def test_resolve_recipe_not_found(self) -> None:
+ result = resolve_recipe("nonexistent_recipe_xyz")
+ assert result is None
diff --git a/tests/skills/__init__.py b/tests/skills/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/skills/test_bundled_skills.py b/tests/skills/test_bundled_skills.py
new file mode 100644
index 00000000..f0498a9d
--- /dev/null
+++ b/tests/skills/test_bundled_skills.py
@@ -0,0 +1,97 @@
+"""Tests for bundled skill TOML files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from openjarvis.skills.loader import load_skill
+
+# Resolve the skills/builtin/ directory relative to the project root.
+BUILTIN_DIR = Path(__file__).resolve().parents[2] / "skills" / "builtin"
+
+# Collect all TOML files once so parametrized IDs are readable.
+_toml_files = sorted(BUILTIN_DIR.glob("*.toml")) if BUILTIN_DIR.is_dir() else []
+
+
+def _load_all():
+ """Load every bundled skill manifest.
+
+ Returns a list of (path, manifest) tuples.
+ """
+ results = []
+ for toml_path in _toml_files:
+ manifest = load_skill(toml_path)
+ results.append((toml_path, manifest))
+ return results
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestBundledSkillsLoad:
+ """Verify every TOML in skills/builtin/ can be loaded without errors."""
+
+ @pytest.mark.parametrize(
+ "toml_path",
+ _toml_files,
+ ids=[p.stem for p in _toml_files],
+ )
+ def test_all_bundled_skills_load(self, toml_path: Path):
+ manifest = load_skill(toml_path)
+ assert manifest is not None
+
+
+class TestBundledSkillsHaveName:
+ """Every bundled skill must declare a non-empty name."""
+
+ @pytest.mark.parametrize(
+ "toml_path",
+ _toml_files,
+ ids=[p.stem for p in _toml_files],
+ )
+ def test_all_skills_have_name(self, toml_path: Path):
+ manifest = load_skill(toml_path)
+ assert manifest.name, f"{toml_path.name} has an empty name"
+ assert len(manifest.name) > 0
+
+
+class TestBundledSkillsHaveSteps:
+ """Every bundled skill must have at least one step."""
+
+ @pytest.mark.parametrize(
+ "toml_path",
+ _toml_files,
+ ids=[p.stem for p in _toml_files],
+ )
+ def test_all_skills_have_steps(self, toml_path: Path):
+ manifest = load_skill(toml_path)
+ assert len(manifest.steps) >= 1, f"{toml_path.name} has no steps"
+
+
+class TestSkillCount:
+ """The builtin directory must contain at least 20 skill files."""
+
+ def test_skill_count(self):
+ assert len(_toml_files) >= 20, (
+ f"Expected at least 20 bundled skills, found {len(_toml_files)}"
+ )
+
+
+class TestStepsHaveToolNames:
+ """Every step in every bundled skill must have a non-empty tool_name."""
+
+ @pytest.mark.parametrize(
+ "toml_path",
+ _toml_files,
+ ids=[p.stem for p in _toml_files],
+ )
+ def test_steps_have_tool_names(self, toml_path: Path):
+ manifest = load_skill(toml_path)
+ for i, step in enumerate(manifest.steps):
+ assert step.tool_name, (
+ f"{toml_path.name} step {i} has empty tool_name"
+ )
diff --git a/tests/templates/__init__.py b/tests/templates/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/templates/test_agent_templates.py b/tests/templates/test_agent_templates.py
new file mode 100644
index 00000000..d63bf24f
--- /dev/null
+++ b/tests/templates/test_agent_templates.py
@@ -0,0 +1,64 @@
+"""Tests for the agent template loader."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from openjarvis.templates.agent_templates import (
+ AgentTemplate,
+ discover_templates,
+ load_template,
+)
+
+TEMPLATES_DIR = Path(__file__).resolve().parents[2] / "templates" / "agents"
+
+VALID_AGENT_TYPES = {"simple", "orchestrator", "native_react"}
+
+
+def test_load_single_template() -> None:
+ """Load a single template and verify its fields are populated."""
+ path = TEMPLATES_DIR / "code-reviewer.toml"
+ tpl = load_template(path)
+
+ assert isinstance(tpl, AgentTemplate)
+ assert tpl.name == "code-reviewer"
+ assert tpl.description != ""
+ assert tpl.system_prompt != ""
+ assert tpl.agent_type == "native_react"
+ assert isinstance(tpl.tools, list)
+ assert len(tpl.tools) > 0
+ assert isinstance(tpl.max_turns, int)
+ assert isinstance(tpl.temperature, float)
+
+
+def test_discover_all_templates() -> None:
+ """Discover all built-in templates and verify count."""
+ templates = discover_templates()
+ assert len(templates) >= 15
+
+
+def test_all_templates_have_required_fields() -> None:
+ """Every template must have name, system_prompt, and agent type."""
+ templates = discover_templates()
+ for tpl in templates:
+ assert tpl.name, f"Template missing name: {tpl}"
+ assert tpl.system_prompt, f"Template '{tpl.name}' missing system_prompt"
+ assert tpl.agent_type, f"Template '{tpl.name}' missing agent type"
+
+
+def test_all_templates_have_valid_agent_type() -> None:
+ """Agent type must be one of the known types."""
+ templates = discover_templates()
+ for tpl in templates:
+ assert tpl.agent_type in VALID_AGENT_TYPES, (
+ f"Template '{tpl.name}' has invalid agent_type '{tpl.agent_type}'. "
+ f"Expected one of {VALID_AGENT_TYPES}"
+ )
+
+
+def test_template_missing_file_raises() -> None:
+ """Loading a non-existent template file raises FileNotFoundError."""
+ with pytest.raises(FileNotFoundError):
+ load_template("/tmp/nonexistent_template_12345.toml")
diff --git a/tests/test_system_learning.py b/tests/test_system_learning.py
new file mode 100644
index 00000000..3f71856a
--- /dev/null
+++ b/tests/test_system_learning.py
@@ -0,0 +1,50 @@
+"""Tests for LearningOrchestrator integration with SystemBuilder."""
+
+
+class TestSystemLearningIntegration:
+ def test_learning_orchestrator_not_created_when_disabled(self):
+ """Default config has training_enabled=False, so no orchestrator."""
+ from openjarvis.core.config import JarvisConfig
+ from openjarvis.system import SystemBuilder
+
+ config = JarvisConfig()
+ assert config.learning.training_enabled is False
+ result = SystemBuilder._setup_learning_orchestrator(config)
+ assert result is None
+
+ def test_learning_orchestrator_created_when_enabled(self):
+ """When training_enabled=True, orchestrator is created."""
+ from openjarvis.core.config import JarvisConfig
+ from openjarvis.learning.learning_orchestrator import LearningOrchestrator
+ from openjarvis.system import SystemBuilder
+
+ config = JarvisConfig()
+ config.learning.training_enabled = True
+ result = SystemBuilder._setup_learning_orchestrator(config)
+ assert isinstance(result, LearningOrchestrator)
+
+ def test_config_has_training_fields(self):
+ """LearningConfig has the training pipeline fields."""
+ from openjarvis.core.config import LearningConfig
+
+ config = LearningConfig()
+ assert config.training_enabled is False
+ assert config.training_schedule == ""
+ assert config.lora_rank == 16
+ assert config.lora_alpha == 32
+ assert config.min_sft_pairs == 50
+ assert config.min_improvement == 0.02
+
+ def test_training_components_exported(self):
+ """Learning package exports all training components."""
+ from openjarvis.learning import (
+ AgentConfigEvolver,
+ LearningOrchestrator,
+ LoRATrainer,
+ TrainingDataMiner,
+ )
+
+ assert TrainingDataMiner is not None
+ assert LoRATrainer is not None
+ assert AgentConfigEvolver is not None
+ assert LearningOrchestrator is not None