diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 00000000..b6590a76 --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,37 @@ +name: Frontend CI + +on: + push: + branches: [main] + paths: + - 'frontend/**' + - '.github/workflows/frontend.yml' + pull_request: + branches: [main] + paths: + - 'frontend/**' + - '.github/workflows/frontend.yml' + workflow_dispatch: + +concurrency: + group: frontend-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + + - run: npm ci + - run: npx tsc --noEmit + - run: npm run build diff --git a/.gitignore b/.gitignore index 6d75fd5f..15351b0e 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ src/openjarvis/server/static/ desktop/node_modules/ desktop/dist/ desktop/src-tauri/target/ + +# Worktrees +.worktrees/ diff --git a/desktop/src-tauri/icons/128x128.png b/desktop/src-tauri/icons/128x128.png index fa602eb3..ed9ac0cd 100644 Binary files a/desktop/src-tauri/icons/128x128.png and b/desktop/src-tauri/icons/128x128.png differ diff --git a/desktop/src-tauri/icons/128x128@2x.png b/desktop/src-tauri/icons/128x128@2x.png index a9e067b4..abc13ba5 100644 Binary files a/desktop/src-tauri/icons/128x128@2x.png and b/desktop/src-tauri/icons/128x128@2x.png differ diff --git a/desktop/src-tauri/icons/256x256.png b/desktop/src-tauri/icons/256x256.png index a9e067b4..abc13ba5 100644 Binary files a/desktop/src-tauri/icons/256x256.png and b/desktop/src-tauri/icons/256x256.png differ diff --git a/desktop/src-tauri/icons/32x32.png b/desktop/src-tauri/icons/32x32.png index a5312a6a..a1533ccc 100644 Binary files a/desktop/src-tauri/icons/32x32.png and b/desktop/src-tauri/icons/32x32.png differ diff --git a/desktop/src-tauri/icons/icon.ico b/desktop/src-tauri/icons/icon.ico index d41d586a..f4545926 100644 Binary files a/desktop/src-tauri/icons/icon.ico and b/desktop/src-tauri/icons/icon.ico differ diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 80ba69b7..55dba784 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -1,5 +1,6 @@ -use serde::Serialize; use tauri::Manager; +use tauri::menu::{MenuBuilder, MenuItemBuilder}; +use tauri::tray::TrayIconBuilder; use tauri_plugin_autostart::MacosLauncher; /// Fetch health status from the OpenJarvis API server. @@ -185,8 +186,45 @@ pub fn run() { } })) .setup(|app| { - // Set up system tray menu - let _tray = app.tray_by_id("main"); + let show = MenuItemBuilder::with_id("show", "Show / Hide") + .build(app)?; + let health = MenuItemBuilder::with_id("health", "Health: checking...") + .enabled(false) + .build(app)?; + let quit = MenuItemBuilder::with_id("quit", "Quit OpenJarvis") + .build(app)?; + + let menu = MenuBuilder::new(app) + .item(&show) + .separator() + .item(&health) + .separator() + .item(&quit) + .build()?; + + let _tray = TrayIconBuilder::with_id("main") + .icon(app.default_window_icon().unwrap().clone()) + .tooltip("OpenJarvis") + .menu(&menu) + .on_menu_event(move |app, event| { + match event.id().as_ref() { + "show" => { + if let Some(window) = app.get_webview_window("main") { + if window.is_visible().unwrap_or(false) { + let _ = window.hide(); + } else { + let _ = window.show(); + let _ = window.set_focus(); + } + } + } + "quit" => { + app.exit(0); + } + _ => {} + } + }) + .build(app)?; Ok(()) }) diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index 592674ea..99f92845 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -23,11 +23,6 @@ "transparent": false } ], - "trayIcon": { - "iconPath": "icons/icon.png", - "iconAsTemplate": true, - "tooltip": "OpenJarvis" - }, "security": { "csp": "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; connect-src 'self' http://localhost:* ws://localhost:*; img-src 'self' data: blob:" } diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index 7cabbd17..d6176971 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -5,8 +5,9 @@ import { TraceDebugger } from './components/TraceDebugger'; import { LearningCurve } from './components/LearningCurve'; import { MemoryBrowser } from './components/MemoryBrowser'; import { AdminPanel } from './components/AdminPanel'; +import { SettingsPanel } from './components/SettingsPanel'; -type TabId = 'energy' | 'traces' | 'learning' | 'memory' | 'admin'; +type TabId = 'energy' | 'traces' | 'learning' | 'memory' | 'admin' | 'settings'; interface Tab { id: TabId; @@ -19,6 +20,7 @@ const TABS: Tab[] = [ { id: 'learning', label: 'Learning' }, { id: 'memory', label: 'Memory' }, { id: 'admin', label: 'Admin' }, + { id: 'settings', label: 'Settings' }, ]; const API_URL = 'http://localhost:8000'; @@ -54,6 +56,7 @@ export function App() { {activeTab === 'learning' && } {activeTab === 'memory' && } {activeTab === 'admin' && } + {activeTab === 'settings' && } ); diff --git a/desktop/src/components/SettingsPanel.tsx b/desktop/src/components/SettingsPanel.tsx new file mode 100644 index 00000000..e5eb9cad --- /dev/null +++ b/desktop/src/components/SettingsPanel.tsx @@ -0,0 +1,205 @@ +import { useState, useEffect } from 'react'; +import type React from 'react'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface Settings { + apiUrl: string; + refreshInterval: number; // seconds + theme: 'dark' | 'light'; +} + +const DEFAULT_SETTINGS: Settings = { + apiUrl: 'http://localhost:8000', + refreshInterval: 5, + theme: 'dark', +}; + +const STORAGE_KEY = 'openjarvis-settings'; + +function loadSettings(): Settings { + try { + const raw = localStorage.getItem(STORAGE_KEY); + if (raw) { + return { ...DEFAULT_SETTINGS, ...JSON.parse(raw) }; + } + } catch { + // ignore corrupt data + } + return { ...DEFAULT_SETTINGS }; +} + +function saveSettings(settings: Settings): void { + localStorage.setItem(STORAGE_KEY, JSON.stringify(settings)); +} + +// --------------------------------------------------------------------------- +// Styles +// --------------------------------------------------------------------------- + +const styles: Record = { + container: { + backgroundColor: '#1e1e2e', + color: '#cdd6f4', + padding: 24, + maxWidth: 600, + }, + heading: { + fontSize: 20, + fontWeight: 600, + marginBottom: 24, + color: '#89b4fa', + }, + fieldGroup: { + marginBottom: 20, + }, + label: { + display: 'block', + fontSize: 13, + fontWeight: 500, + color: '#a6adc8', + marginBottom: 6, + }, + input: { + width: '100%', + padding: '8px 12px', + borderRadius: 6, + border: '1px solid #313244', + backgroundColor: '#181825', + color: '#cdd6f4', + fontSize: 14, + outline: 'none', + boxSizing: 'border-box' as const, + }, + select: { + padding: '8px 12px', + borderRadius: 6, + border: '1px solid #313244', + backgroundColor: '#181825', + color: '#cdd6f4', + fontSize: 14, + outline: 'none', + cursor: 'pointer', + }, + toggleRow: { + display: 'flex', + gap: 8, + }, + toggleButton: { + padding: '8px 16px', + borderRadius: 6, + border: '1px solid #313244', + backgroundColor: 'transparent', + color: '#a6adc8', + cursor: 'pointer', + fontSize: 14, + fontWeight: 500, + transition: 'all 0.15s ease', + }, + toggleActive: { + backgroundColor: '#313244', + color: '#cdd6f4', + borderColor: '#89b4fa', + }, + savedNotice: { + marginTop: 16, + padding: '8px 12px', + borderRadius: 6, + backgroundColor: '#1e3a2f', + color: '#a6e3a1', + fontSize: 13, + }, +}; + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- + +interface SettingsPanelProps { + onSettingsChange?: (settings: Settings) => void; +} + +export function SettingsPanel({ onSettingsChange }: SettingsPanelProps) { + const [settings, setSettings] = useState(loadSettings); + const [saved, setSaved] = useState(false); + + useEffect(() => { + saveSettings(settings); + onSettingsChange?.(settings); + setSaved(true); + const timer = setTimeout(() => setSaved(false), 2000); + return () => clearTimeout(timer); + }, [settings, onSettingsChange]); + + return ( +
+

Settings

+ +
+ + + setSettings((s) => ({ ...s, apiUrl: e.target.value })) + } + placeholder="http://localhost:8000" + /> +
+ +
+ + +
+ +
+ +
+ + +
+
+ + {saved &&
Settings saved
} +
+ ); +} + +export type { Settings }; diff --git a/docs/plans/2026-02-28-experience-polish-design.md b/docs/plans/2026-02-28-experience-polish-design.md new file mode 100644 index 00000000..e20a339f --- /dev/null +++ b/docs/plans/2026-02-28-experience-polish-design.md @@ -0,0 +1,319 @@ +# Experience Polish Design: Eval, CLI, Install, Dashboard + +**Date**: 2026-02-28 +**Status**: Approved +**Approach**: Vertical slices (3 mini-phases, independently shippable) + +## Context + +OpenJarvis has strong infrastructure (multi-vendor energy monitoring, eval framework, Tauri desktop, PWA) but the end-to-end user experience has gaps. This design addresses: eval output quality, CLI polish, installation flow, and dashboard aesthetics. + +### Current State (from audits) + +| Area | Score | Key Gaps | +|------|-------|----------| +| Energy Monitors | 8.5/10 | IPJ/IPW only in evals, not core telemetry | +| Eval Framework | 9/10 | Token stats null, no grouped tables, no trace aggregation | +| CLI Formatting | 9/10 | No logging, no verbose/quiet, bench lacks full stats | +| Installation | 8.5/10 | No quickstart, no auto-suggestions in errors | +| Desktop App | 8/10 | No settings panel, placeholder icon, stub tray | +| Browser/PWA | 6/10 | No CI/CD, no versioning, no error boundary | + +### Key Decisions + +- **Benchmark focus**: GAIA with OpenHands (TerminalBench deferred) +- **IPW** = task_accuracy / avg_power_watts (task-level, NOT per-inference) +- **IPJ** = task_accuracy / avg_energy_joules (task-level, NOT per-inference) +- **tokens_per_joule** = per-inference efficiency metric in core telemetry +- **Table layout**: Grouped panels by default, `--compact` for single dense table +- **Trace detail**: Summary + step-type breakdown by default, `--trace-detail` for full listing +- **Installation**: Non-interactive `jarvis quickstart` (zero prompts) +- **Dashboard style**: Clean, minimalistic (ChatGPT/Claude aesthetic) with unique side panels + +--- + +## Phase 23a: Perfect Eval Run + +Everything needed so `python -m evals run -c config.toml` produces beautiful, complete results. + +### 1. Core Telemetry: tokens_per_joule + +**Files**: +- `src/openjarvis/core/types.py` — Add `tokens_per_joule` to `TelemetryRecord` +- `src/openjarvis/telemetry/instrumented_engine.py` — Compute `tokens_per_joule = completion_tokens / energy_joules` +- `src/openjarvis/telemetry/store.py` — Schema migration for new column +- `src/openjarvis/telemetry/aggregator.py` — Add `avg_tokens_per_joule` to `ModelStats`/`EngineStats` + +**Not** adding IPJ/IPW to core telemetry — they require task accuracy and are inherently eval-level. + +### 2. Eval Metrics: Fix Token Stats + Strengthen IPJ/IPW + +**Files**: +- `evals/core/runner.py`: + - Wire `prompt_tokens` and `completion_tokens` from engine response into `EvalResult` + - Verify IPW = `accuracy / avg(power_watts)` across all scored samples + - Verify IPJ = `accuracy / avg(energy_joules)` across all scored samples + - Ensure `energy_joules` and `power_watts` populated from telemetry for every sample +- `evals/core/types.py`: + - Add `total_energy_joules` (sum over all samples) to `RunSummary` + - Add `avg_power_watts` (mean over all samples) to `RunSummary` + - Add `trace_steps: int`, `trace_energy_joules: float` to `EvalResult` + - Add `trace_step_type_stats: Dict[str, StepTypeStats]` to `RunSummary` + +### 3. Eval Display: Grouped Tables + +Rewrite `evals/core/display.py` with these functions: + +#### `print_accuracy_panel(summary)` +``` +╭─ Accuracy ──────────────────────────────────────────────────╮ +│ Overall Accuracy 42.0% (84/200) │ +│ Level 1 58.8% (40/68) │ +│ Level 2 35.0% (35/100) │ +│ Level 3 28.1% (9/32) │ +╰─────────────────────────────────────────────────────────────╯ +``` + +#### `print_latency_table(summary)` +``` +╭─ Latency & Throughput ──────────────────────────────────────╮ +│ Metric Avg Median Min Max Std │ +│ Latency (s) 15.41 12.30 2.10 89.50 11.20 │ +│ TTFT (ms) 145.2 120.0 45.0 890.0 85.3 │ +│ Throughput (tok/s) 41.9 38.5 12.0 95.0 15.2 │ +│ Avg Input Tokens 1024 890 128 4096 520 │ +│ Avg Output Tokens 256 210 32 2048 180 │ +╰─────────────────────────────────────────────────────────────╯ +``` + +#### `print_energy_table(summary)` +``` +╭─ Energy & Efficiency ───────────────────────────────────────╮ +│ Metric Avg Median Min Max Std │ +│ Energy (J) 46502 38500 4145 410522 89390 │ +│ Power (W) 883.8 870.2 650.0 1050.0 85.0 │ +│ GPU Util (%) 46.4 48.0 12.0 92.0 18.5 │ +│ Energy/Token (mJ) 12.5 10.8 3.2 45.0 8.1 │ +│ Tokens/Joule 80.0 92.6 22.2 312.5 65.0 │ +│ ────────────────────────────────────────────────────────── │ +│ IPW (acc/W) 0.00048 │ +│ IPJ (acc/J) 9.03e-06 │ +│ Total Energy (kJ) 9300.4 │ +╰─────────────────────────────────────────────────────────────╯ +``` + +#### `print_trace_summary(summary)` +``` +╭─ Agentic Trace Summary ────────────────────────────────────╮ +│ Total Steps: 1240 │ Avg Steps/Sample: 6.2 │ +│ │ +│ Step Type Count Avg Duration Avg Energy Avg In Tok Avg Out Tok │ +│ Median/Min/Max/Std for each column │ +│ generate 580 8.2s 38200 J 890 256 │ +│ tool_call 420 3.1s — — — │ +│ retrieve 120 0.8s — — — │ +│ route 120 0.01s — — — │ +╰─────────────────────────────────────────────────────────────╯ +``` + +All metrics in trace summary show avg/median/min/max/std where applicable. + +#### `print_compact_table(summary)` — `--compact` flag +Single dense table with all 17 metrics as rows, columns: avg/median/min/max/std. + +#### CLI flags +- `--compact`: Dense single table +- `--trace-detail`: Full per-step listing for each sample + +### 4. Per-Step Trace Aggregation + +**Files**: +- `src/openjarvis/traces/analyzer.py`: + - Add `TraceSummary.total_energy_joules` — sum of `step.metadata['energy_joules']` + - Add `TraceSummary.total_generate_energy_joules` — sum for GENERATE steps + - Add `TraceSummary.step_type_stats` — dict mapping step type to `{count, avg_duration, median_duration, min_duration, max_duration, std_duration, total_energy, avg_input_tokens, median_input_tokens, min_input_tokens, max_input_tokens, std_input_tokens, avg_output_tokens, median_output_tokens, min_output_tokens, max_output_tokens, std_output_tokens}` +- `evals/core/runner.py` — After each agent eval sample, extract `TraceSummary` and store in `EvalResult.metadata` + +--- + +## Phase 23b: Perfect First Experience + +Everything needed so a new user goes from zero to first eval in one sitting. + +### 5. `jarvis quickstart` Command + +**New file**: `src/openjarvis/cli/quickstart_cmd.py` + +Non-interactive flow (5 numbered steps): +1. Detect hardware (platform, CPU, RAM, GPU vendor/model/VRAM) +2. Write config to `~/.openjarvis/config.toml` (skip if exists, unless `--force`) +3. Check engine health (try auto-detected engine) +4. Verify model availability (list models from engine) +5. Test query ("What is 2+2?") with latency + energy measurement + +Each step prints a status line. If a step fails, print a helpful suggestion and exit gracefully. + +Flags: `--force` (redo everything) + +**Register** in `src/openjarvis/cli/__init__.py`. + +### 6. Error Message Auto-Suggestions + +**New file**: `src/openjarvis/cli/hints.py` + +Centralized hint functions: +- `hint_no_config()` → "Config not found. Run: `jarvis quickstart`" +- `hint_no_engine()` → "No engine responding. Run: `jarvis doctor`" +- `hint_no_model()` → "No model available. Try: `ollama pull qwen3:8b`" + +**Wire into**: `ask.py`, `serve.py`, `bench_cmd.py`, `chat_cmd.py` at failure points. + +### 7. Global Logging & Verbose/Quiet Flags + +**Changes**: +- `src/openjarvis/cli/__init__.py` — Add `--verbose` / `--quiet` to root `cli` group +- **New file**: `src/openjarvis/cli/log_config.py` — Centralized logging setup + - `RichHandler` for console (respects quiet flag) + - `RotatingFileHandler` for `~/.openjarvis/cli.log` (5 MB max, 3 backups) + - Default level: WARNING; `--verbose`: DEBUG; `--quiet`: ERROR + +**Add progress indicators**: +- `ask.py` — Spinner during generation +- `memory_cmd.py` — Progress bar for indexing + +### 8. Bench CLI Full Stats Tables + +**Changes**: +- `src/openjarvis/cli/bench_cmd.py` — Show full stats table (avg/median/min/max/std) matching eval style +- `src/openjarvis/bench/latency.py` — Export all percentile stats (already computes internally) +- `src/openjarvis/bench/throughput.py` — Add stats computation +- `src/openjarvis/bench/energy.py` — Add stats computation + +--- + +## Phase 23c: Perfect Dashboard + +Desktop and browser apps polished for demos and daily use. + +### Design Aesthetic + +**Clean, minimalistic** — inspired by ChatGPT and Claude web interfaces: +- Generous whitespace, muted color palette, subtle borders +- Sans-serif typography, comfortable line heights +- Smooth transitions, no visual clutter + +**Differentiated** via unique side panels: +- Energy dashboard with real-time power charts +- Trace debugger with color-coded timeline +- Learning curve visualization +- Memory browser with relevance scoring + +The side panels are what make OpenJarvis unique vs generic chat UIs. Keep them accessible but not overwhelming — collapsible, clean data density. + +### 9. Desktop: Settings Panel + +**New file**: `desktop/src/components/SettingsPanel.tsx` +- API URL configuration (default: `localhost:8000`) +- Auto-update interval setting +- Theme toggle (dark/light) — currently hardcoded dark +- Persist to `localStorage` + +### 10. Desktop: Windows Icon + +Replace 108-byte placeholder `desktop/src-tauri/icons/icon.ico` with proper multi-resolution icon generated from existing `icon.png`. + +### 11. Desktop: System Tray + +Implement stubbed tray menu in `desktop/src-tauri/src/lib.rs`: +- Show/Hide window toggle +- Health status indicator (green/red dot) +- Quit action + +### 12. Browser/PWA: CI/CD + +**New file**: `.github/workflows/frontend.yml` +- Trigger on changes to `frontend/` +- `npm ci && npm run build` +- Commit built output to `src/openjarvis/server/static/` +- Add `version` field to `manifest.webmanifest` (auto-bumped) + +### 13. Browser/PWA: Error Boundary + Config + +- Wrap `` in React error boundary (graceful crash recovery) +- Support `VITE_API_URL` environment variable (default `localhost:8000`) + +### 14. Browser/PWA: Style Refresh + +Refactor the 12,971-line `App.css`: +- Modularize into component-scoped CSS files +- Clean minimalist aesthetic (ChatGPT/Claude-inspired) +- Catppuccin-based dark theme (keep existing) + light theme option +- Collapsible side panels with smooth transitions + +--- + +## Implementation Order + +### Phase 23a (highest priority — enables GAIA research) +1. Core telemetry: `tokens_per_joule` +2. Eval metrics: fix token stats, strengthen IPJ/IPW +3. Per-step trace aggregation in `TraceAnalyzer` +4. Eval display: grouped tables, `--compact`, `--trace-detail` + +### Phase 23b (high priority — onboarding & CLI) +5. `jarvis quickstart` command +6. Error message auto-suggestions +7. Global logging + verbose/quiet flags +8. Bench CLI full stats tables + +### Phase 23c (lower priority — dashboard polish) +9. Desktop settings panel +10. Windows icon fix +11. System tray implementation +12. PWA CI/CD +13. PWA error boundary + config +14. PWA style refresh + +--- + +## Files Modified (Summary) + +### Phase 23a +| File | Change | +|------|--------| +| `src/openjarvis/core/types.py` | Add `tokens_per_joule` to `TelemetryRecord` | +| `src/openjarvis/telemetry/instrumented_engine.py` | Compute `tokens_per_joule` | +| `src/openjarvis/telemetry/store.py` | Schema migration | +| `src/openjarvis/telemetry/aggregator.py` | Add `avg_tokens_per_joule` | +| `src/openjarvis/traces/analyzer.py` | Add energy aggregation, step-type stats | +| `evals/core/types.py` | Add trace fields, total_energy, avg_power | +| `evals/core/runner.py` | Fix token stats, wire trace data, verify IPJ/IPW | +| `evals/core/display.py` | Rewrite: grouped panels, compact mode | +| `evals/cli.py` | Add `--compact`, `--trace-detail` flags | + +### Phase 23b +| File | Change | +|------|--------| +| `src/openjarvis/cli/quickstart_cmd.py` | **New**: quickstart command | +| `src/openjarvis/cli/hints.py` | **New**: centralized hint system | +| `src/openjarvis/cli/log_config.py` | **New**: logging setup | +| `src/openjarvis/cli/__init__.py` | Register quickstart, add verbose/quiet | +| `src/openjarvis/cli/ask.py` | Add hints, progress spinner | +| `src/openjarvis/cli/serve.py` | Add hints | +| `src/openjarvis/cli/bench_cmd.py` | Full stats tables | +| `src/openjarvis/cli/chat_cmd.py` | Add hints | +| `src/openjarvis/cli/memory_cmd.py` | Progress bar for indexing | +| `src/openjarvis/bench/latency.py` | Export full stats | +| `src/openjarvis/bench/throughput.py` | Add stats computation | +| `src/openjarvis/bench/energy.py` | Add stats computation | + +### Phase 23c +| File | Change | +|------|--------| +| `desktop/src/components/SettingsPanel.tsx` | **New**: settings UI | +| `desktop/src-tauri/icons/icon.ico` | Replace placeholder | +| `desktop/src-tauri/src/lib.rs` | Implement system tray | +| `.github/workflows/frontend.yml` | **New**: PWA CI/CD | +| `frontend/src/App.tsx` | Error boundary wrapper | +| `frontend/vite.config.ts` | VITE_API_URL support | +| `frontend/src/App.css` | Modularize, style refresh | diff --git a/docs/plans/2026-02-28-experience-polish-implementation.md b/docs/plans/2026-02-28-experience-polish-implementation.md new file mode 100644 index 00000000..ae45a606 --- /dev/null +++ b/docs/plans/2026-02-28-experience-polish-implementation.md @@ -0,0 +1,1285 @@ +# Experience Polish Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Make the eval output, CLI, installation, and dashboard experiences clean, complete, and publication-ready across three independently shippable phases. + +**Architecture:** Vertical slices — Phase 23a delivers a "perfect eval run" (grouped tables, IPJ/IPW, trace breakdown), Phase 23b delivers a "perfect first experience" (quickstart, logging, hints), Phase 23c polishes the desktop/browser dashboards. + +**Tech Stack:** Python 3.10+, Rich (tables/panels), Click (CLI), Tauri 2.0 (desktop), React/Vite (PWA), SQLite (telemetry/traces), pytest (testing) + +--- + +## Phase 23a: Perfect Eval Run + +### Task 1: Add `tokens_per_joule` to TelemetryRecord + +**Files:** +- Modify: `src/openjarvis/core/types.py:125-165` +- Test: `tests/telemetry/test_telemetry_record.py` (new) + +**Step 1: Write the failing test** + +Create `tests/telemetry/test_telemetry_record.py`: + +```python +"""Tests for TelemetryRecord fields.""" + +from __future__ import annotations + +from openjarvis.core.types import TelemetryRecord + + +class TestTelemetryRecord: + def test_tokens_per_joule_field_exists(self): + rec = TelemetryRecord(timestamp=1.0, model_id="test") + assert hasattr(rec, "tokens_per_joule") + assert rec.tokens_per_joule == 0.0 + + def test_tokens_per_joule_set(self): + rec = TelemetryRecord( + timestamp=1.0, + model_id="test", + tokens_per_joule=80.0, + ) + assert rec.tokens_per_joule == 80.0 +``` + +**Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/telemetry/test_telemetry_record.py -v` +Expected: FAIL — `TypeError: ... unexpected keyword argument 'tokens_per_joule'` + +**Step 3: Write minimal implementation** + +In `src/openjarvis/core/types.py`, add after `dram_energy_joules` field (around line 163): + +```python + tokens_per_joule: float = 0.0 +``` + +**Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/telemetry/test_telemetry_record.py -v` +Expected: PASS (2 tests) + +**Step 5: Commit** + +```bash +git add src/openjarvis/core/types.py tests/telemetry/test_telemetry_record.py +git commit -m "feat(telemetry): add tokens_per_joule field to TelemetryRecord" +``` + +--- + +### Task 2: Compute `tokens_per_joule` in InstrumentedEngine + +**Files:** +- Modify: `src/openjarvis/telemetry/instrumented_engine.py:154-174` +- Test: `tests/telemetry/test_instrumented_engine.py` (existing — add test) + +**Step 1: Write the failing test** + +Add to `tests/telemetry/test_instrumented_engine.py`: + +```python +class TestTokensPerJoule: + def test_tokens_per_joule_computed(self, mock_engine, bus): + """tokens_per_joule = completion_tokens / energy_joules.""" + mock_engine.generate.return_value = { + "content": "hello", + "usage": {"prompt_tokens": 10, "completion_tokens": 50, "total_tokens": 60}, + } + inst = InstrumentedEngine(mock_engine, bus=bus) + # Mock energy monitor to return known energy + from unittest.mock import MagicMock, patch + monitor = MagicMock() + sample = MagicMock() + sample.energy_joules = 2.5 + sample.mean_power_watts = 100.0 + sample.peak_power_watts = 120.0 + sample.gpu_utilization_pct = 50.0 + sample.gpu_memory_used_gb = 4.0 + sample.gpu_temperature_c = 60.0 + sample.cpu_energy_joules = 0.0 + sample.gpu_energy_joules = 2.5 + sample.dram_energy_joules = 0.0 + sample.energy_method = "test" + sample.vendor = "test" + monitor.sample.return_value.__enter__ = lambda s: sample + monitor.sample.return_value.__exit__ = lambda s, *a: None + inst._energy_monitor = monitor + + inst.generate(messages=[{"role": "user", "content": "hi"}]) + records = [ + e.data for e in bus.history + if hasattr(e, 'event_type') + and e.event_type.value == "telemetry_record" + ] + assert len(records) >= 1 + rec = records[0] + # 50 tokens / 2.5 J = 20.0 tokens/joule + assert rec.tokens_per_joule == pytest.approx(20.0, rel=0.1) + + def test_tokens_per_joule_zero_energy(self, mock_engine, bus): + """tokens_per_joule is 0.0 when energy is zero.""" + mock_engine.generate.return_value = { + "content": "hello", + "usage": {"prompt_tokens": 10, "completion_tokens": 50, "total_tokens": 60}, + } + inst = InstrumentedEngine(mock_engine, bus=bus) + inst.generate(messages=[{"role": "user", "content": "hi"}]) + records = [ + e.data for e in bus.history + if hasattr(e, 'event_type') + and e.event_type.value == "telemetry_record" + ] + rec = records[0] + assert rec.tokens_per_joule == 0.0 +``` + +**Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/telemetry/test_instrumented_engine.py::TestTokensPerJoule -v` +Expected: FAIL — `AttributeError: ... has no attribute 'tokens_per_joule'` on the record + +**Step 3: Write minimal implementation** + +In `src/openjarvis/telemetry/instrumented_engine.py`, in the `generate()` method, after the existing derived metrics block (around line 159, after `throughput_per_watt`), add: + +```python + # Tokens per joule (inverse of energy-per-token, per-inference efficiency) + tokens_per_joule = 0.0 + if energy_j > 0 and completion_tokens > 0: + tokens_per_joule = completion_tokens / energy_j +``` + +Then in the `TelemetryRecord(...)` constructor call (around line 178-204), add: + +```python + tokens_per_joule=tokens_per_joule, +``` + +Do the same in the `stream()` method (around line 369-400). + +**Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/telemetry/test_instrumented_engine.py::TestTokensPerJoule -v` +Expected: PASS (2 tests) + +**Step 5: Run full telemetry tests** + +Run: `uv run pytest tests/telemetry/ -v` +Expected: All pass (no regressions) + +**Step 6: Commit** + +```bash +git add src/openjarvis/telemetry/instrumented_engine.py tests/telemetry/test_instrumented_engine.py +git commit -m "feat(telemetry): compute tokens_per_joule in InstrumentedEngine" +``` + +--- + +### Task 3: Store and aggregate `tokens_per_joule` + +**Files:** +- Modify: `src/openjarvis/telemetry/store.py:13-52` (schema) +- Modify: `src/openjarvis/telemetry/store.py:126-170` (record method) +- Modify: `src/openjarvis/telemetry/aggregator.py:11-33` (ModelStats) +- Modify: `src/openjarvis/telemetry/aggregator.py:36-56` (EngineStats) +- Modify: `src/openjarvis/telemetry/aggregator.py:115-198` (per_model_stats) +- Modify: `src/openjarvis/telemetry/aggregator.py:200-278` (per_engine_stats) +- Test: `tests/telemetry/test_store_tokens_per_joule.py` (new) + +**Step 1: Write the failing test** + +Create `tests/telemetry/test_store_tokens_per_joule.py`: + +```python +"""Tests for tokens_per_joule storage and aggregation.""" + +from __future__ import annotations + +import time + +import pytest + +from openjarvis.core.types import TelemetryRecord +from openjarvis.telemetry.store import TelemetryStore +from openjarvis.telemetry.aggregator import TelemetryAggregator + + +class TestTokensPerJouleStorage: + def test_store_and_retrieve(self, tmp_path): + store = TelemetryStore(db_path=tmp_path / "tel.db") + rec = TelemetryRecord( + timestamp=time.time(), + model_id="test-model", + completion_tokens=50, + energy_joules=2.5, + tokens_per_joule=20.0, + ) + store.record(rec) + agg = TelemetryAggregator(store) + stats = agg.per_model_stats() + assert len(stats) == 1 + assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1) + store.close() + + def test_aggregate_multiple(self, tmp_path): + store = TelemetryStore(db_path=tmp_path / "tel.db") + for tpj in [10.0, 20.0, 30.0]: + rec = TelemetryRecord( + timestamp=time.time(), + model_id="m1", + tokens_per_joule=tpj, + ) + store.record(rec) + agg = TelemetryAggregator(store) + stats = agg.per_model_stats() + assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1) + store.close() +``` + +**Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/telemetry/test_store_tokens_per_joule.py -v` +Expected: FAIL — column or attribute not found + +**Step 3: Write minimal implementation** + +1. In `src/openjarvis/telemetry/store.py` schema list (lines 13-52), add `"tokens_per_joule"` to the columns list. + +2. In `store.py` `record()` method, add `tokens_per_joule` to the INSERT statement and values tuple. + +3. In `store.py` schema migration (`_maybe_add_columns` or equivalent), add migration for the new column. + +4. In `src/openjarvis/telemetry/aggregator.py` `ModelStats`, add: + ```python + avg_tokens_per_joule: float = 0.0 + ``` + +5. In `EngineStats`, add the same field. + +6. In `per_model_stats()` SQL query, add `AVG(tokens_per_joule)` using `_safe_col()`. + +7. In `per_engine_stats()` SQL query, add same. + +**Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/telemetry/test_store_tokens_per_joule.py -v` +Expected: PASS (2 tests) + +**Step 5: Run full telemetry test suite** + +Run: `uv run pytest tests/telemetry/ -v` +Expected: All pass + +**Step 6: Commit** + +```bash +git add src/openjarvis/telemetry/store.py src/openjarvis/telemetry/aggregator.py tests/telemetry/test_store_tokens_per_joule.py +git commit -m "feat(telemetry): store and aggregate tokens_per_joule" +``` + +--- + +### Task 4: Add energy aggregation and step-type stats to TraceAnalyzer + +**Files:** +- Modify: `src/openjarvis/traces/analyzer.py:39-49` (TraceSummary) +- Modify: `src/openjarvis/traces/analyzer.py:62-91` (summary method) +- Test: `tests/traces/test_analyzer_energy.py` (new) + +**Step 1: Write the failing test** + +Create `tests/traces/test_analyzer_energy.py`: + +```python +"""Tests for energy aggregation in TraceAnalyzer.""" + +from __future__ import annotations + +import time + +import pytest + +from openjarvis.core.types import StepType, Trace, TraceStep +from openjarvis.traces.analyzer import StepTypeStats, TraceAnalyzer, TraceSummary +from openjarvis.traces.store import TraceStore + + +def _make_trace(steps: list[TraceStep]) -> Trace: + return Trace( + query="test", + agent="test_agent", + model="test_model", + engine="test_engine", + steps=steps, + started_at=time.time(), + ended_at=time.time() + 10, + total_tokens=100, + total_latency_seconds=10.0, + ) + + +def _gen_step(energy: float = 0.0, duration: float = 1.0, + prompt_tokens: int = 50, completion_tokens: int = 25) -> TraceStep: + return TraceStep( + step_type=StepType.GENERATE, + timestamp=time.time(), + duration_seconds=duration, + output={ + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + metadata={ + "energy_joules": energy, + "power_watts": energy / duration if duration > 0 else 0.0, + }, + ) + + +def _tool_step(duration: float = 0.5) -> TraceStep: + return TraceStep( + step_type=StepType.TOOL_CALL, + timestamp=time.time(), + duration_seconds=duration, + input={"tool": "calculator"}, + output={"success": True}, + ) + + +class TestTraceSummaryEnergyFields: + def test_total_energy_joules(self, tmp_path): + store = TraceStore(db_path=tmp_path / "traces.db") + trace = _make_trace([ + _gen_step(energy=10.0, duration=2.0), + _tool_step(duration=0.5), + _gen_step(energy=15.0, duration=3.0), + ]) + store.save(trace) + analyzer = TraceAnalyzer(store) + summary = analyzer.summary() + assert summary.total_energy_joules == pytest.approx(25.0, rel=0.01) + assert summary.total_generate_energy_joules == pytest.approx(25.0, rel=0.01) + store.close() + + def test_step_type_stats(self, tmp_path): + store = TraceStore(db_path=tmp_path / "traces.db") + trace = _make_trace([ + _gen_step(energy=10.0, duration=2.0, prompt_tokens=100, completion_tokens=50), + _gen_step(energy=20.0, duration=4.0, prompt_tokens=80, completion_tokens=40), + _tool_step(duration=0.5), + _tool_step(duration=1.5), + ]) + store.save(trace) + analyzer = TraceAnalyzer(store) + summary = analyzer.summary() + + assert "generate" in summary.step_type_stats + gen = summary.step_type_stats["generate"] + assert gen.count == 2 + assert gen.avg_duration == pytest.approx(3.0, rel=0.01) + assert gen.total_energy == pytest.approx(30.0, rel=0.01) + assert gen.avg_input_tokens == pytest.approx(90.0, rel=0.01) + assert gen.avg_output_tokens == pytest.approx(45.0, rel=0.01) + assert gen.min_duration == pytest.approx(2.0, rel=0.01) + assert gen.max_duration == pytest.approx(4.0, rel=0.01) + + assert "tool_call" in summary.step_type_stats + tc = summary.step_type_stats["tool_call"] + assert tc.count == 2 + assert tc.avg_duration == pytest.approx(1.0, rel=0.01) + store.close() + + +class TestStepTypeStats: + def test_dataclass_fields(self): + s = StepTypeStats(count=5, avg_duration=2.0, total_energy=10.0) + assert s.count == 5 + assert s.std_duration == 0.0 # default +``` + +**Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/traces/test_analyzer_energy.py -v` +Expected: FAIL — `ImportError: cannot import name 'StepTypeStats'` + +**Step 3: Write minimal implementation** + +In `src/openjarvis/traces/analyzer.py`: + +1. Add new dataclass before `TraceSummary` (after line 37): + +```python +@dataclass(slots=True) +class StepTypeStats: + """Aggregated statistics for a specific step type across traces.""" + + count: int = 0 + avg_duration: float = 0.0 + median_duration: float = 0.0 + min_duration: float = 0.0 + max_duration: float = 0.0 + std_duration: float = 0.0 + total_energy: float = 0.0 + avg_input_tokens: float = 0.0 + median_input_tokens: float = 0.0 + min_input_tokens: float = 0.0 + max_input_tokens: float = 0.0 + std_input_tokens: float = 0.0 + avg_output_tokens: float = 0.0 + median_output_tokens: float = 0.0 + min_output_tokens: float = 0.0 + max_output_tokens: float = 0.0 + std_output_tokens: float = 0.0 +``` + +2. Add fields to `TraceSummary`: + +```python + total_energy_joules: float = 0.0 + total_generate_energy_joules: float = 0.0 + step_type_stats: Dict[str, StepTypeStats] = field(default_factory=dict) +``` + +3. In the `summary()` method, after computing `step_dist`, add energy and step-type stats computation: + +```python + import statistics as stats_mod + + # Compute energy totals and step-type stats + total_energy = 0.0 + generate_energy = 0.0 + step_data: Dict[str, Dict[str, list]] = {} + + for t in traces: + for s in t.steps: + key = _step_type_str(s) + energy = s.metadata.get("energy_joules", 0.0) + total_energy += energy + if key == "generate": + generate_energy += energy + + if key not in step_data: + step_data[key] = { + "durations": [], "energies": [], + "input_tokens": [], "output_tokens": [], + } + step_data[key]["durations"].append(s.duration_seconds) + step_data[key]["energies"].append(energy) + step_data[key]["input_tokens"].append( + s.output.get("prompt_tokens", 0) + ) + step_data[key]["output_tokens"].append( + s.output.get("completion_tokens", 0) + ) + + sts_map: Dict[str, StepTypeStats] = {} + for key, data in step_data.items(): + durations = data["durations"] + in_tok = [float(x) for x in data["input_tokens"]] + out_tok = [float(x) for x in data["output_tokens"]] + sts_map[key] = StepTypeStats( + count=len(durations), + avg_duration=_avg(durations), + median_duration=stats_mod.median(durations) if durations else 0.0, + min_duration=min(durations) if durations else 0.0, + max_duration=max(durations) if durations else 0.0, + std_duration=stats_mod.stdev(durations) if len(durations) > 1 else 0.0, + total_energy=sum(data["energies"]), + avg_input_tokens=_avg(in_tok), + median_input_tokens=stats_mod.median(in_tok) if in_tok else 0.0, + min_input_tokens=min(in_tok) if in_tok else 0.0, + max_input_tokens=max(in_tok) if in_tok else 0.0, + std_input_tokens=stats_mod.stdev(in_tok) if len(in_tok) > 1 else 0.0, + avg_output_tokens=_avg(out_tok), + median_output_tokens=stats_mod.median(out_tok) if out_tok else 0.0, + min_output_tokens=min(out_tok) if out_tok else 0.0, + max_output_tokens=max(out_tok) if out_tok else 0.0, + std_output_tokens=stats_mod.stdev(out_tok) if len(out_tok) > 1 else 0.0, + ) +``` + +Then add the new fields to the returned `TraceSummary`: + +```python + total_energy_joules=total_energy, + total_generate_energy_joules=generate_energy, + step_type_stats=sts_map, +``` + +4. Update `__all__` to include `StepTypeStats`. + +**Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/traces/test_analyzer_energy.py -v` +Expected: PASS (3 tests) + +**Step 5: Run full trace tests** + +Run: `uv run pytest tests/traces/ -v` +Expected: All pass + +**Step 6: Commit** + +```bash +git add src/openjarvis/traces/analyzer.py tests/traces/test_analyzer_energy.py +git commit -m "feat(traces): add energy aggregation and step-type stats to TraceAnalyzer" +``` + +--- + +### Task 5: Add trace fields to eval types + +**Files:** +- Modify: `evals/core/types.py:21-47` (EvalResult) +- Modify: `evals/core/types.py:87-125` (RunSummary) +- Test: `evals/tests/test_types.py` (existing — add tests) + +**Step 1: Write the failing test** + +Add to `evals/tests/test_types.py`: + +```python +class TestEvalResultTraceFields: + def test_trace_fields_exist(self): + r = EvalResult(record_id="test", model_answer="hi") + assert r.trace_steps == 0 + assert r.trace_energy_joules == 0.0 + +class TestRunSummaryTraceFields: + def test_trace_aggregate_fields(self): + s = RunSummary( + benchmark="test", category="test", backend="test", + model="test", total_samples=1, scored_samples=1, + correct=1, accuracy=1.0, errors=0, + mean_latency_seconds=1.0, total_cost_usd=0.0, + ) + assert s.avg_power_watts == 0.0 + assert s.trace_step_type_stats == {} + assert s.total_input_tokens == 0 + assert s.total_output_tokens == 0 +``` + +**Step 2: Run test to verify it fails** + +Run: `uv run pytest evals/tests/test_types.py::TestEvalResultTraceFields -v` +Expected: FAIL — `AttributeError` + +**Step 3: Write minimal implementation** + +In `evals/core/types.py`: + +1. Add to `EvalResult` (after line 46, before the class ends): +```python + trace_steps: int = 0 + trace_energy_joules: float = 0.0 +``` + +2. Add to `RunSummary` (after line 124): +```python + avg_power_watts: float = 0.0 + total_input_tokens: int = 0 + total_output_tokens: int = 0 + trace_step_type_stats: Dict[str, Dict[str, float]] = field(default_factory=dict) +``` + +**Step 4: Run tests** + +Run: `uv run pytest evals/tests/test_types.py -v` +Expected: All pass + +**Step 5: Commit** + +```bash +git add evals/core/types.py evals/tests/test_types.py +git commit -m "feat(evals): add trace and power fields to EvalResult and RunSummary" +``` + +--- + +### Task 6: Wire trace data and strengthen token stats in eval runner + +**Files:** +- Modify: `evals/core/runner.py:134-223` (_process_one) +- Modify: `evals/core/runner.py:267-380` (_compute_summary) +- Test: `evals/tests/test_runner.py` (existing — add tests) + +**Step 1: Write the failing test** + +Add to `evals/tests/test_runner.py`: + +```python +class TestRunnerTokenStats: + def test_summary_has_total_input_output_tokens(self, tmp_path): + """RunSummary should include total token counts.""" + records = [ + EvalRecord(record_id=f"r{i}", problem=f"q{i}", reference="a", category="test") + for i in range(3) + ] + backend = MockBackend() + scorer = MockScorer() + dataset = MockDataset(records=records) + config = RunConfig(benchmark="test", backend="jarvis-direct", model="m") + runner = EvalRunner(config=config, backend=backend, scorer=scorer, dataset=dataset) + + results, summary = runner.run(output_dir=tmp_path) + # MockBackend returns usage with tokens — they should be tallied + assert summary.total_input_tokens >= 0 + assert summary.total_output_tokens >= 0 + # input_token_stats and output_token_stats should be populated + if summary.input_token_stats is not None: + assert summary.input_token_stats.mean >= 0 + + def test_summary_has_avg_power(self, tmp_path): + """RunSummary should include avg_power_watts.""" + records = [ + EvalRecord(record_id="r1", problem="q", reference="a", category="test") + ] + config = RunConfig(benchmark="test", backend="jarvis-direct", model="m") + runner = EvalRunner( + config=config, backend=MockBackend(), scorer=MockScorer(), + dataset=MockDataset(records=records), + ) + results, summary = runner.run(output_dir=tmp_path) + assert hasattr(summary, "avg_power_watts") +``` + +**Step 2: Run test to verify it fails** + +Run: `uv run pytest evals/tests/test_runner.py::TestRunnerTokenStats -v` +Expected: FAIL — `total_input_tokens` not populated or missing + +**Step 3: Write minimal implementation** + +In `evals/core/runner.py` `_compute_summary()` method: + +1. After collecting value lists (around line 343), add: +```python + total_input_tokens = sum(r.prompt_tokens for r in scored) + total_output_tokens = sum(r.completion_tokens for r in scored) + power_values = [r.power_watts for r in scored if r.power_watts > 0] + avg_power = _avg(power_values) if power_values else 0.0 +``` + +2. In the `RunSummary(...)` constructor, add: +```python + total_input_tokens=total_input_tokens, + total_output_tokens=total_output_tokens, + avg_power_watts=avg_power, +``` + +3. For trace data wiring in `_process_one()`: After the agent/backend returns, check if trace data is available in `full.get("_trace")` and extract step count and energy. Add to the returned `EvalResult`: +```python + trace_steps=trace_step_count, + trace_energy_joules=trace_energy, +``` + +**Step 4: Run tests** + +Run: `uv run pytest evals/tests/test_runner.py -v` +Expected: All pass + +**Step 5: Commit** + +```bash +git add evals/core/runner.py evals/tests/test_runner.py +git commit -m "feat(evals): wire trace data and token totals into RunSummary" +``` + +--- + +### Task 7: Rewrite eval display with grouped tables + +This is the largest task in Phase 23a. The current `display.py` has one monolithic `print_metrics_table()`. We replace it with grouped panels. + +**Files:** +- Modify: `evals/core/display.py` (major rewrite) +- Test: `evals/tests/test_display.py` (new) + +**Step 1: Write the failing test** + +Create `evals/tests/test_display.py`: + +```python +"""Tests for eval display functions.""" + +from __future__ import annotations + +from io import StringIO + +from rich.console import Console + +from evals.core.display import ( + print_accuracy_panel, + print_energy_table, + print_latency_table, + print_trace_summary, + print_compact_table, + print_full_results, +) +from evals.core.types import MetricStats, RunSummary + + +def _make_summary(**overrides) -> RunSummary: + defaults = dict( + benchmark="gaia", category="agentic", backend="jarvis-agent", + model="qwen3:8b", total_samples=100, scored_samples=100, + correct=42, accuracy=0.42, errors=0, + mean_latency_seconds=15.0, total_cost_usd=0.0, + total_energy_joules=9300.0, avg_power_watts=880.0, + total_input_tokens=50000, total_output_tokens=12000, + per_subject={"level_1": {"accuracy": 0.58, "correct": 40, "scored": 68}}, + ) + defaults.update(overrides) + return RunSummary(**defaults) + + +def _make_stats(mean=10.0) -> MetricStats: + return MetricStats( + mean=mean, median=mean * 0.9, min=mean * 0.2, + max=mean * 3.0, std=mean * 0.5, p90=mean * 1.5, + p95=mean * 2.0, p99=mean * 2.5, + ) + + +class TestAccuracyPanel: + def test_renders_without_error(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary() + print_accuracy_panel(console, summary) + output = console.file.getvalue() + assert "42.0%" in output or "0.42" in output + + def test_shows_per_subject(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary() + print_accuracy_panel(console, summary) + output = console.file.getvalue() + assert "level_1" in output + + +class TestLatencyTable: + def test_renders_with_stats(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + latency_stats=_make_stats(15.0), + throughput_stats=_make_stats(40.0), + input_token_stats=_make_stats(1024.0), + output_token_stats=_make_stats(256.0), + ) + print_latency_table(console, summary) + output = console.file.getvalue() + assert "Latency" in output + assert "Avg" in output + + +class TestEnergyTable: + def test_renders_ipj_ipw(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + energy_stats=_make_stats(46000.0), + power_stats=_make_stats(880.0), + ipw_stats=_make_stats(0.00048), + ipj_stats=_make_stats(9.0e-6), + ) + print_energy_table(console, summary) + output = console.file.getvalue() + assert "IPW" in output + assert "IPJ" in output + + +class TestTraceSummary: + def test_renders_step_type_breakdown(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + trace_step_type_stats={ + "generate": { + "count": 580, "avg_duration": 8.2, + "total_energy": 22000.0, + "avg_input_tokens": 890.0, "avg_output_tokens": 256.0, + }, + "tool_call": { + "count": 420, "avg_duration": 3.1, + "total_energy": 0.0, + "avg_input_tokens": 0.0, "avg_output_tokens": 0.0, + }, + }, + ) + print_trace_summary(console, summary) + output = console.file.getvalue() + assert "generate" in output + assert "tool_call" in output + + +class TestCompactTable: + def test_renders_all_metrics(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + latency_stats=_make_stats(15.0), + energy_stats=_make_stats(46000.0), + ) + print_compact_table(console, summary) + output = console.file.getvalue() + assert "Latency" in output + assert "Energy" in output + + +class TestFullResults: + def test_renders_all_sections(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + latency_stats=_make_stats(15.0), + energy_stats=_make_stats(46000.0), + power_stats=_make_stats(880.0), + ) + print_full_results(console, summary) + output = console.file.getvalue() + assert "Accuracy" in output +``` + +**Step 2: Run test to verify it fails** + +Run: `uv run pytest evals/tests/test_display.py -v` +Expected: FAIL — `ImportError: cannot import name 'print_accuracy_panel'` + +**Step 3: Write the implementation** + +Rewrite `evals/core/display.py`. Keep existing `print_banner`, `print_section`, `print_run_header`, `print_suite_summary`, `print_completion`. Replace `print_metrics_table` and add new functions: + +```python +def print_accuracy_panel(console: Console, summary: RunSummary) -> None: + """Print accuracy panel with per-subject breakdown.""" + lines = [ + f"[bold]Overall Accuracy {summary.accuracy:.1%}[/bold]" + f" ({summary.correct}/{summary.scored_samples})", + ] + for subj, stats in sorted(summary.per_subject.items()): + acc = stats.get("accuracy", 0.0) + correct = int(stats.get("correct", 0)) + scored = int(stats.get("scored", 0)) + lines.append(f" {subj:<20s} {acc:.1%} ({correct}/{scored})") + body = "\n".join(lines) + panel = Panel(body, title="[bold]Accuracy[/bold]", border_style="green", expand=False) + console.print(panel) + + +def _stats_table(title: str, rows: list[tuple[str, Optional[MetricStats], int]]) -> Table: + """Build a stats table with Avg/Median/Min/Max/Std columns.""" + table = Table( + title=f"[bold]{title}[/bold]", + show_header=True, + header_style="bold bright_white", + border_style="bright_blue", + title_style="bold cyan", + ) + table.add_column("Metric", style="cyan", no_wrap=True) + table.add_column("Avg", justify="right") + table.add_column("Median", justify="right") + table.add_column("Min", justify="right") + table.add_column("Max", justify="right") + table.add_column("Std", justify="right") + for label, stats, decimals in rows: + if stats is not None: + _add_metric_row(table, label, stats, decimals) + return table + + +def print_latency_table(console: Console, summary: RunSummary) -> None: + """Print latency, throughput, and token stats table.""" + table = _stats_table("Latency & Throughput", [ + ("Latency (s)", summary.latency_stats, 2), + ("TTFT (s)", summary.ttft_stats, 3), + ("Throughput (tok/s)", summary.throughput_stats, 1), + ("Avg Input Tokens", summary.input_token_stats, 1), + ("Avg Output Tokens", summary.output_token_stats, 1), + ]) + if table.row_count > 0: + console.print(table) + + +def print_energy_table(console: Console, summary: RunSummary) -> None: + """Print energy, efficiency, and IPJ/IPW table.""" + table = _stats_table("Energy & Efficiency", [ + ("Energy (J)", summary.energy_stats, 1), + ("Power (W)", summary.power_stats, 1), + ("GPU Util (%)", summary.gpu_utilization_stats, 1), + ("Energy/OutTok (J)", summary.energy_per_output_token_stats, 6), + ("Tokens/Joule", None, 1), # placeholder — see note below + ("MFU (%)", summary.mfu_stats, 3), + ("MBU (%)", summary.mbu_stats, 3), + ]) + if table.row_count > 0: + console.print(table) + # Headline: IPW, IPJ, Total Energy + parts = [] + if summary.ipw_stats: + parts.append(f"[bold]IPW (acc/W):[/bold] {summary.ipw_stats.mean:.6f}") + if summary.ipj_stats: + parts.append(f"[bold]IPJ (acc/J):[/bold] {summary.ipj_stats.mean:.2e}") + if summary.total_energy_joules > 0: + val = summary.total_energy_joules + unit = "kJ" if val > 1000 else "J" + display = val / 1000 if val > 1000 else val + parts.append(f"[bold]Total Energy:[/bold] {display:.1f} {unit}") + if summary.avg_power_watts > 0: + parts.append(f"[bold]Avg Power:[/bold] {summary.avg_power_watts:.1f} W") + if parts: + console.print(" ".join(parts)) + + +def print_trace_summary(console: Console, summary: RunSummary) -> None: + """Print agentic trace step-type breakdown.""" + sts = summary.trace_step_type_stats + if not sts: + return + total_steps = sum(s.get("count", 0) for s in sts.values()) + avg_per_sample = total_steps / summary.scored_samples if summary.scored_samples > 0 else 0 + + table = Table( + title="[bold]Agentic Trace Summary[/bold]", + show_header=True, + header_style="bold bright_white", + border_style="bright_blue", + title_style="bold cyan", + caption=f"Total Steps: {total_steps} | Avg Steps/Sample: {avg_per_sample:.1f}", + ) + table.add_column("Step Type", style="cyan", no_wrap=True) + table.add_column("Count", justify="right") + table.add_column("Avg Duration", justify="right") + table.add_column("Avg Energy (J)", justify="right") + table.add_column("Avg In Tokens", justify="right") + table.add_column("Avg Out Tokens", justify="right") + + for stype, data in sorted(sts.items()): + count = data.get("count", 0) + avg_dur = data.get("avg_duration", 0.0) + total_e = data.get("total_energy", 0.0) + avg_e = total_e / count if count > 0 else 0.0 + avg_in = data.get("avg_input_tokens", 0.0) + avg_out = data.get("avg_output_tokens", 0.0) + table.add_row( + stype, + str(count), + f"{avg_dur:.2f}s", + f"{avg_e:.1f}" if avg_e > 0 else "—", + f"{avg_in:.0f}" if avg_in > 0 else "—", + f"{avg_out:.0f}" if avg_out > 0 else "—", + ) + console.print(table) + + +def print_compact_table(console: Console, summary: RunSummary) -> None: + """Print a single dense metrics table (legacy behavior, enhanced).""" + # This is the existing print_metrics_table with updated column headers + print_metrics_table(console, summary) + + +def print_full_results( + console: Console, + summary: RunSummary, + *, + compact: bool = False, + trace_detail: bool = False, +) -> None: + """Orchestrate all result panels.""" + if compact: + print_compact_table(console, summary) + return + print_accuracy_panel(console, summary) + print_latency_table(console, summary) + print_energy_table(console, summary) + print_trace_summary(console, summary) +``` + +Update `__all__` to include all new functions. + +**Step 4: Run tests** + +Run: `uv run pytest evals/tests/test_display.py -v` +Expected: All pass + +**Step 5: Run full eval test suite** + +Run: `uv run pytest evals/tests/ -v` +Expected: All pass + +**Step 6: Commit** + +```bash +git add evals/core/display.py evals/tests/test_display.py +git commit -m "feat(evals): grouped display tables (accuracy, latency, energy, trace)" +``` + +--- + +### Task 8: Add `--compact` and `--trace-detail` flags to eval CLI + +**Files:** +- Modify: `evals/cli.py:117-131` (_print_summary) +- Modify: `evals/cli.py:235-344` (run command flags) +- Test: `evals/tests/test_cli_flags.py` (new) + +**Step 1: Write the failing test** + +Create `evals/tests/test_cli_flags.py`: + +```python +"""Tests for eval CLI display flags.""" + +from __future__ import annotations + +from click.testing import CliRunner + +from evals.cli import cli + + +class TestCompactFlag: + def test_compact_flag_accepted(self): + runner = CliRunner() + result = runner.invoke(cli, ["run", "--help"]) + assert "--compact" in result.output + + def test_trace_detail_flag_accepted(self): + runner = CliRunner() + result = runner.invoke(cli, ["run", "--help"]) + assert "--trace-detail" in result.output +``` + +**Step 2: Run test to verify it fails** + +Run: `uv run pytest evals/tests/test_cli_flags.py -v` +Expected: FAIL — `--compact` not in help output + +**Step 3: Write minimal implementation** + +In `evals/cli.py`: + +1. Add flags to `run` command (around line 270): +```python +@click.option("--compact", is_flag=True, default=False, help="Dense single-table output") +@click.option("--trace-detail", is_flag=True, default=False, help="Full per-step trace listing") +``` + +2. Pass flags through to `_print_summary()`: +```python +def _print_summary(console, summary, per_subject, output_path, traces_dir, + *, compact=False, trace_detail=False): +``` + +3. In `_print_summary`, replace the `print_metrics_table` call with: +```python + print_full_results(console, summary, compact=compact, trace_detail=trace_detail) +``` + +4. Update imports to include new display functions. + +**Step 4: Run tests** + +Run: `uv run pytest evals/tests/test_cli_flags.py -v` +Expected: PASS + +**Step 5: Run full eval test suite** + +Run: `uv run pytest evals/tests/ -v` +Expected: All pass + +**Step 6: Commit** + +```bash +git add evals/cli.py evals/tests/test_cli_flags.py +git commit -m "feat(evals): add --compact and --trace-detail CLI flags" +``` + +--- + +## Phase 23b: Perfect First Experience + +### Task 9: `jarvis quickstart` command + +**Files:** +- Create: `src/openjarvis/cli/quickstart_cmd.py` +- Modify: `src/openjarvis/cli/__init__.py:34-54` (register command) +- Test: `tests/cli/test_quickstart.py` (new) + +**Implementation outline:** +- 5-step flow: detect hardware → write config → check engine → verify model → test query +- Each step prints `[N/5] Description...` with Rich status +- Reuse `detect_hardware()` from `init_cmd.py` +- Reuse `_run_all_checks()` logic from `doctor_cmd.py` +- Skip steps already done (config exists → skip step 2) +- `--force` flag to redo everything +- On failure: print helpful suggestion and exit with code 1 + +**Test approach:** +- Use `CliRunner` with monkeypatched `detect_hardware`, `load_config`, engine checks +- Verify exit code 0 on happy path +- Verify helpful message when engine not found +- Verify `--force` regenerates config + +**Step 1:** Write failing tests for happy path + error paths +**Step 2:** Run tests to verify they fail +**Step 3:** Implement `quickstart_cmd.py` and register in `__init__.py` +**Step 4:** Run tests to verify they pass +**Step 5:** Commit: `"feat(cli): add jarvis quickstart command"` + +--- + +### Task 10: Error message auto-suggestions + +**Files:** +- Create: `src/openjarvis/cli/hints.py` +- Modify: `src/openjarvis/cli/ask.py:215-224,258-259,269-271` +- Modify: `src/openjarvis/cli/serve.py` (startup errors) +- Modify: `src/openjarvis/cli/bench_cmd.py` (engine errors) +- Modify: `src/openjarvis/cli/chat_cmd.py` (startup errors) +- Test: `tests/cli/test_hints.py` (new) + +**Implementation outline:** +- `hints.py` provides `hint_no_config()`, `hint_no_engine()`, `hint_no_model()` functions +- Each returns a Rich-formatted suggestion string +- Wire into existing error handlers in ask, serve, bench, chat commands +- Pattern: catch error → print original message → print hint → exit + +**Test approach:** +- Unit test each hint function returns non-empty string +- CLI integration tests verify hints appear in error output + +**Step 1:** Write failing tests +**Step 2:** Implement `hints.py` and wire into CLI commands +**Step 3:** Run tests, commit: `"feat(cli): add error hints system"` + +--- + +### Task 11: Global logging and verbose/quiet flags + +**Files:** +- Create: `src/openjarvis/cli/log_config.py` +- Modify: `src/openjarvis/cli/__init__.py:28-31` (add flags to root group) +- Test: `tests/cli/test_log_config.py` (new) + +**Implementation outline:** +- Add `--verbose` / `--quiet` flags to root `cli` group via `click.pass_context` +- `log_config.py`: `setup_logging(verbose, quiet)` configures: + - `RichHandler` for console (WARNING default, DEBUG on verbose, ERROR on quiet) + - `RotatingFileHandler` for `~/.openjarvis/cli.log` (5 MB, 3 backups) +- Call `setup_logging()` in root `cli` group callback + +**Test approach:** +- Verify `--verbose` sets DEBUG level +- Verify `--quiet` sets ERROR level +- Verify log file created on verbose mode + +**Step 1-5:** TDD cycle, commit: `"feat(cli): add global verbose/quiet logging flags"` + +--- + +### Task 12: Progress indicators for slow operations + +**Files:** +- Modify: `src/openjarvis/cli/ask.py:320-330` (generation) +- Modify: `src/openjarvis/cli/memory_cmd.py:77-96` (indexing) +- Test: Visual verification only (progress spinners are UI-only) + +**Implementation outline:** +- Wrap generation call in `ask.py` with `console.status("[bold green]Generating..."):` context +- Wrap indexing loop in `memory_cmd.py` with `rich.progress.Progress` bar +- Use `track()` for known-length operations, `status()` for unknown-length + +**Commit:** `"feat(cli): add progress indicators for generation and indexing"` + +--- + +### Task 13: Bench CLI full stats tables + +**Files:** +- Modify: `src/openjarvis/cli/bench_cmd.py:175-209` +- Modify: `src/openjarvis/bench/latency.py:60-87` (return full stats) +- Modify: `src/openjarvis/bench/throughput.py:50-74` (add stats) +- Modify: `src/openjarvis/bench/energy.py:85-123` (add stats) +- Test: `tests/bench/test_bench_stats.py` (new) + +**Implementation outline:** +- Each benchmark returns a `metrics` dict. Currently flat key-value pairs. +- Change to include `_stats` suffixed keys for metrics that have multiple samples: + - latency: `latency_avg`, `latency_median`, `latency_min`, `latency_max`, `latency_std`, `latency_p95` + - throughput: same pattern + - energy: same pattern +- `bench_cmd.py` detects stats keys and renders a Rich stats table (Avg/Median/Min/Max/Std columns) +- Non-stats metrics (single values like `total_energy`, `errors`) shown as before + +**Test approach:** +- Verify benchmark `run()` returns stats-suffixed keys +- Verify CLI renders table with correct columns + +**Step 1-5:** TDD cycle, commit: `"feat(bench): full stats tables in bench CLI"` + +--- + +## Phase 23c: Perfect Dashboard + +### Task 14: Desktop settings panel + +**Files:** +- Create: `desktop/src/components/SettingsPanel.tsx` +- Modify: `desktop/src/App.tsx` (add settings tab) + +**Implementation:** React component with API URL input, auto-update interval selector, dark/light theme toggle. Persists to `localStorage`. Clean minimalist UI matching ChatGPT/Claude aesthetic. + +--- + +### Task 15: Desktop Windows icon + +**Files:** +- Replace: `desktop/src-tauri/icons/icon.ico` + +**Implementation:** Generate proper multi-resolution `.ico` from existing `icon.png` (16, 32, 48, 256px). + +--- + +### Task 16: Desktop system tray + +**Files:** +- Modify: `desktop/src-tauri/src/lib.rs:189-191` + +**Implementation:** Add tray menu items: Show/Hide, Health Status, Quit. Use Tauri's `SystemTray` API. + +--- + +### Task 17: PWA CI/CD and error boundary + +**Files:** +- Create: `.github/workflows/frontend.yml` +- Modify: `frontend/src/App.tsx` (error boundary wrapper) +- Modify: `frontend/vite.config.ts` (VITE_API_URL env var) + +**Implementation:** GitHub Actions workflow triggers on `frontend/` changes, builds with `npm run build`, commits to `src/openjarvis/server/static/`. Error boundary wraps ``. + +--- + +### Task 18: PWA style refresh + +**Files:** +- Modify: `frontend/src/App.css` (modularize) +- Create: `frontend/src/components/Chat/Chat.css` (component-scoped) +- Create: `frontend/src/components/Sidebar/Sidebar.css` (component-scoped) + +**Implementation:** Break 12,971-line `App.css` into component-scoped CSS modules. Clean minimalist aesthetic inspired by ChatGPT/Claude web interfaces. Keep Catppuccin dark theme as default, add light theme option. Generous whitespace, muted color palette, smooth transitions. Side panels (energy, traces, learning, memory) remain the differentiator — collapsible with clean data density. + +--- + +## Summary + +| Phase | Tasks | Commits | Priority | +|-------|-------|---------|----------| +| 23a: Perfect Eval Run | 1-8 | 8 | Highest — enables GAIA research | +| 23b: Perfect First Experience | 9-13 | 5 | High — onboarding & CLI | +| 23c: Perfect Dashboard | 14-18 | 5 | Lower — demo polish | + +**Total: 18 tasks, ~18 commits** + +Start with Phase 23a Task 1. Each task is independently testable and committable. diff --git a/evals/cli.py b/evals/cli.py index 3c548c34..f8d5cc23 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -20,6 +20,7 @@ from rich.progress import ( from evals.core.display import ( print_banner, print_completion, + print_full_results, print_metrics_table, print_run_header, print_section, @@ -119,13 +120,16 @@ def _print_summary( console: Optional[Console] = None, output_path: Optional[Path] = None, traces_dir: Optional[Path] = None, + *, + compact: bool = False, + trace_detail: bool = False, ) -> None: """Print a single run summary using Rich display primitives.""" if console is None: console = Console() print_section(console, "Results") - print_metrics_table(console, summary) - if summary.per_subject and len(summary.per_subject) > 1: + print_full_results(console, summary, compact=compact, trace_detail=trace_detail) + if not compact and summary.per_subject and len(summary.per_subject) > 1: print_subject_table(console, summary.per_subject) print_completion(console, summary, output_path, traces_dir) @@ -266,11 +270,14 @@ def main(): help="Enable telemetry collection during eval") @click.option("--gpu-metrics/--no-gpu-metrics", default=False, help="Enable GPU metrics collection") +@click.option("--compact", is_flag=True, default=False, help="Dense single-table output") +@click.option("--trace-detail", is_flag=True, default=False, help="Full per-step trace listing") @click.option("-v", "--verbose", is_flag=True, help="Verbose logging") @click.pass_context def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, tools, max_samples, max_workers, judge_model, output_path, seed, - dataset_split, temperature, max_tokens, telemetry, gpu_metrics, verbose): + dataset_split, temperature, max_tokens, telemetry, gpu_metrics, + compact, trace_detail, verbose): """Run a single benchmark evaluation, or a full suite from a TOML config.""" _setup_logging(verbose) @@ -340,6 +347,8 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, console=console, output_path=_output_path, traces_dir=_traces_dir, + compact=compact, + trace_detail=trace_detail, ) diff --git a/evals/core/display.py b/evals/core/display.py index e58f6bab..f427d775 100644 --- a/evals/core/display.py +++ b/evals/core/display.py @@ -152,6 +152,155 @@ def print_metrics_table(console: Console, summary: RunSummary) -> None: console.print(headline) +def _stats_table(title: str, rows: list[tuple[str, Optional[MetricStats], int]]) -> Table: + """Build a stats table with Avg/Median/Min/Max/Std columns.""" + table = Table( + title=f"[bold]{title}[/bold]", + show_header=True, + header_style="bold bright_white", + border_style="bright_blue", + title_style="bold cyan", + ) + table.add_column("Metric", style="cyan", no_wrap=True) + table.add_column("Avg", justify="right") + table.add_column("Median", justify="right") + table.add_column("Min", justify="right") + table.add_column("Max", justify="right") + table.add_column("Std", justify="right") + for label, stats, decimals in rows: + if stats is not None: + table.add_row( + label, + _fmt(stats.mean, decimals), + _fmt(stats.median, decimals), + _fmt(stats.min, decimals), + _fmt(stats.max, decimals), + _fmt(stats.std, decimals), + ) + return table + + +def print_accuracy_panel(console: Console, summary: RunSummary) -> None: + """Print accuracy panel with per-subject breakdown.""" + lines = [ + f"[bold]Overall Accuracy {summary.accuracy:.1%}[/bold]" + f" ({summary.correct}/{summary.scored_samples})", + ] + for subj, stats in sorted(summary.per_subject.items()): + acc = stats.get("accuracy", 0.0) + correct = int(stats.get("correct", 0)) + scored = int(stats.get("scored", 0)) + lines.append(f" {subj:<20s} {acc:.1%} ({correct}/{scored})") + body = "\n".join(lines) + panel = Panel(body, title="[bold]Accuracy[/bold]", border_style="green", expand=False) + console.print(panel) + + +def print_latency_table(console: Console, summary: RunSummary) -> None: + """Print latency, throughput, and token stats table.""" + table = _stats_table("Latency & Throughput", [ + ("Latency (s)", summary.latency_stats, 2), + ("TTFT (s)", summary.ttft_stats, 3), + ("Throughput (tok/s)", summary.throughput_stats, 1), + ("Avg Input Tokens", summary.input_token_stats, 1), + ("Avg Output Tokens", summary.output_token_stats, 1), + ]) + if table.row_count > 0: + console.print(table) + + +def print_energy_table(console: Console, summary: RunSummary) -> None: + """Print energy, efficiency, and IPJ/IPW table.""" + table = _stats_table("Energy & Efficiency", [ + ("Energy (J)", summary.energy_stats, 1), + ("Power (W)", summary.power_stats, 1), + ("GPU Util (%)", summary.gpu_utilization_stats, 1), + ("Energy/OutTok (J)", summary.energy_per_output_token_stats, 6), + ("MFU (%)", summary.mfu_stats, 3), + ("MBU (%)", summary.mbu_stats, 3), + ]) + if table.row_count > 0: + console.print(table) + # Headline: IPW, IPJ, Total Energy + parts: list[str] = [] + if summary.ipw_stats: + parts.append(f"[bold]IPW (acc/W):[/bold] {summary.ipw_stats.mean:.6f}") + if summary.ipj_stats: + parts.append(f"[bold]IPJ (acc/J):[/bold] {summary.ipj_stats.mean:.2e}") + if summary.total_energy_joules > 0: + val = summary.total_energy_joules + unit = "kJ" if val > 1000 else "J" + display = val / 1000 if val > 1000 else val + parts.append(f"[bold]Total Energy:[/bold] {display:.1f} {unit}") + if summary.avg_power_watts > 0: + parts.append(f"[bold]Avg Power:[/bold] {summary.avg_power_watts:.1f} W") + if parts: + console.print(" ".join(parts)) + + +def print_trace_summary(console: Console, summary: RunSummary) -> None: + """Print agentic trace step-type breakdown.""" + sts = summary.trace_step_type_stats + if not sts: + return + total_steps = sum(s.get("count", 0) for s in sts.values()) + avg_per_sample = total_steps / summary.scored_samples if summary.scored_samples > 0 else 0 + + table = Table( + title="[bold]Agentic Trace Summary[/bold]", + show_header=True, + header_style="bold bright_white", + border_style="bright_blue", + title_style="bold cyan", + caption=f"Total Steps: {total_steps} | Avg Steps/Sample: {avg_per_sample:.1f}", + ) + table.add_column("Step Type", style="cyan", no_wrap=True) + table.add_column("Count", justify="right") + table.add_column("Avg Duration", justify="right") + table.add_column("Avg Energy (J)", justify="right") + table.add_column("Avg In Tokens", justify="right") + table.add_column("Avg Out Tokens", justify="right") + + for stype, data in sorted(sts.items()): + count = data.get("count", 0) + avg_dur = data.get("avg_duration", 0.0) + total_e = data.get("total_energy", 0.0) + avg_e = total_e / count if count > 0 else 0.0 + avg_in = data.get("avg_input_tokens", 0.0) + avg_out = data.get("avg_output_tokens", 0.0) + table.add_row( + stype, + str(count), + f"{avg_dur:.2f}s", + f"{avg_e:.1f}" if avg_e > 0 else "\u2014", + f"{avg_in:.0f}" if avg_in > 0 else "\u2014", + f"{avg_out:.0f}" if avg_out > 0 else "\u2014", + ) + console.print(table) + + +def print_compact_table(console: Console, summary: RunSummary) -> None: + """Print a single dense metrics table (legacy behavior, enhanced).""" + print_metrics_table(console, summary) + + +def print_full_results( + console: Console, + summary: RunSummary, + *, + compact: bool = False, + trace_detail: bool = False, +) -> None: + """Orchestrate all result panels.""" + if compact: + print_compact_table(console, summary) + return + print_accuracy_panel(console, summary) + print_latency_table(console, summary) + print_energy_table(console, summary) + print_trace_summary(console, summary) + + def print_subject_table( console: Console, per_subject: Dict[str, Dict[str, float]], @@ -248,11 +397,17 @@ def print_completion( __all__ = [ "OPENJARVIS_BANNER", + "print_accuracy_panel", "print_banner", - "print_section", - "print_run_header", + "print_compact_table", + "print_completion", + "print_energy_table", + "print_full_results", + "print_latency_table", "print_metrics_table", + "print_run_header", + "print_section", "print_subject_table", "print_suite_summary", - "print_completion", + "print_trace_summary", ] diff --git a/evals/core/runner.py b/evals/core/runner.py index b3f5adf0..d23fa882 100644 --- a/evals/core/runner.py +++ b/evals/core/runner.py @@ -343,6 +343,9 @@ class EvalRunner: ] total_energy = sum(r.energy_joules for r in results) + total_input_tokens = sum(r.prompt_tokens for r in results) + total_output_tokens = sum(r.completion_tokens for r in results) + avg_power = statistics.mean(power_vals) if power_vals else 0.0 return RunSummary( benchmark=cfg.benchmark, @@ -377,6 +380,9 @@ class EvalRunner: output_token_stats=_metric_stats([float(v) for v in output_tok_vals]), total_energy_joules=round(total_energy, 6), warmup_samples_excluded=cfg.warmup_samples, + avg_power_watts=round(avg_power, 4), + total_input_tokens=total_input_tokens, + total_output_tokens=total_output_tokens, ) @@ -464,6 +470,9 @@ def _summary_to_dict(s: RunSummary) -> Dict[str, Any]: "warmup_samples_excluded": s.warmup_samples_excluded, "steady_state_reached": s.steady_state_reached, "energy_method": s.energy_method, + "avg_power_watts": s.avg_power_watts, + "total_input_tokens": s.total_input_tokens, + "total_output_tokens": s.total_output_tokens, } diff --git a/evals/core/types.py b/evals/core/types.py index 95550e1d..22702dd0 100644 --- a/evals/core/types.py +++ b/evals/core/types.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Dict, List, Optional @@ -44,6 +45,8 @@ class EvalResult: energy_per_output_token_joules: float = 0.0 throughput_per_watt: float = 0.0 mean_itl_ms: float = 0.0 + trace_steps: int = 0 + trace_energy_joules: float = 0.0 @dataclass(slots=True) @@ -122,6 +125,13 @@ class RunSummary: warmup_samples_excluded: int = 0 steady_state_reached: bool = False energy_method: str = "" + avg_power_watts: float = 0.0 + total_input_tokens: int = 0 + total_output_tokens: int = 0 + trace_step_type_stats: Dict[str, Dict[str, float]] = field(default_factory=dict) + # Internal fields set by the runner after construction + _output_path: Optional[Path] = None + _traces_dir: Optional[Path] = None # --------------------------------------------------------------------------- diff --git a/evals/tests/test_cli_flags.py b/evals/tests/test_cli_flags.py new file mode 100644 index 00000000..24b54960 --- /dev/null +++ b/evals/tests/test_cli_flags.py @@ -0,0 +1,19 @@ +"""Tests for eval CLI display flags.""" + +from __future__ import annotations + +from click.testing import CliRunner + +from evals.cli import main + + +class TestCompactFlag: + def test_compact_flag_accepted(self): + runner = CliRunner() + result = runner.invoke(main, ["run", "--help"]) + assert "--compact" in result.output + + def test_trace_detail_flag_accepted(self): + runner = CliRunner() + result = runner.invoke(main, ["run", "--help"]) + assert "--trace-detail" in result.output diff --git a/evals/tests/test_display.py b/evals/tests/test_display.py new file mode 100644 index 00000000..336aede2 --- /dev/null +++ b/evals/tests/test_display.py @@ -0,0 +1,134 @@ +"""Tests for eval display functions.""" + +from __future__ import annotations + +from io import StringIO + +from rich.console import Console + +from evals.core.display import ( + print_accuracy_panel, + print_energy_table, + print_latency_table, + print_trace_summary, + print_compact_table, + print_full_results, +) +from evals.core.types import MetricStats, RunSummary + + +def _make_summary(**overrides) -> RunSummary: + defaults = dict( + benchmark="gaia", category="agentic", backend="jarvis-agent", + model="qwen3:8b", total_samples=100, scored_samples=100, + correct=42, accuracy=0.42, errors=0, + mean_latency_seconds=15.0, total_cost_usd=0.0, + total_energy_joules=9300.0, avg_power_watts=880.0, + total_input_tokens=50000, total_output_tokens=12000, + per_subject={"level_1": {"accuracy": 0.58, "correct": 40, "scored": 68}}, + ) + defaults.update(overrides) + return RunSummary(**defaults) + + +def _make_stats(mean=10.0) -> MetricStats: + return MetricStats( + mean=mean, median=mean * 0.9, min=mean * 0.2, + max=mean * 3.0, std=mean * 0.5, p90=mean * 1.5, + p95=mean * 2.0, p99=mean * 2.5, + ) + + +class TestAccuracyPanel: + def test_renders_without_error(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary() + print_accuracy_panel(console, summary) + output = console.file.getvalue() + assert "42.0%" in output or "0.42" in output + + def test_shows_per_subject(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary() + print_accuracy_panel(console, summary) + output = console.file.getvalue() + assert "level_1" in output + + +class TestLatencyTable: + def test_renders_with_stats(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + latency_stats=_make_stats(15.0), + throughput_stats=_make_stats(40.0), + input_token_stats=_make_stats(1024.0), + output_token_stats=_make_stats(256.0), + ) + print_latency_table(console, summary) + output = console.file.getvalue() + assert "Latency" in output + assert "Avg" in output + + +class TestEnergyTable: + def test_renders_ipj_ipw(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + energy_stats=_make_stats(46000.0), + power_stats=_make_stats(880.0), + ipw_stats=_make_stats(0.00048), + ipj_stats=_make_stats(9.0e-6), + ) + print_energy_table(console, summary) + output = console.file.getvalue() + assert "IPW" in output + assert "IPJ" in output + + +class TestTraceSummary: + def test_renders_step_type_breakdown(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + trace_step_type_stats={ + "generate": { + "count": 580, "avg_duration": 8.2, + "total_energy": 22000.0, + "avg_input_tokens": 890.0, "avg_output_tokens": 256.0, + }, + "tool_call": { + "count": 420, "avg_duration": 3.1, + "total_energy": 0.0, + "avg_input_tokens": 0.0, "avg_output_tokens": 0.0, + }, + }, + ) + print_trace_summary(console, summary) + output = console.file.getvalue() + assert "generate" in output + assert "tool_call" in output + + +class TestCompactTable: + def test_renders_all_metrics(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + latency_stats=_make_stats(15.0), + energy_stats=_make_stats(46000.0), + ) + print_compact_table(console, summary) + output = console.file.getvalue() + assert "Latency" in output + assert "Energy" in output + + +class TestFullResults: + def test_renders_all_sections(self): + console = Console(file=StringIO(), force_terminal=True) + summary = _make_summary( + latency_stats=_make_stats(15.0), + energy_stats=_make_stats(46000.0), + power_stats=_make_stats(880.0), + ) + print_full_results(console, summary) + output = console.file.getvalue() + assert "Accuracy" in output diff --git a/evals/tests/test_runner.py b/evals/tests/test_runner.py index 1f1ef527..f10ec043 100644 --- a/evals/tests/test_runner.py +++ b/evals/tests/test_runner.py @@ -373,6 +373,45 @@ class TestEvalRunner: assert "total_energy_joules" in data +class TestRunnerTokenStats: + def test_summary_has_total_input_output_tokens(self, tmp_path): + """RunSummary should include total token counts.""" + records = [ + EvalRecord(record_id=f"r{i}", problem=f"q{i}", reference="a", category="test") + for i in range(3) + ] + output_path = tmp_path / "results.jsonl" + config = RunConfig( + benchmark="test", backend="mock", model="m", + max_workers=1, output_path=str(output_path), + ) + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=True) + runner = EvalRunner(config, dataset, backend, scorer) + summary = runner.run() + # MockBackend returns prompt_tokens=100, completion_tokens=50 + assert summary.total_input_tokens == 300 # 3 * 100 + assert summary.total_output_tokens == 150 # 3 * 50 + + def test_summary_has_avg_power(self, tmp_path): + """RunSummary should include avg_power_watts.""" + records = [ + EvalRecord(record_id="r1", problem="q", reference="a", category="test") + ] + output_path = tmp_path / "results.jsonl" + config = RunConfig( + benchmark="test", backend="mock", model="m", + max_workers=1, output_path=str(output_path), + ) + dataset = MockDataset(records) + backend = MockBackend() # returns power_watts=250.0 + scorer = MockScorer(result=True) + runner = EvalRunner(config, dataset, backend, scorer) + summary = runner.run() + assert summary.avg_power_watts == 250.0 + + class TestMetricStatsHelpers: def test_metric_stats_empty(self): assert _metric_stats([]) is None @@ -401,4 +440,11 @@ class TestMetricStatsHelpers: def test_metric_stats_to_dict(self): ms = MetricStats(mean=1.0, median=2.0, min=0.5, max=3.0, std=0.8) d = _metric_stats_to_dict(ms) - assert d == {"mean": 1.0, "median": 2.0, "min": 0.5, "max": 3.0, "std": 0.8} + assert d["mean"] == 1.0 + assert d["median"] == 2.0 + assert d["min"] == 0.5 + assert d["max"] == 3.0 + assert d["std"] == 0.8 + assert "p90" in d + assert "p95" in d + assert "p99" in d diff --git a/evals/tests/test_types.py b/evals/tests/test_types.py index eda1e765..31c437aa 100644 --- a/evals/tests/test_types.py +++ b/evals/tests/test_types.py @@ -209,6 +209,35 @@ class TestRunSummary: # --------------------------------------------------------------------------- +class TestEvalResultTraceFields: + def test_trace_fields_exist(self): + r = EvalResult(record_id="test", model_answer="hi") + assert r.trace_steps == 0 + assert r.trace_energy_joules == 0.0 + + def test_trace_fields_set(self): + r = EvalResult( + record_id="test", model_answer="hi", + trace_steps=5, trace_energy_joules=100.0, + ) + assert r.trace_steps == 5 + assert r.trace_energy_joules == 100.0 + + +class TestRunSummaryTraceFields: + def test_trace_aggregate_fields(self): + s = RunSummary( + benchmark="test", category="test", backend="test", + model="test", total_samples=1, scored_samples=1, + correct=1, accuracy=1.0, errors=0, + mean_latency_seconds=1.0, total_cost_usd=0.0, + ) + assert s.avg_power_watts == 0.0 + assert s.trace_step_type_stats == {} + assert s.total_input_tokens == 0 + assert s.total_output_tokens == 0 + + class TestMetaConfig: def test_defaults(self): m = MetaConfig() diff --git a/frontend/src/App.css b/frontend/src/App.css deleted file mode 100644 index faffe3ac..00000000 --- a/frontend/src/App.css +++ /dev/null @@ -1,725 +0,0 @@ -/* === CSS Variables === */ -:root { - --color-primary: #2563eb; - --color-primary-light: #3b82f6; - --color-primary-dark: #1d4ed8; - --color-bg: #ffffff; - --color-bg-sidebar: #f1f5f9; - --color-text: #1e293b; - --color-text-secondary: #64748b; - --color-border: #e2e8f0; - --color-user-bubble: #2563eb; - --color-assistant-bubble: #ffffff; - --color-tool-bg: #f0f9ff; - --color-tool-border: #7dd3fc; - --color-tool-success: #22c55e; - --color-tool-running: #f59e0b; - --color-tool-error: #ef4444; - --color-savings: #22c55e; - --sidebar-width: 300px; -} - -/* === Reset === */ -*, *::before, *::after { - box-sizing: border-box; - margin: 0; - padding: 0; -} - -body { - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif; - background: var(--color-bg); - color: var(--color-text); - height: 100vh; - overflow: hidden; -} - -#root { - height: 100vh; -} - -/* === App Layout === */ -.app { - display: flex; - height: 100vh; - position: relative; -} - -/* === Sidebar Toggle === */ -.sidebar-toggle { - display: none; - position: fixed; - top: 12px; - left: 12px; - z-index: 1000; - background: var(--color-primary); - color: white; - border: none; - border-radius: 8px; - width: 36px; - height: 36px; - font-size: 18px; - cursor: pointer; - align-items: center; - justify-content: center; -} - -/* === Sidebar === */ -.sidebar { - width: var(--sidebar-width); - background: var(--color-bg-sidebar); - border-right: 1px solid var(--color-border); - display: flex; - flex-direction: column; - height: 100vh; - flex-shrink: 0; - transition: transform 0.2s ease; -} - -.sidebar-header { - padding: 16px; - border-bottom: 1px solid var(--color-border); -} - -.sidebar-header h1 { - font-size: 18px; - font-weight: 700; - color: var(--color-primary); - margin-bottom: 12px; -} - -.new-chat-btn { - width: 100%; - padding: 10px 16px; - background: var(--color-primary); - color: white; - border: none; - border-radius: 8px; - font-size: 14px; - font-weight: 500; - cursor: pointer; - transition: background 0.15s; -} - -.new-chat-btn:hover { - background: var(--color-primary-dark); -} - -/* === Model Selector === */ -.model-selector { - padding: 12px 16px; - border-bottom: 1px solid var(--color-border); -} - -.model-selector label { - display: block; - font-size: 11px; - font-weight: 600; - text-transform: uppercase; - letter-spacing: 0.05em; - color: var(--color-text-secondary); - margin-bottom: 6px; -} - -.model-selector select { - width: 100%; - padding: 8px 10px; - border: 1px solid var(--color-border); - border-radius: 6px; - background: white; - font-size: 13px; - color: var(--color-text); - cursor: pointer; -} - -/* === Conversation List === */ -.conversation-list { - flex: 1; - overflow-y: auto; - padding: 8px; -} - -.conversation-item { - display: flex; - align-items: center; - padding: 10px 12px; - border-radius: 8px; - cursor: pointer; - font-size: 13px; - color: var(--color-text); - transition: background 0.15s; - margin-bottom: 2px; -} - -.conversation-item:hover { - background: rgba(0, 0, 0, 0.04); -} - -.conversation-item.active { - background: var(--color-primary); - color: white; -} - -.conversation-item .conv-title { - flex: 1; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -.conversation-item .conv-delete { - opacity: 0; - border: none; - background: none; - color: inherit; - cursor: pointer; - padding: 2px 6px; - border-radius: 4px; - font-size: 16px; - line-height: 1; -} - -.conversation-item:hover .conv-delete { - opacity: 0.6; -} - -.conversation-item .conv-delete:hover { - opacity: 1; - background: rgba(0, 0, 0, 0.1); -} - -.conversation-item.active .conv-delete:hover { - background: rgba(255, 255, 255, 0.2); -} - -/* === Savings Panel === */ -.savings-panel { - padding: 16px; - border-top: 2px solid var(--color-primary); - background: linear-gradient(to bottom, #eff6ff, var(--color-bg-sidebar)); -} - -.savings-panel h3 { - font-size: 13px; - font-weight: 700; - text-transform: uppercase; - letter-spacing: 0.05em; - color: var(--color-primary); - margin-bottom: 8px; -} - -.savings-models { - display: flex; - flex-direction: column; - gap: 6px; - margin-bottom: 12px; -} - -.savings-model-card { - background: white; - border: 1px solid var(--color-border); - border-radius: 8px; - padding: 8px 10px; - border-left: 3px solid var(--color-border); -} - -.savings-model-card.local { - border-left-color: #22c55e; -} - -.savings-model-card.cloud { - border-left-color: #f59e0b; -} - -.savings-model-card-label { - display: block; - font-size: 9px; - font-weight: 700; - letter-spacing: 0.08em; - color: var(--color-text-secondary); - margin-bottom: 2px; -} - -.savings-model-card-name { - display: block; - font-family: monospace; - font-size: 11px; - color: var(--color-text); - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -.savings-grid { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 8px; -} - -.savings-item { - background: white; - border-radius: 8px; - padding: 10px; - border: 1px solid var(--color-border); -} - -.savings-item .savings-label { - font-size: 10px; - text-transform: uppercase; - letter-spacing: 0.04em; - color: var(--color-text-secondary); - margin-bottom: 4px; -} - -.savings-item .savings-value { - font-size: 18px; - font-weight: 800; - color: var(--color-savings); - font-variant-numeric: tabular-nums; -} - -.savings-item.calls .savings-value { - color: var(--color-primary); -} - -/* === Chat Area === */ -.chat-area { - flex: 1; - display: flex; - flex-direction: column; - height: 100vh; - min-width: 0; -} - -.chat-header { - padding: 12px 24px; - border-bottom: 1px solid var(--color-border); - background: var(--color-bg); - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - flex-wrap: wrap; -} - -.chat-header-title { - font-size: 14px; - font-weight: 600; - color: var(--color-text); - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -.chat-header-meta { - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; -} - -.chat-header-info { - display: flex; - align-items: center; - gap: 6px; -} - -.header-badge { - display: inline-block; - padding: 3px 8px; - border-radius: 4px; - font-size: 11px; - font-weight: 600; - font-family: monospace; -} - -.model-badge { - background: #eff6ff; - color: var(--color-primary); - border: 1px solid #bfdbfe; -} - -.agent-badge { - background: #f0fdf4; - color: #16a34a; - border: 1px solid #bbf7d0; -} - -.chat-header-tokens { - font-size: 11px; - color: var(--color-text-secondary); - font-variant-numeric: tabular-nums; - white-space: nowrap; -} - -/* === Message List === */ -.message-list { - flex: 1; - overflow-y: auto; - padding: 24px; - display: flex; - flex-direction: column; - gap: 16px; -} - -.message-list-empty { - flex: 1; - display: flex; - align-items: center; - justify-content: center; - color: var(--color-text-secondary); - font-size: 15px; -} - -/* === Message Bubble === */ -.message-bubble { - max-width: 75%; - position: relative; - group: message; -} - -.message-bubble.user { - align-self: flex-end; -} - -.message-bubble.assistant { - align-self: flex-start; -} - -.message-content { - padding: 12px 16px; - border-radius: 16px; - font-size: 14px; - line-height: 1.6; - white-space: pre-wrap; - word-break: break-word; -} - -.message-bubble.user .message-content { - background: var(--color-user-bubble); - color: white; - border-bottom-right-radius: 4px; -} - -.message-bubble.assistant .message-content { - background: var(--color-assistant-bubble); - color: var(--color-text); - border: 1px solid var(--color-border); - border-bottom-left-radius: 4px; -} - -.message-meta { - display: flex; - align-items: center; - gap: 8px; - margin-top: 4px; - padding: 0 4px; -} - -.message-time { - font-size: 11px; - color: var(--color-text-secondary); -} - -.message-tokens { - font-size: 10px; - color: var(--color-text-secondary); - font-variant-numeric: tabular-nums; - opacity: 0.7; -} - -.message-bubble.user .message-meta { - justify-content: flex-end; -} - -/* === Copy Button === */ -.copy-btn { - position: absolute; - top: 8px; - right: 8px; - opacity: 0; - background: rgba(0, 0, 0, 0.06); - border: none; - border-radius: 6px; - padding: 4px 8px; - cursor: pointer; - font-size: 12px; - color: var(--color-text-secondary); - transition: opacity 0.15s; -} - -.message-bubble:hover .copy-btn { - opacity: 1; -} - -.copy-btn:hover { - background: rgba(0, 0, 0, 0.1); -} - -.message-bubble.user .copy-btn { - color: rgba(255, 255, 255, 0.7); - background: rgba(255, 255, 255, 0.15); -} - -.message-bubble.user .copy-btn:hover { - background: rgba(255, 255, 255, 0.25); -} - -/* === Tool Call Indicator === */ -.tool-calls { - margin-top: 8px; - display: flex; - flex-direction: column; - gap: 6px; -} - -.tool-call { - display: flex; - align-items: center; - gap: 8px; - padding: 8px 12px; - background: var(--color-tool-bg); - border: 1px solid var(--color-tool-border); - border-radius: 8px; - font-size: 13px; -} - -.tool-call .tool-status { - width: 8px; - height: 8px; - border-radius: 50%; - flex-shrink: 0; -} - -.tool-call .tool-status.running { - background: var(--color-tool-running); - animation: pulse 1.5s infinite; -} - -.tool-call .tool-status.success { - background: var(--color-tool-success); -} - -.tool-call .tool-status.error { - background: var(--color-tool-error); -} - -.tool-call .tool-name { - font-weight: 600; - color: var(--color-text); -} - -.tool-call .tool-latency { - margin-left: auto; - font-size: 11px; - color: var(--color-text-secondary); -} - -@keyframes pulse { - 0%, 100% { opacity: 1; } - 50% { opacity: 0.4; } -} - -/* === Streaming Indicator === */ -.streaming-indicator { - display: flex; - flex-direction: column; - gap: 8px; - padding: 12px 16px; - margin: 0 24px; -} - -.streaming-tool-calls { - display: flex; - flex-wrap: wrap; - gap: 6px; -} - -.streaming-progress-row { - display: flex; - align-items: center; - gap: 12px; -} - -.streaming-bar-container { - flex: 1; - height: 4px; - background: var(--color-border); - border-radius: 2px; - overflow: hidden; -} - -.streaming-bar { - height: 100%; - background: var(--color-primary); - border-radius: 2px; - animation: stream-progress 2s ease-in-out infinite; -} - -@keyframes stream-progress { - 0% { width: 0%; margin-left: 0%; } - 50% { width: 40%; margin-left: 30%; } - 100% { width: 0%; margin-left: 100%; } -} - -.streaming-phase { - font-size: 12px; - color: var(--color-text-secondary); - white-space: nowrap; -} - -.streaming-elapsed { - font-size: 12px; - color: var(--color-text-secondary); - font-variant-numeric: tabular-nums; - white-space: nowrap; -} - -/* === Input Area === */ -.input-area { - padding: 16px 24px 24px; - border-top: 1px solid var(--color-border); - background: var(--color-bg); -} - -.input-attachment-row { - margin-bottom: 8px; -} - -.input-container { - display: flex; - gap: 8px; - align-items: flex-end; -} - -.input-container textarea { - flex: 1; - resize: none; - border: 1px solid var(--color-border); - border-radius: 12px; - padding: 12px 16px; - font-size: 14px; - font-family: inherit; - line-height: 1.5; - min-height: 72px; - max-height: 200px; - outline: none; - transition: border-color 0.15s; -} - -.input-container textarea:focus { - border-color: var(--color-primary); -} - -.send-btn, .stop-btn { - padding: 10px 20px; - border: none; - border-radius: 12px; - font-size: 14px; - font-weight: 500; - cursor: pointer; - flex-shrink: 0; - transition: background 0.15s; -} - -.send-btn { - background: var(--color-primary); - color: white; -} - -.send-btn:hover:not(:disabled) { - background: var(--color-primary-dark); -} - -.send-btn:disabled { - opacity: 0.5; - cursor: not-allowed; -} - -.stop-btn { - background: var(--color-tool-error); - color: white; -} - -.stop-btn:hover { - background: #dc2626; -} - -/* === Pasted text pill === */ -.pasted-pill { - display: inline-flex; - align-items: center; - gap: 8px; - background: var(--color-surface); - border: 1px solid var(--color-border); - border-radius: 10px; - padding: 10px 14px; - color: var(--color-text-secondary); - font-size: 13px; - max-width: 100%; -} - -.pasted-pill svg { - flex-shrink: 0; - opacity: 0.6; -} - -.pasted-pill-text { - font-weight: 500; - color: var(--color-text); -} - -.pasted-pill-size { - color: var(--color-text-secondary); - font-size: 12px; -} - -.pasted-pill-action { - background: none; - border: 1px solid var(--color-border); - border-radius: 6px; - padding: 2px 8px; - font-size: 12px; - color: var(--color-text-secondary); - cursor: pointer; - transition: background 0.15s, color 0.15s; -} - -.pasted-pill-action:hover { - background: var(--color-border); - color: var(--color-text); -} - -.pasted-pill-remove { - font-size: 16px; - line-height: 1; - padding: 2px 6px; -} - -.pasted-pill-remove:hover { - color: var(--color-tool-error); - border-color: var(--color-tool-error); -} - -/* === Responsive === */ -@media (max-width: 768px) { - .sidebar-toggle { - display: flex; - } - - .sidebar { - position: fixed; - left: 0; - top: 0; - z-index: 999; - transform: translateX(-100%); - } - - .sidebar.open { - transform: translateX(0); - } - - .message-bubble { - max-width: 90%; - } -} diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index f7a64d21..de9a0c6d 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,6 +1,6 @@ import type { ModelInfo, SavingsData, ServerInfo } from '../types'; -const BASE = ''; // relative to same origin +const BASE = import.meta.env.VITE_API_URL || ''; // relative to same origin by default export async function fetchModels(): Promise { const res = await fetch(`${BASE}/v1/models`); diff --git a/frontend/src/components/ErrorBoundary.tsx b/frontend/src/components/ErrorBoundary.tsx new file mode 100644 index 00000000..c370c090 --- /dev/null +++ b/frontend/src/components/ErrorBoundary.tsx @@ -0,0 +1,86 @@ +import { Component } from 'react'; +import type { ErrorInfo, ReactNode } from 'react'; + +interface Props { + children: ReactNode; +} + +interface State { + hasError: boolean; + error: Error | null; +} + +export class ErrorBoundary extends Component { + constructor(props: Props) { + super(props); + this.state = { hasError: false, error: null }; + } + + static getDerivedStateFromError(error: Error): State { + return { hasError: true, error }; + } + + componentDidCatch(error: Error, info: ErrorInfo) { + console.error('ErrorBoundary caught:', error, info.componentStack); + } + + render() { + if (this.state.hasError) { + return ( +
+
+

Something went wrong

+

+ {this.state.error?.message || 'An unexpected error occurred.'} +

+ +
+
+ ); + } + return this.props.children; + } +} + +const styles: Record = { + container: { + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + height: '100vh', + backgroundColor: '#1a1a1e', + color: '#e2e8f0', + fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif', + }, + card: { + textAlign: 'center' as const, + padding: 32, + maxWidth: 420, + }, + heading: { + fontSize: 20, + fontWeight: 600, + marginBottom: 12, + }, + message: { + fontSize: 14, + color: '#94a3b8', + marginBottom: 24, + lineHeight: 1.5, + }, + button: { + padding: '10px 24px', + borderRadius: 8, + border: 'none', + backgroundColor: '#2563eb', + color: 'white', + fontSize: 14, + fontWeight: 500, + cursor: 'pointer', + }, +}; diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index fad42318..0bfeeefc 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -1,10 +1,17 @@ import { StrictMode } from 'react'; import { createRoot } from 'react-dom/client'; +import { ErrorBoundary } from './components/ErrorBoundary'; import App from './App'; -import './App.css'; +import './styles/variables.css'; +import './styles/base.css'; +import './styles/sidebar.css'; +import './styles/chat.css'; +import './styles/input.css'; createRoot(document.getElementById('root')!).render( - + + + , ); diff --git a/frontend/src/styles/base.css b/frontend/src/styles/base.css new file mode 100644 index 00000000..d13f70fd --- /dev/null +++ b/frontend/src/styles/base.css @@ -0,0 +1,64 @@ +/* Reset & layout */ +*, *::before, *::after { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif; + background: var(--color-bg); + color: var(--color-text); + height: 100vh; + overflow: hidden; +} + +#root { + height: 100vh; +} + +.app { + display: flex; + height: 100vh; + position: relative; +} + +.sidebar-toggle { + display: none; + position: fixed; + top: 12px; + left: 12px; + z-index: 1000; + background: var(--color-primary); + color: var(--ctp-crust); + border: none; + border-radius: 8px; + width: 36px; + height: 36px; + font-size: 18px; + cursor: pointer; + align-items: center; + justify-content: center; +} + +@media (max-width: 768px) { + .sidebar-toggle { + display: flex; + } + + .sidebar { + position: fixed; + left: 0; + top: 0; + z-index: 999; + transform: translateX(-100%); + } + + .sidebar.open { + transform: translateX(0); + } + + .message-bubble { + max-width: 90%; + } +} diff --git a/frontend/src/styles/chat.css b/frontend/src/styles/chat.css new file mode 100644 index 00000000..23353b43 --- /dev/null +++ b/frontend/src/styles/chat.css @@ -0,0 +1,291 @@ +/* Chat Area */ +.chat-area { + flex: 1; + display: flex; + flex-direction: column; + height: 100vh; + min-width: 0; +} + +.chat-header { + padding: 12px 24px; + border-bottom: 1px solid var(--color-border); + background: var(--color-bg); + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; +} + +.chat-header-title { + font-size: 14px; + font-weight: 600; + color: var(--color-text); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.chat-header-meta { + display: flex; + align-items: center; + gap: 10px; + flex-shrink: 0; +} + +.chat-header-info { + display: flex; + align-items: center; + gap: 6px; +} + +.header-badge { + display: inline-block; + padding: 3px 8px; + border-radius: 4px; + font-size: 11px; + font-weight: 600; + font-family: monospace; +} + +.model-badge { + background: rgba(137, 180, 250, 0.12); + color: var(--ctp-blue); + border: 1px solid rgba(137, 180, 250, 0.25); +} + +.agent-badge { + background: rgba(166, 227, 161, 0.12); + color: var(--ctp-green); + border: 1px solid rgba(166, 227, 161, 0.25); +} + +.chat-header-tokens { + font-size: 11px; + color: var(--color-text-secondary); + font-variant-numeric: tabular-nums; + white-space: nowrap; +} + +/* Message List */ +.message-list { + flex: 1; + overflow-y: auto; + padding: 24px; + display: flex; + flex-direction: column; + gap: 16px; +} + +.message-list-empty { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + color: var(--color-text-secondary); + font-size: 15px; +} + +/* Message Bubble */ +.message-bubble { + max-width: 75%; + position: relative; +} + +.message-bubble.user { + align-self: flex-end; +} + +.message-bubble.assistant { + align-self: flex-start; +} + +.message-content { + padding: 12px 16px; + border-radius: 16px; + font-size: 14px; + line-height: 1.6; + white-space: pre-wrap; + word-break: break-word; +} + +.message-bubble.user .message-content { + background: var(--color-user-bubble); + color: var(--ctp-crust); + border-bottom-right-radius: 4px; +} + +.message-bubble.assistant .message-content { + background: var(--color-assistant-bubble); + color: var(--color-text); + border: 1px solid var(--ctp-surface1); + border-bottom-left-radius: 4px; +} + +.message-meta { + display: flex; + align-items: center; + gap: 8px; + margin-top: 4px; + padding: 0 4px; +} + +.message-time { + font-size: 11px; + color: var(--color-text-secondary); +} + +.message-tokens { + font-size: 10px; + color: var(--color-text-secondary); + font-variant-numeric: tabular-nums; + opacity: 0.7; +} + +.message-bubble.user .message-meta { + justify-content: flex-end; +} + +/* Copy Button */ +.copy-btn { + position: absolute; + top: 8px; + right: 8px; + opacity: 0; + background: rgba(255, 255, 255, 0.06); + border: none; + border-radius: 6px; + padding: 4px 8px; + cursor: pointer; + font-size: 12px; + color: var(--color-text-secondary); + transition: opacity 0.15s; +} + +.message-bubble:hover .copy-btn { + opacity: 1; +} + +.copy-btn:hover { + background: rgba(255, 255, 255, 0.1); +} + +.message-bubble.user .copy-btn { + color: rgba(0, 0, 0, 0.5); + background: rgba(0, 0, 0, 0.1); +} + +.message-bubble.user .copy-btn:hover { + background: rgba(0, 0, 0, 0.2); +} + +/* Tool Call Indicator */ +.tool-calls { + margin-top: 8px; + display: flex; + flex-direction: column; + gap: 6px; +} + +.tool-call { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px; + background: var(--color-tool-bg); + border: 1px solid var(--color-tool-border); + border-radius: 8px; + font-size: 13px; +} + +.tool-call .tool-status { + width: 8px; + height: 8px; + border-radius: 50%; + flex-shrink: 0; +} + +.tool-call .tool-status.running { + background: var(--color-tool-running); + animation: pulse 1.5s infinite; +} + +.tool-call .tool-status.success { + background: var(--color-tool-success); +} + +.tool-call .tool-status.error { + background: var(--color-tool-error); +} + +.tool-call .tool-name { + font-weight: 600; + color: var(--color-text); +} + +.tool-call .tool-latency { + margin-left: auto; + font-size: 11px; + color: var(--color-text-secondary); +} + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.4; } +} + +/* Streaming Indicator */ +.streaming-indicator { + display: flex; + flex-direction: column; + gap: 8px; + padding: 12px 16px; + margin: 0 24px; +} + +.streaming-tool-calls { + display: flex; + flex-wrap: wrap; + gap: 6px; +} + +.streaming-progress-row { + display: flex; + align-items: center; + gap: 12px; +} + +.streaming-bar-container { + flex: 1; + height: 4px; + background: var(--color-border); + border-radius: 2px; + overflow: hidden; +} + +.streaming-bar { + height: 100%; + background: var(--color-primary); + border-radius: 2px; + animation: stream-progress 2s ease-in-out infinite; +} + +@keyframes stream-progress { + 0% { width: 0%; margin-left: 0%; } + 50% { width: 40%; margin-left: 30%; } + 100% { width: 0%; margin-left: 100%; } +} + +.streaming-phase { + font-size: 12px; + color: var(--color-text-secondary); + white-space: nowrap; +} + +.streaming-elapsed { + font-size: 12px; + color: var(--color-text-secondary); + font-variant-numeric: tabular-nums; + white-space: nowrap; +} diff --git a/frontend/src/styles/input.css b/frontend/src/styles/input.css new file mode 100644 index 00000000..4398ef52 --- /dev/null +++ b/frontend/src/styles/input.css @@ -0,0 +1,131 @@ +/* Input Area */ +.input-area { + padding: 16px 24px 24px; + border-top: 1px solid var(--color-border); + background: var(--color-bg); +} + +.input-attachment-row { + margin-bottom: 8px; +} + +.input-container { + display: flex; + gap: 8px; + align-items: flex-end; +} + +.input-container textarea { + flex: 1; + resize: none; + border: 1px solid var(--color-border); + border-radius: 12px; + padding: 12px 16px; + font-size: 14px; + font-family: inherit; + line-height: 1.5; + min-height: 72px; + max-height: 200px; + outline: none; + background: var(--ctp-surface0); + color: var(--color-text); + transition: border-color 0.15s; +} + +.input-container textarea:focus { + border-color: var(--color-primary); +} + +.input-container textarea::placeholder { + color: var(--ctp-overlay0); +} + +.send-btn, .stop-btn { + padding: 10px 20px; + border: none; + border-radius: 12px; + font-size: 14px; + font-weight: 500; + cursor: pointer; + flex-shrink: 0; + transition: background 0.15s; +} + +.send-btn { + background: var(--color-primary); + color: var(--ctp-crust); +} + +.send-btn:hover:not(:disabled) { + background: var(--color-primary-dark); +} + +.send-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.stop-btn { + background: var(--color-tool-error); + color: var(--ctp-crust); +} + +.stop-btn:hover { + background: var(--ctp-maroon); +} + +/* Pasted text pill */ +.pasted-pill { + display: inline-flex; + align-items: center; + gap: 8px; + background: var(--ctp-surface0); + border: 1px solid var(--color-border); + border-radius: 10px; + padding: 10px 14px; + color: var(--color-text-secondary); + font-size: 13px; + max-width: 100%; +} + +.pasted-pill svg { + flex-shrink: 0; + opacity: 0.6; +} + +.pasted-pill-text { + font-weight: 500; + color: var(--color-text); +} + +.pasted-pill-size { + color: var(--color-text-secondary); + font-size: 12px; +} + +.pasted-pill-action { + background: none; + border: 1px solid var(--color-border); + border-radius: 6px; + padding: 2px 8px; + font-size: 12px; + color: var(--color-text-secondary); + cursor: pointer; + transition: background 0.15s, color 0.15s; +} + +.pasted-pill-action:hover { + background: var(--ctp-surface1); + color: var(--color-text); +} + +.pasted-pill-remove { + font-size: 16px; + line-height: 1; + padding: 2px 6px; +} + +.pasted-pill-remove:hover { + color: var(--color-tool-error); + border-color: var(--color-tool-error); +} diff --git a/frontend/src/styles/sidebar.css b/frontend/src/styles/sidebar.css new file mode 100644 index 00000000..1043494c --- /dev/null +++ b/frontend/src/styles/sidebar.css @@ -0,0 +1,217 @@ +/* Sidebar */ +.sidebar { + width: var(--sidebar-width); + background: var(--color-bg-sidebar); + border-right: 1px solid var(--color-border); + display: flex; + flex-direction: column; + height: 100vh; + flex-shrink: 0; + transition: transform 0.2s ease; +} + +.sidebar-header { + padding: 16px; + border-bottom: 1px solid var(--color-border); +} + +.sidebar-header h1 { + font-size: 18px; + font-weight: 700; + color: var(--color-primary); + margin-bottom: 12px; +} + +.new-chat-btn { + width: 100%; + padding: 10px 16px; + background: var(--color-primary); + color: var(--ctp-crust); + border: none; + border-radius: 8px; + font-size: 14px; + font-weight: 500; + cursor: pointer; + transition: background 0.15s; +} + +.new-chat-btn:hover { + background: var(--color-primary-dark); +} + +/* Model Selector */ +.model-selector { + padding: 12px 16px; + border-bottom: 1px solid var(--color-border); +} + +.model-selector label { + display: block; + font-size: 11px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--color-text-secondary); + margin-bottom: 6px; +} + +.model-selector select { + width: 100%; + padding: 8px 10px; + border: 1px solid var(--color-border); + border-radius: 6px; + background: var(--ctp-surface0); + font-size: 13px; + color: var(--color-text); + cursor: pointer; +} + +/* Conversation List */ +.conversation-list { + flex: 1; + overflow-y: auto; + padding: 8px; +} + +.conversation-item { + display: flex; + align-items: center; + padding: 10px 12px; + border-radius: 8px; + cursor: pointer; + font-size: 13px; + color: var(--color-text); + transition: background 0.15s; + margin-bottom: 2px; +} + +.conversation-item:hover { + background: var(--ctp-surface0); +} + +.conversation-item.active { + background: var(--color-primary); + color: var(--ctp-crust); +} + +.conversation-item .conv-title { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.conversation-item .conv-delete { + opacity: 0; + border: none; + background: none; + color: inherit; + cursor: pointer; + padding: 2px 6px; + border-radius: 4px; + font-size: 16px; + line-height: 1; +} + +.conversation-item:hover .conv-delete { + opacity: 0.6; +} + +.conversation-item .conv-delete:hover { + opacity: 1; + background: rgba(255, 255, 255, 0.08); +} + +.conversation-item.active .conv-delete:hover { + background: rgba(0, 0, 0, 0.2); +} + +/* Savings Panel */ +.savings-panel { + padding: 16px; + border-top: 1px solid var(--color-primary); + background: var(--ctp-crust); +} + +.savings-panel h3 { + font-size: 13px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--color-primary); + margin-bottom: 8px; +} + +.savings-models { + display: flex; + flex-direction: column; + gap: 6px; + margin-bottom: 12px; +} + +.savings-model-card { + background: var(--ctp-surface0); + border: 1px solid var(--color-border); + border-radius: 8px; + padding: 8px 10px; + border-left: 3px solid var(--color-border); +} + +.savings-model-card.local { + border-left-color: var(--ctp-green); +} + +.savings-model-card.cloud { + border-left-color: var(--ctp-yellow); +} + +.savings-model-card-label { + display: block; + font-size: 9px; + font-weight: 700; + letter-spacing: 0.08em; + color: var(--color-text-secondary); + margin-bottom: 2px; +} + +.savings-model-card-name { + display: block; + font-family: monospace; + font-size: 11px; + color: var(--color-text); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.savings-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} + +.savings-item { + background: var(--ctp-surface0); + border-radius: 8px; + padding: 10px; + border: 1px solid var(--color-border); +} + +.savings-item .savings-label { + font-size: 10px; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--color-text-secondary); + margin-bottom: 4px; +} + +.savings-item .savings-value { + font-size: 18px; + font-weight: 800; + color: var(--color-savings); + font-variant-numeric: tabular-nums; +} + +.savings-item.calls .savings-value { + color: var(--color-primary); +} diff --git a/frontend/src/styles/variables.css b/frontend/src/styles/variables.css new file mode 100644 index 00000000..d7f80218 --- /dev/null +++ b/frontend/src/styles/variables.css @@ -0,0 +1,50 @@ +/* Catppuccin Mocha theme tokens */ +:root { + /* Base palette */ + --ctp-rosewater: #f5e0dc; + --ctp-flamingo: #f2cdcd; + --ctp-pink: #f5c2e7; + --ctp-mauve: #cba6f7; + --ctp-red: #f38ba8; + --ctp-maroon: #eba0ac; + --ctp-peach: #fab387; + --ctp-yellow: #f9e2af; + --ctp-green: #a6e3a1; + --ctp-teal: #94e2d5; + --ctp-sky: #89dceb; + --ctp-sapphire: #74c7ec; + --ctp-blue: #89b4fa; + --ctp-lavender: #b4befe; + --ctp-text: #cdd6f4; + --ctp-subtext1: #bac2de; + --ctp-subtext0: #a6adc8; + --ctp-overlay2: #9399b2; + --ctp-overlay1: #7f849c; + --ctp-overlay0: #6c7086; + --ctp-surface2: #585b70; + --ctp-surface1: #45475a; + --ctp-surface0: #313244; + --ctp-base: #1e1e2e; + --ctp-mantle: #181825; + --ctp-crust: #11111b; + + /* Semantic tokens */ + --color-primary: var(--ctp-blue); + --color-primary-light: var(--ctp-sapphire); + --color-primary-dark: var(--ctp-lavender); + --color-bg: var(--ctp-base); + --color-bg-sidebar: var(--ctp-mantle); + --color-bg-surface: var(--ctp-surface0); + --color-text: var(--ctp-text); + --color-text-secondary: var(--ctp-subtext0); + --color-border: var(--ctp-surface0); + --color-user-bubble: var(--ctp-blue); + --color-assistant-bubble: var(--ctp-surface0); + --color-tool-bg: rgba(137, 180, 250, 0.08); + --color-tool-border: var(--ctp-surface1); + --color-tool-success: var(--ctp-green); + --color-tool-running: var(--ctp-yellow); + --color-tool-error: var(--ctp-red); + --color-savings: var(--ctp-green); + --sidebar-width: 280px; +} diff --git a/frontend/src/vite-env.d.ts b/frontend/src/vite-env.d.ts new file mode 100644 index 00000000..29c29a18 --- /dev/null +++ b/frontend/src/vite-env.d.ts @@ -0,0 +1,9 @@ +/// + +interface ImportMetaEnv { + readonly VITE_API_URL: string; +} + +interface ImportMeta { + readonly env: ImportMetaEnv; +} diff --git a/src/openjarvis/bench/latency.py b/src/openjarvis/bench/latency.py index 7d9252fd..d7928193 100644 --- a/src/openjarvis/bench/latency.py +++ b/src/openjarvis/bench/latency.py @@ -80,6 +80,9 @@ class LatencyBenchmark(BaseBenchmark): "p95_latency": _percentile(latencies, 0.95), "min_latency": min(latencies), "max_latency": max(latencies), + "std_latency": ( + statistics.stdev(latencies) if len(latencies) > 1 else 0.0 + ), }, samples=num_samples, errors=errors, diff --git a/src/openjarvis/cli/__init__.py b/src/openjarvis/cli/__init__.py index 965cc25b..936a6239 100644 --- a/src/openjarvis/cli/__init__.py +++ b/src/openjarvis/cli/__init__.py @@ -17,6 +17,7 @@ from openjarvis.cli.doctor_cmd import doctor from openjarvis.cli.init_cmd import init from openjarvis.cli.memory_cmd import memory from openjarvis.cli.model import model +from openjarvis.cli.quickstart_cmd import quickstart from openjarvis.cli.scheduler_cmd import scheduler from openjarvis.cli.serve import serve from openjarvis.cli.skill_cmd import skill @@ -27,8 +28,17 @@ from openjarvis.cli.workflow_cmd import workflow @click.group(help="OpenJarvis — modular AI assistant backend") @click.version_option(version=openjarvis.__version__, prog_name="jarvis") -def cli() -> None: +@click.option("--verbose", is_flag=True, default=False, help="Enable debug logging") +@click.option("--quiet", is_flag=True, default=False, help="Suppress non-error output") +@click.pass_context +def cli(ctx: click.Context, verbose: bool, quiet: bool) -> None: """Top-level CLI group.""" + from openjarvis.cli.log_config import setup_logging + + ctx.ensure_object(dict) + ctx.obj["verbose"] = verbose + ctx.obj["quiet"] = quiet + setup_logging(verbose=verbose, quiet=quiet) cli.add_command(init, "init") @@ -52,6 +62,7 @@ cli.add_command(status, "status") cli.add_command(vault, "vault") cli.add_command(add, "add") cli.add_command(operators, "operators") +cli.add_command(quickstart, "quickstart") def main() -> None: diff --git a/src/openjarvis/cli/ask.py b/src/openjarvis/cli/ask.py index 9811f99a..8897b06a 100644 --- a/src/openjarvis/cli/ask.py +++ b/src/openjarvis/cli/ask.py @@ -8,6 +8,7 @@ import sys import click from rich.console import Console +from openjarvis.cli.hints import hint_no_engine from openjarvis.core.config import load_config from openjarvis.core.events import EventBus from openjarvis.core.types import Message, Role @@ -268,6 +269,7 @@ def ask( ) except EngineConnectionError as exc: console.print(f"[red]Engine error:[/red] {exc}") + console.print(hint_no_engine()) sys.exit(1) if output_json: @@ -321,14 +323,16 @@ def ask( # Generate (InstrumentedEngine handles telemetry + energy recording) try: - result = engine.generate( - messages, - model=model_name, - temperature=temperature, - max_tokens=max_tokens, - ) + with console.status("[bold green]Generating...[/bold green]"): + result = engine.generate( + messages, + model=model_name, + temperature=temperature, + max_tokens=max_tokens, + ) except EngineConnectionError as exc: console.print(f"[red]Engine error:[/red] {exc}") + console.print(hint_no_engine()) sys.exit(1) # Output diff --git a/src/openjarvis/cli/bench_cmd.py b/src/openjarvis/cli/bench_cmd.py index 294c7e4a..c520f436 100644 --- a/src/openjarvis/cli/bench_cmd.py +++ b/src/openjarvis/cli/bench_cmd.py @@ -11,7 +11,12 @@ from rich.panel import Panel from rich.rule import Rule from rich.table import Table +from typing import TYPE_CHECKING + from openjarvis.core.config import load_config + +if TYPE_CHECKING: + from openjarvis.bench._stubs import BenchmarkResult from openjarvis.engine import get_engine _BANNER = r""" @@ -38,6 +43,90 @@ def _section(console: Console, title: str) -> None: console.print(Rule(title, style="bright_blue")) +# -- Stats-aware rendering ---------------------------------------------------- + +_STATS_PREFIXES = {"mean_", "p50_", "p95_", "min_", "max_", "std_"} + + +def _detect_stat_groups(metrics: dict[str, float]) -> dict[str, dict[str, float]]: + """Detect metrics following the stats pattern (mean_X, p50_X, ...). + + Returns ``{metric_base: {prefix: value}}`` for grouped metrics. + """ + groups: dict[str, dict[str, float]] = {} + for key, val in metrics.items(): + for pfx in _STATS_PREFIXES: + if key.startswith(pfx): + base = key[len(pfx):] + groups.setdefault(base, {})[pfx.rstrip("_")] = val + break + return groups + + +def _render_stats_table(console: Console, result: BenchmarkResult) -> None: + """Render benchmark result as a stats table when stats keys are present.""" + groups = _detect_stat_groups(result.metrics) + + # Determine which keys are consumed by stat groups + consumed: set[str] = set() + for base, prefixes in groups.items(): + for pfx in prefixes: + consumed.add(f"{pfx}_{base}") + + # Stat groups → multi-column table + if groups: + table = Table( + title=( + f"[bold]{result.benchmark_name}[/bold]" + f" ({result.samples} samples, {result.errors} errors)" + ), + show_header=True, + header_style="bold bright_white", + border_style="bright_blue", + title_style="bold cyan", + ) + table.add_column("Metric", style="cyan", no_wrap=True) + table.add_column("Avg", justify="right") + table.add_column("Median", justify="right") + table.add_column("Min", justify="right") + table.add_column("Max", justify="right") + table.add_column("Std", justify="right") + table.add_column("P95", justify="right") + + for base, vals in sorted(groups.items()): + table.add_row( + base, + f"{vals.get('mean', 0):.4f}", + f"{vals.get('p50', 0):.4f}", + f"{vals.get('min', 0):.4f}", + f"{vals.get('max', 0):.4f}", + f"{vals.get('std', 0):.4f}", + f"{vals.get('p95', 0):.4f}", + ) + console.print(table) + + # Remaining non-stats metrics → simple key-value table + remaining = {k: v for k, v in result.metrics.items() if k not in consumed} + if remaining or result.total_energy_joules > 0: + kv_table = Table( + show_header=True, + header_style="bold bright_white", + border_style="bright_blue", + ) + kv_table.add_column("Metric", style="cyan", no_wrap=True) + kv_table.add_column("Value", justify="right", style="green") + for k, v in remaining.items(): + kv_table.add_row(k, f"{v:.4f}") + if result.total_energy_joules > 0: + kv_table.add_row("Total Energy (J)", f"{result.total_energy_joules:.4f}") + kv_table.add_row("Energy Method", str(result.energy_method)) + if result.energy_per_token_joules > 0: + kv_table.add_row( + "Energy/Token (J)", f"{result.energy_per_token_joules:.6f}", + ) + console.print(kv_table) + + @click.group() def bench() -> None: """Run inference benchmarks.""" @@ -175,30 +264,7 @@ def run( # Pretty-print results as Rich tables _section(console, "Results") for r in results: - table = Table( - title=( - f"[bold]{r.benchmark_name}[/bold]" - f" ({r.samples} samples, {r.errors} errors)" - ), - show_header=True, - header_style="bold bright_white", - border_style="bright_blue", - title_style="bold cyan", - ) - table.add_column("Metric", style="cyan", no_wrap=True) - table.add_column("Value", justify="right", style="green") - - for k, v in r.metrics.items(): - table.add_row(k, f"{v:.4f}") - if r.total_energy_joules > 0: - table.add_row("Total Energy (J)", f"{r.total_energy_joules:.4f}") - table.add_row("Energy Method", str(r.energy_method)) - if r.energy_per_token_joules > 0: - table.add_row( - "Energy/Token (J)", f"{r.energy_per_token_joules:.6f}", - ) - - console.print(table) + _render_stats_table(console, r) # Cleanup energy monitor if energy_monitor is not None: diff --git a/src/openjarvis/cli/hints.py b/src/openjarvis/cli/hints.py new file mode 100644 index 00000000..41924b91 --- /dev/null +++ b/src/openjarvis/cli/hints.py @@ -0,0 +1,41 @@ +"""Rich-formatted error hints for common CLI failure modes.""" + +from __future__ import annotations + +from typing import Optional + + +def hint_no_config() -> str: + """Return a suggestion when no config file is found.""" + return ( + "[yellow]Hint:[/yellow] No config file found.\n" + " Run [bold]jarvis init[/bold] to detect hardware and generate " + "[cyan]~/.openjarvis/config.toml[/cyan].\n" + " Or run [bold]jarvis quickstart[/bold] for a guided setup." + ) + + +def hint_no_engine(engine_name: Optional[str] = None) -> str: + """Return a suggestion when the inference engine is unreachable.""" + name = engine_name or "ollama" + return ( + f"[yellow]Hint:[/yellow] Engine '{name}' is not reachable.\n" + f" Make sure the {name} server is running.\n" + " Run [bold]jarvis doctor[/bold] to check all engines.\n" + " Run [bold]jarvis quickstart[/bold] for guided setup." + ) + + +def hint_no_model(model_name: Optional[str] = None) -> str: + """Return a suggestion when no model is available.""" + if model_name: + return ( + f"[yellow]Hint:[/yellow] Model '{model_name}' not found.\n" + f" Try: [bold]ollama pull {model_name}[/bold]\n" + " Run [bold]jarvis model list[/bold] to see available models." + ) + return ( + "[yellow]Hint:[/yellow] No models available.\n" + " Pull a model first: [bold]ollama pull qwen3:8b[/bold]\n" + " Run [bold]jarvis model list[/bold] to see available models." + ) diff --git a/src/openjarvis/cli/log_config.py b/src/openjarvis/cli/log_config.py new file mode 100644 index 00000000..6cf6fa15 --- /dev/null +++ b/src/openjarvis/cli/log_config.py @@ -0,0 +1,70 @@ +"""Global logging configuration for the OpenJarvis CLI.""" + +from __future__ import annotations + +import logging +from logging.handlers import RotatingFileHandler +from pathlib import Path +from typing import Optional, Union + + +def setup_logging( + verbose: bool = False, + quiet: bool = False, + log_file: Optional[Union[str, Path]] = None, +) -> logging.Logger: + """Configure the ``openjarvis`` logger. + + Parameters + ---------- + verbose: + Set log level to DEBUG. + quiet: + Set log level to ERROR (overrides verbose if both set). + log_file: + Path for a rotating file handler. When *verbose* is ``True`` + and no *log_file* is given, defaults to + ``~/.openjarvis/cli.log``. + + Returns + ------- + The configured ``openjarvis`` logger. + """ + logger = logging.getLogger("openjarvis") + + # Clear existing handlers to avoid duplication across calls + logger.handlers.clear() + + if quiet: + level = logging.ERROR + elif verbose: + level = logging.DEBUG + else: + level = logging.WARNING + + logger.setLevel(level) + + # Console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(level) + fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s") + console_handler.setFormatter(fmt) + logger.addHandler(console_handler) + + # File handler (verbose or explicit path) + if verbose or log_file is not None: + if log_file is None: + log_dir = Path.home() / ".openjarvis" + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / "cli.log" + file_handler = RotatingFileHandler( + str(log_file), maxBytes=5 * 1024 * 1024, backupCount=3, + ) + file_handler.setLevel(logging.DEBUG) + file_fmt = logging.Formatter( + "%(asctime)s %(levelname)s %(name)s: %(message)s" + ) + file_handler.setFormatter(file_fmt) + logger.addHandler(file_handler) + + return logger diff --git a/src/openjarvis/cli/memory_cmd.py b/src/openjarvis/cli/memory_cmd.py index 854f1631..ccc0f09b 100644 --- a/src/openjarvis/cli/memory_cmd.py +++ b/src/openjarvis/cli/memory_cmd.py @@ -7,6 +7,7 @@ from pathlib import Path import click from rich.console import Console +from rich.progress import track from rich.table import Table from openjarvis.core.config import load_config @@ -82,7 +83,7 @@ def index( mem = _get_backend(backend) try: - for chunk in chunks: + for chunk in track(chunks, description="Storing chunks...", console=console): mem.store( chunk.content, source=chunk.source, diff --git a/src/openjarvis/cli/quickstart_cmd.py b/src/openjarvis/cli/quickstart_cmd.py new file mode 100644 index 00000000..936f4bd2 --- /dev/null +++ b/src/openjarvis/cli/quickstart_cmd.py @@ -0,0 +1,123 @@ +"""``jarvis quickstart`` — guided 5-step setup for new users.""" + +from __future__ import annotations + +import click +from rich.console import Console + +from openjarvis.core.config import ( + DEFAULT_CONFIG_DIR, + DEFAULT_CONFIG_PATH, + detect_hardware, + generate_default_toml, + recommend_engine, +) + + +def _check_engine_health(engine_key: str) -> bool: + """Return True if the recommended engine is reachable.""" + try: + from openjarvis.core.config import load_config + from openjarvis.core.registry import EngineRegistry + from openjarvis.engine import _discovery + + import openjarvis.engine # noqa: F401 — trigger registration + + config = load_config() + if engine_key not in EngineRegistry.keys(): + return False + engine = _discovery._make_engine(engine_key, config) + return engine.health() + except Exception: + return False + + +def _check_model_available(engine_key: str) -> bool: + """Return True if at least one model is available on the engine.""" + try: + from openjarvis.core.config import load_config + from openjarvis.core.registry import EngineRegistry + from openjarvis.engine import _discovery + + config = load_config() + if engine_key not in EngineRegistry.keys(): + return False + engine = _discovery._make_engine(engine_key, config) + return bool(engine.list_models()) + except Exception: + return False + + +def _test_query(engine_key: str) -> str: + """Run a quick test query and return the response text.""" + try: + from openjarvis import Jarvis + + j = Jarvis(engine_key=engine_key) + response = j.ask("Say hello in one sentence.") + j.close() + return response + except Exception as exc: + return f"(query failed: {exc})" + + +@click.command() +@click.option("--force", is_flag=True, help="Redo all steps even if already done.") +def quickstart(force: bool) -> None: + """Guided 5-step setup for new users.""" + console = Console() + + # Step 1: Detect hardware + console.print("[bold cyan][1/5][/bold cyan] Detecting hardware...") + hw = detect_hardware() + console.print(f" Platform : {hw.platform}") + console.print(f" CPU : {hw.cpu_brand} ({hw.cpu_count} cores)") + console.print(f" RAM : {hw.ram_gb} GB") + if hw.gpu: + console.print( + f" GPU : {hw.gpu.name} ({hw.gpu.vram_gb} GB VRAM, x{hw.gpu.count})" + ) + else: + console.print(" GPU : none detected") + + engine_key = recommend_engine(hw) + + # Step 2: Write config + console.print() + console.print("[bold cyan][2/5][/bold cyan] Writing config...") + if DEFAULT_CONFIG_PATH.exists() and not force: + console.print(f" [dim]Config already exists at {DEFAULT_CONFIG_PATH} (skip)[/dim]") + else: + toml_content = generate_default_toml(hw) + DEFAULT_CONFIG_DIR.mkdir(parents=True, exist_ok=True) + DEFAULT_CONFIG_PATH.write_text(toml_content) + console.print(f" [green]Config written to {DEFAULT_CONFIG_PATH}[/green]") + + # Step 3: Check engine + console.print() + console.print(f"[bold cyan][3/5][/bold cyan] Checking engine ({engine_key})...") + if not _check_engine_health(engine_key): + console.print(f" [red bold]Engine '{engine_key}' is not reachable.[/red bold]") + console.print() + console.print(f" Start the {engine_key} server and try again.") + console.print(f" Run [bold]jarvis doctor[/bold] for detailed diagnostics.") + raise SystemExit(1) + console.print(f" [green]Engine '{engine_key}' is healthy.[/green]") + + # Step 4: Verify model + console.print() + console.print("[bold cyan][4/5][/bold cyan] Checking for available models...") + if not _check_model_available(engine_key): + console.print(" [yellow]No models found.[/yellow]") + console.print(" Pull a model first (e.g. [bold]ollama pull qwen3:8b[/bold]).") + raise SystemExit(1) + console.print(" [green]Models available.[/green]") + + # Step 5: Test query + console.print() + console.print("[bold cyan][5/5][/bold cyan] Running test query...") + response = _test_query(engine_key) + console.print(f" [green]Response:[/green] {response[:200]}") + + console.print() + console.print("[bold green]Setup complete![/bold green] Try: [bold]jarvis ask \"Hello\"[/bold]") diff --git a/src/openjarvis/core/types.py b/src/openjarvis/core/types.py index 623aed40..3da6bba3 100644 --- a/src/openjarvis/core/types.py +++ b/src/openjarvis/core/types.py @@ -162,6 +162,7 @@ class TelemetryRecord: cpu_energy_joules: float = 0.0 gpu_energy_joules: float = 0.0 dram_energy_joules: float = 0.0 + tokens_per_joule: float = 0.0 metadata: Dict[str, Any] = field(default_factory=dict) diff --git a/src/openjarvis/telemetry/aggregator.py b/src/openjarvis/telemetry/aggregator.py index f68b49be..986cdf9f 100644 --- a/src/openjarvis/telemetry/aggregator.py +++ b/src/openjarvis/telemetry/aggregator.py @@ -24,6 +24,7 @@ class ModelStats: total_energy_joules: float = 0.0 avg_gpu_utilization_pct: float = 0.0 avg_throughput_tok_per_sec: float = 0.0 + avg_tokens_per_joule: float = 0.0 avg_energy_per_output_token_joules: float = 0.0 avg_throughput_per_watt: float = 0.0 total_prefill_energy_joules: float = 0.0 @@ -47,6 +48,7 @@ class EngineStats: total_energy_joules: float = 0.0 avg_gpu_utilization_pct: float = 0.0 avg_throughput_tok_per_sec: float = 0.0 + avg_tokens_per_joule: float = 0.0 avg_energy_per_output_token_joules: float = 0.0 avg_throughput_per_watt: float = 0.0 total_prefill_energy_joules: float = 0.0 @@ -122,10 +124,15 @@ class TelemetryAggregator: # Build optional columns for new fields (graceful on old DBs) extra_cols = "" + has_tpj = self._safe_col("tokens_per_joule") has_derived = self._safe_col("energy_per_output_token_joules") has_phase = self._safe_col("prefill_energy_joules") has_itl = self._safe_col("mean_itl_ms") + if has_tpj: + extra_cols += ( + ", AVG(tokens_per_joule) AS avg_tokens_per_joule" + ) if has_derived: extra_cols += ( ", AVG(energy_per_output_token_joules)" @@ -178,6 +185,8 @@ class TelemetryAggregator: avg_gpu_utilization_pct=r["avg_gpu_utilization_pct"] or 0.0, avg_throughput_tok_per_sec=r["avg_throughput_tok_per_sec"] or 0.0, ) + if has_tpj: + ms.avg_tokens_per_joule = r["avg_tokens_per_joule"] or 0.0 if has_derived: ms.avg_energy_per_output_token_joules = ( r["avg_energy_per_output_token_joules"] or 0.0 @@ -206,10 +215,15 @@ class TelemetryAggregator: where, params = self._time_filter(since, until) extra_cols = "" + has_tpj = self._safe_col("tokens_per_joule") has_derived = self._safe_col("energy_per_output_token_joules") has_phase = self._safe_col("prefill_energy_joules") has_itl = self._safe_col("mean_itl_ms") + if has_tpj: + extra_cols += ( + ", AVG(tokens_per_joule) AS avg_tokens_per_joule" + ) if has_derived: extra_cols += ( ", AVG(energy_per_output_token_joules)" @@ -258,6 +272,8 @@ class TelemetryAggregator: avg_gpu_utilization_pct=r["avg_gpu_utilization_pct"] or 0.0, avg_throughput_tok_per_sec=r["avg_throughput_tok_per_sec"] or 0.0, ) + if has_tpj: + es.avg_tokens_per_joule = r["avg_tokens_per_joule"] or 0.0 if has_derived: es.avg_energy_per_output_token_joules = ( r["avg_energy_per_output_token_joules"] or 0.0 diff --git a/src/openjarvis/telemetry/instrumented_engine.py b/src/openjarvis/telemetry/instrumented_engine.py index bc01a03c..ea0d5a00 100644 --- a/src/openjarvis/telemetry/instrumented_engine.py +++ b/src/openjarvis/telemetry/instrumented_engine.py @@ -173,6 +173,12 @@ class InstrumentedEngine(InferenceEngine): if completion_tokens > 0 and decode_latency > 0 else 0.0 ) + # --- Tier 4: Per-inference efficiency --- + tokens_per_joule = ( + completion_tokens / energy_joules + if energy_joules > 0 and completion_tokens > 0 else 0.0 + ) + engine_id = getattr(self._inner, "engine_id", "unknown") record = TelemetryRecord( @@ -201,6 +207,7 @@ class InstrumentedEngine(InferenceEngine): cpu_energy_joules=cpu_energy_joules, gpu_energy_joules=gpu_energy_joules, dram_energy_joules=dram_energy_joules, + tokens_per_joule=tokens_per_joule, ) event_data = { @@ -364,6 +371,12 @@ class InstrumentedEngine(InferenceEngine): prefill_energy = energy_joules * prefill_frac decode_energy = energy_joules * (1.0 - prefill_frac) + # Per-inference efficiency + tokens_per_joule = ( + token_count / energy_joules + if energy_joules > 0 and token_count > 0 else 0.0 + ) + engine_id = getattr(self._inner, "engine_id", "unknown") record = TelemetryRecord( @@ -397,6 +410,7 @@ class InstrumentedEngine(InferenceEngine): cpu_energy_joules=cpu_energy_joules, gpu_energy_joules=gpu_energy_joules, dram_energy_joules=dram_energy_joules, + tokens_per_joule=tokens_per_joule, ) event_data = { diff --git a/src/openjarvis/telemetry/store.py b/src/openjarvis/telemetry/store.py index 01cccd90..c7601134 100644 --- a/src/openjarvis/telemetry/store.py +++ b/src/openjarvis/telemetry/store.py @@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS telemetry ( cpu_energy_joules REAL NOT NULL DEFAULT 0.0, gpu_energy_joules REAL NOT NULL DEFAULT 0.0, dram_energy_joules REAL NOT NULL DEFAULT 0.0, + tokens_per_joule REAL NOT NULL DEFAULT 0.0, energy_per_output_token_joules REAL NOT NULL DEFAULT 0.0, throughput_per_watt REAL NOT NULL DEFAULT 0.0, prefill_energy_joules REAL NOT NULL DEFAULT 0.0, @@ -61,6 +62,7 @@ INSERT INTO telemetry ( throughput_tok_per_sec, prefill_latency_seconds, decode_latency_seconds, energy_method, energy_vendor, batch_id, is_warmup, cpu_energy_joules, gpu_energy_joules, dram_energy_joules, + tokens_per_joule, energy_per_output_token_joules, throughput_per_watt, prefill_energy_joules, decode_energy_joules, mean_itl_ms, median_itl_ms, p90_itl_ms, p95_itl_ms, p99_itl_ms, std_itl_ms, @@ -70,7 +72,7 @@ INSERT INTO telemetry ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ) """ @@ -88,6 +90,7 @@ _MIGRATE_COLUMNS = [ ("cpu_energy_joules", "REAL NOT NULL DEFAULT 0.0"), ("gpu_energy_joules", "REAL NOT NULL DEFAULT 0.0"), ("dram_energy_joules", "REAL NOT NULL DEFAULT 0.0"), + ("tokens_per_joule", "REAL NOT NULL DEFAULT 0.0"), ("energy_per_output_token_joules", "REAL NOT NULL DEFAULT 0.0"), ("throughput_per_watt", "REAL NOT NULL DEFAULT 0.0"), ("prefill_energy_joules", "REAL NOT NULL DEFAULT 0.0"), @@ -153,6 +156,7 @@ class TelemetryStore: rec.cpu_energy_joules, rec.gpu_energy_joules, rec.dram_energy_joules, + rec.tokens_per_joule, rec.energy_per_output_token_joules, rec.throughput_per_watt, rec.prefill_energy_joules, diff --git a/src/openjarvis/traces/analyzer.py b/src/openjarvis/traces/analyzer.py index 3d7e7b38..9f2927a1 100644 --- a/src/openjarvis/traces/analyzer.py +++ b/src/openjarvis/traces/analyzer.py @@ -6,6 +6,7 @@ routing policies, tool selection strategies, and memory configuration. from __future__ import annotations +import statistics as stats_mod from dataclasses import dataclass, field from typing import Any, Dict, List, Optional @@ -36,6 +37,29 @@ class ToolStats: success_rate: float = 0.0 +@dataclass(slots=True) +class StepTypeStats: + """Aggregated statistics for a specific step type across traces.""" + + count: int = 0 + avg_duration: float = 0.0 + median_duration: float = 0.0 + min_duration: float = 0.0 + max_duration: float = 0.0 + std_duration: float = 0.0 + total_energy: float = 0.0 + avg_input_tokens: float = 0.0 + median_input_tokens: float = 0.0 + min_input_tokens: float = 0.0 + max_input_tokens: float = 0.0 + std_input_tokens: float = 0.0 + avg_output_tokens: float = 0.0 + median_output_tokens: float = 0.0 + min_output_tokens: float = 0.0 + max_output_tokens: float = 0.0 + std_output_tokens: float = 0.0 + + @dataclass(slots=True) class TraceSummary: """Overall summary statistics across all traces.""" @@ -47,6 +71,9 @@ class TraceSummary: avg_tokens: float = 0.0 success_rate: float = 0.0 step_type_distribution: Dict[str, int] = field(default_factory=dict) + total_energy_joules: float = 0.0 + total_generate_energy_joules: float = 0.0 + step_type_stats: Dict[str, StepTypeStats] = field(default_factory=dict) class TraceAnalyzer: @@ -75,11 +102,59 @@ class TraceAnalyzer: successes = [t for t in evaluated if t.outcome == "success"] step_dist: Dict[str, int] = {} + total_energy = 0.0 + generate_energy = 0.0 + step_data: Dict[str, Dict[str, list]] = {} + for t in traces: for s in t.steps: key = _step_type_str(s) step_dist[key] = step_dist.get(key, 0) + 1 + energy = s.metadata.get("energy_joules", 0.0) + total_energy += energy + if key == "generate": + generate_energy += energy + + if key not in step_data: + step_data[key] = { + "durations": [], "energies": [], + "input_tokens": [], "output_tokens": [], + } + step_data[key]["durations"].append(s.duration_seconds) + step_data[key]["energies"].append(energy) + step_data[key]["input_tokens"].append( + s.output.get("prompt_tokens", 0) + ) + step_data[key]["output_tokens"].append( + s.output.get("completion_tokens", 0) + ) + + sts_map: Dict[str, StepTypeStats] = {} + for key, data in step_data.items(): + durations = data["durations"] + in_tok = [float(x) for x in data["input_tokens"]] + out_tok = [float(x) for x in data["output_tokens"]] + sts_map[key] = StepTypeStats( + count=len(durations), + avg_duration=_avg(durations), + median_duration=stats_mod.median(durations) if durations else 0.0, + min_duration=min(durations) if durations else 0.0, + max_duration=max(durations) if durations else 0.0, + std_duration=stats_mod.stdev(durations) if len(durations) > 1 else 0.0, + total_energy=sum(data["energies"]), + avg_input_tokens=_avg(in_tok), + median_input_tokens=stats_mod.median(in_tok) if in_tok else 0.0, + min_input_tokens=min(in_tok) if in_tok else 0.0, + max_input_tokens=max(in_tok) if in_tok else 0.0, + std_input_tokens=stats_mod.stdev(in_tok) if len(in_tok) > 1 else 0.0, + avg_output_tokens=_avg(out_tok), + median_output_tokens=stats_mod.median(out_tok) if out_tok else 0.0, + min_output_tokens=min(out_tok) if out_tok else 0.0, + max_output_tokens=max(out_tok) if out_tok else 0.0, + std_output_tokens=stats_mod.stdev(out_tok) if len(out_tok) > 1 else 0.0, + ) + return TraceSummary( total_traces=len(traces), total_steps=total_steps, @@ -88,6 +163,9 @@ class TraceAnalyzer: avg_tokens=_avg([float(t.total_tokens) for t in traces]), success_rate=len(successes) / len(evaluated) if evaluated else 0.0, step_type_distribution=step_dist, + total_energy_joules=total_energy, + total_generate_energy_joules=generate_energy, + step_type_stats=sts_map, ) def per_route_stats( @@ -243,4 +321,4 @@ def _trace_to_dict(trace: Trace) -> Dict[str, Any]: } -__all__ = ["RouteStats", "ToolStats", "TraceAnalyzer", "TraceSummary"] +__all__ = ["RouteStats", "StepTypeStats", "ToolStats", "TraceAnalyzer", "TraceSummary"] diff --git a/tests/bench/test_bench_stats.py b/tests/bench/test_bench_stats.py new file mode 100644 index 00000000..ecd47df4 --- /dev/null +++ b/tests/bench/test_bench_stats.py @@ -0,0 +1,69 @@ +"""Tests for benchmark stats output and Rich table rendering.""" + +from __future__ import annotations + +from io import StringIO +from unittest.mock import MagicMock + +from rich.console import Console + +from openjarvis.bench._stubs import BenchmarkResult +from openjarvis.cli.bench_cmd import _render_stats_table + + +class TestLatencyBenchmarkStats: + def test_latency_includes_std(self): + """LatencyBenchmark should include std_latency in metrics.""" + from openjarvis.bench.latency import LatencyBenchmark + + bench = LatencyBenchmark() + engine = MagicMock() + engine.engine_id = "mock" + engine.generate.return_value = {"content": "hi", "usage": {}} + result = bench.run(engine, model="test", num_samples=5) + assert "std_latency" in result.metrics + + +class TestRenderStatsTable: + def test_renders_stats_columns(self): + """_render_stats_table should produce Avg/Median/Min/Max/Std columns.""" + result = BenchmarkResult( + benchmark_name="latency", + model="test", + engine="mock", + metrics={ + "mean_latency": 1.5, + "p50_latency": 1.4, + "p95_latency": 2.1, + "min_latency": 0.8, + "max_latency": 3.0, + "std_latency": 0.5, + }, + samples=10, + errors=0, + ) + console = Console(file=StringIO(), force_terminal=True) + _render_stats_table(console, result) + output = console.file.getvalue() + assert "Avg" in output + assert "Median" in output + assert "Min" in output + assert "Max" in output + + def test_falls_back_for_non_stats_metrics(self): + """Non-stats metrics should still render as simple key-value pairs.""" + result = BenchmarkResult( + benchmark_name="throughput", + model="test", + engine="mock", + metrics={ + "tokens_per_second": 42.5, + "total_time_seconds": 10.0, + }, + samples=5, + errors=0, + ) + console = Console(file=StringIO(), force_terminal=True) + _render_stats_table(console, result) + output = console.file.getvalue() + assert "tokens_per_second" in output or "42.5" in output diff --git a/tests/bench/test_latency.py b/tests/bench/test_latency.py index 378e1cf4..109da0f6 100644 --- a/tests/bench/test_latency.py +++ b/tests/bench/test_latency.py @@ -57,7 +57,7 @@ class TestLatencyBenchmark: result = b.run(engine, "test-model", num_samples=3) expected_keys = { "mean_latency", "p50_latency", "p95_latency", - "min_latency", "max_latency", + "min_latency", "max_latency", "std_latency", } assert set(result.metrics.keys()) == expected_keys diff --git a/tests/cli/test_hints.py b/tests/cli/test_hints.py new file mode 100644 index 00000000..0ba4797a --- /dev/null +++ b/tests/cli/test_hints.py @@ -0,0 +1,37 @@ +"""Tests for CLI error hint functions.""" + +from __future__ import annotations + +from openjarvis.cli.hints import ( + hint_no_config, + hint_no_engine, + hint_no_model, +) + + +class TestHintFunctions: + def test_hint_no_config_returns_string(self): + msg = hint_no_config() + assert isinstance(msg, str) + assert len(msg) > 0 + assert "init" in msg.lower() or "config" in msg.lower() + + def test_hint_no_engine_returns_string(self): + msg = hint_no_engine() + assert isinstance(msg, str) + assert len(msg) > 0 + assert "engine" in msg.lower() or "ollama" in msg.lower() + + def test_hint_no_engine_with_name(self): + msg = hint_no_engine("vllm") + assert "vllm" in msg.lower() + + def test_hint_no_model_returns_string(self): + msg = hint_no_model() + assert isinstance(msg, str) + assert len(msg) > 0 + assert "model" in msg.lower() or "pull" in msg.lower() + + def test_hint_no_model_with_name(self): + msg = hint_no_model("qwen3:8b") + assert "qwen3:8b" in msg diff --git a/tests/cli/test_log_config.py b/tests/cli/test_log_config.py new file mode 100644 index 00000000..7bacfef0 --- /dev/null +++ b/tests/cli/test_log_config.py @@ -0,0 +1,40 @@ +"""Tests for CLI log configuration.""" + +from __future__ import annotations + +import logging + +from openjarvis.cli.log_config import setup_logging + + +class TestSetupLogging: + def test_default_level_is_warning(self): + logger = setup_logging(verbose=False, quiet=False) + assert logger.level == logging.WARNING + + def test_verbose_sets_debug(self): + logger = setup_logging(verbose=True, quiet=False) + assert logger.level == logging.DEBUG + + def test_quiet_sets_error(self): + logger = setup_logging(verbose=False, quiet=True) + assert logger.level == logging.ERROR + + def test_returns_logger(self): + logger = setup_logging(verbose=False, quiet=False) + assert isinstance(logger, logging.Logger) + assert logger.name == "openjarvis" + + def test_log_file_handler_on_verbose(self, tmp_path): + log_file = tmp_path / "cli.log" + logger = setup_logging(verbose=True, quiet=False, log_file=log_file) + # Should have at least one file handler + file_handlers = [ + h for h in logger.handlers + if hasattr(h, "baseFilename") + ] + assert len(file_handlers) >= 1 + # Clean up + for h in logger.handlers[:]: + logger.removeHandler(h) + h.close() diff --git a/tests/cli/test_quickstart.py b/tests/cli/test_quickstart.py new file mode 100644 index 00000000..3b1e5541 --- /dev/null +++ b/tests/cli/test_quickstart.py @@ -0,0 +1,119 @@ +"""Tests for ``jarvis quickstart`` command.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from click.testing import CliRunner + +from openjarvis.cli import cli + + +class TestQuickstartCommand: + def test_registered(self): + """quickstart should be a registered CLI command.""" + runner = CliRunner() + result = runner.invoke(cli, ["quickstart", "--help"]) + assert result.exit_code == 0 + assert "quickstart" in result.output.lower() or "--help" in result.output + + def test_happy_path(self, tmp_path): + """Full quickstart succeeds when hardware detected and engine healthy.""" + config_path = tmp_path / "config.toml" + hw = MagicMock() + hw.platform = "linux" + hw.cpu_brand = "Test CPU" + hw.cpu_count = 8 + hw.ram_gb = 32 + hw.gpu = MagicMock(name="Test GPU", vram_gb=24, count=1, vendor="nvidia") + + with ( + patch("openjarvis.cli.quickstart_cmd.detect_hardware", return_value=hw), + patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path), + patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path), + patch("openjarvis.cli.quickstart_cmd.generate_default_toml", return_value="[engine]\n"), + patch("openjarvis.cli.quickstart_cmd.recommend_engine", return_value="ollama"), + patch("openjarvis.cli.quickstart_cmd._check_engine_health", return_value=True), + patch("openjarvis.cli.quickstart_cmd._check_model_available", return_value=True), + patch("openjarvis.cli.quickstart_cmd._test_query", return_value="Hello!"), + ): + runner = CliRunner() + result = runner.invoke(cli, ["quickstart"]) + assert result.exit_code == 0 + assert "1/5" in result.output + assert "5/5" in result.output + + def test_skips_config_if_exists(self, tmp_path): + """Config step is skipped when config already exists.""" + config_path = tmp_path / "config.toml" + config_path.write_text("[engine]\n") + hw = MagicMock() + hw.platform = "linux" + hw.cpu_brand = "Test CPU" + hw.cpu_count = 8 + hw.ram_gb = 32 + hw.gpu = None + + with ( + patch("openjarvis.cli.quickstart_cmd.detect_hardware", return_value=hw), + patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path), + patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path), + patch("openjarvis.cli.quickstart_cmd.generate_default_toml", return_value="[engine]\n"), + patch("openjarvis.cli.quickstart_cmd.recommend_engine", return_value="ollama"), + patch("openjarvis.cli.quickstart_cmd._check_engine_health", return_value=True), + patch("openjarvis.cli.quickstart_cmd._check_model_available", return_value=True), + patch("openjarvis.cli.quickstart_cmd._test_query", return_value="Hello!"), + ): + runner = CliRunner() + result = runner.invoke(cli, ["quickstart"]) + assert result.exit_code == 0 + assert "already exists" in result.output.lower() or "skip" in result.output.lower() + + def test_force_regenerates_config(self, tmp_path): + """--force should regenerate config even if it exists.""" + config_path = tmp_path / "config.toml" + config_path.write_text("[old]\n") + hw = MagicMock() + hw.platform = "linux" + hw.cpu_brand = "Test CPU" + hw.cpu_count = 8 + hw.ram_gb = 32 + hw.gpu = None + + with ( + patch("openjarvis.cli.quickstart_cmd.detect_hardware", return_value=hw), + patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path), + patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path), + patch("openjarvis.cli.quickstart_cmd.generate_default_toml", return_value="[engine]\nnew = true\n"), + patch("openjarvis.cli.quickstart_cmd.recommend_engine", return_value="ollama"), + patch("openjarvis.cli.quickstart_cmd._check_engine_health", return_value=True), + patch("openjarvis.cli.quickstart_cmd._check_model_available", return_value=True), + patch("openjarvis.cli.quickstart_cmd._test_query", return_value="Hello!"), + ): + runner = CliRunner() + result = runner.invoke(cli, ["quickstart", "--force"]) + assert result.exit_code == 0 + assert "new = true" in config_path.read_text() + + def test_engine_not_found(self, tmp_path): + """Helpful message when engine is unreachable.""" + config_path = tmp_path / "config.toml" + hw = MagicMock() + hw.platform = "linux" + hw.cpu_brand = "Test CPU" + hw.cpu_count = 8 + hw.ram_gb = 32 + hw.gpu = None + + with ( + patch("openjarvis.cli.quickstart_cmd.detect_hardware", return_value=hw), + patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path), + patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path), + patch("openjarvis.cli.quickstart_cmd.generate_default_toml", return_value="[engine]\n"), + patch("openjarvis.cli.quickstart_cmd.recommend_engine", return_value="ollama"), + patch("openjarvis.cli.quickstart_cmd._check_engine_health", return_value=False), + ): + runner = CliRunner() + result = runner.invoke(cli, ["quickstart"]) + assert result.exit_code == 1 + assert "engine" in result.output.lower() or "not reachable" in result.output.lower() diff --git a/tests/telemetry/test_instrumented_engine.py b/tests/telemetry/test_instrumented_engine.py index b708ff90..2795ad9e 100644 --- a/tests/telemetry/test_instrumented_engine.py +++ b/tests/telemetry/test_instrumented_engine.py @@ -120,3 +120,45 @@ class TestInstrumentedEngine: def test_engine_id_attribute(self, mock_engine, bus): ie = InstrumentedEngine(mock_engine, bus) assert ie.engine_id == "instrumented" + + +class TestTokensPerJoule: + def test_tokens_per_joule_zero_without_energy(self, mock_engine, bus): + """tokens_per_joule is 0.0 when no energy monitor is available.""" + ie = InstrumentedEngine(mock_engine, bus) + messages = [Message(role=Role.USER, content="Hi")] + ie.generate(messages, model="test") + + tel_events = [ + e for e in bus.history + if e.event_type == EventType.TELEMETRY_RECORD + ] + record = tel_events[0].data["record"] + assert record.tokens_per_joule == 0.0 + + def test_tokens_per_joule_formula_via_record(self): + """Verify the formula: tokens_per_joule = completion_tokens / energy_joules.""" + from openjarvis.core.types import TelemetryRecord + + # Direct construction — verifies the field accepts computed values + rec = TelemetryRecord( + timestamp=1.0, + model_id="test", + completion_tokens=50, + energy_joules=2.5, + tokens_per_joule=50.0 / 2.5, # = 20.0 + ) + assert rec.tokens_per_joule == pytest.approx(20.0) + + def test_tokens_per_joule_zero_when_no_tokens(self): + """tokens_per_joule is 0.0 when completion_tokens is 0.""" + from openjarvis.core.types import TelemetryRecord + + rec = TelemetryRecord( + timestamp=1.0, + model_id="test", + completion_tokens=0, + energy_joules=5.0, + tokens_per_joule=0.0, + ) + assert rec.tokens_per_joule == 0.0 diff --git a/tests/telemetry/test_store_tokens_per_joule.py b/tests/telemetry/test_store_tokens_per_joule.py new file mode 100644 index 00000000..3f3d3971 --- /dev/null +++ b/tests/telemetry/test_store_tokens_per_joule.py @@ -0,0 +1,65 @@ +"""Tests for tokens_per_joule storage and aggregation.""" + +from __future__ import annotations + +import time + +import pytest + +from openjarvis.core.types import TelemetryRecord +from openjarvis.telemetry.store import TelemetryStore +from openjarvis.telemetry.aggregator import TelemetryAggregator + + +class TestTokensPerJouleStorage: + def test_store_and_retrieve(self, tmp_path): + db = tmp_path / "tel.db" + store = TelemetryStore(db_path=db) + rec = TelemetryRecord( + timestamp=time.time(), + model_id="test-model", + completion_tokens=50, + energy_joules=2.5, + tokens_per_joule=20.0, + ) + store.record(rec) + store.close() + agg = TelemetryAggregator(db) + stats = agg.per_model_stats() + assert len(stats) == 1 + assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1) + agg.close() + + def test_aggregate_multiple(self, tmp_path): + db = tmp_path / "tel.db" + store = TelemetryStore(db_path=db) + for tpj in [10.0, 20.0, 30.0]: + rec = TelemetryRecord( + timestamp=time.time(), + model_id="m1", + tokens_per_joule=tpj, + ) + store.record(rec) + store.close() + agg = TelemetryAggregator(db) + stats = agg.per_model_stats() + assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1) + agg.close() + + def test_engine_stats_aggregate(self, tmp_path): + db = tmp_path / "tel.db" + store = TelemetryStore(db_path=db) + for tpj in [15.0, 25.0]: + rec = TelemetryRecord( + timestamp=time.time(), + model_id="m1", + engine="ollama", + tokens_per_joule=tpj, + ) + store.record(rec) + store.close() + agg = TelemetryAggregator(db) + stats = agg.per_engine_stats() + assert len(stats) == 1 + assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1) + agg.close() diff --git a/tests/telemetry/test_telemetry_record.py b/tests/telemetry/test_telemetry_record.py new file mode 100644 index 00000000..51ebc085 --- /dev/null +++ b/tests/telemetry/test_telemetry_record.py @@ -0,0 +1,20 @@ +"""Tests for TelemetryRecord fields.""" + +from __future__ import annotations + +from openjarvis.core.types import TelemetryRecord + + +class TestTelemetryRecord: + def test_tokens_per_joule_field_exists(self): + rec = TelemetryRecord(timestamp=1.0, model_id="test") + assert hasattr(rec, "tokens_per_joule") + assert rec.tokens_per_joule == 0.0 + + def test_tokens_per_joule_set(self): + rec = TelemetryRecord( + timestamp=1.0, + model_id="test", + tokens_per_joule=80.0, + ) + assert rec.tokens_per_joule == 80.0 diff --git a/tests/traces/test_analyzer_energy.py b/tests/traces/test_analyzer_energy.py new file mode 100644 index 00000000..f11585eb --- /dev/null +++ b/tests/traces/test_analyzer_energy.py @@ -0,0 +1,104 @@ +"""Tests for energy aggregation in TraceAnalyzer.""" + +from __future__ import annotations + +import time + +import pytest + +from openjarvis.core.types import StepType, Trace, TraceStep +from openjarvis.traces.analyzer import StepTypeStats, TraceAnalyzer, TraceSummary +from openjarvis.traces.store import TraceStore + + +def _make_trace(steps: list[TraceStep]) -> Trace: + return Trace( + query="test", + agent="test_agent", + model="test_model", + engine="test_engine", + steps=steps, + started_at=time.time(), + ended_at=time.time() + 10, + total_tokens=100, + total_latency_seconds=10.0, + ) + + +def _gen_step(energy: float = 0.0, duration: float = 1.0, + prompt_tokens: int = 50, completion_tokens: int = 25) -> TraceStep: + return TraceStep( + step_type=StepType.GENERATE, + timestamp=time.time(), + duration_seconds=duration, + output={ + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + metadata={ + "energy_joules": energy, + "power_watts": energy / duration if duration > 0 else 0.0, + }, + ) + + +def _tool_step(duration: float = 0.5) -> TraceStep: + return TraceStep( + step_type=StepType.TOOL_CALL, + timestamp=time.time(), + duration_seconds=duration, + input={"tool": "calculator"}, + output={"success": True}, + ) + + +class TestTraceSummaryEnergyFields: + def test_total_energy_joules(self, tmp_path): + store = TraceStore(db_path=tmp_path / "traces.db") + trace = _make_trace([ + _gen_step(energy=10.0, duration=2.0), + _tool_step(duration=0.5), + _gen_step(energy=15.0, duration=3.0), + ]) + store.save(trace) + analyzer = TraceAnalyzer(store) + summary = analyzer.summary() + assert summary.total_energy_joules == pytest.approx(25.0, rel=0.01) + assert summary.total_generate_energy_joules == pytest.approx(25.0, rel=0.01) + store.close() + + def test_step_type_stats(self, tmp_path): + store = TraceStore(db_path=tmp_path / "traces.db") + trace = _make_trace([ + _gen_step(energy=10.0, duration=2.0, prompt_tokens=100, completion_tokens=50), + _gen_step(energy=20.0, duration=4.0, prompt_tokens=80, completion_tokens=40), + _tool_step(duration=0.5), + _tool_step(duration=1.5), + ]) + store.save(trace) + analyzer = TraceAnalyzer(store) + summary = analyzer.summary() + + assert "generate" in summary.step_type_stats + gen = summary.step_type_stats["generate"] + assert gen.count == 2 + assert gen.avg_duration == pytest.approx(3.0, rel=0.01) + assert gen.total_energy == pytest.approx(30.0, rel=0.01) + assert gen.avg_input_tokens == pytest.approx(90.0, rel=0.01) + assert gen.avg_output_tokens == pytest.approx(45.0, rel=0.01) + assert gen.min_duration == pytest.approx(2.0, rel=0.01) + assert gen.max_duration == pytest.approx(4.0, rel=0.01) + + assert "tool_call" in summary.step_type_stats + tc = summary.step_type_stats["tool_call"] + assert tc.count == 2 + assert tc.avg_duration == pytest.approx(1.0, rel=0.01) + store.close() + + +class TestStepTypeStats: + def test_dataclass_fields(self): + s = StepTypeStats(count=5, avg_duration=2.0, total_energy=10.0) + assert s.count == 5 + assert s.std_duration == 0.0 # default