mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Merge pull request #1 from HazyResearch/feat/experience-polish
Experience polish: eval display, first-run UX, dashboard improvements
This commit is contained in:
Jon Saad-Falcon
@@ -0,0 +1,37 @@
|
||||
name: Frontend CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'frontend/**'
|
||||
- '.github/workflows/frontend.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'frontend/**'
|
||||
- '.github/workflows/frontend.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: frontend-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- run: npm ci
|
||||
- run: npx tsc --noEmit
|
||||
- run: npm run build
|
||||
@@ -56,3 +56,6 @@ src/openjarvis/server/static/
|
||||
desktop/node_modules/
|
||||
desktop/dist/
|
||||
desktop/src-tauri/target/
|
||||
|
||||
# Worktrees
|
||||
.worktrees/
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 360 B After Width: | Height: | Size: 394 B |
Binary file not shown.
|
Before Width: | Height: | Size: 856 B After Width: | Height: | Size: 858 B |
Binary file not shown.
|
Before Width: | Height: | Size: 856 B After Width: | Height: | Size: 858 B |
Binary file not shown.
|
Before Width: | Height: | Size: 104 B After Width: | Height: | Size: 108 B |
Binary file not shown.
|
Before Width: | Height: | Size: 108 B After Width: | Height: | Size: 1.2 KiB |
@@ -1,5 +1,6 @@
|
||||
use serde::Serialize;
|
||||
use tauri::Manager;
|
||||
use tauri::menu::{MenuBuilder, MenuItemBuilder};
|
||||
use tauri::tray::TrayIconBuilder;
|
||||
use tauri_plugin_autostart::MacosLauncher;
|
||||
|
||||
/// Fetch health status from the OpenJarvis API server.
|
||||
@@ -185,8 +186,45 @@ pub fn run() {
|
||||
}
|
||||
}))
|
||||
.setup(|app| {
|
||||
// Set up system tray menu
|
||||
let _tray = app.tray_by_id("main");
|
||||
let show = MenuItemBuilder::with_id("show", "Show / Hide")
|
||||
.build(app)?;
|
||||
let health = MenuItemBuilder::with_id("health", "Health: checking...")
|
||||
.enabled(false)
|
||||
.build(app)?;
|
||||
let quit = MenuItemBuilder::with_id("quit", "Quit OpenJarvis")
|
||||
.build(app)?;
|
||||
|
||||
let menu = MenuBuilder::new(app)
|
||||
.item(&show)
|
||||
.separator()
|
||||
.item(&health)
|
||||
.separator()
|
||||
.item(&quit)
|
||||
.build()?;
|
||||
|
||||
let _tray = TrayIconBuilder::with_id("main")
|
||||
.icon(app.default_window_icon().unwrap().clone())
|
||||
.tooltip("OpenJarvis")
|
||||
.menu(&menu)
|
||||
.on_menu_event(move |app, event| {
|
||||
match event.id().as_ref() {
|
||||
"show" => {
|
||||
if let Some(window) = app.get_webview_window("main") {
|
||||
if window.is_visible().unwrap_or(false) {
|
||||
let _ = window.hide();
|
||||
} else {
|
||||
let _ = window.show();
|
||||
let _ = window.set_focus();
|
||||
}
|
||||
}
|
||||
}
|
||||
"quit" => {
|
||||
app.exit(0);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
})
|
||||
.build(app)?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
|
||||
@@ -23,11 +23,6 @@
|
||||
"transparent": false
|
||||
}
|
||||
],
|
||||
"trayIcon": {
|
||||
"iconPath": "icons/icon.png",
|
||||
"iconAsTemplate": true,
|
||||
"tooltip": "OpenJarvis"
|
||||
},
|
||||
"security": {
|
||||
"csp": "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; connect-src 'self' http://localhost:* ws://localhost:*; img-src 'self' data: blob:"
|
||||
}
|
||||
|
||||
+4
-1
@@ -5,8 +5,9 @@ import { TraceDebugger } from './components/TraceDebugger';
|
||||
import { LearningCurve } from './components/LearningCurve';
|
||||
import { MemoryBrowser } from './components/MemoryBrowser';
|
||||
import { AdminPanel } from './components/AdminPanel';
|
||||
import { SettingsPanel } from './components/SettingsPanel';
|
||||
|
||||
type TabId = 'energy' | 'traces' | 'learning' | 'memory' | 'admin';
|
||||
type TabId = 'energy' | 'traces' | 'learning' | 'memory' | 'admin' | 'settings';
|
||||
|
||||
interface Tab {
|
||||
id: TabId;
|
||||
@@ -19,6 +20,7 @@ const TABS: Tab[] = [
|
||||
{ id: 'learning', label: 'Learning' },
|
||||
{ id: 'memory', label: 'Memory' },
|
||||
{ id: 'admin', label: 'Admin' },
|
||||
{ id: 'settings', label: 'Settings' },
|
||||
];
|
||||
|
||||
const API_URL = 'http://localhost:8000';
|
||||
@@ -54,6 +56,7 @@ export function App() {
|
||||
{activeTab === 'learning' && <LearningCurve apiUrl={API_URL} />}
|
||||
{activeTab === 'memory' && <MemoryBrowser apiUrl={API_URL} />}
|
||||
{activeTab === 'admin' && <AdminPanel apiUrl={API_URL} />}
|
||||
{activeTab === 'settings' && <SettingsPanel />}
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import type React from 'react';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface Settings {
|
||||
apiUrl: string;
|
||||
refreshInterval: number; // seconds
|
||||
theme: 'dark' | 'light';
|
||||
}
|
||||
|
||||
const DEFAULT_SETTINGS: Settings = {
|
||||
apiUrl: 'http://localhost:8000',
|
||||
refreshInterval: 5,
|
||||
theme: 'dark',
|
||||
};
|
||||
|
||||
const STORAGE_KEY = 'openjarvis-settings';
|
||||
|
||||
function loadSettings(): Settings {
|
||||
try {
|
||||
const raw = localStorage.getItem(STORAGE_KEY);
|
||||
if (raw) {
|
||||
return { ...DEFAULT_SETTINGS, ...JSON.parse(raw) };
|
||||
}
|
||||
} catch {
|
||||
// ignore corrupt data
|
||||
}
|
||||
return { ...DEFAULT_SETTINGS };
|
||||
}
|
||||
|
||||
function saveSettings(settings: Settings): void {
|
||||
localStorage.setItem(STORAGE_KEY, JSON.stringify(settings));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Styles
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const styles: Record<string, React.CSSProperties> = {
|
||||
container: {
|
||||
backgroundColor: '#1e1e2e',
|
||||
color: '#cdd6f4',
|
||||
padding: 24,
|
||||
maxWidth: 600,
|
||||
},
|
||||
heading: {
|
||||
fontSize: 20,
|
||||
fontWeight: 600,
|
||||
marginBottom: 24,
|
||||
color: '#89b4fa',
|
||||
},
|
||||
fieldGroup: {
|
||||
marginBottom: 20,
|
||||
},
|
||||
label: {
|
||||
display: 'block',
|
||||
fontSize: 13,
|
||||
fontWeight: 500,
|
||||
color: '#a6adc8',
|
||||
marginBottom: 6,
|
||||
},
|
||||
input: {
|
||||
width: '100%',
|
||||
padding: '8px 12px',
|
||||
borderRadius: 6,
|
||||
border: '1px solid #313244',
|
||||
backgroundColor: '#181825',
|
||||
color: '#cdd6f4',
|
||||
fontSize: 14,
|
||||
outline: 'none',
|
||||
boxSizing: 'border-box' as const,
|
||||
},
|
||||
select: {
|
||||
padding: '8px 12px',
|
||||
borderRadius: 6,
|
||||
border: '1px solid #313244',
|
||||
backgroundColor: '#181825',
|
||||
color: '#cdd6f4',
|
||||
fontSize: 14,
|
||||
outline: 'none',
|
||||
cursor: 'pointer',
|
||||
},
|
||||
toggleRow: {
|
||||
display: 'flex',
|
||||
gap: 8,
|
||||
},
|
||||
toggleButton: {
|
||||
padding: '8px 16px',
|
||||
borderRadius: 6,
|
||||
border: '1px solid #313244',
|
||||
backgroundColor: 'transparent',
|
||||
color: '#a6adc8',
|
||||
cursor: 'pointer',
|
||||
fontSize: 14,
|
||||
fontWeight: 500,
|
||||
transition: 'all 0.15s ease',
|
||||
},
|
||||
toggleActive: {
|
||||
backgroundColor: '#313244',
|
||||
color: '#cdd6f4',
|
||||
borderColor: '#89b4fa',
|
||||
},
|
||||
savedNotice: {
|
||||
marginTop: 16,
|
||||
padding: '8px 12px',
|
||||
borderRadius: 6,
|
||||
backgroundColor: '#1e3a2f',
|
||||
color: '#a6e3a1',
|
||||
fontSize: 13,
|
||||
},
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Component
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface SettingsPanelProps {
|
||||
onSettingsChange?: (settings: Settings) => void;
|
||||
}
|
||||
|
||||
export function SettingsPanel({ onSettingsChange }: SettingsPanelProps) {
|
||||
const [settings, setSettings] = useState<Settings>(loadSettings);
|
||||
const [saved, setSaved] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
saveSettings(settings);
|
||||
onSettingsChange?.(settings);
|
||||
setSaved(true);
|
||||
const timer = setTimeout(() => setSaved(false), 2000);
|
||||
return () => clearTimeout(timer);
|
||||
}, [settings, onSettingsChange]);
|
||||
|
||||
return (
|
||||
<div style={styles.container}>
|
||||
<h2 style={styles.heading}>Settings</h2>
|
||||
|
||||
<div style={styles.fieldGroup}>
|
||||
<label style={styles.label}>API URL</label>
|
||||
<input
|
||||
style={styles.input}
|
||||
type="text"
|
||||
value={settings.apiUrl}
|
||||
onChange={(e) =>
|
||||
setSettings((s) => ({ ...s, apiUrl: e.target.value }))
|
||||
}
|
||||
placeholder="http://localhost:8000"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div style={styles.fieldGroup}>
|
||||
<label style={styles.label}>Auto-refresh interval</label>
|
||||
<select
|
||||
style={styles.select}
|
||||
value={settings.refreshInterval}
|
||||
onChange={(e) =>
|
||||
setSettings((s) => ({
|
||||
...s,
|
||||
refreshInterval: Number(e.target.value),
|
||||
}))
|
||||
}
|
||||
>
|
||||
<option value={1}>1 second</option>
|
||||
<option value={2}>2 seconds</option>
|
||||
<option value={5}>5 seconds</option>
|
||||
<option value={10}>10 seconds</option>
|
||||
<option value={30}>30 seconds</option>
|
||||
<option value={60}>60 seconds</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div style={styles.fieldGroup}>
|
||||
<label style={styles.label}>Theme</label>
|
||||
<div style={styles.toggleRow}>
|
||||
<button
|
||||
type="button"
|
||||
style={{
|
||||
...styles.toggleButton,
|
||||
...(settings.theme === 'dark' ? styles.toggleActive : {}),
|
||||
}}
|
||||
onClick={() => setSettings((s) => ({ ...s, theme: 'dark' }))}
|
||||
>
|
||||
Dark
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
style={{
|
||||
...styles.toggleButton,
|
||||
...(settings.theme === 'light' ? styles.toggleActive : {}),
|
||||
}}
|
||||
onClick={() => setSettings((s) => ({ ...s, theme: 'light' }))}
|
||||
>
|
||||
Light
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{saved && <div style={styles.savedNotice}>Settings saved</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export type { Settings };
|
||||
@@ -0,0 +1,319 @@
|
||||
# Experience Polish Design: Eval, CLI, Install, Dashboard
|
||||
|
||||
**Date**: 2026-02-28
|
||||
**Status**: Approved
|
||||
**Approach**: Vertical slices (3 mini-phases, independently shippable)
|
||||
|
||||
## Context
|
||||
|
||||
OpenJarvis has strong infrastructure (multi-vendor energy monitoring, eval framework, Tauri desktop, PWA) but the end-to-end user experience has gaps. This design addresses: eval output quality, CLI polish, installation flow, and dashboard aesthetics.
|
||||
|
||||
### Current State (from audits)
|
||||
|
||||
| Area | Score | Key Gaps |
|
||||
|------|-------|----------|
|
||||
| Energy Monitors | 8.5/10 | IPJ/IPW only in evals, not core telemetry |
|
||||
| Eval Framework | 9/10 | Token stats null, no grouped tables, no trace aggregation |
|
||||
| CLI Formatting | 9/10 | No logging, no verbose/quiet, bench lacks full stats |
|
||||
| Installation | 8.5/10 | No quickstart, no auto-suggestions in errors |
|
||||
| Desktop App | 8/10 | No settings panel, placeholder icon, stub tray |
|
||||
| Browser/PWA | 6/10 | No CI/CD, no versioning, no error boundary |
|
||||
|
||||
### Key Decisions
|
||||
|
||||
- **Benchmark focus**: GAIA with OpenHands (TerminalBench deferred)
|
||||
- **IPW** = task_accuracy / avg_power_watts (task-level, NOT per-inference)
|
||||
- **IPJ** = task_accuracy / avg_energy_joules (task-level, NOT per-inference)
|
||||
- **tokens_per_joule** = per-inference efficiency metric in core telemetry
|
||||
- **Table layout**: Grouped panels by default, `--compact` for single dense table
|
||||
- **Trace detail**: Summary + step-type breakdown by default, `--trace-detail` for full listing
|
||||
- **Installation**: Non-interactive `jarvis quickstart` (zero prompts)
|
||||
- **Dashboard style**: Clean, minimalistic (ChatGPT/Claude aesthetic) with unique side panels
|
||||
|
||||
---
|
||||
|
||||
## Phase 23a: Perfect Eval Run
|
||||
|
||||
Everything needed so `python -m evals run -c config.toml` produces beautiful, complete results.
|
||||
|
||||
### 1. Core Telemetry: tokens_per_joule
|
||||
|
||||
**Files**:
|
||||
- `src/openjarvis/core/types.py` — Add `tokens_per_joule` to `TelemetryRecord`
|
||||
- `src/openjarvis/telemetry/instrumented_engine.py` — Compute `tokens_per_joule = completion_tokens / energy_joules`
|
||||
- `src/openjarvis/telemetry/store.py` — Schema migration for new column
|
||||
- `src/openjarvis/telemetry/aggregator.py` — Add `avg_tokens_per_joule` to `ModelStats`/`EngineStats`
|
||||
|
||||
**Not** adding IPJ/IPW to core telemetry — they require task accuracy and are inherently eval-level.
|
||||
|
||||
### 2. Eval Metrics: Fix Token Stats + Strengthen IPJ/IPW
|
||||
|
||||
**Files**:
|
||||
- `evals/core/runner.py`:
|
||||
- Wire `prompt_tokens` and `completion_tokens` from engine response into `EvalResult`
|
||||
- Verify IPW = `accuracy / avg(power_watts)` across all scored samples
|
||||
- Verify IPJ = `accuracy / avg(energy_joules)` across all scored samples
|
||||
- Ensure `energy_joules` and `power_watts` populated from telemetry for every sample
|
||||
- `evals/core/types.py`:
|
||||
- Add `total_energy_joules` (sum over all samples) to `RunSummary`
|
||||
- Add `avg_power_watts` (mean over all samples) to `RunSummary`
|
||||
- Add `trace_steps: int`, `trace_energy_joules: float` to `EvalResult`
|
||||
- Add `trace_step_type_stats: Dict[str, StepTypeStats]` to `RunSummary`
|
||||
|
||||
### 3. Eval Display: Grouped Tables
|
||||
|
||||
Rewrite `evals/core/display.py` with these functions:
|
||||
|
||||
#### `print_accuracy_panel(summary)`
|
||||
```
|
||||
╭─ Accuracy ──────────────────────────────────────────────────╮
|
||||
│ Overall Accuracy 42.0% (84/200) │
|
||||
│ Level 1 58.8% (40/68) │
|
||||
│ Level 2 35.0% (35/100) │
|
||||
│ Level 3 28.1% (9/32) │
|
||||
╰─────────────────────────────────────────────────────────────╯
|
||||
```
|
||||
|
||||
#### `print_latency_table(summary)`
|
||||
```
|
||||
╭─ Latency & Throughput ──────────────────────────────────────╮
|
||||
│ Metric Avg Median Min Max Std │
|
||||
│ Latency (s) 15.41 12.30 2.10 89.50 11.20 │
|
||||
│ TTFT (ms) 145.2 120.0 45.0 890.0 85.3 │
|
||||
│ Throughput (tok/s) 41.9 38.5 12.0 95.0 15.2 │
|
||||
│ Avg Input Tokens 1024 890 128 4096 520 │
|
||||
│ Avg Output Tokens 256 210 32 2048 180 │
|
||||
╰─────────────────────────────────────────────────────────────╯
|
||||
```
|
||||
|
||||
#### `print_energy_table(summary)`
|
||||
```
|
||||
╭─ Energy & Efficiency ───────────────────────────────────────╮
|
||||
│ Metric Avg Median Min Max Std │
|
||||
│ Energy (J) 46502 38500 4145 410522 89390 │
|
||||
│ Power (W) 883.8 870.2 650.0 1050.0 85.0 │
|
||||
│ GPU Util (%) 46.4 48.0 12.0 92.0 18.5 │
|
||||
│ Energy/Token (mJ) 12.5 10.8 3.2 45.0 8.1 │
|
||||
│ Tokens/Joule 80.0 92.6 22.2 312.5 65.0 │
|
||||
│ ────────────────────────────────────────────────────────── │
|
||||
│ IPW (acc/W) 0.00048 │
|
||||
│ IPJ (acc/J) 9.03e-06 │
|
||||
│ Total Energy (kJ) 9300.4 │
|
||||
╰─────────────────────────────────────────────────────────────╯
|
||||
```
|
||||
|
||||
#### `print_trace_summary(summary)`
|
||||
```
|
||||
╭─ Agentic Trace Summary ────────────────────────────────────╮
|
||||
│ Total Steps: 1240 │ Avg Steps/Sample: 6.2 │
|
||||
│ │
|
||||
│ Step Type Count Avg Duration Avg Energy Avg In Tok Avg Out Tok │
|
||||
│ Median/Min/Max/Std for each column │
|
||||
│ generate 580 8.2s 38200 J 890 256 │
|
||||
│ tool_call 420 3.1s — — — │
|
||||
│ retrieve 120 0.8s — — — │
|
||||
│ route 120 0.01s — — — │
|
||||
╰─────────────────────────────────────────────────────────────╯
|
||||
```
|
||||
|
||||
All metrics in trace summary show avg/median/min/max/std where applicable.
|
||||
|
||||
#### `print_compact_table(summary)` — `--compact` flag
|
||||
Single dense table with all 17 metrics as rows, columns: avg/median/min/max/std.
|
||||
|
||||
#### CLI flags
|
||||
- `--compact`: Dense single table
|
||||
- `--trace-detail`: Full per-step listing for each sample
|
||||
|
||||
### 4. Per-Step Trace Aggregation
|
||||
|
||||
**Files**:
|
||||
- `src/openjarvis/traces/analyzer.py`:
|
||||
- Add `TraceSummary.total_energy_joules` — sum of `step.metadata['energy_joules']`
|
||||
- Add `TraceSummary.total_generate_energy_joules` — sum for GENERATE steps
|
||||
- Add `TraceSummary.step_type_stats` — dict mapping step type to `{count, avg_duration, median_duration, min_duration, max_duration, std_duration, total_energy, avg_input_tokens, median_input_tokens, min_input_tokens, max_input_tokens, std_input_tokens, avg_output_tokens, median_output_tokens, min_output_tokens, max_output_tokens, std_output_tokens}`
|
||||
- `evals/core/runner.py` — After each agent eval sample, extract `TraceSummary` and store in `EvalResult.metadata`
|
||||
|
||||
---
|
||||
|
||||
## Phase 23b: Perfect First Experience
|
||||
|
||||
Everything needed so a new user goes from zero to first eval in one sitting.
|
||||
|
||||
### 5. `jarvis quickstart` Command
|
||||
|
||||
**New file**: `src/openjarvis/cli/quickstart_cmd.py`
|
||||
|
||||
Non-interactive flow (5 numbered steps):
|
||||
1. Detect hardware (platform, CPU, RAM, GPU vendor/model/VRAM)
|
||||
2. Write config to `~/.openjarvis/config.toml` (skip if exists, unless `--force`)
|
||||
3. Check engine health (try auto-detected engine)
|
||||
4. Verify model availability (list models from engine)
|
||||
5. Test query ("What is 2+2?") with latency + energy measurement
|
||||
|
||||
Each step prints a status line. If a step fails, print a helpful suggestion and exit gracefully.
|
||||
|
||||
Flags: `--force` (redo everything)
|
||||
|
||||
**Register** in `src/openjarvis/cli/__init__.py`.
|
||||
|
||||
### 6. Error Message Auto-Suggestions
|
||||
|
||||
**New file**: `src/openjarvis/cli/hints.py`
|
||||
|
||||
Centralized hint functions:
|
||||
- `hint_no_config()` → "Config not found. Run: `jarvis quickstart`"
|
||||
- `hint_no_engine()` → "No engine responding. Run: `jarvis doctor`"
|
||||
- `hint_no_model()` → "No model available. Try: `ollama pull qwen3:8b`"
|
||||
|
||||
**Wire into**: `ask.py`, `serve.py`, `bench_cmd.py`, `chat_cmd.py` at failure points.
|
||||
|
||||
### 7. Global Logging & Verbose/Quiet Flags
|
||||
|
||||
**Changes**:
|
||||
- `src/openjarvis/cli/__init__.py` — Add `--verbose` / `--quiet` to root `cli` group
|
||||
- **New file**: `src/openjarvis/cli/log_config.py` — Centralized logging setup
|
||||
- `RichHandler` for console (respects quiet flag)
|
||||
- `RotatingFileHandler` for `~/.openjarvis/cli.log` (5 MB max, 3 backups)
|
||||
- Default level: WARNING; `--verbose`: DEBUG; `--quiet`: ERROR
|
||||
|
||||
**Add progress indicators**:
|
||||
- `ask.py` — Spinner during generation
|
||||
- `memory_cmd.py` — Progress bar for indexing
|
||||
|
||||
### 8. Bench CLI Full Stats Tables
|
||||
|
||||
**Changes**:
|
||||
- `src/openjarvis/cli/bench_cmd.py` — Show full stats table (avg/median/min/max/std) matching eval style
|
||||
- `src/openjarvis/bench/latency.py` — Export all percentile stats (already computes internally)
|
||||
- `src/openjarvis/bench/throughput.py` — Add stats computation
|
||||
- `src/openjarvis/bench/energy.py` — Add stats computation
|
||||
|
||||
---
|
||||
|
||||
## Phase 23c: Perfect Dashboard
|
||||
|
||||
Desktop and browser apps polished for demos and daily use.
|
||||
|
||||
### Design Aesthetic
|
||||
|
||||
**Clean, minimalistic** — inspired by ChatGPT and Claude web interfaces:
|
||||
- Generous whitespace, muted color palette, subtle borders
|
||||
- Sans-serif typography, comfortable line heights
|
||||
- Smooth transitions, no visual clutter
|
||||
|
||||
**Differentiated** via unique side panels:
|
||||
- Energy dashboard with real-time power charts
|
||||
- Trace debugger with color-coded timeline
|
||||
- Learning curve visualization
|
||||
- Memory browser with relevance scoring
|
||||
|
||||
The side panels are what make OpenJarvis unique vs generic chat UIs. Keep them accessible but not overwhelming — collapsible, clean data density.
|
||||
|
||||
### 9. Desktop: Settings Panel
|
||||
|
||||
**New file**: `desktop/src/components/SettingsPanel.tsx`
|
||||
- API URL configuration (default: `localhost:8000`)
|
||||
- Auto-update interval setting
|
||||
- Theme toggle (dark/light) — currently hardcoded dark
|
||||
- Persist to `localStorage`
|
||||
|
||||
### 10. Desktop: Windows Icon
|
||||
|
||||
Replace 108-byte placeholder `desktop/src-tauri/icons/icon.ico` with proper multi-resolution icon generated from existing `icon.png`.
|
||||
|
||||
### 11. Desktop: System Tray
|
||||
|
||||
Implement stubbed tray menu in `desktop/src-tauri/src/lib.rs`:
|
||||
- Show/Hide window toggle
|
||||
- Health status indicator (green/red dot)
|
||||
- Quit action
|
||||
|
||||
### 12. Browser/PWA: CI/CD
|
||||
|
||||
**New file**: `.github/workflows/frontend.yml`
|
||||
- Trigger on changes to `frontend/`
|
||||
- `npm ci && npm run build`
|
||||
- Commit built output to `src/openjarvis/server/static/`
|
||||
- Add `version` field to `manifest.webmanifest` (auto-bumped)
|
||||
|
||||
### 13. Browser/PWA: Error Boundary + Config
|
||||
|
||||
- Wrap `<App />` in React error boundary (graceful crash recovery)
|
||||
- Support `VITE_API_URL` environment variable (default `localhost:8000`)
|
||||
|
||||
### 14. Browser/PWA: Style Refresh
|
||||
|
||||
Refactor the 12,971-line `App.css`:
|
||||
- Modularize into component-scoped CSS files
|
||||
- Clean minimalist aesthetic (ChatGPT/Claude-inspired)
|
||||
- Catppuccin-based dark theme (keep existing) + light theme option
|
||||
- Collapsible side panels with smooth transitions
|
||||
|
||||
---
|
||||
|
||||
## Implementation Order
|
||||
|
||||
### Phase 23a (highest priority — enables GAIA research)
|
||||
1. Core telemetry: `tokens_per_joule`
|
||||
2. Eval metrics: fix token stats, strengthen IPJ/IPW
|
||||
3. Per-step trace aggregation in `TraceAnalyzer`
|
||||
4. Eval display: grouped tables, `--compact`, `--trace-detail`
|
||||
|
||||
### Phase 23b (high priority — onboarding & CLI)
|
||||
5. `jarvis quickstart` command
|
||||
6. Error message auto-suggestions
|
||||
7. Global logging + verbose/quiet flags
|
||||
8. Bench CLI full stats tables
|
||||
|
||||
### Phase 23c (lower priority — dashboard polish)
|
||||
9. Desktop settings panel
|
||||
10. Windows icon fix
|
||||
11. System tray implementation
|
||||
12. PWA CI/CD
|
||||
13. PWA error boundary + config
|
||||
14. PWA style refresh
|
||||
|
||||
---
|
||||
|
||||
## Files Modified (Summary)
|
||||
|
||||
### Phase 23a
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `src/openjarvis/core/types.py` | Add `tokens_per_joule` to `TelemetryRecord` |
|
||||
| `src/openjarvis/telemetry/instrumented_engine.py` | Compute `tokens_per_joule` |
|
||||
| `src/openjarvis/telemetry/store.py` | Schema migration |
|
||||
| `src/openjarvis/telemetry/aggregator.py` | Add `avg_tokens_per_joule` |
|
||||
| `src/openjarvis/traces/analyzer.py` | Add energy aggregation, step-type stats |
|
||||
| `evals/core/types.py` | Add trace fields, total_energy, avg_power |
|
||||
| `evals/core/runner.py` | Fix token stats, wire trace data, verify IPJ/IPW |
|
||||
| `evals/core/display.py` | Rewrite: grouped panels, compact mode |
|
||||
| `evals/cli.py` | Add `--compact`, `--trace-detail` flags |
|
||||
|
||||
### Phase 23b
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `src/openjarvis/cli/quickstart_cmd.py` | **New**: quickstart command |
|
||||
| `src/openjarvis/cli/hints.py` | **New**: centralized hint system |
|
||||
| `src/openjarvis/cli/log_config.py` | **New**: logging setup |
|
||||
| `src/openjarvis/cli/__init__.py` | Register quickstart, add verbose/quiet |
|
||||
| `src/openjarvis/cli/ask.py` | Add hints, progress spinner |
|
||||
| `src/openjarvis/cli/serve.py` | Add hints |
|
||||
| `src/openjarvis/cli/bench_cmd.py` | Full stats tables |
|
||||
| `src/openjarvis/cli/chat_cmd.py` | Add hints |
|
||||
| `src/openjarvis/cli/memory_cmd.py` | Progress bar for indexing |
|
||||
| `src/openjarvis/bench/latency.py` | Export full stats |
|
||||
| `src/openjarvis/bench/throughput.py` | Add stats computation |
|
||||
| `src/openjarvis/bench/energy.py` | Add stats computation |
|
||||
|
||||
### Phase 23c
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `desktop/src/components/SettingsPanel.tsx` | **New**: settings UI |
|
||||
| `desktop/src-tauri/icons/icon.ico` | Replace placeholder |
|
||||
| `desktop/src-tauri/src/lib.rs` | Implement system tray |
|
||||
| `.github/workflows/frontend.yml` | **New**: PWA CI/CD |
|
||||
| `frontend/src/App.tsx` | Error boundary wrapper |
|
||||
| `frontend/vite.config.ts` | VITE_API_URL support |
|
||||
| `frontend/src/App.css` | Modularize, style refresh |
|
||||
File diff suppressed because it is too large
Load Diff
+12
-3
@@ -20,6 +20,7 @@ from rich.progress import (
|
||||
from evals.core.display import (
|
||||
print_banner,
|
||||
print_completion,
|
||||
print_full_results,
|
||||
print_metrics_table,
|
||||
print_run_header,
|
||||
print_section,
|
||||
@@ -119,13 +120,16 @@ def _print_summary(
|
||||
console: Optional[Console] = None,
|
||||
output_path: Optional[Path] = None,
|
||||
traces_dir: Optional[Path] = None,
|
||||
*,
|
||||
compact: bool = False,
|
||||
trace_detail: bool = False,
|
||||
) -> None:
|
||||
"""Print a single run summary using Rich display primitives."""
|
||||
if console is None:
|
||||
console = Console()
|
||||
print_section(console, "Results")
|
||||
print_metrics_table(console, summary)
|
||||
if summary.per_subject and len(summary.per_subject) > 1:
|
||||
print_full_results(console, summary, compact=compact, trace_detail=trace_detail)
|
||||
if not compact and summary.per_subject and len(summary.per_subject) > 1:
|
||||
print_subject_table(console, summary.per_subject)
|
||||
print_completion(console, summary, output_path, traces_dir)
|
||||
|
||||
@@ -266,11 +270,14 @@ def main():
|
||||
help="Enable telemetry collection during eval")
|
||||
@click.option("--gpu-metrics/--no-gpu-metrics", default=False,
|
||||
help="Enable GPU metrics collection")
|
||||
@click.option("--compact", is_flag=True, default=False, help="Dense single-table output")
|
||||
@click.option("--trace-detail", is_flag=True, default=False, help="Full per-step trace listing")
|
||||
@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
|
||||
@click.pass_context
|
||||
def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name,
|
||||
tools, max_samples, max_workers, judge_model, output_path, seed,
|
||||
dataset_split, temperature, max_tokens, telemetry, gpu_metrics, verbose):
|
||||
dataset_split, temperature, max_tokens, telemetry, gpu_metrics,
|
||||
compact, trace_detail, verbose):
|
||||
"""Run a single benchmark evaluation, or a full suite from a TOML config."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
@@ -340,6 +347,8 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name,
|
||||
console=console,
|
||||
output_path=_output_path,
|
||||
traces_dir=_traces_dir,
|
||||
compact=compact,
|
||||
trace_detail=trace_detail,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+158
-3
@@ -152,6 +152,155 @@ def print_metrics_table(console: Console, summary: RunSummary) -> None:
|
||||
console.print(headline)
|
||||
|
||||
|
||||
def _stats_table(title: str, rows: list[tuple[str, Optional[MetricStats], int]]) -> Table:
|
||||
"""Build a stats table with Avg/Median/Min/Max/Std columns."""
|
||||
table = Table(
|
||||
title=f"[bold]{title}[/bold]",
|
||||
show_header=True,
|
||||
header_style="bold bright_white",
|
||||
border_style="bright_blue",
|
||||
title_style="bold cyan",
|
||||
)
|
||||
table.add_column("Metric", style="cyan", no_wrap=True)
|
||||
table.add_column("Avg", justify="right")
|
||||
table.add_column("Median", justify="right")
|
||||
table.add_column("Min", justify="right")
|
||||
table.add_column("Max", justify="right")
|
||||
table.add_column("Std", justify="right")
|
||||
for label, stats, decimals in rows:
|
||||
if stats is not None:
|
||||
table.add_row(
|
||||
label,
|
||||
_fmt(stats.mean, decimals),
|
||||
_fmt(stats.median, decimals),
|
||||
_fmt(stats.min, decimals),
|
||||
_fmt(stats.max, decimals),
|
||||
_fmt(stats.std, decimals),
|
||||
)
|
||||
return table
|
||||
|
||||
|
||||
def print_accuracy_panel(console: Console, summary: RunSummary) -> None:
|
||||
"""Print accuracy panel with per-subject breakdown."""
|
||||
lines = [
|
||||
f"[bold]Overall Accuracy {summary.accuracy:.1%}[/bold]"
|
||||
f" ({summary.correct}/{summary.scored_samples})",
|
||||
]
|
||||
for subj, stats in sorted(summary.per_subject.items()):
|
||||
acc = stats.get("accuracy", 0.0)
|
||||
correct = int(stats.get("correct", 0))
|
||||
scored = int(stats.get("scored", 0))
|
||||
lines.append(f" {subj:<20s} {acc:.1%} ({correct}/{scored})")
|
||||
body = "\n".join(lines)
|
||||
panel = Panel(body, title="[bold]Accuracy[/bold]", border_style="green", expand=False)
|
||||
console.print(panel)
|
||||
|
||||
|
||||
def print_latency_table(console: Console, summary: RunSummary) -> None:
|
||||
"""Print latency, throughput, and token stats table."""
|
||||
table = _stats_table("Latency & Throughput", [
|
||||
("Latency (s)", summary.latency_stats, 2),
|
||||
("TTFT (s)", summary.ttft_stats, 3),
|
||||
("Throughput (tok/s)", summary.throughput_stats, 1),
|
||||
("Avg Input Tokens", summary.input_token_stats, 1),
|
||||
("Avg Output Tokens", summary.output_token_stats, 1),
|
||||
])
|
||||
if table.row_count > 0:
|
||||
console.print(table)
|
||||
|
||||
|
||||
def print_energy_table(console: Console, summary: RunSummary) -> None:
|
||||
"""Print energy, efficiency, and IPJ/IPW table."""
|
||||
table = _stats_table("Energy & Efficiency", [
|
||||
("Energy (J)", summary.energy_stats, 1),
|
||||
("Power (W)", summary.power_stats, 1),
|
||||
("GPU Util (%)", summary.gpu_utilization_stats, 1),
|
||||
("Energy/OutTok (J)", summary.energy_per_output_token_stats, 6),
|
||||
("MFU (%)", summary.mfu_stats, 3),
|
||||
("MBU (%)", summary.mbu_stats, 3),
|
||||
])
|
||||
if table.row_count > 0:
|
||||
console.print(table)
|
||||
# Headline: IPW, IPJ, Total Energy
|
||||
parts: list[str] = []
|
||||
if summary.ipw_stats:
|
||||
parts.append(f"[bold]IPW (acc/W):[/bold] {summary.ipw_stats.mean:.6f}")
|
||||
if summary.ipj_stats:
|
||||
parts.append(f"[bold]IPJ (acc/J):[/bold] {summary.ipj_stats.mean:.2e}")
|
||||
if summary.total_energy_joules > 0:
|
||||
val = summary.total_energy_joules
|
||||
unit = "kJ" if val > 1000 else "J"
|
||||
display = val / 1000 if val > 1000 else val
|
||||
parts.append(f"[bold]Total Energy:[/bold] {display:.1f} {unit}")
|
||||
if summary.avg_power_watts > 0:
|
||||
parts.append(f"[bold]Avg Power:[/bold] {summary.avg_power_watts:.1f} W")
|
||||
if parts:
|
||||
console.print(" ".join(parts))
|
||||
|
||||
|
||||
def print_trace_summary(console: Console, summary: RunSummary) -> None:
|
||||
"""Print agentic trace step-type breakdown."""
|
||||
sts = summary.trace_step_type_stats
|
||||
if not sts:
|
||||
return
|
||||
total_steps = sum(s.get("count", 0) for s in sts.values())
|
||||
avg_per_sample = total_steps / summary.scored_samples if summary.scored_samples > 0 else 0
|
||||
|
||||
table = Table(
|
||||
title="[bold]Agentic Trace Summary[/bold]",
|
||||
show_header=True,
|
||||
header_style="bold bright_white",
|
||||
border_style="bright_blue",
|
||||
title_style="bold cyan",
|
||||
caption=f"Total Steps: {total_steps} | Avg Steps/Sample: {avg_per_sample:.1f}",
|
||||
)
|
||||
table.add_column("Step Type", style="cyan", no_wrap=True)
|
||||
table.add_column("Count", justify="right")
|
||||
table.add_column("Avg Duration", justify="right")
|
||||
table.add_column("Avg Energy (J)", justify="right")
|
||||
table.add_column("Avg In Tokens", justify="right")
|
||||
table.add_column("Avg Out Tokens", justify="right")
|
||||
|
||||
for stype, data in sorted(sts.items()):
|
||||
count = data.get("count", 0)
|
||||
avg_dur = data.get("avg_duration", 0.0)
|
||||
total_e = data.get("total_energy", 0.0)
|
||||
avg_e = total_e / count if count > 0 else 0.0
|
||||
avg_in = data.get("avg_input_tokens", 0.0)
|
||||
avg_out = data.get("avg_output_tokens", 0.0)
|
||||
table.add_row(
|
||||
stype,
|
||||
str(count),
|
||||
f"{avg_dur:.2f}s",
|
||||
f"{avg_e:.1f}" if avg_e > 0 else "\u2014",
|
||||
f"{avg_in:.0f}" if avg_in > 0 else "\u2014",
|
||||
f"{avg_out:.0f}" if avg_out > 0 else "\u2014",
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
def print_compact_table(console: Console, summary: RunSummary) -> None:
|
||||
"""Print a single dense metrics table (legacy behavior, enhanced)."""
|
||||
print_metrics_table(console, summary)
|
||||
|
||||
|
||||
def print_full_results(
|
||||
console: Console,
|
||||
summary: RunSummary,
|
||||
*,
|
||||
compact: bool = False,
|
||||
trace_detail: bool = False,
|
||||
) -> None:
|
||||
"""Orchestrate all result panels."""
|
||||
if compact:
|
||||
print_compact_table(console, summary)
|
||||
return
|
||||
print_accuracy_panel(console, summary)
|
||||
print_latency_table(console, summary)
|
||||
print_energy_table(console, summary)
|
||||
print_trace_summary(console, summary)
|
||||
|
||||
|
||||
def print_subject_table(
|
||||
console: Console,
|
||||
per_subject: Dict[str, Dict[str, float]],
|
||||
@@ -248,11 +397,17 @@ def print_completion(
|
||||
|
||||
__all__ = [
|
||||
"OPENJARVIS_BANNER",
|
||||
"print_accuracy_panel",
|
||||
"print_banner",
|
||||
"print_section",
|
||||
"print_run_header",
|
||||
"print_compact_table",
|
||||
"print_completion",
|
||||
"print_energy_table",
|
||||
"print_full_results",
|
||||
"print_latency_table",
|
||||
"print_metrics_table",
|
||||
"print_run_header",
|
||||
"print_section",
|
||||
"print_subject_table",
|
||||
"print_suite_summary",
|
||||
"print_completion",
|
||||
"print_trace_summary",
|
||||
]
|
||||
|
||||
@@ -343,6 +343,9 @@ class EvalRunner:
|
||||
]
|
||||
|
||||
total_energy = sum(r.energy_joules for r in results)
|
||||
total_input_tokens = sum(r.prompt_tokens for r in results)
|
||||
total_output_tokens = sum(r.completion_tokens for r in results)
|
||||
avg_power = statistics.mean(power_vals) if power_vals else 0.0
|
||||
|
||||
return RunSummary(
|
||||
benchmark=cfg.benchmark,
|
||||
@@ -377,6 +380,9 @@ class EvalRunner:
|
||||
output_token_stats=_metric_stats([float(v) for v in output_tok_vals]),
|
||||
total_energy_joules=round(total_energy, 6),
|
||||
warmup_samples_excluded=cfg.warmup_samples,
|
||||
avg_power_watts=round(avg_power, 4),
|
||||
total_input_tokens=total_input_tokens,
|
||||
total_output_tokens=total_output_tokens,
|
||||
)
|
||||
|
||||
|
||||
@@ -464,6 +470,9 @@ def _summary_to_dict(s: RunSummary) -> Dict[str, Any]:
|
||||
"warmup_samples_excluded": s.warmup_samples_excluded,
|
||||
"steady_state_reached": s.steady_state_reached,
|
||||
"energy_method": s.energy_method,
|
||||
"avg_power_watts": s.avg_power_watts,
|
||||
"total_input_tokens": s.total_input_tokens,
|
||||
"total_output_tokens": s.total_output_tokens,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
@@ -44,6 +45,8 @@ class EvalResult:
|
||||
energy_per_output_token_joules: float = 0.0
|
||||
throughput_per_watt: float = 0.0
|
||||
mean_itl_ms: float = 0.0
|
||||
trace_steps: int = 0
|
||||
trace_energy_joules: float = 0.0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -122,6 +125,13 @@ class RunSummary:
|
||||
warmup_samples_excluded: int = 0
|
||||
steady_state_reached: bool = False
|
||||
energy_method: str = ""
|
||||
avg_power_watts: float = 0.0
|
||||
total_input_tokens: int = 0
|
||||
total_output_tokens: int = 0
|
||||
trace_step_type_stats: Dict[str, Dict[str, float]] = field(default_factory=dict)
|
||||
# Internal fields set by the runner after construction
|
||||
_output_path: Optional[Path] = None
|
||||
_traces_dir: Optional[Path] = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Tests for eval CLI display flags."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from evals.cli import main
|
||||
|
||||
|
||||
class TestCompactFlag:
|
||||
def test_compact_flag_accepted(self):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(main, ["run", "--help"])
|
||||
assert "--compact" in result.output
|
||||
|
||||
def test_trace_detail_flag_accepted(self):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(main, ["run", "--help"])
|
||||
assert "--trace-detail" in result.output
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Tests for eval display functions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import StringIO
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from evals.core.display import (
|
||||
print_accuracy_panel,
|
||||
print_energy_table,
|
||||
print_latency_table,
|
||||
print_trace_summary,
|
||||
print_compact_table,
|
||||
print_full_results,
|
||||
)
|
||||
from evals.core.types import MetricStats, RunSummary
|
||||
|
||||
|
||||
def _make_summary(**overrides) -> RunSummary:
|
||||
defaults = dict(
|
||||
benchmark="gaia", category="agentic", backend="jarvis-agent",
|
||||
model="qwen3:8b", total_samples=100, scored_samples=100,
|
||||
correct=42, accuracy=0.42, errors=0,
|
||||
mean_latency_seconds=15.0, total_cost_usd=0.0,
|
||||
total_energy_joules=9300.0, avg_power_watts=880.0,
|
||||
total_input_tokens=50000, total_output_tokens=12000,
|
||||
per_subject={"level_1": {"accuracy": 0.58, "correct": 40, "scored": 68}},
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return RunSummary(**defaults)
|
||||
|
||||
|
||||
def _make_stats(mean=10.0) -> MetricStats:
|
||||
return MetricStats(
|
||||
mean=mean, median=mean * 0.9, min=mean * 0.2,
|
||||
max=mean * 3.0, std=mean * 0.5, p90=mean * 1.5,
|
||||
p95=mean * 2.0, p99=mean * 2.5,
|
||||
)
|
||||
|
||||
|
||||
class TestAccuracyPanel:
|
||||
def test_renders_without_error(self):
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
summary = _make_summary()
|
||||
print_accuracy_panel(console, summary)
|
||||
output = console.file.getvalue()
|
||||
assert "42.0%" in output or "0.42" in output
|
||||
|
||||
def test_shows_per_subject(self):
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
summary = _make_summary()
|
||||
print_accuracy_panel(console, summary)
|
||||
output = console.file.getvalue()
|
||||
assert "level_1" in output
|
||||
|
||||
|
||||
class TestLatencyTable:
|
||||
def test_renders_with_stats(self):
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
summary = _make_summary(
|
||||
latency_stats=_make_stats(15.0),
|
||||
throughput_stats=_make_stats(40.0),
|
||||
input_token_stats=_make_stats(1024.0),
|
||||
output_token_stats=_make_stats(256.0),
|
||||
)
|
||||
print_latency_table(console, summary)
|
||||
output = console.file.getvalue()
|
||||
assert "Latency" in output
|
||||
assert "Avg" in output
|
||||
|
||||
|
||||
class TestEnergyTable:
|
||||
def test_renders_ipj_ipw(self):
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
summary = _make_summary(
|
||||
energy_stats=_make_stats(46000.0),
|
||||
power_stats=_make_stats(880.0),
|
||||
ipw_stats=_make_stats(0.00048),
|
||||
ipj_stats=_make_stats(9.0e-6),
|
||||
)
|
||||
print_energy_table(console, summary)
|
||||
output = console.file.getvalue()
|
||||
assert "IPW" in output
|
||||
assert "IPJ" in output
|
||||
|
||||
|
||||
class TestTraceSummary:
|
||||
def test_renders_step_type_breakdown(self):
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
summary = _make_summary(
|
||||
trace_step_type_stats={
|
||||
"generate": {
|
||||
"count": 580, "avg_duration": 8.2,
|
||||
"total_energy": 22000.0,
|
||||
"avg_input_tokens": 890.0, "avg_output_tokens": 256.0,
|
||||
},
|
||||
"tool_call": {
|
||||
"count": 420, "avg_duration": 3.1,
|
||||
"total_energy": 0.0,
|
||||
"avg_input_tokens": 0.0, "avg_output_tokens": 0.0,
|
||||
},
|
||||
},
|
||||
)
|
||||
print_trace_summary(console, summary)
|
||||
output = console.file.getvalue()
|
||||
assert "generate" in output
|
||||
assert "tool_call" in output
|
||||
|
||||
|
||||
class TestCompactTable:
|
||||
def test_renders_all_metrics(self):
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
summary = _make_summary(
|
||||
latency_stats=_make_stats(15.0),
|
||||
energy_stats=_make_stats(46000.0),
|
||||
)
|
||||
print_compact_table(console, summary)
|
||||
output = console.file.getvalue()
|
||||
assert "Latency" in output
|
||||
assert "Energy" in output
|
||||
|
||||
|
||||
class TestFullResults:
|
||||
def test_renders_all_sections(self):
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
summary = _make_summary(
|
||||
latency_stats=_make_stats(15.0),
|
||||
energy_stats=_make_stats(46000.0),
|
||||
power_stats=_make_stats(880.0),
|
||||
)
|
||||
print_full_results(console, summary)
|
||||
output = console.file.getvalue()
|
||||
assert "Accuracy" in output
|
||||
@@ -373,6 +373,45 @@ class TestEvalRunner:
|
||||
assert "total_energy_joules" in data
|
||||
|
||||
|
||||
class TestRunnerTokenStats:
|
||||
def test_summary_has_total_input_output_tokens(self, tmp_path):
|
||||
"""RunSummary should include total token counts."""
|
||||
records = [
|
||||
EvalRecord(record_id=f"r{i}", problem=f"q{i}", reference="a", category="test")
|
||||
for i in range(3)
|
||||
]
|
||||
output_path = tmp_path / "results.jsonl"
|
||||
config = RunConfig(
|
||||
benchmark="test", backend="mock", model="m",
|
||||
max_workers=1, output_path=str(output_path),
|
||||
)
|
||||
dataset = MockDataset(records)
|
||||
backend = MockBackend()
|
||||
scorer = MockScorer(result=True)
|
||||
runner = EvalRunner(config, dataset, backend, scorer)
|
||||
summary = runner.run()
|
||||
# MockBackend returns prompt_tokens=100, completion_tokens=50
|
||||
assert summary.total_input_tokens == 300 # 3 * 100
|
||||
assert summary.total_output_tokens == 150 # 3 * 50
|
||||
|
||||
def test_summary_has_avg_power(self, tmp_path):
|
||||
"""RunSummary should include avg_power_watts."""
|
||||
records = [
|
||||
EvalRecord(record_id="r1", problem="q", reference="a", category="test")
|
||||
]
|
||||
output_path = tmp_path / "results.jsonl"
|
||||
config = RunConfig(
|
||||
benchmark="test", backend="mock", model="m",
|
||||
max_workers=1, output_path=str(output_path),
|
||||
)
|
||||
dataset = MockDataset(records)
|
||||
backend = MockBackend() # returns power_watts=250.0
|
||||
scorer = MockScorer(result=True)
|
||||
runner = EvalRunner(config, dataset, backend, scorer)
|
||||
summary = runner.run()
|
||||
assert summary.avg_power_watts == 250.0
|
||||
|
||||
|
||||
class TestMetricStatsHelpers:
|
||||
def test_metric_stats_empty(self):
|
||||
assert _metric_stats([]) is None
|
||||
@@ -401,4 +440,11 @@ class TestMetricStatsHelpers:
|
||||
def test_metric_stats_to_dict(self):
|
||||
ms = MetricStats(mean=1.0, median=2.0, min=0.5, max=3.0, std=0.8)
|
||||
d = _metric_stats_to_dict(ms)
|
||||
assert d == {"mean": 1.0, "median": 2.0, "min": 0.5, "max": 3.0, "std": 0.8}
|
||||
assert d["mean"] == 1.0
|
||||
assert d["median"] == 2.0
|
||||
assert d["min"] == 0.5
|
||||
assert d["max"] == 3.0
|
||||
assert d["std"] == 0.8
|
||||
assert "p90" in d
|
||||
assert "p95" in d
|
||||
assert "p99" in d
|
||||
|
||||
@@ -209,6 +209,35 @@ class TestRunSummary:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEvalResultTraceFields:
|
||||
def test_trace_fields_exist(self):
|
||||
r = EvalResult(record_id="test", model_answer="hi")
|
||||
assert r.trace_steps == 0
|
||||
assert r.trace_energy_joules == 0.0
|
||||
|
||||
def test_trace_fields_set(self):
|
||||
r = EvalResult(
|
||||
record_id="test", model_answer="hi",
|
||||
trace_steps=5, trace_energy_joules=100.0,
|
||||
)
|
||||
assert r.trace_steps == 5
|
||||
assert r.trace_energy_joules == 100.0
|
||||
|
||||
|
||||
class TestRunSummaryTraceFields:
|
||||
def test_trace_aggregate_fields(self):
|
||||
s = RunSummary(
|
||||
benchmark="test", category="test", backend="test",
|
||||
model="test", total_samples=1, scored_samples=1,
|
||||
correct=1, accuracy=1.0, errors=0,
|
||||
mean_latency_seconds=1.0, total_cost_usd=0.0,
|
||||
)
|
||||
assert s.avg_power_watts == 0.0
|
||||
assert s.trace_step_type_stats == {}
|
||||
assert s.total_input_tokens == 0
|
||||
assert s.total_output_tokens == 0
|
||||
|
||||
|
||||
class TestMetaConfig:
|
||||
def test_defaults(self):
|
||||
m = MetaConfig()
|
||||
|
||||
@@ -1,725 +0,0 @@
|
||||
/* === CSS Variables === */
|
||||
:root {
|
||||
--color-primary: #2563eb;
|
||||
--color-primary-light: #3b82f6;
|
||||
--color-primary-dark: #1d4ed8;
|
||||
--color-bg: #ffffff;
|
||||
--color-bg-sidebar: #f1f5f9;
|
||||
--color-text: #1e293b;
|
||||
--color-text-secondary: #64748b;
|
||||
--color-border: #e2e8f0;
|
||||
--color-user-bubble: #2563eb;
|
||||
--color-assistant-bubble: #ffffff;
|
||||
--color-tool-bg: #f0f9ff;
|
||||
--color-tool-border: #7dd3fc;
|
||||
--color-tool-success: #22c55e;
|
||||
--color-tool-running: #f59e0b;
|
||||
--color-tool-error: #ef4444;
|
||||
--color-savings: #22c55e;
|
||||
--sidebar-width: 300px;
|
||||
}
|
||||
|
||||
/* === Reset === */
|
||||
*, *::before, *::after {
|
||||
box-sizing: border-box;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
|
||||
background: var(--color-bg);
|
||||
color: var(--color-text);
|
||||
height: 100vh;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
#root {
|
||||
height: 100vh;
|
||||
}
|
||||
|
||||
/* === App Layout === */
|
||||
.app {
|
||||
display: flex;
|
||||
height: 100vh;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
/* === Sidebar Toggle === */
|
||||
.sidebar-toggle {
|
||||
display: none;
|
||||
position: fixed;
|
||||
top: 12px;
|
||||
left: 12px;
|
||||
z-index: 1000;
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
font-size: 18px;
|
||||
cursor: pointer;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
/* === Sidebar === */
|
||||
.sidebar {
|
||||
width: var(--sidebar-width);
|
||||
background: var(--color-bg-sidebar);
|
||||
border-right: 1px solid var(--color-border);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100vh;
|
||||
flex-shrink: 0;
|
||||
transition: transform 0.2s ease;
|
||||
}
|
||||
|
||||
.sidebar-header {
|
||||
padding: 16px;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.sidebar-header h1 {
|
||||
font-size: 18px;
|
||||
font-weight: 700;
|
||||
color: var(--color-primary);
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.new-chat-btn {
|
||||
width: 100%;
|
||||
padding: 10px 16px;
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.new-chat-btn:hover {
|
||||
background: var(--color-primary-dark);
|
||||
}
|
||||
|
||||
/* === Model Selector === */
|
||||
.model-selector {
|
||||
padding: 12px 16px;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.model-selector label {
|
||||
display: block;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
color: var(--color-text-secondary);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.model-selector select {
|
||||
width: 100%;
|
||||
padding: 8px 10px;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
background: white;
|
||||
font-size: 13px;
|
||||
color: var(--color-text);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* === Conversation List === */
|
||||
.conversation-list {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.conversation-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 10px 12px;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
font-size: 13px;
|
||||
color: var(--color-text);
|
||||
transition: background 0.15s;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.conversation-item:hover {
|
||||
background: rgba(0, 0, 0, 0.04);
|
||||
}
|
||||
|
||||
.conversation-item.active {
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.conversation-item .conv-title {
|
||||
flex: 1;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.conversation-item .conv-delete {
|
||||
opacity: 0;
|
||||
border: none;
|
||||
background: none;
|
||||
color: inherit;
|
||||
cursor: pointer;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.conversation-item:hover .conv-delete {
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.conversation-item .conv-delete:hover {
|
||||
opacity: 1;
|
||||
background: rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.conversation-item.active .conv-delete:hover {
|
||||
background: rgba(255, 255, 255, 0.2);
|
||||
}
|
||||
|
||||
/* === Savings Panel === */
|
||||
.savings-panel {
|
||||
padding: 16px;
|
||||
border-top: 2px solid var(--color-primary);
|
||||
background: linear-gradient(to bottom, #eff6ff, var(--color-bg-sidebar));
|
||||
}
|
||||
|
||||
.savings-panel h3 {
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
color: var(--color-primary);
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.savings-models {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.savings-model-card {
|
||||
background: white;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 8px;
|
||||
padding: 8px 10px;
|
||||
border-left: 3px solid var(--color-border);
|
||||
}
|
||||
|
||||
.savings-model-card.local {
|
||||
border-left-color: #22c55e;
|
||||
}
|
||||
|
||||
.savings-model-card.cloud {
|
||||
border-left-color: #f59e0b;
|
||||
}
|
||||
|
||||
.savings-model-card-label {
|
||||
display: block;
|
||||
font-size: 9px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.08em;
|
||||
color: var(--color-text-secondary);
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.savings-model-card-name {
|
||||
display: block;
|
||||
font-family: monospace;
|
||||
font-size: 11px;
|
||||
color: var(--color-text);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.savings-grid {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.savings-item {
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
padding: 10px;
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.savings-item .savings-label {
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
color: var(--color-text-secondary);
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.savings-item .savings-value {
|
||||
font-size: 18px;
|
||||
font-weight: 800;
|
||||
color: var(--color-savings);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.savings-item.calls .savings-value {
|
||||
color: var(--color-primary);
|
||||
}
|
||||
|
||||
/* === Chat Area === */
|
||||
.chat-area {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100vh;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.chat-header {
|
||||
padding: 12px 24px;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
background: var(--color-bg);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.chat-header-title {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: var(--color-text);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.chat-header-meta {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.chat-header-info {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.header-badge {
|
||||
display: inline-block;
|
||||
padding: 3px 8px;
|
||||
border-radius: 4px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
.model-badge {
|
||||
background: #eff6ff;
|
||||
color: var(--color-primary);
|
||||
border: 1px solid #bfdbfe;
|
||||
}
|
||||
|
||||
.agent-badge {
|
||||
background: #f0fdf4;
|
||||
color: #16a34a;
|
||||
border: 1px solid #bbf7d0;
|
||||
}
|
||||
|
||||
.chat-header-tokens {
|
||||
font-size: 11px;
|
||||
color: var(--color-text-secondary);
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* === Message List === */
|
||||
.message-list {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 24px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.message-list-empty {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: var(--color-text-secondary);
|
||||
font-size: 15px;
|
||||
}
|
||||
|
||||
/* === Message Bubble === */
|
||||
.message-bubble {
|
||||
max-width: 75%;
|
||||
position: relative;
|
||||
group: message;
|
||||
}
|
||||
|
||||
.message-bubble.user {
|
||||
align-self: flex-end;
|
||||
}
|
||||
|
||||
.message-bubble.assistant {
|
||||
align-self: flex-start;
|
||||
}
|
||||
|
||||
.message-content {
|
||||
padding: 12px 16px;
|
||||
border-radius: 16px;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.message-bubble.user .message-content {
|
||||
background: var(--color-user-bubble);
|
||||
color: white;
|
||||
border-bottom-right-radius: 4px;
|
||||
}
|
||||
|
||||
.message-bubble.assistant .message-content {
|
||||
background: var(--color-assistant-bubble);
|
||||
color: var(--color-text);
|
||||
border: 1px solid var(--color-border);
|
||||
border-bottom-left-radius: 4px;
|
||||
}
|
||||
|
||||
.message-meta {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-top: 4px;
|
||||
padding: 0 4px;
|
||||
}
|
||||
|
||||
.message-time {
|
||||
font-size: 11px;
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
|
||||
.message-tokens {
|
||||
font-size: 10px;
|
||||
color: var(--color-text-secondary);
|
||||
font-variant-numeric: tabular-nums;
|
||||
opacity: 0.7;
|
||||
}
|
||||
|
||||
.message-bubble.user .message-meta {
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
/* === Copy Button === */
|
||||
.copy-btn {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
opacity: 0;
|
||||
background: rgba(0, 0, 0, 0.06);
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
padding: 4px 8px;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
color: var(--color-text-secondary);
|
||||
transition: opacity 0.15s;
|
||||
}
|
||||
|
||||
.message-bubble:hover .copy-btn {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.copy-btn:hover {
|
||||
background: rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.message-bubble.user .copy-btn {
|
||||
color: rgba(255, 255, 255, 0.7);
|
||||
background: rgba(255, 255, 255, 0.15);
|
||||
}
|
||||
|
||||
.message-bubble.user .copy-btn:hover {
|
||||
background: rgba(255, 255, 255, 0.25);
|
||||
}
|
||||
|
||||
/* === Tool Call Indicator === */
|
||||
.tool-calls {
|
||||
margin-top: 8px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.tool-call {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 8px 12px;
|
||||
background: var(--color-tool-bg);
|
||||
border: 1px solid var(--color-tool-border);
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.tool-call .tool-status {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.tool-call .tool-status.running {
|
||||
background: var(--color-tool-running);
|
||||
animation: pulse 1.5s infinite;
|
||||
}
|
||||
|
||||
.tool-call .tool-status.success {
|
||||
background: var(--color-tool-success);
|
||||
}
|
||||
|
||||
.tool-call .tool-status.error {
|
||||
background: var(--color-tool-error);
|
||||
}
|
||||
|
||||
.tool-call .tool-name {
|
||||
font-weight: 600;
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.tool-call .tool-latency {
|
||||
margin-left: auto;
|
||||
font-size: 11px;
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.4; }
|
||||
}
|
||||
|
||||
/* === Streaming Indicator === */
|
||||
.streaming-indicator {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
padding: 12px 16px;
|
||||
margin: 0 24px;
|
||||
}
|
||||
|
||||
.streaming-tool-calls {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.streaming-progress-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.streaming-bar-container {
|
||||
flex: 1;
|
||||
height: 4px;
|
||||
background: var(--color-border);
|
||||
border-radius: 2px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.streaming-bar {
|
||||
height: 100%;
|
||||
background: var(--color-primary);
|
||||
border-radius: 2px;
|
||||
animation: stream-progress 2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes stream-progress {
|
||||
0% { width: 0%; margin-left: 0%; }
|
||||
50% { width: 40%; margin-left: 30%; }
|
||||
100% { width: 0%; margin-left: 100%; }
|
||||
}
|
||||
|
||||
.streaming-phase {
|
||||
font-size: 12px;
|
||||
color: var(--color-text-secondary);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.streaming-elapsed {
|
||||
font-size: 12px;
|
||||
color: var(--color-text-secondary);
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* === Input Area === */
|
||||
.input-area {
|
||||
padding: 16px 24px 24px;
|
||||
border-top: 1px solid var(--color-border);
|
||||
background: var(--color-bg);
|
||||
}
|
||||
|
||||
.input-attachment-row {
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.input-container {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: flex-end;
|
||||
}
|
||||
|
||||
.input-container textarea {
|
||||
flex: 1;
|
||||
resize: none;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 12px;
|
||||
padding: 12px 16px;
|
||||
font-size: 14px;
|
||||
font-family: inherit;
|
||||
line-height: 1.5;
|
||||
min-height: 72px;
|
||||
max-height: 200px;
|
||||
outline: none;
|
||||
transition: border-color 0.15s;
|
||||
}
|
||||
|
||||
.input-container textarea:focus {
|
||||
border-color: var(--color-primary);
|
||||
}
|
||||
|
||||
.send-btn, .stop-btn {
|
||||
padding: 10px 20px;
|
||||
border: none;
|
||||
border-radius: 12px;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
flex-shrink: 0;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.send-btn {
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.send-btn:hover:not(:disabled) {
|
||||
background: var(--color-primary-dark);
|
||||
}
|
||||
|
||||
.send-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.stop-btn {
|
||||
background: var(--color-tool-error);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.stop-btn:hover {
|
||||
background: #dc2626;
|
||||
}
|
||||
|
||||
/* === Pasted text pill === */
|
||||
.pasted-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
background: var(--color-surface);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 10px;
|
||||
padding: 10px 14px;
|
||||
color: var(--color-text-secondary);
|
||||
font-size: 13px;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.pasted-pill svg {
|
||||
flex-shrink: 0;
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.pasted-pill-text {
|
||||
font-weight: 500;
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.pasted-pill-size {
|
||||
color: var(--color-text-secondary);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.pasted-pill-action {
|
||||
background: none;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
padding: 2px 8px;
|
||||
font-size: 12px;
|
||||
color: var(--color-text-secondary);
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, color 0.15s;
|
||||
}
|
||||
|
||||
.pasted-pill-action:hover {
|
||||
background: var(--color-border);
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.pasted-pill-remove {
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
padding: 2px 6px;
|
||||
}
|
||||
|
||||
.pasted-pill-remove:hover {
|
||||
color: var(--color-tool-error);
|
||||
border-color: var(--color-tool-error);
|
||||
}
|
||||
|
||||
/* === Responsive === */
|
||||
@media (max-width: 768px) {
|
||||
.sidebar-toggle {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.sidebar {
|
||||
position: fixed;
|
||||
left: 0;
|
||||
top: 0;
|
||||
z-index: 999;
|
||||
transform: translateX(-100%);
|
||||
}
|
||||
|
||||
.sidebar.open {
|
||||
transform: translateX(0);
|
||||
}
|
||||
|
||||
.message-bubble {
|
||||
max-width: 90%;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { ModelInfo, SavingsData, ServerInfo } from '../types';
|
||||
|
||||
const BASE = ''; // relative to same origin
|
||||
const BASE = import.meta.env.VITE_API_URL || ''; // relative to same origin by default
|
||||
|
||||
export async function fetchModels(): Promise<ModelInfo[]> {
|
||||
const res = await fetch(`${BASE}/v1/models`);
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { Component } from 'react';
|
||||
import type { ErrorInfo, ReactNode } from 'react';
|
||||
|
||||
interface Props {
|
||||
children: ReactNode;
|
||||
}
|
||||
|
||||
interface State {
|
||||
hasError: boolean;
|
||||
error: Error | null;
|
||||
}
|
||||
|
||||
export class ErrorBoundary extends Component<Props, State> {
|
||||
constructor(props: Props) {
|
||||
super(props);
|
||||
this.state = { hasError: false, error: null };
|
||||
}
|
||||
|
||||
static getDerivedStateFromError(error: Error): State {
|
||||
return { hasError: true, error };
|
||||
}
|
||||
|
||||
componentDidCatch(error: Error, info: ErrorInfo) {
|
||||
console.error('ErrorBoundary caught:', error, info.componentStack);
|
||||
}
|
||||
|
||||
render() {
|
||||
if (this.state.hasError) {
|
||||
return (
|
||||
<div style={styles.container}>
|
||||
<div style={styles.card}>
|
||||
<h2 style={styles.heading}>Something went wrong</h2>
|
||||
<p style={styles.message}>
|
||||
{this.state.error?.message || 'An unexpected error occurred.'}
|
||||
</p>
|
||||
<button
|
||||
style={styles.button}
|
||||
onClick={() => this.setState({ hasError: false, error: null })}
|
||||
>
|
||||
Try again
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return this.props.children;
|
||||
}
|
||||
}
|
||||
|
||||
const styles: Record<string, React.CSSProperties> = {
|
||||
container: {
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
height: '100vh',
|
||||
backgroundColor: '#1a1a1e',
|
||||
color: '#e2e8f0',
|
||||
fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif',
|
||||
},
|
||||
card: {
|
||||
textAlign: 'center' as const,
|
||||
padding: 32,
|
||||
maxWidth: 420,
|
||||
},
|
||||
heading: {
|
||||
fontSize: 20,
|
||||
fontWeight: 600,
|
||||
marginBottom: 12,
|
||||
},
|
||||
message: {
|
||||
fontSize: 14,
|
||||
color: '#94a3b8',
|
||||
marginBottom: 24,
|
||||
lineHeight: 1.5,
|
||||
},
|
||||
button: {
|
||||
padding: '10px 24px',
|
||||
borderRadius: 8,
|
||||
border: 'none',
|
||||
backgroundColor: '#2563eb',
|
||||
color: 'white',
|
||||
fontSize: 14,
|
||||
fontWeight: 500,
|
||||
cursor: 'pointer',
|
||||
},
|
||||
};
|
||||
@@ -1,10 +1,17 @@
|
||||
import { StrictMode } from 'react';
|
||||
import { createRoot } from 'react-dom/client';
|
||||
import { ErrorBoundary } from './components/ErrorBoundary';
|
||||
import App from './App';
|
||||
import './App.css';
|
||||
import './styles/variables.css';
|
||||
import './styles/base.css';
|
||||
import './styles/sidebar.css';
|
||||
import './styles/chat.css';
|
||||
import './styles/input.css';
|
||||
|
||||
createRoot(document.getElementById('root')!).render(
|
||||
<StrictMode>
|
||||
<App />
|
||||
<ErrorBoundary>
|
||||
<App />
|
||||
</ErrorBoundary>
|
||||
</StrictMode>,
|
||||
);
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
/* Reset & layout */
|
||||
*, *::before, *::after {
|
||||
box-sizing: border-box;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
|
||||
background: var(--color-bg);
|
||||
color: var(--color-text);
|
||||
height: 100vh;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
#root {
|
||||
height: 100vh;
|
||||
}
|
||||
|
||||
.app {
|
||||
display: flex;
|
||||
height: 100vh;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.sidebar-toggle {
|
||||
display: none;
|
||||
position: fixed;
|
||||
top: 12px;
|
||||
left: 12px;
|
||||
z-index: 1000;
|
||||
background: var(--color-primary);
|
||||
color: var(--ctp-crust);
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
font-size: 18px;
|
||||
cursor: pointer;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.sidebar-toggle {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.sidebar {
|
||||
position: fixed;
|
||||
left: 0;
|
||||
top: 0;
|
||||
z-index: 999;
|
||||
transform: translateX(-100%);
|
||||
}
|
||||
|
||||
.sidebar.open {
|
||||
transform: translateX(0);
|
||||
}
|
||||
|
||||
.message-bubble {
|
||||
max-width: 90%;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
/* Chat Area */
|
||||
.chat-area {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100vh;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.chat-header {
|
||||
padding: 12px 24px;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
background: var(--color-bg);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.chat-header-title {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: var(--color-text);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.chat-header-meta {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.chat-header-info {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.header-badge {
|
||||
display: inline-block;
|
||||
padding: 3px 8px;
|
||||
border-radius: 4px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
.model-badge {
|
||||
background: rgba(137, 180, 250, 0.12);
|
||||
color: var(--ctp-blue);
|
||||
border: 1px solid rgba(137, 180, 250, 0.25);
|
||||
}
|
||||
|
||||
.agent-badge {
|
||||
background: rgba(166, 227, 161, 0.12);
|
||||
color: var(--ctp-green);
|
||||
border: 1px solid rgba(166, 227, 161, 0.25);
|
||||
}
|
||||
|
||||
.chat-header-tokens {
|
||||
font-size: 11px;
|
||||
color: var(--color-text-secondary);
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* Message List */
|
||||
.message-list {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 24px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.message-list-empty {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: var(--color-text-secondary);
|
||||
font-size: 15px;
|
||||
}
|
||||
|
||||
/* Message Bubble */
|
||||
.message-bubble {
|
||||
max-width: 75%;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.message-bubble.user {
|
||||
align-self: flex-end;
|
||||
}
|
||||
|
||||
.message-bubble.assistant {
|
||||
align-self: flex-start;
|
||||
}
|
||||
|
||||
.message-content {
|
||||
padding: 12px 16px;
|
||||
border-radius: 16px;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.message-bubble.user .message-content {
|
||||
background: var(--color-user-bubble);
|
||||
color: var(--ctp-crust);
|
||||
border-bottom-right-radius: 4px;
|
||||
}
|
||||
|
||||
.message-bubble.assistant .message-content {
|
||||
background: var(--color-assistant-bubble);
|
||||
color: var(--color-text);
|
||||
border: 1px solid var(--ctp-surface1);
|
||||
border-bottom-left-radius: 4px;
|
||||
}
|
||||
|
||||
.message-meta {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-top: 4px;
|
||||
padding: 0 4px;
|
||||
}
|
||||
|
||||
.message-time {
|
||||
font-size: 11px;
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
|
||||
.message-tokens {
|
||||
font-size: 10px;
|
||||
color: var(--color-text-secondary);
|
||||
font-variant-numeric: tabular-nums;
|
||||
opacity: 0.7;
|
||||
}
|
||||
|
||||
.message-bubble.user .message-meta {
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
/* Copy Button */
|
||||
.copy-btn {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
opacity: 0;
|
||||
background: rgba(255, 255, 255, 0.06);
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
padding: 4px 8px;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
color: var(--color-text-secondary);
|
||||
transition: opacity 0.15s;
|
||||
}
|
||||
|
||||
.message-bubble:hover .copy-btn {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.copy-btn:hover {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
|
||||
.message-bubble.user .copy-btn {
|
||||
color: rgba(0, 0, 0, 0.5);
|
||||
background: rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.message-bubble.user .copy-btn:hover {
|
||||
background: rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
||||
/* Tool Call Indicator */
|
||||
.tool-calls {
|
||||
margin-top: 8px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.tool-call {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 8px 12px;
|
||||
background: var(--color-tool-bg);
|
||||
border: 1px solid var(--color-tool-border);
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.tool-call .tool-status {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.tool-call .tool-status.running {
|
||||
background: var(--color-tool-running);
|
||||
animation: pulse 1.5s infinite;
|
||||
}
|
||||
|
||||
.tool-call .tool-status.success {
|
||||
background: var(--color-tool-success);
|
||||
}
|
||||
|
||||
.tool-call .tool-status.error {
|
||||
background: var(--color-tool-error);
|
||||
}
|
||||
|
||||
.tool-call .tool-name {
|
||||
font-weight: 600;
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.tool-call .tool-latency {
|
||||
margin-left: auto;
|
||||
font-size: 11px;
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.4; }
|
||||
}
|
||||
|
||||
/* Streaming Indicator */
|
||||
.streaming-indicator {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
padding: 12px 16px;
|
||||
margin: 0 24px;
|
||||
}
|
||||
|
||||
.streaming-tool-calls {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.streaming-progress-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.streaming-bar-container {
|
||||
flex: 1;
|
||||
height: 4px;
|
||||
background: var(--color-border);
|
||||
border-radius: 2px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.streaming-bar {
|
||||
height: 100%;
|
||||
background: var(--color-primary);
|
||||
border-radius: 2px;
|
||||
animation: stream-progress 2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes stream-progress {
|
||||
0% { width: 0%; margin-left: 0%; }
|
||||
50% { width: 40%; margin-left: 30%; }
|
||||
100% { width: 0%; margin-left: 100%; }
|
||||
}
|
||||
|
||||
.streaming-phase {
|
||||
font-size: 12px;
|
||||
color: var(--color-text-secondary);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.streaming-elapsed {
|
||||
font-size: 12px;
|
||||
color: var(--color-text-secondary);
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/* Input Area */
|
||||
.input-area {
|
||||
padding: 16px 24px 24px;
|
||||
border-top: 1px solid var(--color-border);
|
||||
background: var(--color-bg);
|
||||
}
|
||||
|
||||
.input-attachment-row {
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.input-container {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: flex-end;
|
||||
}
|
||||
|
||||
.input-container textarea {
|
||||
flex: 1;
|
||||
resize: none;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 12px;
|
||||
padding: 12px 16px;
|
||||
font-size: 14px;
|
||||
font-family: inherit;
|
||||
line-height: 1.5;
|
||||
min-height: 72px;
|
||||
max-height: 200px;
|
||||
outline: none;
|
||||
background: var(--ctp-surface0);
|
||||
color: var(--color-text);
|
||||
transition: border-color 0.15s;
|
||||
}
|
||||
|
||||
.input-container textarea:focus {
|
||||
border-color: var(--color-primary);
|
||||
}
|
||||
|
||||
.input-container textarea::placeholder {
|
||||
color: var(--ctp-overlay0);
|
||||
}
|
||||
|
||||
.send-btn, .stop-btn {
|
||||
padding: 10px 20px;
|
||||
border: none;
|
||||
border-radius: 12px;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
flex-shrink: 0;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.send-btn {
|
||||
background: var(--color-primary);
|
||||
color: var(--ctp-crust);
|
||||
}
|
||||
|
||||
.send-btn:hover:not(:disabled) {
|
||||
background: var(--color-primary-dark);
|
||||
}
|
||||
|
||||
.send-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.stop-btn {
|
||||
background: var(--color-tool-error);
|
||||
color: var(--ctp-crust);
|
||||
}
|
||||
|
||||
.stop-btn:hover {
|
||||
background: var(--ctp-maroon);
|
||||
}
|
||||
|
||||
/* Pasted text pill */
|
||||
.pasted-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
background: var(--ctp-surface0);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 10px;
|
||||
padding: 10px 14px;
|
||||
color: var(--color-text-secondary);
|
||||
font-size: 13px;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.pasted-pill svg {
|
||||
flex-shrink: 0;
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.pasted-pill-text {
|
||||
font-weight: 500;
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.pasted-pill-size {
|
||||
color: var(--color-text-secondary);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.pasted-pill-action {
|
||||
background: none;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
padding: 2px 8px;
|
||||
font-size: 12px;
|
||||
color: var(--color-text-secondary);
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, color 0.15s;
|
||||
}
|
||||
|
||||
.pasted-pill-action:hover {
|
||||
background: var(--ctp-surface1);
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.pasted-pill-remove {
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
padding: 2px 6px;
|
||||
}
|
||||
|
||||
.pasted-pill-remove:hover {
|
||||
color: var(--color-tool-error);
|
||||
border-color: var(--color-tool-error);
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
/* Sidebar */
|
||||
.sidebar {
|
||||
width: var(--sidebar-width);
|
||||
background: var(--color-bg-sidebar);
|
||||
border-right: 1px solid var(--color-border);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100vh;
|
||||
flex-shrink: 0;
|
||||
transition: transform 0.2s ease;
|
||||
}
|
||||
|
||||
.sidebar-header {
|
||||
padding: 16px;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.sidebar-header h1 {
|
||||
font-size: 18px;
|
||||
font-weight: 700;
|
||||
color: var(--color-primary);
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.new-chat-btn {
|
||||
width: 100%;
|
||||
padding: 10px 16px;
|
||||
background: var(--color-primary);
|
||||
color: var(--ctp-crust);
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.new-chat-btn:hover {
|
||||
background: var(--color-primary-dark);
|
||||
}
|
||||
|
||||
/* Model Selector */
|
||||
.model-selector {
|
||||
padding: 12px 16px;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.model-selector label {
|
||||
display: block;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
color: var(--color-text-secondary);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.model-selector select {
|
||||
width: 100%;
|
||||
padding: 8px 10px;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
background: var(--ctp-surface0);
|
||||
font-size: 13px;
|
||||
color: var(--color-text);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Conversation List */
|
||||
.conversation-list {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.conversation-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 10px 12px;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
font-size: 13px;
|
||||
color: var(--color-text);
|
||||
transition: background 0.15s;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.conversation-item:hover {
|
||||
background: var(--ctp-surface0);
|
||||
}
|
||||
|
||||
.conversation-item.active {
|
||||
background: var(--color-primary);
|
||||
color: var(--ctp-crust);
|
||||
}
|
||||
|
||||
.conversation-item .conv-title {
|
||||
flex: 1;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.conversation-item .conv-delete {
|
||||
opacity: 0;
|
||||
border: none;
|
||||
background: none;
|
||||
color: inherit;
|
||||
cursor: pointer;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.conversation-item:hover .conv-delete {
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.conversation-item .conv-delete:hover {
|
||||
opacity: 1;
|
||||
background: rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.conversation-item.active .conv-delete:hover {
|
||||
background: rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
||||
/* Savings Panel */
|
||||
.savings-panel {
|
||||
padding: 16px;
|
||||
border-top: 1px solid var(--color-primary);
|
||||
background: var(--ctp-crust);
|
||||
}
|
||||
|
||||
.savings-panel h3 {
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
color: var(--color-primary);
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.savings-models {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.savings-model-card {
|
||||
background: var(--ctp-surface0);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 8px;
|
||||
padding: 8px 10px;
|
||||
border-left: 3px solid var(--color-border);
|
||||
}
|
||||
|
||||
.savings-model-card.local {
|
||||
border-left-color: var(--ctp-green);
|
||||
}
|
||||
|
||||
.savings-model-card.cloud {
|
||||
border-left-color: var(--ctp-yellow);
|
||||
}
|
||||
|
||||
.savings-model-card-label {
|
||||
display: block;
|
||||
font-size: 9px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.08em;
|
||||
color: var(--color-text-secondary);
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.savings-model-card-name {
|
||||
display: block;
|
||||
font-family: monospace;
|
||||
font-size: 11px;
|
||||
color: var(--color-text);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.savings-grid {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.savings-item {
|
||||
background: var(--ctp-surface0);
|
||||
border-radius: 8px;
|
||||
padding: 10px;
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.savings-item .savings-label {
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
color: var(--color-text-secondary);
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.savings-item .savings-value {
|
||||
font-size: 18px;
|
||||
font-weight: 800;
|
||||
color: var(--color-savings);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.savings-item.calls .savings-value {
|
||||
color: var(--color-primary);
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/* Catppuccin Mocha theme tokens */
|
||||
:root {
|
||||
/* Base palette */
|
||||
--ctp-rosewater: #f5e0dc;
|
||||
--ctp-flamingo: #f2cdcd;
|
||||
--ctp-pink: #f5c2e7;
|
||||
--ctp-mauve: #cba6f7;
|
||||
--ctp-red: #f38ba8;
|
||||
--ctp-maroon: #eba0ac;
|
||||
--ctp-peach: #fab387;
|
||||
--ctp-yellow: #f9e2af;
|
||||
--ctp-green: #a6e3a1;
|
||||
--ctp-teal: #94e2d5;
|
||||
--ctp-sky: #89dceb;
|
||||
--ctp-sapphire: #74c7ec;
|
||||
--ctp-blue: #89b4fa;
|
||||
--ctp-lavender: #b4befe;
|
||||
--ctp-text: #cdd6f4;
|
||||
--ctp-subtext1: #bac2de;
|
||||
--ctp-subtext0: #a6adc8;
|
||||
--ctp-overlay2: #9399b2;
|
||||
--ctp-overlay1: #7f849c;
|
||||
--ctp-overlay0: #6c7086;
|
||||
--ctp-surface2: #585b70;
|
||||
--ctp-surface1: #45475a;
|
||||
--ctp-surface0: #313244;
|
||||
--ctp-base: #1e1e2e;
|
||||
--ctp-mantle: #181825;
|
||||
--ctp-crust: #11111b;
|
||||
|
||||
/* Semantic tokens */
|
||||
--color-primary: var(--ctp-blue);
|
||||
--color-primary-light: var(--ctp-sapphire);
|
||||
--color-primary-dark: var(--ctp-lavender);
|
||||
--color-bg: var(--ctp-base);
|
||||
--color-bg-sidebar: var(--ctp-mantle);
|
||||
--color-bg-surface: var(--ctp-surface0);
|
||||
--color-text: var(--ctp-text);
|
||||
--color-text-secondary: var(--ctp-subtext0);
|
||||
--color-border: var(--ctp-surface0);
|
||||
--color-user-bubble: var(--ctp-blue);
|
||||
--color-assistant-bubble: var(--ctp-surface0);
|
||||
--color-tool-bg: rgba(137, 180, 250, 0.08);
|
||||
--color-tool-border: var(--ctp-surface1);
|
||||
--color-tool-success: var(--ctp-green);
|
||||
--color-tool-running: var(--ctp-yellow);
|
||||
--color-tool-error: var(--ctp-red);
|
||||
--color-savings: var(--ctp-green);
|
||||
--sidebar-width: 280px;
|
||||
}
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
interface ImportMetaEnv {
|
||||
readonly VITE_API_URL: string;
|
||||
}
|
||||
|
||||
interface ImportMeta {
|
||||
readonly env: ImportMetaEnv;
|
||||
}
|
||||
@@ -80,6 +80,9 @@ class LatencyBenchmark(BaseBenchmark):
|
||||
"p95_latency": _percentile(latencies, 0.95),
|
||||
"min_latency": min(latencies),
|
||||
"max_latency": max(latencies),
|
||||
"std_latency": (
|
||||
statistics.stdev(latencies) if len(latencies) > 1 else 0.0
|
||||
),
|
||||
},
|
||||
samples=num_samples,
|
||||
errors=errors,
|
||||
|
||||
@@ -17,6 +17,7 @@ from openjarvis.cli.doctor_cmd import doctor
|
||||
from openjarvis.cli.init_cmd import init
|
||||
from openjarvis.cli.memory_cmd import memory
|
||||
from openjarvis.cli.model import model
|
||||
from openjarvis.cli.quickstart_cmd import quickstart
|
||||
from openjarvis.cli.scheduler_cmd import scheduler
|
||||
from openjarvis.cli.serve import serve
|
||||
from openjarvis.cli.skill_cmd import skill
|
||||
@@ -27,8 +28,17 @@ from openjarvis.cli.workflow_cmd import workflow
|
||||
|
||||
@click.group(help="OpenJarvis — modular AI assistant backend")
|
||||
@click.version_option(version=openjarvis.__version__, prog_name="jarvis")
|
||||
def cli() -> None:
|
||||
@click.option("--verbose", is_flag=True, default=False, help="Enable debug logging")
|
||||
@click.option("--quiet", is_flag=True, default=False, help="Suppress non-error output")
|
||||
@click.pass_context
|
||||
def cli(ctx: click.Context, verbose: bool, quiet: bool) -> None:
|
||||
"""Top-level CLI group."""
|
||||
from openjarvis.cli.log_config import setup_logging
|
||||
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["verbose"] = verbose
|
||||
ctx.obj["quiet"] = quiet
|
||||
setup_logging(verbose=verbose, quiet=quiet)
|
||||
|
||||
|
||||
cli.add_command(init, "init")
|
||||
@@ -52,6 +62,7 @@ cli.add_command(status, "status")
|
||||
cli.add_command(vault, "vault")
|
||||
cli.add_command(add, "add")
|
||||
cli.add_command(operators, "operators")
|
||||
cli.add_command(quickstart, "quickstart")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -8,6 +8,7 @@ import sys
|
||||
import click
|
||||
from rich.console import Console
|
||||
|
||||
from openjarvis.cli.hints import hint_no_engine
|
||||
from openjarvis.core.config import load_config
|
||||
from openjarvis.core.events import EventBus
|
||||
from openjarvis.core.types import Message, Role
|
||||
@@ -268,6 +269,7 @@ def ask(
|
||||
)
|
||||
except EngineConnectionError as exc:
|
||||
console.print(f"[red]Engine error:[/red] {exc}")
|
||||
console.print(hint_no_engine())
|
||||
sys.exit(1)
|
||||
|
||||
if output_json:
|
||||
@@ -321,14 +323,16 @@ def ask(
|
||||
|
||||
# Generate (InstrumentedEngine handles telemetry + energy recording)
|
||||
try:
|
||||
result = engine.generate(
|
||||
messages,
|
||||
model=model_name,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
with console.status("[bold green]Generating...[/bold green]"):
|
||||
result = engine.generate(
|
||||
messages,
|
||||
model=model_name,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
except EngineConnectionError as exc:
|
||||
console.print(f"[red]Engine error:[/red] {exc}")
|
||||
console.print(hint_no_engine())
|
||||
sys.exit(1)
|
||||
|
||||
# Output
|
||||
|
||||
@@ -11,7 +11,12 @@ from rich.panel import Panel
|
||||
from rich.rule import Rule
|
||||
from rich.table import Table
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from openjarvis.core.config import load_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openjarvis.bench._stubs import BenchmarkResult
|
||||
from openjarvis.engine import get_engine
|
||||
|
||||
_BANNER = r"""
|
||||
@@ -38,6 +43,90 @@ def _section(console: Console, title: str) -> None:
|
||||
console.print(Rule(title, style="bright_blue"))
|
||||
|
||||
|
||||
# -- Stats-aware rendering ----------------------------------------------------
|
||||
|
||||
_STATS_PREFIXES = {"mean_", "p50_", "p95_", "min_", "max_", "std_"}
|
||||
|
||||
|
||||
def _detect_stat_groups(metrics: dict[str, float]) -> dict[str, dict[str, float]]:
|
||||
"""Detect metrics following the stats pattern (mean_X, p50_X, ...).
|
||||
|
||||
Returns ``{metric_base: {prefix: value}}`` for grouped metrics.
|
||||
"""
|
||||
groups: dict[str, dict[str, float]] = {}
|
||||
for key, val in metrics.items():
|
||||
for pfx in _STATS_PREFIXES:
|
||||
if key.startswith(pfx):
|
||||
base = key[len(pfx):]
|
||||
groups.setdefault(base, {})[pfx.rstrip("_")] = val
|
||||
break
|
||||
return groups
|
||||
|
||||
|
||||
def _render_stats_table(console: Console, result: BenchmarkResult) -> None:
|
||||
"""Render benchmark result as a stats table when stats keys are present."""
|
||||
groups = _detect_stat_groups(result.metrics)
|
||||
|
||||
# Determine which keys are consumed by stat groups
|
||||
consumed: set[str] = set()
|
||||
for base, prefixes in groups.items():
|
||||
for pfx in prefixes:
|
||||
consumed.add(f"{pfx}_{base}")
|
||||
|
||||
# Stat groups → multi-column table
|
||||
if groups:
|
||||
table = Table(
|
||||
title=(
|
||||
f"[bold]{result.benchmark_name}[/bold]"
|
||||
f" ({result.samples} samples, {result.errors} errors)"
|
||||
),
|
||||
show_header=True,
|
||||
header_style="bold bright_white",
|
||||
border_style="bright_blue",
|
||||
title_style="bold cyan",
|
||||
)
|
||||
table.add_column("Metric", style="cyan", no_wrap=True)
|
||||
table.add_column("Avg", justify="right")
|
||||
table.add_column("Median", justify="right")
|
||||
table.add_column("Min", justify="right")
|
||||
table.add_column("Max", justify="right")
|
||||
table.add_column("Std", justify="right")
|
||||
table.add_column("P95", justify="right")
|
||||
|
||||
for base, vals in sorted(groups.items()):
|
||||
table.add_row(
|
||||
base,
|
||||
f"{vals.get('mean', 0):.4f}",
|
||||
f"{vals.get('p50', 0):.4f}",
|
||||
f"{vals.get('min', 0):.4f}",
|
||||
f"{vals.get('max', 0):.4f}",
|
||||
f"{vals.get('std', 0):.4f}",
|
||||
f"{vals.get('p95', 0):.4f}",
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
# Remaining non-stats metrics → simple key-value table
|
||||
remaining = {k: v for k, v in result.metrics.items() if k not in consumed}
|
||||
if remaining or result.total_energy_joules > 0:
|
||||
kv_table = Table(
|
||||
show_header=True,
|
||||
header_style="bold bright_white",
|
||||
border_style="bright_blue",
|
||||
)
|
||||
kv_table.add_column("Metric", style="cyan", no_wrap=True)
|
||||
kv_table.add_column("Value", justify="right", style="green")
|
||||
for k, v in remaining.items():
|
||||
kv_table.add_row(k, f"{v:.4f}")
|
||||
if result.total_energy_joules > 0:
|
||||
kv_table.add_row("Total Energy (J)", f"{result.total_energy_joules:.4f}")
|
||||
kv_table.add_row("Energy Method", str(result.energy_method))
|
||||
if result.energy_per_token_joules > 0:
|
||||
kv_table.add_row(
|
||||
"Energy/Token (J)", f"{result.energy_per_token_joules:.6f}",
|
||||
)
|
||||
console.print(kv_table)
|
||||
|
||||
|
||||
@click.group()
|
||||
def bench() -> None:
|
||||
"""Run inference benchmarks."""
|
||||
@@ -175,30 +264,7 @@ def run(
|
||||
# Pretty-print results as Rich tables
|
||||
_section(console, "Results")
|
||||
for r in results:
|
||||
table = Table(
|
||||
title=(
|
||||
f"[bold]{r.benchmark_name}[/bold]"
|
||||
f" ({r.samples} samples, {r.errors} errors)"
|
||||
),
|
||||
show_header=True,
|
||||
header_style="bold bright_white",
|
||||
border_style="bright_blue",
|
||||
title_style="bold cyan",
|
||||
)
|
||||
table.add_column("Metric", style="cyan", no_wrap=True)
|
||||
table.add_column("Value", justify="right", style="green")
|
||||
|
||||
for k, v in r.metrics.items():
|
||||
table.add_row(k, f"{v:.4f}")
|
||||
if r.total_energy_joules > 0:
|
||||
table.add_row("Total Energy (J)", f"{r.total_energy_joules:.4f}")
|
||||
table.add_row("Energy Method", str(r.energy_method))
|
||||
if r.energy_per_token_joules > 0:
|
||||
table.add_row(
|
||||
"Energy/Token (J)", f"{r.energy_per_token_joules:.6f}",
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
_render_stats_table(console, r)
|
||||
|
||||
# Cleanup energy monitor
|
||||
if energy_monitor is not None:
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Rich-formatted error hints for common CLI failure modes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def hint_no_config() -> str:
|
||||
"""Return a suggestion when no config file is found."""
|
||||
return (
|
||||
"[yellow]Hint:[/yellow] No config file found.\n"
|
||||
" Run [bold]jarvis init[/bold] to detect hardware and generate "
|
||||
"[cyan]~/.openjarvis/config.toml[/cyan].\n"
|
||||
" Or run [bold]jarvis quickstart[/bold] for a guided setup."
|
||||
)
|
||||
|
||||
|
||||
def hint_no_engine(engine_name: Optional[str] = None) -> str:
|
||||
"""Return a suggestion when the inference engine is unreachable."""
|
||||
name = engine_name or "ollama"
|
||||
return (
|
||||
f"[yellow]Hint:[/yellow] Engine '{name}' is not reachable.\n"
|
||||
f" Make sure the {name} server is running.\n"
|
||||
" Run [bold]jarvis doctor[/bold] to check all engines.\n"
|
||||
" Run [bold]jarvis quickstart[/bold] for guided setup."
|
||||
)
|
||||
|
||||
|
||||
def hint_no_model(model_name: Optional[str] = None) -> str:
|
||||
"""Return a suggestion when no model is available."""
|
||||
if model_name:
|
||||
return (
|
||||
f"[yellow]Hint:[/yellow] Model '{model_name}' not found.\n"
|
||||
f" Try: [bold]ollama pull {model_name}[/bold]\n"
|
||||
" Run [bold]jarvis model list[/bold] to see available models."
|
||||
)
|
||||
return (
|
||||
"[yellow]Hint:[/yellow] No models available.\n"
|
||||
" Pull a model first: [bold]ollama pull qwen3:8b[/bold]\n"
|
||||
" Run [bold]jarvis model list[/bold] to see available models."
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Global logging configuration for the OpenJarvis CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
|
||||
def setup_logging(
|
||||
verbose: bool = False,
|
||||
quiet: bool = False,
|
||||
log_file: Optional[Union[str, Path]] = None,
|
||||
) -> logging.Logger:
|
||||
"""Configure the ``openjarvis`` logger.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
verbose:
|
||||
Set log level to DEBUG.
|
||||
quiet:
|
||||
Set log level to ERROR (overrides verbose if both set).
|
||||
log_file:
|
||||
Path for a rotating file handler. When *verbose* is ``True``
|
||||
and no *log_file* is given, defaults to
|
||||
``~/.openjarvis/cli.log``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
The configured ``openjarvis`` logger.
|
||||
"""
|
||||
logger = logging.getLogger("openjarvis")
|
||||
|
||||
# Clear existing handlers to avoid duplication across calls
|
||||
logger.handlers.clear()
|
||||
|
||||
if quiet:
|
||||
level = logging.ERROR
|
||||
elif verbose:
|
||||
level = logging.DEBUG
|
||||
else:
|
||||
level = logging.WARNING
|
||||
|
||||
logger.setLevel(level)
|
||||
|
||||
# Console handler
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(level)
|
||||
fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
|
||||
console_handler.setFormatter(fmt)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# File handler (verbose or explicit path)
|
||||
if verbose or log_file is not None:
|
||||
if log_file is None:
|
||||
log_dir = Path.home() / ".openjarvis"
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_file = log_dir / "cli.log"
|
||||
file_handler = RotatingFileHandler(
|
||||
str(log_file), maxBytes=5 * 1024 * 1024, backupCount=3,
|
||||
)
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
file_fmt = logging.Formatter(
|
||||
"%(asctime)s %(levelname)s %(name)s: %(message)s"
|
||||
)
|
||||
file_handler.setFormatter(file_fmt)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
return logger
|
||||
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.progress import track
|
||||
from rich.table import Table
|
||||
|
||||
from openjarvis.core.config import load_config
|
||||
@@ -82,7 +83,7 @@ def index(
|
||||
|
||||
mem = _get_backend(backend)
|
||||
try:
|
||||
for chunk in chunks:
|
||||
for chunk in track(chunks, description="Storing chunks...", console=console):
|
||||
mem.store(
|
||||
chunk.content,
|
||||
source=chunk.source,
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""``jarvis quickstart`` — guided 5-step setup for new users."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
|
||||
from openjarvis.core.config import (
|
||||
DEFAULT_CONFIG_DIR,
|
||||
DEFAULT_CONFIG_PATH,
|
||||
detect_hardware,
|
||||
generate_default_toml,
|
||||
recommend_engine,
|
||||
)
|
||||
|
||||
|
||||
def _check_engine_health(engine_key: str) -> bool:
|
||||
"""Return True if the recommended engine is reachable."""
|
||||
try:
|
||||
from openjarvis.core.config import load_config
|
||||
from openjarvis.core.registry import EngineRegistry
|
||||
from openjarvis.engine import _discovery
|
||||
|
||||
import openjarvis.engine # noqa: F401 — trigger registration
|
||||
|
||||
config = load_config()
|
||||
if engine_key not in EngineRegistry.keys():
|
||||
return False
|
||||
engine = _discovery._make_engine(engine_key, config)
|
||||
return engine.health()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _check_model_available(engine_key: str) -> bool:
|
||||
"""Return True if at least one model is available on the engine."""
|
||||
try:
|
||||
from openjarvis.core.config import load_config
|
||||
from openjarvis.core.registry import EngineRegistry
|
||||
from openjarvis.engine import _discovery
|
||||
|
||||
config = load_config()
|
||||
if engine_key not in EngineRegistry.keys():
|
||||
return False
|
||||
engine = _discovery._make_engine(engine_key, config)
|
||||
return bool(engine.list_models())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _test_query(engine_key: str) -> str:
|
||||
"""Run a quick test query and return the response text."""
|
||||
try:
|
||||
from openjarvis import Jarvis
|
||||
|
||||
j = Jarvis(engine_key=engine_key)
|
||||
response = j.ask("Say hello in one sentence.")
|
||||
j.close()
|
||||
return response
|
||||
except Exception as exc:
|
||||
return f"(query failed: {exc})"
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--force", is_flag=True, help="Redo all steps even if already done.")
|
||||
def quickstart(force: bool) -> None:
|
||||
"""Guided 5-step setup for new users."""
|
||||
console = Console()
|
||||
|
||||
# Step 1: Detect hardware
|
||||
console.print("[bold cyan][1/5][/bold cyan] Detecting hardware...")
|
||||
hw = detect_hardware()
|
||||
console.print(f" Platform : {hw.platform}")
|
||||
console.print(f" CPU : {hw.cpu_brand} ({hw.cpu_count} cores)")
|
||||
console.print(f" RAM : {hw.ram_gb} GB")
|
||||
if hw.gpu:
|
||||
console.print(
|
||||
f" GPU : {hw.gpu.name} ({hw.gpu.vram_gb} GB VRAM, x{hw.gpu.count})"
|
||||
)
|
||||
else:
|
||||
console.print(" GPU : none detected")
|
||||
|
||||
engine_key = recommend_engine(hw)
|
||||
|
||||
# Step 2: Write config
|
||||
console.print()
|
||||
console.print("[bold cyan][2/5][/bold cyan] Writing config...")
|
||||
if DEFAULT_CONFIG_PATH.exists() and not force:
|
||||
console.print(f" [dim]Config already exists at {DEFAULT_CONFIG_PATH} (skip)[/dim]")
|
||||
else:
|
||||
toml_content = generate_default_toml(hw)
|
||||
DEFAULT_CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
DEFAULT_CONFIG_PATH.write_text(toml_content)
|
||||
console.print(f" [green]Config written to {DEFAULT_CONFIG_PATH}[/green]")
|
||||
|
||||
# Step 3: Check engine
|
||||
console.print()
|
||||
console.print(f"[bold cyan][3/5][/bold cyan] Checking engine ({engine_key})...")
|
||||
if not _check_engine_health(engine_key):
|
||||
console.print(f" [red bold]Engine '{engine_key}' is not reachable.[/red bold]")
|
||||
console.print()
|
||||
console.print(f" Start the {engine_key} server and try again.")
|
||||
console.print(f" Run [bold]jarvis doctor[/bold] for detailed diagnostics.")
|
||||
raise SystemExit(1)
|
||||
console.print(f" [green]Engine '{engine_key}' is healthy.[/green]")
|
||||
|
||||
# Step 4: Verify model
|
||||
console.print()
|
||||
console.print("[bold cyan][4/5][/bold cyan] Checking for available models...")
|
||||
if not _check_model_available(engine_key):
|
||||
console.print(" [yellow]No models found.[/yellow]")
|
||||
console.print(" Pull a model first (e.g. [bold]ollama pull qwen3:8b[/bold]).")
|
||||
raise SystemExit(1)
|
||||
console.print(" [green]Models available.[/green]")
|
||||
|
||||
# Step 5: Test query
|
||||
console.print()
|
||||
console.print("[bold cyan][5/5][/bold cyan] Running test query...")
|
||||
response = _test_query(engine_key)
|
||||
console.print(f" [green]Response:[/green] {response[:200]}")
|
||||
|
||||
console.print()
|
||||
console.print("[bold green]Setup complete![/bold green] Try: [bold]jarvis ask \"Hello\"[/bold]")
|
||||
@@ -162,6 +162,7 @@ class TelemetryRecord:
|
||||
cpu_energy_joules: float = 0.0
|
||||
gpu_energy_joules: float = 0.0
|
||||
dram_energy_joules: float = 0.0
|
||||
tokens_per_joule: float = 0.0
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ class ModelStats:
|
||||
total_energy_joules: float = 0.0
|
||||
avg_gpu_utilization_pct: float = 0.0
|
||||
avg_throughput_tok_per_sec: float = 0.0
|
||||
avg_tokens_per_joule: float = 0.0
|
||||
avg_energy_per_output_token_joules: float = 0.0
|
||||
avg_throughput_per_watt: float = 0.0
|
||||
total_prefill_energy_joules: float = 0.0
|
||||
@@ -47,6 +48,7 @@ class EngineStats:
|
||||
total_energy_joules: float = 0.0
|
||||
avg_gpu_utilization_pct: float = 0.0
|
||||
avg_throughput_tok_per_sec: float = 0.0
|
||||
avg_tokens_per_joule: float = 0.0
|
||||
avg_energy_per_output_token_joules: float = 0.0
|
||||
avg_throughput_per_watt: float = 0.0
|
||||
total_prefill_energy_joules: float = 0.0
|
||||
@@ -122,10 +124,15 @@ class TelemetryAggregator:
|
||||
|
||||
# Build optional columns for new fields (graceful on old DBs)
|
||||
extra_cols = ""
|
||||
has_tpj = self._safe_col("tokens_per_joule")
|
||||
has_derived = self._safe_col("energy_per_output_token_joules")
|
||||
has_phase = self._safe_col("prefill_energy_joules")
|
||||
has_itl = self._safe_col("mean_itl_ms")
|
||||
|
||||
if has_tpj:
|
||||
extra_cols += (
|
||||
", AVG(tokens_per_joule) AS avg_tokens_per_joule"
|
||||
)
|
||||
if has_derived:
|
||||
extra_cols += (
|
||||
", AVG(energy_per_output_token_joules)"
|
||||
@@ -178,6 +185,8 @@ class TelemetryAggregator:
|
||||
avg_gpu_utilization_pct=r["avg_gpu_utilization_pct"] or 0.0,
|
||||
avg_throughput_tok_per_sec=r["avg_throughput_tok_per_sec"] or 0.0,
|
||||
)
|
||||
if has_tpj:
|
||||
ms.avg_tokens_per_joule = r["avg_tokens_per_joule"] or 0.0
|
||||
if has_derived:
|
||||
ms.avg_energy_per_output_token_joules = (
|
||||
r["avg_energy_per_output_token_joules"] or 0.0
|
||||
@@ -206,10 +215,15 @@ class TelemetryAggregator:
|
||||
where, params = self._time_filter(since, until)
|
||||
|
||||
extra_cols = ""
|
||||
has_tpj = self._safe_col("tokens_per_joule")
|
||||
has_derived = self._safe_col("energy_per_output_token_joules")
|
||||
has_phase = self._safe_col("prefill_energy_joules")
|
||||
has_itl = self._safe_col("mean_itl_ms")
|
||||
|
||||
if has_tpj:
|
||||
extra_cols += (
|
||||
", AVG(tokens_per_joule) AS avg_tokens_per_joule"
|
||||
)
|
||||
if has_derived:
|
||||
extra_cols += (
|
||||
", AVG(energy_per_output_token_joules)"
|
||||
@@ -258,6 +272,8 @@ class TelemetryAggregator:
|
||||
avg_gpu_utilization_pct=r["avg_gpu_utilization_pct"] or 0.0,
|
||||
avg_throughput_tok_per_sec=r["avg_throughput_tok_per_sec"] or 0.0,
|
||||
)
|
||||
if has_tpj:
|
||||
es.avg_tokens_per_joule = r["avg_tokens_per_joule"] or 0.0
|
||||
if has_derived:
|
||||
es.avg_energy_per_output_token_joules = (
|
||||
r["avg_energy_per_output_token_joules"] or 0.0
|
||||
|
||||
@@ -173,6 +173,12 @@ class InstrumentedEngine(InferenceEngine):
|
||||
if completion_tokens > 0 and decode_latency > 0 else 0.0
|
||||
)
|
||||
|
||||
# --- Tier 4: Per-inference efficiency ---
|
||||
tokens_per_joule = (
|
||||
completion_tokens / energy_joules
|
||||
if energy_joules > 0 and completion_tokens > 0 else 0.0
|
||||
)
|
||||
|
||||
engine_id = getattr(self._inner, "engine_id", "unknown")
|
||||
|
||||
record = TelemetryRecord(
|
||||
@@ -201,6 +207,7 @@ class InstrumentedEngine(InferenceEngine):
|
||||
cpu_energy_joules=cpu_energy_joules,
|
||||
gpu_energy_joules=gpu_energy_joules,
|
||||
dram_energy_joules=dram_energy_joules,
|
||||
tokens_per_joule=tokens_per_joule,
|
||||
)
|
||||
|
||||
event_data = {
|
||||
@@ -364,6 +371,12 @@ class InstrumentedEngine(InferenceEngine):
|
||||
prefill_energy = energy_joules * prefill_frac
|
||||
decode_energy = energy_joules * (1.0 - prefill_frac)
|
||||
|
||||
# Per-inference efficiency
|
||||
tokens_per_joule = (
|
||||
token_count / energy_joules
|
||||
if energy_joules > 0 and token_count > 0 else 0.0
|
||||
)
|
||||
|
||||
engine_id = getattr(self._inner, "engine_id", "unknown")
|
||||
|
||||
record = TelemetryRecord(
|
||||
@@ -397,6 +410,7 @@ class InstrumentedEngine(InferenceEngine):
|
||||
cpu_energy_joules=cpu_energy_joules,
|
||||
gpu_energy_joules=gpu_energy_joules,
|
||||
dram_energy_joules=dram_energy_joules,
|
||||
tokens_per_joule=tokens_per_joule,
|
||||
)
|
||||
|
||||
event_data = {
|
||||
|
||||
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS telemetry (
|
||||
cpu_energy_joules REAL NOT NULL DEFAULT 0.0,
|
||||
gpu_energy_joules REAL NOT NULL DEFAULT 0.0,
|
||||
dram_energy_joules REAL NOT NULL DEFAULT 0.0,
|
||||
tokens_per_joule REAL NOT NULL DEFAULT 0.0,
|
||||
energy_per_output_token_joules REAL NOT NULL DEFAULT 0.0,
|
||||
throughput_per_watt REAL NOT NULL DEFAULT 0.0,
|
||||
prefill_energy_joules REAL NOT NULL DEFAULT 0.0,
|
||||
@@ -61,6 +62,7 @@ INSERT INTO telemetry (
|
||||
throughput_tok_per_sec, prefill_latency_seconds, decode_latency_seconds,
|
||||
energy_method, energy_vendor, batch_id, is_warmup,
|
||||
cpu_energy_joules, gpu_energy_joules, dram_energy_joules,
|
||||
tokens_per_joule,
|
||||
energy_per_output_token_joules, throughput_per_watt,
|
||||
prefill_energy_joules, decode_energy_joules,
|
||||
mean_itl_ms, median_itl_ms, p90_itl_ms, p95_itl_ms, p99_itl_ms, std_itl_ms,
|
||||
@@ -70,7 +72,7 @@ INSERT INTO telemetry (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -88,6 +90,7 @@ _MIGRATE_COLUMNS = [
|
||||
("cpu_energy_joules", "REAL NOT NULL DEFAULT 0.0"),
|
||||
("gpu_energy_joules", "REAL NOT NULL DEFAULT 0.0"),
|
||||
("dram_energy_joules", "REAL NOT NULL DEFAULT 0.0"),
|
||||
("tokens_per_joule", "REAL NOT NULL DEFAULT 0.0"),
|
||||
("energy_per_output_token_joules", "REAL NOT NULL DEFAULT 0.0"),
|
||||
("throughput_per_watt", "REAL NOT NULL DEFAULT 0.0"),
|
||||
("prefill_energy_joules", "REAL NOT NULL DEFAULT 0.0"),
|
||||
@@ -153,6 +156,7 @@ class TelemetryStore:
|
||||
rec.cpu_energy_joules,
|
||||
rec.gpu_energy_joules,
|
||||
rec.dram_energy_joules,
|
||||
rec.tokens_per_joule,
|
||||
rec.energy_per_output_token_joules,
|
||||
rec.throughput_per_watt,
|
||||
rec.prefill_energy_joules,
|
||||
|
||||
@@ -6,6 +6,7 @@ routing policies, tool selection strategies, and memory configuration.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import statistics as stats_mod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -36,6 +37,29 @@ class ToolStats:
|
||||
success_rate: float = 0.0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class StepTypeStats:
|
||||
"""Aggregated statistics for a specific step type across traces."""
|
||||
|
||||
count: int = 0
|
||||
avg_duration: float = 0.0
|
||||
median_duration: float = 0.0
|
||||
min_duration: float = 0.0
|
||||
max_duration: float = 0.0
|
||||
std_duration: float = 0.0
|
||||
total_energy: float = 0.0
|
||||
avg_input_tokens: float = 0.0
|
||||
median_input_tokens: float = 0.0
|
||||
min_input_tokens: float = 0.0
|
||||
max_input_tokens: float = 0.0
|
||||
std_input_tokens: float = 0.0
|
||||
avg_output_tokens: float = 0.0
|
||||
median_output_tokens: float = 0.0
|
||||
min_output_tokens: float = 0.0
|
||||
max_output_tokens: float = 0.0
|
||||
std_output_tokens: float = 0.0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TraceSummary:
|
||||
"""Overall summary statistics across all traces."""
|
||||
@@ -47,6 +71,9 @@ class TraceSummary:
|
||||
avg_tokens: float = 0.0
|
||||
success_rate: float = 0.0
|
||||
step_type_distribution: Dict[str, int] = field(default_factory=dict)
|
||||
total_energy_joules: float = 0.0
|
||||
total_generate_energy_joules: float = 0.0
|
||||
step_type_stats: Dict[str, StepTypeStats] = field(default_factory=dict)
|
||||
|
||||
|
||||
class TraceAnalyzer:
|
||||
@@ -75,11 +102,59 @@ class TraceAnalyzer:
|
||||
successes = [t for t in evaluated if t.outcome == "success"]
|
||||
|
||||
step_dist: Dict[str, int] = {}
|
||||
total_energy = 0.0
|
||||
generate_energy = 0.0
|
||||
step_data: Dict[str, Dict[str, list]] = {}
|
||||
|
||||
for t in traces:
|
||||
for s in t.steps:
|
||||
key = _step_type_str(s)
|
||||
step_dist[key] = step_dist.get(key, 0) + 1
|
||||
|
||||
energy = s.metadata.get("energy_joules", 0.0)
|
||||
total_energy += energy
|
||||
if key == "generate":
|
||||
generate_energy += energy
|
||||
|
||||
if key not in step_data:
|
||||
step_data[key] = {
|
||||
"durations": [], "energies": [],
|
||||
"input_tokens": [], "output_tokens": [],
|
||||
}
|
||||
step_data[key]["durations"].append(s.duration_seconds)
|
||||
step_data[key]["energies"].append(energy)
|
||||
step_data[key]["input_tokens"].append(
|
||||
s.output.get("prompt_tokens", 0)
|
||||
)
|
||||
step_data[key]["output_tokens"].append(
|
||||
s.output.get("completion_tokens", 0)
|
||||
)
|
||||
|
||||
sts_map: Dict[str, StepTypeStats] = {}
|
||||
for key, data in step_data.items():
|
||||
durations = data["durations"]
|
||||
in_tok = [float(x) for x in data["input_tokens"]]
|
||||
out_tok = [float(x) for x in data["output_tokens"]]
|
||||
sts_map[key] = StepTypeStats(
|
||||
count=len(durations),
|
||||
avg_duration=_avg(durations),
|
||||
median_duration=stats_mod.median(durations) if durations else 0.0,
|
||||
min_duration=min(durations) if durations else 0.0,
|
||||
max_duration=max(durations) if durations else 0.0,
|
||||
std_duration=stats_mod.stdev(durations) if len(durations) > 1 else 0.0,
|
||||
total_energy=sum(data["energies"]),
|
||||
avg_input_tokens=_avg(in_tok),
|
||||
median_input_tokens=stats_mod.median(in_tok) if in_tok else 0.0,
|
||||
min_input_tokens=min(in_tok) if in_tok else 0.0,
|
||||
max_input_tokens=max(in_tok) if in_tok else 0.0,
|
||||
std_input_tokens=stats_mod.stdev(in_tok) if len(in_tok) > 1 else 0.0,
|
||||
avg_output_tokens=_avg(out_tok),
|
||||
median_output_tokens=stats_mod.median(out_tok) if out_tok else 0.0,
|
||||
min_output_tokens=min(out_tok) if out_tok else 0.0,
|
||||
max_output_tokens=max(out_tok) if out_tok else 0.0,
|
||||
std_output_tokens=stats_mod.stdev(out_tok) if len(out_tok) > 1 else 0.0,
|
||||
)
|
||||
|
||||
return TraceSummary(
|
||||
total_traces=len(traces),
|
||||
total_steps=total_steps,
|
||||
@@ -88,6 +163,9 @@ class TraceAnalyzer:
|
||||
avg_tokens=_avg([float(t.total_tokens) for t in traces]),
|
||||
success_rate=len(successes) / len(evaluated) if evaluated else 0.0,
|
||||
step_type_distribution=step_dist,
|
||||
total_energy_joules=total_energy,
|
||||
total_generate_energy_joules=generate_energy,
|
||||
step_type_stats=sts_map,
|
||||
)
|
||||
|
||||
def per_route_stats(
|
||||
@@ -243,4 +321,4 @@ def _trace_to_dict(trace: Trace) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["RouteStats", "ToolStats", "TraceAnalyzer", "TraceSummary"]
|
||||
__all__ = ["RouteStats", "StepTypeStats", "ToolStats", "TraceAnalyzer", "TraceSummary"]
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Tests for benchmark stats output and Rich table rendering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import StringIO
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from openjarvis.bench._stubs import BenchmarkResult
|
||||
from openjarvis.cli.bench_cmd import _render_stats_table
|
||||
|
||||
|
||||
class TestLatencyBenchmarkStats:
|
||||
def test_latency_includes_std(self):
|
||||
"""LatencyBenchmark should include std_latency in metrics."""
|
||||
from openjarvis.bench.latency import LatencyBenchmark
|
||||
|
||||
bench = LatencyBenchmark()
|
||||
engine = MagicMock()
|
||||
engine.engine_id = "mock"
|
||||
engine.generate.return_value = {"content": "hi", "usage": {}}
|
||||
result = bench.run(engine, model="test", num_samples=5)
|
||||
assert "std_latency" in result.metrics
|
||||
|
||||
|
||||
class TestRenderStatsTable:
|
||||
def test_renders_stats_columns(self):
|
||||
"""_render_stats_table should produce Avg/Median/Min/Max/Std columns."""
|
||||
result = BenchmarkResult(
|
||||
benchmark_name="latency",
|
||||
model="test",
|
||||
engine="mock",
|
||||
metrics={
|
||||
"mean_latency": 1.5,
|
||||
"p50_latency": 1.4,
|
||||
"p95_latency": 2.1,
|
||||
"min_latency": 0.8,
|
||||
"max_latency": 3.0,
|
||||
"std_latency": 0.5,
|
||||
},
|
||||
samples=10,
|
||||
errors=0,
|
||||
)
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
_render_stats_table(console, result)
|
||||
output = console.file.getvalue()
|
||||
assert "Avg" in output
|
||||
assert "Median" in output
|
||||
assert "Min" in output
|
||||
assert "Max" in output
|
||||
|
||||
def test_falls_back_for_non_stats_metrics(self):
|
||||
"""Non-stats metrics should still render as simple key-value pairs."""
|
||||
result = BenchmarkResult(
|
||||
benchmark_name="throughput",
|
||||
model="test",
|
||||
engine="mock",
|
||||
metrics={
|
||||
"tokens_per_second": 42.5,
|
||||
"total_time_seconds": 10.0,
|
||||
},
|
||||
samples=5,
|
||||
errors=0,
|
||||
)
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
_render_stats_table(console, result)
|
||||
output = console.file.getvalue()
|
||||
assert "tokens_per_second" in output or "42.5" in output
|
||||
@@ -57,7 +57,7 @@ class TestLatencyBenchmark:
|
||||
result = b.run(engine, "test-model", num_samples=3)
|
||||
expected_keys = {
|
||||
"mean_latency", "p50_latency", "p95_latency",
|
||||
"min_latency", "max_latency",
|
||||
"min_latency", "max_latency", "std_latency",
|
||||
}
|
||||
assert set(result.metrics.keys()) == expected_keys
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Tests for CLI error hint functions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openjarvis.cli.hints import (
|
||||
hint_no_config,
|
||||
hint_no_engine,
|
||||
hint_no_model,
|
||||
)
|
||||
|
||||
|
||||
class TestHintFunctions:
|
||||
def test_hint_no_config_returns_string(self):
|
||||
msg = hint_no_config()
|
||||
assert isinstance(msg, str)
|
||||
assert len(msg) > 0
|
||||
assert "init" in msg.lower() or "config" in msg.lower()
|
||||
|
||||
def test_hint_no_engine_returns_string(self):
|
||||
msg = hint_no_engine()
|
||||
assert isinstance(msg, str)
|
||||
assert len(msg) > 0
|
||||
assert "engine" in msg.lower() or "ollama" in msg.lower()
|
||||
|
||||
def test_hint_no_engine_with_name(self):
|
||||
msg = hint_no_engine("vllm")
|
||||
assert "vllm" in msg.lower()
|
||||
|
||||
def test_hint_no_model_returns_string(self):
|
||||
msg = hint_no_model()
|
||||
assert isinstance(msg, str)
|
||||
assert len(msg) > 0
|
||||
assert "model" in msg.lower() or "pull" in msg.lower()
|
||||
|
||||
def test_hint_no_model_with_name(self):
|
||||
msg = hint_no_model("qwen3:8b")
|
||||
assert "qwen3:8b" in msg
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Tests for CLI log configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from openjarvis.cli.log_config import setup_logging
|
||||
|
||||
|
||||
class TestSetupLogging:
|
||||
def test_default_level_is_warning(self):
|
||||
logger = setup_logging(verbose=False, quiet=False)
|
||||
assert logger.level == logging.WARNING
|
||||
|
||||
def test_verbose_sets_debug(self):
|
||||
logger = setup_logging(verbose=True, quiet=False)
|
||||
assert logger.level == logging.DEBUG
|
||||
|
||||
def test_quiet_sets_error(self):
|
||||
logger = setup_logging(verbose=False, quiet=True)
|
||||
assert logger.level == logging.ERROR
|
||||
|
||||
def test_returns_logger(self):
|
||||
logger = setup_logging(verbose=False, quiet=False)
|
||||
assert isinstance(logger, logging.Logger)
|
||||
assert logger.name == "openjarvis"
|
||||
|
||||
def test_log_file_handler_on_verbose(self, tmp_path):
|
||||
log_file = tmp_path / "cli.log"
|
||||
logger = setup_logging(verbose=True, quiet=False, log_file=log_file)
|
||||
# Should have at least one file handler
|
||||
file_handlers = [
|
||||
h for h in logger.handlers
|
||||
if hasattr(h, "baseFilename")
|
||||
]
|
||||
assert len(file_handlers) >= 1
|
||||
# Clean up
|
||||
for h in logger.handlers[:]:
|
||||
logger.removeHandler(h)
|
||||
h.close()
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Tests for ``jarvis quickstart`` command."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from openjarvis.cli import cli
|
||||
|
||||
|
||||
class TestQuickstartCommand:
|
||||
def test_registered(self):
|
||||
"""quickstart should be a registered CLI command."""
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["quickstart", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "quickstart" in result.output.lower() or "--help" in result.output
|
||||
|
||||
def test_happy_path(self, tmp_path):
|
||||
"""Full quickstart succeeds when hardware detected and engine healthy."""
|
||||
config_path = tmp_path / "config.toml"
|
||||
hw = MagicMock()
|
||||
hw.platform = "linux"
|
||||
hw.cpu_brand = "Test CPU"
|
||||
hw.cpu_count = 8
|
||||
hw.ram_gb = 32
|
||||
hw.gpu = MagicMock(name="Test GPU", vram_gb=24, count=1, vendor="nvidia")
|
||||
|
||||
with (
|
||||
patch("openjarvis.cli.quickstart_cmd.detect_hardware", return_value=hw),
|
||||
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
|
||||
patch("openjarvis.cli.quickstart_cmd.generate_default_toml", return_value="[engine]\n"),
|
||||
patch("openjarvis.cli.quickstart_cmd.recommend_engine", return_value="ollama"),
|
||||
patch("openjarvis.cli.quickstart_cmd._check_engine_health", return_value=True),
|
||||
patch("openjarvis.cli.quickstart_cmd._check_model_available", return_value=True),
|
||||
patch("openjarvis.cli.quickstart_cmd._test_query", return_value="Hello!"),
|
||||
):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["quickstart"])
|
||||
assert result.exit_code == 0
|
||||
assert "1/5" in result.output
|
||||
assert "5/5" in result.output
|
||||
|
||||
def test_skips_config_if_exists(self, tmp_path):
|
||||
"""Config step is skipped when config already exists."""
|
||||
config_path = tmp_path / "config.toml"
|
||||
config_path.write_text("[engine]\n")
|
||||
hw = MagicMock()
|
||||
hw.platform = "linux"
|
||||
hw.cpu_brand = "Test CPU"
|
||||
hw.cpu_count = 8
|
||||
hw.ram_gb = 32
|
||||
hw.gpu = None
|
||||
|
||||
with (
|
||||
patch("openjarvis.cli.quickstart_cmd.detect_hardware", return_value=hw),
|
||||
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
|
||||
patch("openjarvis.cli.quickstart_cmd.generate_default_toml", return_value="[engine]\n"),
|
||||
patch("openjarvis.cli.quickstart_cmd.recommend_engine", return_value="ollama"),
|
||||
patch("openjarvis.cli.quickstart_cmd._check_engine_health", return_value=True),
|
||||
patch("openjarvis.cli.quickstart_cmd._check_model_available", return_value=True),
|
||||
patch("openjarvis.cli.quickstart_cmd._test_query", return_value="Hello!"),
|
||||
):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["quickstart"])
|
||||
assert result.exit_code == 0
|
||||
assert "already exists" in result.output.lower() or "skip" in result.output.lower()
|
||||
|
||||
def test_force_regenerates_config(self, tmp_path):
|
||||
"""--force should regenerate config even if it exists."""
|
||||
config_path = tmp_path / "config.toml"
|
||||
config_path.write_text("[old]\n")
|
||||
hw = MagicMock()
|
||||
hw.platform = "linux"
|
||||
hw.cpu_brand = "Test CPU"
|
||||
hw.cpu_count = 8
|
||||
hw.ram_gb = 32
|
||||
hw.gpu = None
|
||||
|
||||
with (
|
||||
patch("openjarvis.cli.quickstart_cmd.detect_hardware", return_value=hw),
|
||||
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
|
||||
patch("openjarvis.cli.quickstart_cmd.generate_default_toml", return_value="[engine]\nnew = true\n"),
|
||||
patch("openjarvis.cli.quickstart_cmd.recommend_engine", return_value="ollama"),
|
||||
patch("openjarvis.cli.quickstart_cmd._check_engine_health", return_value=True),
|
||||
patch("openjarvis.cli.quickstart_cmd._check_model_available", return_value=True),
|
||||
patch("openjarvis.cli.quickstart_cmd._test_query", return_value="Hello!"),
|
||||
):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["quickstart", "--force"])
|
||||
assert result.exit_code == 0
|
||||
assert "new = true" in config_path.read_text()
|
||||
|
||||
def test_engine_not_found(self, tmp_path):
|
||||
"""Helpful message when engine is unreachable."""
|
||||
config_path = tmp_path / "config.toml"
|
||||
hw = MagicMock()
|
||||
hw.platform = "linux"
|
||||
hw.cpu_brand = "Test CPU"
|
||||
hw.cpu_count = 8
|
||||
hw.ram_gb = 32
|
||||
hw.gpu = None
|
||||
|
||||
with (
|
||||
patch("openjarvis.cli.quickstart_cmd.detect_hardware", return_value=hw),
|
||||
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
|
||||
patch("openjarvis.cli.quickstart_cmd.generate_default_toml", return_value="[engine]\n"),
|
||||
patch("openjarvis.cli.quickstart_cmd.recommend_engine", return_value="ollama"),
|
||||
patch("openjarvis.cli.quickstart_cmd._check_engine_health", return_value=False),
|
||||
):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["quickstart"])
|
||||
assert result.exit_code == 1
|
||||
assert "engine" in result.output.lower() or "not reachable" in result.output.lower()
|
||||
@@ -120,3 +120,45 @@ class TestInstrumentedEngine:
|
||||
def test_engine_id_attribute(self, mock_engine, bus):
|
||||
ie = InstrumentedEngine(mock_engine, bus)
|
||||
assert ie.engine_id == "instrumented"
|
||||
|
||||
|
||||
class TestTokensPerJoule:
|
||||
def test_tokens_per_joule_zero_without_energy(self, mock_engine, bus):
|
||||
"""tokens_per_joule is 0.0 when no energy monitor is available."""
|
||||
ie = InstrumentedEngine(mock_engine, bus)
|
||||
messages = [Message(role=Role.USER, content="Hi")]
|
||||
ie.generate(messages, model="test")
|
||||
|
||||
tel_events = [
|
||||
e for e in bus.history
|
||||
if e.event_type == EventType.TELEMETRY_RECORD
|
||||
]
|
||||
record = tel_events[0].data["record"]
|
||||
assert record.tokens_per_joule == 0.0
|
||||
|
||||
def test_tokens_per_joule_formula_via_record(self):
|
||||
"""Verify the formula: tokens_per_joule = completion_tokens / energy_joules."""
|
||||
from openjarvis.core.types import TelemetryRecord
|
||||
|
||||
# Direct construction — verifies the field accepts computed values
|
||||
rec = TelemetryRecord(
|
||||
timestamp=1.0,
|
||||
model_id="test",
|
||||
completion_tokens=50,
|
||||
energy_joules=2.5,
|
||||
tokens_per_joule=50.0 / 2.5, # = 20.0
|
||||
)
|
||||
assert rec.tokens_per_joule == pytest.approx(20.0)
|
||||
|
||||
def test_tokens_per_joule_zero_when_no_tokens(self):
|
||||
"""tokens_per_joule is 0.0 when completion_tokens is 0."""
|
||||
from openjarvis.core.types import TelemetryRecord
|
||||
|
||||
rec = TelemetryRecord(
|
||||
timestamp=1.0,
|
||||
model_id="test",
|
||||
completion_tokens=0,
|
||||
energy_joules=5.0,
|
||||
tokens_per_joule=0.0,
|
||||
)
|
||||
assert rec.tokens_per_joule == 0.0
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Tests for tokens_per_joule storage and aggregation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.core.types import TelemetryRecord
|
||||
from openjarvis.telemetry.store import TelemetryStore
|
||||
from openjarvis.telemetry.aggregator import TelemetryAggregator
|
||||
|
||||
|
||||
class TestTokensPerJouleStorage:
|
||||
def test_store_and_retrieve(self, tmp_path):
|
||||
db = tmp_path / "tel.db"
|
||||
store = TelemetryStore(db_path=db)
|
||||
rec = TelemetryRecord(
|
||||
timestamp=time.time(),
|
||||
model_id="test-model",
|
||||
completion_tokens=50,
|
||||
energy_joules=2.5,
|
||||
tokens_per_joule=20.0,
|
||||
)
|
||||
store.record(rec)
|
||||
store.close()
|
||||
agg = TelemetryAggregator(db)
|
||||
stats = agg.per_model_stats()
|
||||
assert len(stats) == 1
|
||||
assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1)
|
||||
agg.close()
|
||||
|
||||
def test_aggregate_multiple(self, tmp_path):
|
||||
db = tmp_path / "tel.db"
|
||||
store = TelemetryStore(db_path=db)
|
||||
for tpj in [10.0, 20.0, 30.0]:
|
||||
rec = TelemetryRecord(
|
||||
timestamp=time.time(),
|
||||
model_id="m1",
|
||||
tokens_per_joule=tpj,
|
||||
)
|
||||
store.record(rec)
|
||||
store.close()
|
||||
agg = TelemetryAggregator(db)
|
||||
stats = agg.per_model_stats()
|
||||
assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1)
|
||||
agg.close()
|
||||
|
||||
def test_engine_stats_aggregate(self, tmp_path):
|
||||
db = tmp_path / "tel.db"
|
||||
store = TelemetryStore(db_path=db)
|
||||
for tpj in [15.0, 25.0]:
|
||||
rec = TelemetryRecord(
|
||||
timestamp=time.time(),
|
||||
model_id="m1",
|
||||
engine="ollama",
|
||||
tokens_per_joule=tpj,
|
||||
)
|
||||
store.record(rec)
|
||||
store.close()
|
||||
agg = TelemetryAggregator(db)
|
||||
stats = agg.per_engine_stats()
|
||||
assert len(stats) == 1
|
||||
assert stats[0].avg_tokens_per_joule == pytest.approx(20.0, rel=0.1)
|
||||
agg.close()
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Tests for TelemetryRecord fields."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openjarvis.core.types import TelemetryRecord
|
||||
|
||||
|
||||
class TestTelemetryRecord:
|
||||
def test_tokens_per_joule_field_exists(self):
|
||||
rec = TelemetryRecord(timestamp=1.0, model_id="test")
|
||||
assert hasattr(rec, "tokens_per_joule")
|
||||
assert rec.tokens_per_joule == 0.0
|
||||
|
||||
def test_tokens_per_joule_set(self):
|
||||
rec = TelemetryRecord(
|
||||
timestamp=1.0,
|
||||
model_id="test",
|
||||
tokens_per_joule=80.0,
|
||||
)
|
||||
assert rec.tokens_per_joule == 80.0
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Tests for energy aggregation in TraceAnalyzer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.core.types import StepType, Trace, TraceStep
|
||||
from openjarvis.traces.analyzer import StepTypeStats, TraceAnalyzer, TraceSummary
|
||||
from openjarvis.traces.store import TraceStore
|
||||
|
||||
|
||||
def _make_trace(steps: list[TraceStep]) -> Trace:
|
||||
return Trace(
|
||||
query="test",
|
||||
agent="test_agent",
|
||||
model="test_model",
|
||||
engine="test_engine",
|
||||
steps=steps,
|
||||
started_at=time.time(),
|
||||
ended_at=time.time() + 10,
|
||||
total_tokens=100,
|
||||
total_latency_seconds=10.0,
|
||||
)
|
||||
|
||||
|
||||
def _gen_step(energy: float = 0.0, duration: float = 1.0,
|
||||
prompt_tokens: int = 50, completion_tokens: int = 25) -> TraceStep:
|
||||
return TraceStep(
|
||||
step_type=StepType.GENERATE,
|
||||
timestamp=time.time(),
|
||||
duration_seconds=duration,
|
||||
output={
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens,
|
||||
},
|
||||
metadata={
|
||||
"energy_joules": energy,
|
||||
"power_watts": energy / duration if duration > 0 else 0.0,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _tool_step(duration: float = 0.5) -> TraceStep:
|
||||
return TraceStep(
|
||||
step_type=StepType.TOOL_CALL,
|
||||
timestamp=time.time(),
|
||||
duration_seconds=duration,
|
||||
input={"tool": "calculator"},
|
||||
output={"success": True},
|
||||
)
|
||||
|
||||
|
||||
class TestTraceSummaryEnergyFields:
|
||||
def test_total_energy_joules(self, tmp_path):
|
||||
store = TraceStore(db_path=tmp_path / "traces.db")
|
||||
trace = _make_trace([
|
||||
_gen_step(energy=10.0, duration=2.0),
|
||||
_tool_step(duration=0.5),
|
||||
_gen_step(energy=15.0, duration=3.0),
|
||||
])
|
||||
store.save(trace)
|
||||
analyzer = TraceAnalyzer(store)
|
||||
summary = analyzer.summary()
|
||||
assert summary.total_energy_joules == pytest.approx(25.0, rel=0.01)
|
||||
assert summary.total_generate_energy_joules == pytest.approx(25.0, rel=0.01)
|
||||
store.close()
|
||||
|
||||
def test_step_type_stats(self, tmp_path):
|
||||
store = TraceStore(db_path=tmp_path / "traces.db")
|
||||
trace = _make_trace([
|
||||
_gen_step(energy=10.0, duration=2.0, prompt_tokens=100, completion_tokens=50),
|
||||
_gen_step(energy=20.0, duration=4.0, prompt_tokens=80, completion_tokens=40),
|
||||
_tool_step(duration=0.5),
|
||||
_tool_step(duration=1.5),
|
||||
])
|
||||
store.save(trace)
|
||||
analyzer = TraceAnalyzer(store)
|
||||
summary = analyzer.summary()
|
||||
|
||||
assert "generate" in summary.step_type_stats
|
||||
gen = summary.step_type_stats["generate"]
|
||||
assert gen.count == 2
|
||||
assert gen.avg_duration == pytest.approx(3.0, rel=0.01)
|
||||
assert gen.total_energy == pytest.approx(30.0, rel=0.01)
|
||||
assert gen.avg_input_tokens == pytest.approx(90.0, rel=0.01)
|
||||
assert gen.avg_output_tokens == pytest.approx(45.0, rel=0.01)
|
||||
assert gen.min_duration == pytest.approx(2.0, rel=0.01)
|
||||
assert gen.max_duration == pytest.approx(4.0, rel=0.01)
|
||||
|
||||
assert "tool_call" in summary.step_type_stats
|
||||
tc = summary.step_type_stats["tool_call"]
|
||||
assert tc.count == 2
|
||||
assert tc.avg_duration == pytest.approx(1.0, rel=0.01)
|
||||
store.close()
|
||||
|
||||
|
||||
class TestStepTypeStats:
|
||||
def test_dataclass_fields(self):
|
||||
s = StepTypeStats(count=5, avg_duration=2.0, total_energy=10.0)
|
||||
assert s.count == 5
|
||||
assert s.std_duration == 0.0 # default
|
||||
Reference in New Issue
Block a user