diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 73bf7301..d7ac0248 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -405,6 +405,15 @@ async fn run_jarvis_command(args: Vec) -> Result { } } +#[tauri::command] +async fn fetch_savings(api_url: String) -> Result { + let base = if api_url.is_empty() { api_base() } else { api_url }; + let resp = reqwest::get(format!("{}/v1/savings", base)) + .await + .map_err(|e| format!("Connection failed: {}", e))?; + resp.json().await.map_err(|e| format!("Invalid response: {}", e)) +} + /// Transcribe audio via the speech API endpoint. #[tauri::command] async fn transcribe_audio( @@ -541,6 +550,7 @@ pub fn run() { fetch_agents, fetch_models, run_jarvis_command, + fetch_savings, transcribe_audio, speech_health, ]) diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index d6176971..906d3414 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -1,5 +1,6 @@ import React, { useState } from 'react'; import { UpdateChecker } from './components/UpdateChecker'; +import { SavingsDashboard } from './components/SavingsDashboard'; import { EnergyDashboard } from './components/EnergyDashboard'; import { TraceDebugger } from './components/TraceDebugger'; import { LearningCurve } from './components/LearningCurve'; @@ -7,7 +8,7 @@ import { MemoryBrowser } from './components/MemoryBrowser'; import { AdminPanel } from './components/AdminPanel'; import { SettingsPanel } from './components/SettingsPanel'; -type TabId = 'energy' | 'traces' | 'learning' | 'memory' | 'admin' | 'settings'; +type TabId = 'savings' | 'energy' | 'traces' | 'learning' | 'memory' | 'admin' | 'settings'; interface Tab { id: TabId; @@ -15,6 +16,7 @@ interface Tab { } const TABS: Tab[] = [ + { id: 'savings', label: 'Savings' }, { id: 'energy', label: 'Energy' }, { id: 'traces', label: 'Traces' }, { id: 'learning', label: 'Learning' }, @@ -26,7 +28,7 @@ const TABS: Tab[] = [ const API_URL = 'http://localhost:8000'; export function App() { - const [activeTab, setActiveTab] = useState('energy'); + const [activeTab, setActiveTab] = useState('savings'); return (
@@ -51,6 +53,7 @@ export function App() {
+ {activeTab === 'savings' && } {activeTab === 'energy' && } {activeTab === 'traces' && } {activeTab === 'learning' && } diff --git a/desktop/src/components/SavingsDashboard.tsx b/desktop/src/components/SavingsDashboard.tsx new file mode 100644 index 00000000..233921ff --- /dev/null +++ b/desktop/src/components/SavingsDashboard.tsx @@ -0,0 +1,432 @@ +import { useState, useEffect, useCallback } from 'react'; +import type React from 'react'; +import { invoke } from '@tauri-apps/api/core'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface ProviderSavings { + provider: string; + label: string; + input_cost: number; + output_cost: number; + total_cost: number; + energy_wh: number; + energy_joules: number; + flops: number; +} + +interface SavingsData { + total_calls: number; + total_prompt_tokens: number; + total_completion_tokens: number; + total_tokens: number; + local_cost: number; + per_provider: ProviderSavings[]; + monthly_projection: Record; + session_start_ts: number; + session_duration_hours: number; + avg_cost_per_query: Record; + cloud_agent_equivalent: { + moderate_low: number; + moderate_high: number; + heavy_low: number; + heavy_high: number; + }; +} + +// --------------------------------------------------------------------------- +// Styles (Catppuccin) +// --------------------------------------------------------------------------- + +const colors = { + bg: '#1e1e2e', + surface: '#282840', + text: '#cdd6f4', + textMuted: '#a6adc8', + accent: '#89b4fa', + green: '#a6e3a1', + yellow: '#f9e2af', + red: '#f38ba8', + purple: '#cba6f7', + border: '#45475a', +} as const; + +const styles: Record = { + container: { + background: colors.bg, + color: colors.text, + padding: 24, + fontFamily: "'Inter', 'Segoe UI', system-ui, sans-serif", + height: '100%', + overflowY: 'auto', + boxSizing: 'border-box', + }, + header: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + marginBottom: 24, + }, + title: { + fontSize: 22, + fontWeight: 600, + margin: 0, + color: colors.text, + }, + liveBadge: { + display: 'inline-flex', + alignItems: 'center', + gap: 6, + fontSize: 12, + color: colors.green, + background: 'rgba(166,227,161,0.1)', + padding: '4px 10px', + borderRadius: 12, + fontWeight: 500, + }, + liveDot: { + width: 6, + height: 6, + borderRadius: '50%', + background: colors.green, + animation: 'pulse 2s infinite', + }, + statsGrid: { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(180px, 1fr))', + gap: 16, + marginBottom: 24, + }, + statCard: { + background: colors.surface, + borderRadius: 10, + padding: 16, + border: `1px solid ${colors.border}`, + }, + statLabel: { + fontSize: 12, + color: colors.textMuted, + marginBottom: 6, + textTransform: 'uppercase' as const, + letterSpacing: '0.05em', + }, + statValue: { + fontSize: 26, + fontWeight: 700, + color: colors.accent, + lineHeight: 1.1, + }, + statUnit: { + fontSize: 13, + fontWeight: 400, + color: colors.textMuted, + marginLeft: 4, + }, + sectionHeading: { + fontSize: 14, + fontWeight: 600, + color: colors.textMuted, + textTransform: 'uppercase' as const, + letterSpacing: '0.05em', + marginBottom: 12, + }, + providersGrid: { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(260px, 1fr))', + gap: 16, + marginBottom: 24, + }, + providerCard: { + background: colors.surface, + borderRadius: 10, + padding: 20, + border: `1px solid ${colors.border}`, + position: 'relative' as const, + overflow: 'hidden' as const, + }, + providerName: { + fontSize: 14, + fontWeight: 600, + marginBottom: 4, + }, + providerModel: { + fontSize: 12, + color: colors.textMuted, + marginBottom: 14, + }, + savingsAmount: { + fontSize: 32, + fontWeight: 700, + color: colors.green, + marginBottom: 8, + }, + breakdown: { + display: 'grid', + gridTemplateColumns: '1fr 1fr', + gap: 12, + marginTop: 14, + paddingTop: 14, + borderTop: `1px solid ${colors.border}`, + }, + breakdownLabel: { + fontSize: 11, + color: colors.textMuted, + textTransform: 'uppercase' as const, + letterSpacing: '0.04em', + }, + breakdownValue: { + fontSize: 16, + fontWeight: 600, + marginTop: 2, + }, + cloudAgentCard: { + background: colors.surface, + borderRadius: 10, + padding: 20, + border: `1px solid ${colors.border}`, + borderTop: `3px solid ${colors.purple}`, + marginBottom: 24, + }, + cloudAgentGrid: { + display: 'grid', + gridTemplateColumns: '1fr 1fr 1fr', + gap: 20, + marginTop: 16, + }, + emptyState: { + display: 'flex', + flexDirection: 'column' as const, + alignItems: 'center', + justifyContent: 'center', + padding: 64, + color: colors.textMuted, + gap: 12, + }, + emptyText: { + fontSize: 15, + textAlign: 'center' as const, + }, + errorBanner: { + background: 'rgba(243,139,168,0.1)', + border: `1px solid ${colors.red}`, + borderRadius: 8, + padding: '10px 16px', + marginBottom: 16, + fontSize: 13, + color: colors.red, + }, +}; + +const PROVIDER_COLORS: Record = { + 'gpt-5.3': colors.green, + 'claude-opus-4.6': colors.yellow, + 'gemini-3.1-pro': colors.accent, +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function fmtDollar(n: number): string { + if (n >= 1000) return '$' + n.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + if (n >= 1) return '$' + n.toFixed(2); + if (n >= 0.01) return '$' + n.toFixed(3); + if (n > 0) return '$' + n.toFixed(4); + return '$0.00'; +} + +function fmtDuration(hours: number): string { + if (hours < 1) return `${Math.round(hours * 60)}m`; + if (hours < 24) return `${hours.toFixed(1)}h`; + return `${(hours / 24).toFixed(1)}d`; +} + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- + +const REFRESH_INTERVAL_MS = 5000; + +export function SavingsDashboard({ apiUrl }: { apiUrl: string }) { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + + const fetchData = useCallback(async () => { + try { + const result = await invoke('fetch_savings', { apiUrl }); + setData(result); + setError(null); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + setError(message); + setData(null); + } finally { + setLoading(false); + } + }, [apiUrl]); + + useEffect(() => { + fetchData(); + const timer = setInterval(fetchData, REFRESH_INTERVAL_MS); + return () => clearInterval(timer); + }, [fetchData]); + + if (!loading && !data && !error) { + return ( +
+
+

Savings Dashboard

+
+
+
$
+
+ No savings data available.
+ Start making inference requests to see savings vs cloud providers. +
+
+
+ ); + } + + const providers = data?.per_provider ?? []; + const projection = data?.monthly_projection ?? {}; + const cloudAgent = data?.cloud_agent_equivalent; + + return ( +
+ + + {/* Header */} +
+

Savings Dashboard

+ + + Live - {REFRESH_INTERVAL_MS / 1000}s + +
+ + {error &&
{error}
} + + {/* Stat cards row */} +
+
+
Total Requests
+
+ {(data?.total_calls ?? 0).toLocaleString()} +
+
+
+
Total Tokens
+
+ {(data?.total_tokens ?? 0).toLocaleString()} +
+
+
+
Session Duration
+
+ {fmtDuration(data?.session_duration_hours ?? 0)} +
+
+
+
Local Cost
+
+ {fmtDollar(data?.local_cost ?? 0)} +
+
+
+ + {/* Provider savings cards */} +
Savings vs Cloud Providers
+
+ {providers.map((p) => ( +
+
{p.label}
+
{p.provider}
+
{fmtDollar(p.total_cost)}
+
+
+
Input Saved
+
{fmtDollar(p.input_cost)}
+
+
+
Output Saved
+
{fmtDollar(p.output_cost)}
+
+
+
+ ))} +
+ + {/* Cloud Agent Platforms */} + {cloudAgent && ( + <> +
vs Cloud Agent Platforms
+
+
Typical Cloud Agent Platform
+
based on published API pricing tiers
+
+
+
MODERATE USE
+
+ ${cloudAgent.moderate_low}–{cloudAgent.moderate_high}/mo +
+
+
+
HEAVY USE
+
+ ${cloudAgent.heavy_low}–{cloudAgent.heavy_high}+/mo +
+
+
+
YOUR COST
+
+ $0.00 +
+
+ local inference +
+
+
+
+ + )} + + {/* Monthly Projection */} +
Monthly Projection
+
+ {providers.map((p) => ( +
+
vs {p.label}
+
projected monthly savings
+
+ {fmtDollar(projection[p.provider] ?? 0)} +
+
+ per month at current rate +
+
+ ))} +
+
+ ); +} diff --git a/rust/crates/openjarvis-python/src/lib.rs b/rust/crates/openjarvis-python/src/lib.rs index 559951d1..91fea929 100644 --- a/rust/crates/openjarvis-python/src/lib.rs +++ b/rust/crates/openjarvis-python/src/lib.rs @@ -100,6 +100,12 @@ fn openjarvis_rust(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + // --- Telemetry (new session/phase/ITL/FLOPs classes) --- + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; // --- Traces --- m.add_class::()?; diff --git a/rust/crates/openjarvis-python/src/telemetry.rs b/rust/crates/openjarvis-python/src/telemetry.rs index 68febec6..c560457a 100644 --- a/rust/crates/openjarvis-python/src/telemetry.rs +++ b/rust/crates/openjarvis-python/src/telemetry.rs @@ -91,3 +91,291 @@ impl PyInstrumentedEngine { self.inner.engine_id() } } + +// --- New telemetry session classes --- + +/// Python wrapper for TelemetrySample. +#[pyclass(name = "TelemetrySample")] +#[derive(Clone)] +pub struct PyTelemetrySample { + pub timestamp_ns: u64, + pub gpu_power_w: f64, + pub cpu_power_w: f64, + pub gpu_energy_j: f64, + pub cpu_energy_j: f64, + pub gpu_util_pct: f64, + pub gpu_temp_c: f64, + pub gpu_mem_gb: f64, +} + +#[pymethods] +impl PyTelemetrySample { + #[new] + #[pyo3(signature = (timestamp_ns, gpu_power_w=0.0, cpu_power_w=0.0, gpu_energy_j=0.0, cpu_energy_j=0.0, gpu_util_pct=0.0, gpu_temp_c=0.0, gpu_mem_gb=0.0))] + fn new( + timestamp_ns: u64, + gpu_power_w: f64, + cpu_power_w: f64, + gpu_energy_j: f64, + cpu_energy_j: f64, + gpu_util_pct: f64, + gpu_temp_c: f64, + gpu_mem_gb: f64, + ) -> Self { + Self { + timestamp_ns, + gpu_power_w, + cpu_power_w, + gpu_energy_j, + cpu_energy_j, + gpu_util_pct, + gpu_temp_c, + gpu_mem_gb, + } + } + + #[getter] + fn timestamp_ns(&self) -> u64 { + self.timestamp_ns + } + #[getter] + fn gpu_power_w(&self) -> f64 { + self.gpu_power_w + } + #[getter] + fn cpu_power_w(&self) -> f64 { + self.cpu_power_w + } + #[getter] + fn gpu_energy_j(&self) -> f64 { + self.gpu_energy_j + } + #[getter] + fn cpu_energy_j(&self) -> f64 { + self.cpu_energy_j + } + #[getter] + fn gpu_util_pct(&self) -> f64 { + self.gpu_util_pct + } + #[getter] + fn gpu_temp_c(&self) -> f64 { + self.gpu_temp_c + } + #[getter] + fn gpu_mem_gb(&self) -> f64 { + self.gpu_mem_gb + } +} + +/// Python wrapper for TelemetrySessionCore (ring buffer). +#[pyclass(name = "TelemetrySessionCore")] +pub struct PyTelemetrySessionCore { + inner: openjarvis_telemetry::session::TelemetrySessionCore, +} + +#[pymethods] +impl PyTelemetrySessionCore { + #[new] + #[pyo3(signature = (capacity=100000, sampling_interval_ms=100))] + fn new(capacity: usize, sampling_interval_ms: u64) -> Self { + Self { + inner: openjarvis_telemetry::session::TelemetrySessionCore::new( + capacity, + sampling_interval_ms, + ), + } + } + + fn add_sample(&self, sample: &PyTelemetrySample) { + let s = openjarvis_telemetry::session::TelemetrySample { + timestamp_ns: sample.timestamp_ns, + gpu_power_w: sample.gpu_power_w, + cpu_power_w: sample.cpu_power_w, + gpu_energy_j: sample.gpu_energy_j, + cpu_energy_j: sample.cpu_energy_j, + gpu_util_pct: sample.gpu_util_pct, + gpu_temp_c: sample.gpu_temp_c, + gpu_mem_gb: sample.gpu_mem_gb, + }; + self.inner.add_sample(s); + } + + fn window(&self, start_ns: u64, end_ns: u64) -> Vec { + self.inner + .window(start_ns, end_ns) + .into_iter() + .map(|s| PyTelemetrySample { + timestamp_ns: s.timestamp_ns, + gpu_power_w: s.gpu_power_w, + cpu_power_w: s.cpu_power_w, + gpu_energy_j: s.gpu_energy_j, + cpu_energy_j: s.cpu_energy_j, + gpu_util_pct: s.gpu_util_pct, + gpu_temp_c: s.gpu_temp_c, + gpu_mem_gb: s.gpu_mem_gb, + }) + .collect() + } + + fn compute_energy_delta(&self, start_ns: u64, end_ns: u64) -> (f64, f64) { + self.inner.compute_energy_delta(start_ns, end_ns) + } + + fn compute_avg_power(&self, start_ns: u64, end_ns: u64) -> (f64, f64) { + self.inner.compute_avg_power(start_ns, end_ns) + } + + fn len(&self) -> usize { + self.inner.len() + } + + fn clear(&self) { + self.inner.clear(); + } +} + +/// Python wrapper for ITL stats computation. +#[pyclass(name = "ItlStats")] +pub struct PyItlStats; + +#[pymethods] +impl PyItlStats { + #[new] + fn new() -> Self { + Self + } + + #[staticmethod] + fn compute(token_timestamps_ms: Vec) -> PyResult> { + let stats = openjarvis_telemetry::itl::compute_itl_stats(&token_timestamps_ms); + pyo3::Python::with_gil(|py| { + let dict = pyo3::types::PyDict::new(py); + dict.set_item("p50_ms", stats.p50_ms)?; + dict.set_item("p90_ms", stats.p90_ms)?; + dict.set_item("p95_ms", stats.p95_ms)?; + dict.set_item("p99_ms", stats.p99_ms)?; + dict.set_item("mean_ms", stats.mean_ms)?; + dict.set_item("min_ms", stats.min_ms)?; + dict.set_item("max_ms", stats.max_ms)?; + Ok(dict.into()) + }) + } +} + +/// Python wrapper for FLOPs estimation. +#[pyclass(name = "FlopsEstimator")] +pub struct PyFlopsEstimator; + +#[pymethods] +impl PyFlopsEstimator { + #[new] + fn new() -> Self { + Self + } + + #[staticmethod] + fn estimate_flops(model: &str, input_tokens: u64, output_tokens: u64) -> (f64, f64) { + openjarvis_telemetry::flops::estimate_flops(model, input_tokens, output_tokens) + } + + #[staticmethod] + #[pyo3(signature = (flops, duration_s, gpu_name, num_gpus=1))] + fn compute_mfu(flops: f64, duration_s: f64, gpu_name: &str, num_gpus: u32) -> f64 { + openjarvis_telemetry::flops::compute_mfu(flops, duration_s, gpu_name, num_gpus) + } +} + +/// Python wrapper for phase metrics. +#[pyclass(name = "PhaseMetrics")] +pub struct PyPhaseMetrics; + +#[pymethods] +impl PyPhaseMetrics { + #[new] + fn new() -> Self { + Self + } + + #[staticmethod] + fn compute( + samples: Vec, + start_ns: u64, + end_ns: u64, + tokens: u64, + ) -> PyResult> { + let rust_samples: Vec = samples + .iter() + .map(|s| openjarvis_telemetry::session::TelemetrySample { + timestamp_ns: s.timestamp_ns, + gpu_power_w: s.gpu_power_w, + cpu_power_w: s.cpu_power_w, + gpu_energy_j: s.gpu_energy_j, + cpu_energy_j: s.cpu_energy_j, + gpu_util_pct: s.gpu_util_pct, + gpu_temp_c: s.gpu_temp_c, + gpu_mem_gb: s.gpu_mem_gb, + }) + .collect(); + let metrics = + openjarvis_telemetry::phase::compute_phase_metrics(&rust_samples, start_ns, end_ns, tokens); + pyo3::Python::with_gil(|py| { + let dict = pyo3::types::PyDict::new(py); + dict.set_item("energy_j", metrics.energy_j)?; + dict.set_item("mean_power_w", metrics.mean_power_w)?; + dict.set_item("duration_s", metrics.duration_s)?; + dict.set_item("energy_per_token_j", metrics.energy_per_token_j)?; + dict.set_item("tokens", metrics.tokens)?; + Ok(dict.into()) + }) + } + + #[staticmethod] + fn split_at_ttft( + samples: Vec, + start_ns: u64, + ttft_ns: u64, + end_ns: u64, + input_tokens: u64, + output_tokens: u64, + ) -> PyResult<(pyo3::Py, pyo3::Py)> { + let rust_samples: Vec = samples + .iter() + .map(|s| openjarvis_telemetry::session::TelemetrySample { + timestamp_ns: s.timestamp_ns, + gpu_power_w: s.gpu_power_w, + cpu_power_w: s.cpu_power_w, + gpu_energy_j: s.gpu_energy_j, + cpu_energy_j: s.cpu_energy_j, + gpu_util_pct: s.gpu_util_pct, + gpu_temp_c: s.gpu_temp_c, + gpu_mem_gb: s.gpu_mem_gb, + }) + .collect(); + let (prefill, decode) = openjarvis_telemetry::phase::split_at_ttft( + &rust_samples, + start_ns, + ttft_ns, + end_ns, + input_tokens, + output_tokens, + ); + pyo3::Python::with_gil(|py| { + let prefill_dict = pyo3::types::PyDict::new(py); + prefill_dict.set_item("energy_j", prefill.energy_j)?; + prefill_dict.set_item("mean_power_w", prefill.mean_power_w)?; + prefill_dict.set_item("duration_s", prefill.duration_s)?; + prefill_dict.set_item("energy_per_token_j", prefill.energy_per_token_j)?; + prefill_dict.set_item("tokens", prefill.tokens)?; + + let decode_dict = pyo3::types::PyDict::new(py); + decode_dict.set_item("energy_j", decode.energy_j)?; + decode_dict.set_item("mean_power_w", decode.mean_power_w)?; + decode_dict.set_item("duration_s", decode.duration_s)?; + decode_dict.set_item("energy_per_token_j", decode.energy_per_token_j)?; + decode_dict.set_item("tokens", decode.tokens)?; + + Ok((prefill_dict.into(), decode_dict.into())) + }) + } +} diff --git a/rust/crates/openjarvis-telemetry/src/flops.rs b/rust/crates/openjarvis-telemetry/src/flops.rs new file mode 100644 index 00000000..b82bea8f --- /dev/null +++ b/rust/crates/openjarvis-telemetry/src/flops.rs @@ -0,0 +1,197 @@ +//! FLOPs estimation and Model FLOPs Utilization (MFU) computation. + +/// Peak TFLOPS (FP16/BF16) for common GPU/accelerator models. +pub const GPU_PEAK_TFLOPS: &[(&str, f64)] = &[ + ("H100", 989.0), + ("H200", 989.0), + ("A100", 312.0), + ("A10G", 31.2), + ("L4", 30.3), + ("L40", 181.0), + ("L40S", 362.0), + ("T4", 65.1), + ("V100", 125.0), + ("4090", 82.6), + ("4080", 48.7), + ("3090", 35.6), + ("M3 Max", 14.2), + ("M3 Ultra", 27.0), + ("M4 Max", 18.0), +]; + +/// Approximate parameter counts (billions) for common models. +pub const MODEL_PARAMS: &[(&str, f64)] = &[ + ("qwen3:8b", 8.0), + ("llama-3.1-70b", 70.0), + ("llama-3.1-8b", 8.0), + ("mistral-7b", 7.0), + ("mixtral-8x7b", 47.0), + ("gpt-4o", 200.0), + ("claude-opus", 137.0), + ("gemini-pro", 137.0), +]; + +/// Look up a value in a `&[(&str, f64)]` table using case-insensitive substring matching. +fn lookup(table: &[(&str, f64)], key: &str) -> Option { + let key_lower = key.to_lowercase(); + // Try exact match first (case-insensitive) + for &(name, val) in table { + if name.to_lowercase() == key_lower { + return Some(val); + } + } + // Fall back to substring match + for &(name, val) in table { + if key_lower.contains(&name.to_lowercase()) || name.to_lowercase().contains(&key_lower) { + return Some(val); + } + } + None +} + +/// Estimate FLOPs for a model inference using the `2 * params * tokens` approximation. +/// +/// Returns `(total_flops, flops_per_token)`. If the model is not found in `MODEL_PARAMS`, +/// returns `(0.0, 0.0)`. +pub fn estimate_flops(model: &str, input_tokens: u64, output_tokens: u64) -> (f64, f64) { + let params_b = match lookup(MODEL_PARAMS, model) { + Some(p) => p, + None => return (0.0, 0.0), + }; + + let total_tokens = input_tokens + output_tokens; + if total_tokens == 0 { + return (0.0, 0.0); + } + + let params = params_b * 1e9; + // 2 * params * tokens approximation for transformer FLOPs + let total_flops = 2.0 * params * total_tokens as f64; + let flops_per_token = 2.0 * params; + + (total_flops, flops_per_token) +} + +/// Compute Model FLOPs Utilization (MFU). +/// +/// MFU = actual_flops / (peak_tflops * 1e12 * duration_s * num_gpus) +/// +/// Returns 0.0 if the GPU is not found or inputs are zero. +pub fn compute_mfu(flops: f64, duration_s: f64, gpu_name: &str, num_gpus: u32) -> f64 { + if duration_s <= 0.0 || num_gpus == 0 || flops <= 0.0 { + return 0.0; + } + + let peak_tflops = match lookup(GPU_PEAK_TFLOPS, gpu_name) { + Some(p) => p, + None => return 0.0, + }; + + let peak_flops_per_sec = peak_tflops * 1e12 * num_gpus as f64; + let theoretical_flops = peak_flops_per_sec * duration_s; + + flops / theoretical_flops +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn estimate_flops_known_model() { + let (total, per_token) = estimate_flops("qwen3:8b", 100, 50); + // 2 * 8e9 * 150 = 2.4e12 + assert!((total - 2.4e12).abs() < 1e3); + // 2 * 8e9 = 16e9 + assert!((per_token - 16e9).abs() < 1e3); + } + + #[test] + fn estimate_flops_unknown_model() { + let (total, per_token) = estimate_flops("unknown-model", 100, 50); + assert_eq!(total, 0.0); + assert_eq!(per_token, 0.0); + } + + #[test] + fn estimate_flops_zero_tokens() { + let (total, per_token) = estimate_flops("qwen3:8b", 0, 0); + assert_eq!(total, 0.0); + assert_eq!(per_token, 0.0); + } + + #[test] + fn estimate_flops_large_model() { + let (total, _) = estimate_flops("llama-3.1-70b", 1000, 500); + // 2 * 70e9 * 1500 = 210e12 + assert!((total - 210e12).abs() < 1e3); + } + + #[test] + fn compute_mfu_basic() { + // 1e12 FLOPs in 1 second on a single H100 (989 TFLOPS) + let mfu = compute_mfu(1e12, 1.0, "H100", 1); + // MFU = 1e12 / (989e12) ~= 0.001011 + let expected = 1e12 / (989.0 * 1e12); + assert!((mfu - expected).abs() < 1e-9, "mfu = {mfu}, expected = {expected}"); + } + + #[test] + fn compute_mfu_multi_gpu() { + let mfu_1 = compute_mfu(1e12, 1.0, "A100", 1); + let mfu_4 = compute_mfu(1e12, 1.0, "A100", 4); + // 4 GPUs -> 4x theoretical -> 1/4 MFU for same actual FLOPs + assert!((mfu_1 - 4.0 * mfu_4).abs() < 1e-12); + } + + #[test] + fn compute_mfu_unknown_gpu() { + let mfu = compute_mfu(1e12, 1.0, "unknown-gpu", 1); + assert_eq!(mfu, 0.0); + } + + #[test] + fn compute_mfu_zero_duration() { + let mfu = compute_mfu(1e12, 0.0, "H100", 1); + assert_eq!(mfu, 0.0); + } + + #[test] + fn compute_mfu_zero_gpus() { + let mfu = compute_mfu(1e12, 1.0, "H100", 0); + assert_eq!(mfu, 0.0); + } + + #[test] + fn compute_mfu_zero_flops() { + let mfu = compute_mfu(0.0, 1.0, "H100", 1); + assert_eq!(mfu, 0.0); + } + + #[test] + fn lookup_case_insensitive() { + assert!(lookup(GPU_PEAK_TFLOPS, "h100").is_some()); + assert!(lookup(GPU_PEAK_TFLOPS, "H100").is_some()); + } + + #[test] + fn lookup_substring_match() { + // "NVIDIA RTX 4090" should match "4090" + assert!(lookup(GPU_PEAK_TFLOPS, "NVIDIA RTX 4090").is_some()); + assert!(lookup(MODEL_PARAMS, "mistral-7b-instruct").is_some()); + } + + #[test] + fn all_gpus_have_positive_tflops() { + for &(name, tflops) in GPU_PEAK_TFLOPS { + assert!(tflops > 0.0, "{name} has non-positive TFLOPS: {tflops}"); + } + } + + #[test] + fn all_models_have_positive_params() { + for &(name, params) in MODEL_PARAMS { + assert!(params > 0.0, "{name} has non-positive params: {params}"); + } + } +} diff --git a/rust/crates/openjarvis-telemetry/src/itl.rs b/rust/crates/openjarvis-telemetry/src/itl.rs new file mode 100644 index 00000000..60ae2721 --- /dev/null +++ b/rust/crates/openjarvis-telemetry/src/itl.rs @@ -0,0 +1,156 @@ +//! Inter-token latency (ITL) statistics — percentiles, mean, min, max. + +/// Aggregated inter-token latency statistics. +#[derive(Debug, Clone, Default)] +pub struct ItlStats { + pub p50_ms: f64, + pub p90_ms: f64, + pub p95_ms: f64, + pub p99_ms: f64, + pub mean_ms: f64, + pub min_ms: f64, + pub max_ms: f64, +} + +/// Compute inter-token latency statistics from token arrival timestamps. +/// +/// `token_timestamps_ms` should be a slice of monotonically increasing timestamps +/// in milliseconds. At least 2 timestamps are needed to compute any latencies. +pub fn compute_itl_stats(token_timestamps_ms: &[f64]) -> ItlStats { + if token_timestamps_ms.len() < 2 { + return ItlStats::default(); + } + + // Compute inter-token latencies (consecutive diffs) + let mut itls: Vec = token_timestamps_ms + .windows(2) + .map(|w| w[1] - w[0]) + .collect(); + + // Sort for percentile computation + itls.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + let n = itls.len(); + let mean_ms = itls.iter().sum::() / n as f64; + let min_ms = itls[0]; + let max_ms = itls[n - 1]; + + ItlStats { + p50_ms: percentile(&itls, 50.0), + p90_ms: percentile(&itls, 90.0), + p95_ms: percentile(&itls, 95.0), + p99_ms: percentile(&itls, 99.0), + mean_ms, + min_ms, + max_ms, + } +} + +/// Compute a percentile from a sorted slice using linear interpolation. +fn percentile(sorted: &[f64], pct: f64) -> f64 { + assert!(!sorted.is_empty()); + if sorted.len() == 1 { + return sorted[0]; + } + let rank = pct / 100.0 * (sorted.len() - 1) as f64; + let lo = rank.floor() as usize; + let hi = rank.ceil() as usize; + if lo == hi { + sorted[lo] + } else { + let frac = rank - lo as f64; + sorted[lo] * (1.0 - frac) + sorted[hi] * frac + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn itl_stats_basic() { + // 5 tokens at 0, 10, 20, 30, 40 ms -> ITLs = [10, 10, 10, 10] + let timestamps = vec![0.0, 10.0, 20.0, 30.0, 40.0]; + let stats = compute_itl_stats(×tamps); + + assert!((stats.mean_ms - 10.0).abs() < 1e-9); + assert!((stats.min_ms - 10.0).abs() < 1e-9); + assert!((stats.max_ms - 10.0).abs() < 1e-9); + assert!((stats.p50_ms - 10.0).abs() < 1e-9); + assert!((stats.p99_ms - 10.0).abs() < 1e-9); + } + + #[test] + fn itl_stats_varying_latencies() { + // ITLs: [5, 10, 15, 20] + let timestamps = vec![0.0, 5.0, 15.0, 30.0, 50.0]; + let stats = compute_itl_stats(×tamps); + // ITLs: [5, 10, 15, 20], sorted: [5, 10, 15, 20] + + assert!((stats.mean_ms - 12.5).abs() < 1e-9); + assert!((stats.min_ms - 5.0).abs() < 1e-9); + assert!((stats.max_ms - 20.0).abs() < 1e-9); + + // p50: rank = 0.5 * 3 = 1.5 -> interpolate between index 1 and 2 + // = 10 * 0.5 + 15 * 0.5 = 12.5 + assert!((stats.p50_ms - 12.5).abs() < 1e-9); + } + + #[test] + fn itl_stats_empty() { + let stats = compute_itl_stats(&[]); + assert_eq!(stats.mean_ms, 0.0); + assert_eq!(stats.p50_ms, 0.0); + } + + #[test] + fn itl_stats_single_timestamp() { + let stats = compute_itl_stats(&[100.0]); + assert_eq!(stats.mean_ms, 0.0); + } + + #[test] + fn itl_stats_two_timestamps() { + let stats = compute_itl_stats(&[0.0, 42.0]); + // Single ITL of 42ms + assert!((stats.mean_ms - 42.0).abs() < 1e-9); + assert!((stats.min_ms - 42.0).abs() < 1e-9); + assert!((stats.max_ms - 42.0).abs() < 1e-9); + assert!((stats.p50_ms - 42.0).abs() < 1e-9); + assert!((stats.p90_ms - 42.0).abs() < 1e-9); + assert!((stats.p95_ms - 42.0).abs() < 1e-9); + assert!((stats.p99_ms - 42.0).abs() < 1e-9); + } + + #[test] + fn itl_stats_percentile_interpolation() { + // 10 ITLs: 1..=10 + let timestamps: Vec = { + let mut ts = vec![0.0]; + for i in 1..=10 { + ts.push(ts.last().unwrap() + i as f64); + } + ts + }; + let stats = compute_itl_stats(×tamps); + // ITLs sorted: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + assert!((stats.min_ms - 1.0).abs() < 1e-9); + assert!((stats.max_ms - 10.0).abs() < 1e-9); + assert!((stats.mean_ms - 5.5).abs() < 1e-9); + + // p50: rank = 0.5 * 9 = 4.5 -> interpolate idx 4 and 5 -> (5+6)/2 = 5.5 + assert!((stats.p50_ms - 5.5).abs() < 1e-9); + + // p90: rank = 0.9 * 9 = 8.1 -> interpolate idx 8 and 9 -> 9*0.9 + 10*0.1 = 9.1 + assert!((stats.p90_ms - 9.1).abs() < 1e-9); + } + + #[test] + fn percentile_function_edge_cases() { + assert!((percentile(&[5.0], 50.0) - 5.0).abs() < 1e-9); + assert!((percentile(&[1.0, 2.0], 0.0) - 1.0).abs() < 1e-9); + assert!((percentile(&[1.0, 2.0], 100.0) - 2.0).abs() < 1e-9); + assert!((percentile(&[1.0, 2.0], 50.0) - 1.5).abs() < 1e-9); + } +} diff --git a/rust/crates/openjarvis-telemetry/src/lib.rs b/rust/crates/openjarvis-telemetry/src/lib.rs index 5c906dba..f31a3ab2 100644 --- a/rust/crates/openjarvis-telemetry/src/lib.rs +++ b/rust/crates/openjarvis-telemetry/src/lib.rs @@ -2,7 +2,11 @@ pub mod aggregator; pub mod energy; +pub mod flops; pub mod instrumented; +pub mod itl; +pub mod phase; +pub mod session; pub mod store; pub use aggregator::TelemetryAggregator; diff --git a/rust/crates/openjarvis-telemetry/src/phase.rs b/rust/crates/openjarvis-telemetry/src/phase.rs new file mode 100644 index 00000000..2b545a52 --- /dev/null +++ b/rust/crates/openjarvis-telemetry/src/phase.rs @@ -0,0 +1,201 @@ +//! Phase-level metrics: energy, power, and per-token efficiency for prefill/decode phases. + +use crate::session::TelemetrySample; + +/// Aggregated metrics for a single inference phase (e.g., prefill or decode). +#[derive(Debug, Clone, Default)] +pub struct PhaseMetrics { + pub energy_j: f64, + pub mean_power_w: f64, + pub duration_s: f64, + pub energy_per_token_j: f64, + pub tokens: u64, +} + +/// Compute phase metrics from telemetry samples within a time range. +/// +/// Uses trapezoidal integration for energy and simple averaging for power. +/// `tokens` is used to compute per-token energy. +pub fn compute_phase_metrics( + samples: &[TelemetrySample], + start_ns: u64, + end_ns: u64, + tokens: u64, +) -> PhaseMetrics { + let duration_s = (end_ns.saturating_sub(start_ns)) as f64 / 1e9; + + // Filter samples within the window + let in_range: Vec<&TelemetrySample> = samples + .iter() + .filter(|s| s.timestamp_ns >= start_ns && s.timestamp_ns <= end_ns) + .collect(); + + if in_range.is_empty() { + return PhaseMetrics { + duration_s, + tokens, + ..Default::default() + }; + } + + // Trapezoidal integration for total energy (GPU + CPU combined) + let mut energy_j = 0.0; + for pair in in_range.windows(2) { + let dt_s = (pair[1].timestamp_ns - pair[0].timestamp_ns) as f64 / 1e9; + let total_power_0 = pair[0].gpu_power_w + pair[0].cpu_power_w; + let total_power_1 = pair[1].gpu_power_w + pair[1].cpu_power_w; + energy_j += (total_power_0 + total_power_1) / 2.0 * dt_s; + } + + // Mean total power + let mean_power_w: f64 = in_range + .iter() + .map(|s| s.gpu_power_w + s.cpu_power_w) + .sum::() + / in_range.len() as f64; + + let energy_per_token_j = if tokens > 0 { + energy_j / tokens as f64 + } else { + 0.0 + }; + + PhaseMetrics { + energy_j, + mean_power_w, + duration_s, + energy_per_token_j, + tokens, + } +} + +/// Split samples at the TTFT boundary into prefill and decode phases. +/// +/// - Prefill phase: [start_ns, ttft_ns] with `input_tokens` +/// - Decode phase: (ttft_ns, end_ns] with `output_tokens` +/// +/// Returns `(prefill_metrics, decode_metrics)`. +pub fn split_at_ttft( + samples: &[TelemetrySample], + start_ns: u64, + ttft_ns: u64, + end_ns: u64, + input_tokens: u64, + output_tokens: u64, +) -> (PhaseMetrics, PhaseMetrics) { + let prefill = compute_phase_metrics(samples, start_ns, ttft_ns, input_tokens); + let decode = compute_phase_metrics(samples, ttft_ns, end_ns, output_tokens); + (prefill, decode) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_sample(ts_ns: u64, gpu_w: f64, cpu_w: f64) -> TelemetrySample { + TelemetrySample { + timestamp_ns: ts_ns, + gpu_power_w: gpu_w, + cpu_power_w: cpu_w, + ..Default::default() + } + } + + #[test] + fn phase_metrics_constant_power() { + let samples = vec![ + make_sample(0, 200.0, 100.0), + make_sample(500_000_000, 200.0, 100.0), + make_sample(1_000_000_000, 200.0, 100.0), + ]; + let m = compute_phase_metrics(&samples, 0, 1_000_000_000, 100); + + assert!((m.duration_s - 1.0).abs() < 1e-9); + assert!((m.energy_j - 300.0).abs() < 1e-9); // 300W * 1s + assert!((m.mean_power_w - 300.0).abs() < 1e-9); + assert!((m.energy_per_token_j - 3.0).abs() < 1e-9); // 300J / 100 tokens + assert_eq!(m.tokens, 100); + } + + #[test] + fn phase_metrics_empty_samples() { + let m = compute_phase_metrics(&[], 0, 1_000_000_000, 10); + assert!((m.duration_s - 1.0).abs() < 1e-9); + assert_eq!(m.energy_j, 0.0); + assert_eq!(m.mean_power_w, 0.0); + assert_eq!(m.energy_per_token_j, 0.0); + } + + #[test] + fn phase_metrics_zero_tokens() { + let samples = vec![ + make_sample(0, 100.0, 50.0), + make_sample(1_000_000_000, 100.0, 50.0), + ]; + let m = compute_phase_metrics(&samples, 0, 1_000_000_000, 0); + assert_eq!(m.energy_per_token_j, 0.0); + } + + #[test] + fn phase_metrics_single_sample() { + let samples = vec![make_sample(500_000_000, 200.0, 100.0)]; + let m = compute_phase_metrics(&samples, 0, 1_000_000_000, 50); + // Single sample -> no trapezoids, energy = 0 + assert_eq!(m.energy_j, 0.0); + assert!((m.mean_power_w - 300.0).abs() < 1e-9); + } + + #[test] + fn phase_metrics_ramp() { + // GPU: 0 -> 200W, CPU: 0 -> 100W over 1 second + let samples = vec![ + make_sample(0, 0.0, 0.0), + make_sample(1_000_000_000, 200.0, 100.0), + ]; + let m = compute_phase_metrics(&samples, 0, 1_000_000_000, 10); + // Trapezoidal: (0+300)/2 * 1.0 = 150 J + assert!((m.energy_j - 150.0).abs() < 1e-9); + assert!((m.energy_per_token_j - 15.0).abs() < 1e-9); + } + + #[test] + fn split_at_ttft_basic() { + let samples = vec![ + make_sample(0, 300.0, 100.0), // prefill start + make_sample(500_000_000, 300.0, 100.0), // prefill end / TTFT + make_sample(500_000_000, 200.0, 80.0), // decode start (at TTFT) + make_sample(1_000_000_000, 200.0, 80.0), // decode mid + make_sample(2_000_000_000, 200.0, 80.0), // decode end + ]; + + let (prefill, decode) = split_at_ttft( + &samples, + 0, + 500_000_000, + 2_000_000_000, + 128, + 256, + ); + + assert!((prefill.duration_s - 0.5).abs() < 1e-9); + assert_eq!(prefill.tokens, 128); + + assert!((decode.duration_s - 1.5).abs() < 1e-9); + assert_eq!(decode.tokens, 256); + assert!(decode.energy_j > 0.0); + } + + #[test] + fn phase_metrics_filters_outside_window() { + let samples = vec![ + make_sample(0, 999.0, 999.0), // before window + make_sample(1_000_000_000, 100.0, 50.0), // in window + make_sample(2_000_000_000, 100.0, 50.0), // in window + make_sample(9_000_000_000, 999.0, 999.0), // after window + ]; + let m = compute_phase_metrics(&samples, 1_000_000_000, 2_000_000_000, 20); + // Only the two in-window samples contribute + assert!((m.energy_j - 150.0).abs() < 1e-9); // (150+150)/2 * 1s + assert!((m.mean_power_w - 150.0).abs() < 1e-9); + } +} diff --git a/rust/crates/openjarvis-telemetry/src/session.rs b/rust/crates/openjarvis-telemetry/src/session.rs new file mode 100644 index 00000000..7e76ff49 --- /dev/null +++ b/rust/crates/openjarvis-telemetry/src/session.rs @@ -0,0 +1,391 @@ +//! Ring-buffer backed telemetry session for high-frequency power/energy sampling. + +use parking_lot::RwLock; + +/// Single telemetry sample with nanosecond timestamp. +#[derive(Debug, Clone, Default)] +pub struct TelemetrySample { + pub timestamp_ns: u64, + pub gpu_power_w: f64, + pub cpu_power_w: f64, + pub gpu_energy_j: f64, + pub cpu_energy_j: f64, + pub gpu_util_pct: f64, + pub gpu_temp_c: f64, + pub gpu_mem_gb: f64, +} + +/// Fixed-capacity ring buffer with O(1) push and O(log n) window queries. +pub struct RingBuffer { + data: Vec>, + head: usize, + len: usize, + cap: usize, +} + +impl RingBuffer { + pub fn new(capacity: usize) -> Self { + assert!(capacity > 0, "RingBuffer capacity must be > 0"); + Self { + data: (0..capacity).map(|_| None).collect(), + head: 0, + len: 0, + cap: capacity, + } + } + + /// Push an item, overwriting the oldest when full. + pub fn push(&mut self, item: T) { + self.data[self.head] = Some(item); + self.head = (self.head + 1) % self.cap; + if self.len < self.cap { + self.len += 1; + } + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn clear(&mut self) { + for slot in self.data.iter_mut() { + *slot = None; + } + self.head = 0; + self.len = 0; + } + + /// Return items ordered oldest to newest. + pub fn as_ordered(&self) -> Vec<&T> { + if self.len == 0 { + return Vec::new(); + } + let mut result = Vec::with_capacity(self.len); + // Start index is where the oldest element lives + let start = if self.len < self.cap { + 0 + } else { + self.head // head points to the next write position = oldest element when full + }; + for i in 0..self.len { + let idx = (start + i) % self.cap; + if let Some(ref item) = self.data[idx] { + result.push(item); + } + } + result + } +} + +/// Core telemetry session using ring buffer. +pub struct TelemetrySessionCore { + buffer: RwLock>, + sampling_interval_ms: u64, +} + +impl TelemetrySessionCore { + pub fn new(capacity: usize, sampling_interval_ms: u64) -> Self { + Self { + buffer: RwLock::new(RingBuffer::new(capacity)), + sampling_interval_ms, + } + } + + pub fn sampling_interval_ms(&self) -> u64 { + self.sampling_interval_ms + } + + pub fn add_sample(&self, sample: TelemetrySample) { + self.buffer.write().push(sample); + } + + /// Return samples within [start_ns, end_ns] using binary search on monotonic timestamps. + pub fn window(&self, start_ns: u64, end_ns: u64) -> Vec { + let buf = self.buffer.read(); + let ordered = buf.as_ordered(); + if ordered.is_empty() { + return Vec::new(); + } + + // Binary search for first element >= start_ns + let lo = match ordered.binary_search_by(|s| s.timestamp_ns.cmp(&start_ns)) { + Ok(i) => i, + Err(i) => i, + }; + + // Binary search for last element <= end_ns + let hi = match ordered.binary_search_by(|s| s.timestamp_ns.cmp(&end_ns)) { + Ok(i) => i + 1, + Err(i) => i, + }; + + ordered[lo..hi].iter().map(|s| (*s).clone()).collect() + } + + /// Compute energy delta via trapezoidal integration of power samples. + /// Returns (gpu_energy_j, cpu_energy_j). + pub fn compute_energy_delta(&self, start_ns: u64, end_ns: u64) -> (f64, f64) { + let samples = self.window(start_ns, end_ns); + if samples.len() < 2 { + return (0.0, 0.0); + } + + let mut gpu_j = 0.0; + let mut cpu_j = 0.0; + for pair in samples.windows(2) { + let dt_s = (pair[1].timestamp_ns - pair[0].timestamp_ns) as f64 / 1e9; + // Trapezoidal rule: area = (a + b) / 2 * dt + gpu_j += (pair[0].gpu_power_w + pair[1].gpu_power_w) / 2.0 * dt_s; + cpu_j += (pair[0].cpu_power_w + pair[1].cpu_power_w) / 2.0 * dt_s; + } + + (gpu_j, cpu_j) + } + + /// Average power (gpu_w, cpu_w) in the given time window. + pub fn compute_avg_power(&self, start_ns: u64, end_ns: u64) -> (f64, f64) { + let samples = self.window(start_ns, end_ns); + if samples.is_empty() { + return (0.0, 0.0); + } + let n = samples.len() as f64; + let gpu_sum: f64 = samples.iter().map(|s| s.gpu_power_w).sum(); + let cpu_sum: f64 = samples.iter().map(|s| s.cpu_power_w).sum(); + (gpu_sum / n, cpu_sum / n) + } + + pub fn len(&self) -> usize { + self.buffer.read().len() + } + + pub fn is_empty(&self) -> bool { + self.buffer.read().is_empty() + } + + pub fn clear(&self) { + self.buffer.write().clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ---- RingBuffer tests ---- + + #[test] + fn ring_buffer_basic_push_and_len() { + let mut rb: RingBuffer = RingBuffer::new(4); + assert!(rb.is_empty()); + assert_eq!(rb.len(), 0); + + rb.push(1); + rb.push(2); + rb.push(3); + assert_eq!(rb.len(), 3); + assert!(!rb.is_empty()); + } + + #[test] + fn ring_buffer_ordered_no_wrap() { + let mut rb: RingBuffer = RingBuffer::new(5); + for i in 0..3 { + rb.push(i); + } + let ordered: Vec = rb.as_ordered().into_iter().cloned().collect(); + assert_eq!(ordered, vec![0, 1, 2]); + } + + #[test] + fn ring_buffer_wrap_around() { + let mut rb: RingBuffer = RingBuffer::new(3); + rb.push(1); + rb.push(2); + rb.push(3); + rb.push(4); // overwrites 1 + rb.push(5); // overwrites 2 + + assert_eq!(rb.len(), 3); + let ordered: Vec = rb.as_ordered().into_iter().cloned().collect(); + assert_eq!(ordered, vec![3, 4, 5]); + } + + #[test] + fn ring_buffer_clear() { + let mut rb: RingBuffer = RingBuffer::new(3); + rb.push(10); + rb.push(20); + rb.clear(); + assert!(rb.is_empty()); + assert_eq!(rb.len(), 0); + assert!(rb.as_ordered().is_empty()); + } + + #[test] + fn ring_buffer_single_capacity() { + let mut rb: RingBuffer = RingBuffer::new(1); + rb.push(42); + assert_eq!(rb.len(), 1); + rb.push(99); + assert_eq!(rb.len(), 1); + let ordered: Vec = rb.as_ordered().into_iter().cloned().collect(); + assert_eq!(ordered, vec![99]); + } + + #[test] + #[should_panic(expected = "capacity must be > 0")] + fn ring_buffer_zero_capacity_panics() { + let _rb: RingBuffer = RingBuffer::new(0); + } + + #[test] + fn ring_buffer_exact_fill() { + let mut rb: RingBuffer = RingBuffer::new(4); + for i in 0..4 { + rb.push(i); + } + assert_eq!(rb.len(), 4); + let ordered: Vec = rb.as_ordered().into_iter().cloned().collect(); + assert_eq!(ordered, vec![0, 1, 2, 3]); + } + + // ---- TelemetrySessionCore tests ---- + + fn make_sample(ts_ns: u64, gpu_w: f64, cpu_w: f64) -> TelemetrySample { + TelemetrySample { + timestamp_ns: ts_ns, + gpu_power_w: gpu_w, + cpu_power_w: cpu_w, + ..Default::default() + } + } + + #[test] + fn session_add_and_len() { + let session = TelemetrySessionCore::new(100, 10); + assert_eq!(session.len(), 0); + assert!(session.is_empty()); + + session.add_sample(make_sample(1_000_000, 100.0, 50.0)); + session.add_sample(make_sample(2_000_000, 110.0, 55.0)); + assert_eq!(session.len(), 2); + assert!(!session.is_empty()); + } + + #[test] + fn session_window_returns_correct_range() { + let session = TelemetrySessionCore::new(100, 10); + for i in 0..10 { + session.add_sample(make_sample(i * 1_000_000, 100.0, 50.0)); + } + + let win = session.window(3_000_000, 6_000_000); + assert_eq!(win.len(), 4); // timestamps 3, 4, 5, 6 (inclusive) + assert_eq!(win[0].timestamp_ns, 3_000_000); + assert_eq!(win[3].timestamp_ns, 6_000_000); + } + + #[test] + fn session_window_empty() { + let session = TelemetrySessionCore::new(100, 10); + let win = session.window(0, 1_000_000); + assert!(win.is_empty()); + } + + #[test] + fn session_window_no_match() { + let session = TelemetrySessionCore::new(100, 10); + session.add_sample(make_sample(1_000_000, 100.0, 50.0)); + session.add_sample(make_sample(2_000_000, 100.0, 50.0)); + + let win = session.window(5_000_000, 10_000_000); + assert!(win.is_empty()); + } + + #[test] + fn session_energy_delta_trapezoidal() { + let session = TelemetrySessionCore::new(100, 10); + // 100W GPU, 50W CPU constant for 1 second (two samples 1s apart) + session.add_sample(make_sample(0, 100.0, 50.0)); + session.add_sample(make_sample(1_000_000_000, 100.0, 50.0)); // 1s later + + let (gpu_j, cpu_j) = session.compute_energy_delta(0, 1_000_000_000); + assert!((gpu_j - 100.0).abs() < 1e-9, "gpu_j = {gpu_j}"); + assert!((cpu_j - 50.0).abs() < 1e-9, "cpu_j = {cpu_j}"); + } + + #[test] + fn session_energy_delta_linear_ramp() { + let session = TelemetrySessionCore::new(100, 10); + // GPU ramps from 0W to 200W over 1 second -> trapezoidal area = 100 J + session.add_sample(make_sample(0, 0.0, 0.0)); + session.add_sample(make_sample(1_000_000_000, 200.0, 100.0)); + + let (gpu_j, cpu_j) = session.compute_energy_delta(0, 1_000_000_000); + assert!((gpu_j - 100.0).abs() < 1e-9, "gpu_j = {gpu_j}"); + assert!((cpu_j - 50.0).abs() < 1e-9, "cpu_j = {cpu_j}"); + } + + #[test] + fn session_energy_delta_single_sample() { + let session = TelemetrySessionCore::new(100, 10); + session.add_sample(make_sample(0, 100.0, 50.0)); + + let (gpu_j, cpu_j) = session.compute_energy_delta(0, 0); + assert_eq!(gpu_j, 0.0); + assert_eq!(cpu_j, 0.0); + } + + #[test] + fn session_avg_power() { + let session = TelemetrySessionCore::new(100, 10); + session.add_sample(make_sample(0, 100.0, 50.0)); + session.add_sample(make_sample(1_000_000, 200.0, 100.0)); + session.add_sample(make_sample(2_000_000, 150.0, 75.0)); + + let (gpu_avg, cpu_avg) = session.compute_avg_power(0, 2_000_000); + assert!((gpu_avg - 150.0).abs() < 1e-9); + assert!((cpu_avg - 75.0).abs() < 1e-9); + } + + #[test] + fn session_avg_power_empty() { + let session = TelemetrySessionCore::new(100, 10); + let (gpu_avg, cpu_avg) = session.compute_avg_power(0, 1_000_000); + assert_eq!(gpu_avg, 0.0); + assert_eq!(cpu_avg, 0.0); + } + + #[test] + fn session_clear() { + let session = TelemetrySessionCore::new(100, 10); + session.add_sample(make_sample(0, 100.0, 50.0)); + session.clear(); + assert!(session.is_empty()); + assert_eq!(session.len(), 0); + } + + #[test] + fn session_ring_buffer_overflow() { + let session = TelemetrySessionCore::new(3, 10); + for i in 0..5 { + session.add_sample(make_sample(i * 1_000_000, 100.0, 50.0)); + } + assert_eq!(session.len(), 3); + // Only the last 3 samples should remain (ts: 2, 3, 4 million) + let win = session.window(0, 10_000_000); + assert_eq!(win.len(), 3); + assert_eq!(win[0].timestamp_ns, 2_000_000); + assert_eq!(win[2].timestamp_ns, 4_000_000); + } + + #[test] + fn session_sampling_interval() { + let session = TelemetrySessionCore::new(100, 42); + assert_eq!(session.sampling_interval_ms(), 42); + } +} diff --git a/scripts/run_use_case_comparison.sh b/scripts/run_use_case_comparison.sh new file mode 100755 index 00000000..583ce2d7 --- /dev/null +++ b/scripts/run_use_case_comparison.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# ============================================================================= +# run_use_case_comparison.sh +# +# Orchestrates the 11-model × 2-agent × 5-benchmark eval comparison. +# +# Open-source models run sequentially (one vLLM server at a time). +# Cloud models run via API (no GPU needed). +# +# Usage: +# ./scripts/run_use_case_comparison.sh # Full comparison +# ./scripts/run_use_case_comparison.sh --cloud-only # Cloud models only +# ./scripts/run_use_case_comparison.sh --oss-only # Open-source models only +# ./scripts/run_use_case_comparison.sh --model "GLM-4.7-Flash" # Single OSS model +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$PROJECT_DIR" + +# === Configuration === +EVAL_CMD="uv run jarvis eval run" +CONFIGS_DIR="src/openjarvis/evals/configs" +VLLM_PORT=8000 +VLLM_HEALTH_URL="http://localhost:${VLLM_PORT}/health" +VLLM_STARTUP_TIMEOUT=300 # seconds +VLLM_PID="" +LOG_DIR="logs/eval-comparison" + +# Open-source models: name|hf_id|tensor_parallel_size +OSS_MODELS=( + "GLM-4.7-Flash|unsloth/GLM-4.7-Flash-GGUF|2" + "Qwen3.5-122B|unsloth/Qwen3.5-122B-A10B-GGUF|4" + "gpt-oss-120b|unsloth/gpt-oss-120b-GGUF|4" + "GLM-5|unsloth/GLM-5-GGUF|4" + "Qwen3.5-397B|unsloth/Qwen3.5-397B-A17B-GGUF|8" +) + +# === Helpers === + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" +} + +ensure_dirs() { + mkdir -p "$LOG_DIR" + mkdir -p results/use-cases-opensource-orchestrator + mkdir -p results/use-cases-opensource-openhands + mkdir -p results/use-cases-cloud-orchestrator + mkdir -p results/use-cases-cloud-openhands +} + +start_vllm() { + local model_id="$1" + local tp_size="$2" + local log_file="$3" + + log "Starting vLLM: model=$model_id tp=$tp_size" + vllm serve "$model_id" \ + --tensor-parallel-size "$tp_size" \ + --port "$VLLM_PORT" \ + --disable-log-requests \ + > "$log_file" 2>&1 & + VLLM_PID=$! + log "vLLM PID: $VLLM_PID" +} + +wait_for_vllm() { + local elapsed=0 + log "Waiting for vLLM to be ready at $VLLM_HEALTH_URL ..." + while [ $elapsed -lt $VLLM_STARTUP_TIMEOUT ]; do + if curl -sf "$VLLM_HEALTH_URL" > /dev/null 2>&1; then + log "vLLM is ready (${elapsed}s)" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + log "ERROR: vLLM failed to start within ${VLLM_STARTUP_TIMEOUT}s" + stop_vllm + return 1 +} + +stop_vllm() { + if [ -n "$VLLM_PID" ] && kill -0 "$VLLM_PID" 2>/dev/null; then + log "Stopping vLLM (PID $VLLM_PID)" + kill "$VLLM_PID" 2>/dev/null || true + wait "$VLLM_PID" 2>/dev/null || true + VLLM_PID="" + fi + # Also kill any lingering vllm serve processes on our port + pkill -f "vllm serve.*--port $VLLM_PORT" 2>/dev/null || true + sleep 2 +} + +run_eval() { + local config="$1" + local filter="${2:-}" + local start_time end_time + + start_time=$(date +%s) + log "Running: $config ${filter:+(filter: $filter)}" + + if [ -n "$filter" ]; then + $EVAL_CMD -c "$config" --model-filter "$filter" -v || { + log "WARNING: eval run failed for $config (filter: $filter)" + return 1 + } + else + $EVAL_CMD -c "$config" -v || { + log "WARNING: eval run failed for $config" + return 1 + } + fi + + end_time=$(date +%s) + log "Completed in $((end_time - start_time))s: $config ${filter:+(filter: $filter)}" +} + +# === Execution Phases === + +run_oss_models() { + local model_filter="${1:-}" + + for entry in "${OSS_MODELS[@]}"; do + IFS='|' read -r name model_id tp_size <<< "$entry" + + # Skip if --model flag set and doesn't match + if [ -n "$model_filter" ] && [[ "$name" != *"$model_filter"* ]]; then + continue + fi + + log "========================================" + log "OSS MODEL: $name ($model_id, TP=$tp_size)" + log "========================================" + + local log_file="$LOG_DIR/vllm_${name}.log" + + # Start vLLM for this model + start_vllm "$model_id" "$tp_size" "$log_file" + if ! wait_for_vllm; then + log "Skipping $name — vLLM failed to start" + continue + fi + + # Run orchestrator config (filtered to this model) + run_eval "$CONFIGS_DIR/use_case_opensource_orchestrator.toml" "$name" || true + + # Run openhands config (filtered to this model) + run_eval "$CONFIGS_DIR/use_case_opensource_openhands.toml" "$name" || true + + # Stop vLLM + stop_vllm + + log "Completed all benchmarks for $name" + done +} + +run_cloud_models() { + log "========================================" + log "CLOUD MODELS" + log "========================================" + + # Source .env for API keys + if [ -f "$PROJECT_DIR/.env" ]; then + # shellcheck disable=SC1091 + set -a && source "$PROJECT_DIR/.env" && set +a + log "Loaded API keys from .env" + else + log "WARNING: .env not found — cloud runs may fail" + fi + + # Orchestrator agent + run_eval "$CONFIGS_DIR/use_case_cloud_orchestrator.toml" || true + + # CodeAct agent + run_eval "$CONFIGS_DIR/use_case_cloud_openhands.toml" || true + + log "Completed cloud model runs" +} + +run_comparison() { + log "========================================" + log "COMPARISON" + log "========================================" + + local result_files=() + for dir in results/use-cases-{opensource-orchestrator,opensource-openhands,cloud-orchestrator,cloud-openhands}; do + if [ -d "$dir" ]; then + while IFS= read -r -d '' f; do + result_files+=("$f") + done < <(find "$dir" -name '*.jsonl' -print0 2>/dev/null) + fi + done + + if [ ${#result_files[@]} -eq 0 ]; then + log "No result files found — skipping comparison" + return + fi + + log "Comparing ${#result_files[@]} result files" + uv run jarvis eval compare "${result_files[@]}" || { + log "WARNING: comparison failed" + } +} + +# === Main === + +main() { + local mode="all" + local model_filter="" + + while [[ $# -gt 0 ]]; do + case "$1" in + --cloud-only) mode="cloud"; shift ;; + --oss-only) mode="oss"; shift ;; + --model) model_filter="$2"; mode="oss"; shift 2 ;; + --compare) mode="compare"; shift ;; + -h|--help) + echo "Usage: $0 [--cloud-only|--oss-only|--model NAME|--compare]" + echo "" + echo "Options:" + echo " --cloud-only Run cloud models only (no GPU needed)" + echo " --oss-only Run open-source models only (needs GPU + vLLM)" + echo " --model NAME Run a single open-source model by name substring" + echo " --compare Only run comparison on existing results" + exit 0 + ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac + done + + log "Starting eval comparison (mode=$mode)" + ensure_dirs + + # Trap to ensure vLLM is stopped on exit + trap stop_vllm EXIT + + case "$mode" in + all) + run_oss_models "" + run_cloud_models + run_comparison + ;; + oss) + run_oss_models "$model_filter" + ;; + cloud) + run_cloud_models + ;; + compare) + run_comparison + ;; + esac + + log "Done!" +} + +main "$@" diff --git a/src/openjarvis/cli/eval_cmd.py b/src/openjarvis/cli/eval_cmd.py index 3a9dc8dc..3f03fb30 100644 --- a/src/openjarvis/cli/eval_cmd.py +++ b/src/openjarvis/cli/eval_cmd.py @@ -34,6 +34,26 @@ KNOWN_BENCHMARKS = { "category": "agentic", "description": "TerminalBench Native (Docker)", }, + "email_triage": { + "category": "use-case", + "description": "Email triage classification + draft", + }, + "morning_brief": { + "category": "use-case", + "description": "Morning briefing generation", + }, + "research_mining": { + "category": "use-case", + "description": "Research synthesis + accuracy", + }, + "knowledge_base": { + "category": "use-case", + "description": "Document-grounded retrieval QA", + }, + "coding_task": { + "category": "use-case", + "description": "Function-level code generation", + }, } KNOWN_BACKENDS = { @@ -101,6 +121,10 @@ def eval_list() -> None: "--agent", "agent_name", default=None, help="Agent name for jarvis-agent backend.", ) +@click.option( + "-e", "--engine", "engine_key", default=None, + help="Engine key (ollama, vllm, cloud, ...).", +) @click.option( "--tools", "tools", default="", help="Comma-separated tool names.", @@ -109,6 +133,26 @@ def eval_list() -> None: "--telemetry/--no-telemetry", "telemetry", default=False, help="Enable telemetry collection during eval.", ) +@click.option( + "--gpu-metrics/--no-gpu-metrics", "gpu_metrics", default=False, + help="Enable GPU metrics collection.", +) +@click.option( + "--seed", "seed", type=int, default=42, + help="Random seed.", +) +@click.option( + "--temperature", "temperature", type=float, default=0.0, + help="Generation temperature.", +) +@click.option( + "--max-tokens", "max_tokens", type=int, default=2048, + help="Max output tokens.", +) +@click.option( + "--model-filter", "model_filter", default=None, + help="Filter models by name substring (for multi-model configs).", +) @click.option( "-o", "--output", "output_path", default=None, type=click.Path(), help="Output JSONL path.", @@ -152,8 +196,14 @@ def eval_run( max_samples: Optional[int], backend: str, agent_name: Optional[str], + engine_key: Optional[str], tools: str, telemetry: bool, + gpu_metrics: bool, + seed: int, + temperature: float, + max_tokens: int, + model_filter: Optional[str], output_path: Optional[str], wandb_project: str, wandb_entity: str, @@ -185,6 +235,17 @@ def eval_run( console.print(f"[red]Error loading config: {exc}[/red]") sys.exit(1) + # Filter by model name substring if requested + if model_filter: + run_configs = [ + rc for rc in run_configs if model_filter in rc.model + ] + if not run_configs: + console.print( + f"[red]No models match filter '{model_filter}'[/red]" + ) + sys.exit(1) + console.print( f"[cyan]Suite:[/cyan] {suite.meta.name or Path(config_path).stem}" ) @@ -248,9 +309,14 @@ def eval_run( model=model, max_samples=max_samples, agent_name=agent_name, + engine_key=engine_key, tools=tool_list, output_path=output_path, + seed=seed, + temperature=temperature, + max_tokens=max_tokens, telemetry=telemetry, + gpu_metrics=gpu_metrics, wandb_project=wandb_project, wandb_entity=wandb_entity, wandb_tags=wandb_tags, diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.py index 9442718a..0c60d0ca 100644 --- a/src/openjarvis/engine/_openai_compat.py +++ b/src/openjarvis/engine/_openai_compat.py @@ -22,7 +22,7 @@ class _OpenAICompatibleEngine(InferenceEngine): engine_id: str = "" _default_host: str = "http://localhost:8000" - def __init__(self, host: str | None = None, *, timeout: float = 120.0) -> None: + def __init__(self, host: str | None = None, *, timeout: float = 600.0) -> None: self._host = (host or self._default_host).rstrip("/") self._client = httpx.Client(base_url=self._host, timeout=timeout) diff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.py index b52a3e83..35b81711 100644 --- a/src/openjarvis/engine/cloud.py +++ b/src/openjarvis/engine/cloud.py @@ -21,6 +21,7 @@ PRICING: Dict[str, tuple[float, float]] = { "gpt-4o": (2.50, 10.00), "gpt-4o-mini": (0.15, 0.60), "gpt-5": (10.00, 30.00), + "gpt-5.4": (15.00, 60.00), "gpt-5-mini": (0.25, 2.00), "o3-mini": (1.10, 4.40), "claude-sonnet-4-20250514": (3.00, 15.00), @@ -33,10 +34,14 @@ PRICING: Dict[str, tuple[float, float]] = { "gemini-2.5-flash": (0.30, 2.50), "gemini-3-pro": (2.00, 12.00), "gemini-3-flash": (0.50, 3.00), + "gemini-3.1-pro-preview": (2.50, 15.00), + "gemini-3-flash-preview": (0.50, 3.00), } # Well-known model IDs per provider -_OPENAI_MODELS = ["gpt-4o", "gpt-4o-mini", "gpt-5", "gpt-5-mini", "o3-mini"] +_OPENAI_MODELS = [ + "gpt-4o", "gpt-4o-mini", "gpt-5", "gpt-5.4", "gpt-5-mini", "o3-mini", +] _ANTHROPIC_MODELS = [ "claude-sonnet-4-20250514", "claude-opus-4-20250514", @@ -50,6 +55,8 @@ _GOOGLE_MODELS = [ "gemini-2.5-flash", "gemini-3-pro", "gemini-3-flash", + "gemini-3.1-pro-preview", + "gemini-3-flash-preview", ] @@ -64,8 +71,10 @@ def _is_google_model(model: str) -> bool: def _is_openai_reasoning_model(model: str) -> bool: """Check if model is an OpenAI reasoning model that restricts temperature.""" m = model.lower() - # o1/o3 series and gpt-5-mini dated snapshots are reasoning models - return m.startswith(("o1", "o3")) or "gpt-5-mini-" in m + # o1/o3 series and gpt-5-mini (all variants) are reasoning models + if m.startswith(("o1", "o3")): + return True + return m == "gpt-5-mini" or m.startswith("gpt-5-mini-") def estimate_cost(model: str, prompt_tokens: int, completion_tokens: int) -> float: diff --git a/src/openjarvis/evals/backends/jarvis_agent.py b/src/openjarvis/evals/backends/jarvis_agent.py index 6f9fca1c..ea1caabf 100644 --- a/src/openjarvis/evals/backends/jarvis_agent.py +++ b/src/openjarvis/evals/backends/jarvis_agent.py @@ -24,6 +24,7 @@ class JarvisAgentBackend(InferenceBackend): tools: Optional[List[str]] = None, telemetry: bool = False, gpu_metrics: bool = False, + model: Optional[str] = None, ) -> None: from openjarvis.system import SystemBuilder @@ -35,6 +36,8 @@ class JarvisAgentBackend(InferenceBackend): builder = SystemBuilder() if engine_key: builder.engine(engine_key) + if model: + builder.model(model) builder.agent(agent_name) if tools: builder.tools(tools) @@ -95,6 +98,12 @@ class JarvisAgentBackend(InferenceBackend): "throughput_tok_per_sec": telemetry_data.get("throughput_tok_per_sec", 0.0), } + def set_task_metadata(self, metadata: dict) -> None: + """Forward task environment metadata to the underlying agent.""" + agent = getattr(self._system, "_agent", None) + if agent and hasattr(agent, "set_task_metadata"): + agent.set_task_metadata(metadata) + def close(self) -> None: self._system.close() diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index 11b35741..eaedb367 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -49,6 +49,26 @@ BENCHMARKS = { "category": "agentic", "description": "TerminalBench Native (Docker)", }, + "email_triage": { + "category": "use-case", + "description": "Email triage classification + draft", + }, + "morning_brief": { + "category": "use-case", + "description": "Morning briefing generation", + }, + "research_mining": { + "category": "use-case", + "description": "Research synthesis + accuracy", + }, + "knowledge_base": { + "category": "use-case", + "description": "Document-grounded retrieval QA", + }, + "coding_task": { + "category": "use-case", + "description": "Function-level code generation", + }, } BACKENDS = { @@ -68,7 +88,8 @@ def _setup_logging(verbose: bool) -> None: def _build_backend(backend_name: str, engine_key: Optional[str], agent_name: str, tools: list[str], - telemetry: bool = False, gpu_metrics: bool = False): + telemetry: bool = False, gpu_metrics: bool = False, + model: Optional[str] = None): """Construct the appropriate backend.""" if backend_name == "jarvis-agent": from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend @@ -78,6 +99,7 @@ def _build_backend(backend_name: str, engine_key: Optional[str], tools=tools, telemetry=telemetry, gpu_metrics=gpu_metrics, + model=model, ) else: from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend @@ -137,6 +159,21 @@ def _build_dataset(benchmark: str): TerminalBenchNativeDataset, ) return TerminalBenchNativeDataset() + elif benchmark == "email_triage": + from openjarvis.evals.datasets.email_triage import EmailTriageDataset + return EmailTriageDataset() + elif benchmark == "morning_brief": + from openjarvis.evals.datasets.morning_brief import MorningBriefDataset + return MorningBriefDataset() + elif benchmark == "research_mining": + from openjarvis.evals.datasets.research_mining import ResearchMiningDataset + return ResearchMiningDataset() + elif benchmark == "knowledge_base": + from openjarvis.evals.datasets.knowledge_base import KnowledgeBaseDataset + return KnowledgeBaseDataset() + elif benchmark == "coding_task": + from openjarvis.evals.datasets.coding_task import CodingTaskDataset + return CodingTaskDataset() else: raise click.ClickException(f"Unknown benchmark: {benchmark}") @@ -187,6 +224,21 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str): TerminalBenchNativeScorer, ) return TerminalBenchNativeScorer(judge_backend, judge_model) + elif benchmark == "email_triage": + from openjarvis.evals.scorers.email_triage import EmailTriageScorer + return EmailTriageScorer(judge_backend, judge_model) + elif benchmark == "morning_brief": + from openjarvis.evals.scorers.morning_brief import MorningBriefScorer + return MorningBriefScorer(judge_backend, judge_model) + elif benchmark == "research_mining": + from openjarvis.evals.scorers.research_mining import ResearchMiningScorer + return ResearchMiningScorer(judge_backend, judge_model) + elif benchmark == "knowledge_base": + from openjarvis.evals.scorers.knowledge_base import KnowledgeBaseScorer + return KnowledgeBaseScorer(judge_backend, judge_model) + elif benchmark == "coding_task": + from openjarvis.evals.scorers.coding_task import CodingTaskScorer + return CodingTaskScorer(judge_backend, judge_model) else: raise click.ClickException(f"Unknown benchmark: {benchmark}") @@ -263,6 +315,7 @@ def _run_single(config, console: Optional[Console] = None) -> object: config.tools, telemetry=getattr(config, "telemetry", False), gpu_metrics=getattr(config, "gpu_metrics", False), + model=config.model, ) dataset = _build_dataset(config.benchmark) judge_backend = _build_judge_backend(config.judge_model) @@ -297,7 +350,217 @@ def _run_single(config, console: Optional[Console] = None) -> object: judge_backend.close() -def _run_from_config(config_path: str, verbose: bool) -> None: +def _run_agentic( + config, + console: Optional[Console] = None, + *, + concurrency: int = 1, + query_timeout: Optional[float] = None, +) -> None: + """Run an agentic evaluation using AgenticRunner with trace + energy capture.""" + import asyncio + from pathlib import Path as _Path + + from openjarvis.evals.core.agentic_runner import AgenticRunner + from openjarvis.evals.core.event_recorder import EventRecorder + from openjarvis.evals.core.export import ( + export_artifacts_manifest, + export_jsonl, + export_summary_json, + ) + + if console is None: + console = Console() + + # Build dataset + dataset = _build_dataset(config.benchmark) + dataset.load( + max_samples=config.max_samples, + split=config.dataset_split, + seed=config.seed, + ) + + # Build agent via SystemBuilder + from openjarvis.system import SystemBuilder + + builder = SystemBuilder() + if config.engine_key: + builder.engine(config.engine_key) + builder.model(config.model) + agent_name = config.agent_name or "orchestrator" + builder.agent(agent_name) + tool_list = config.tools or [] + if tool_list: + builder.tools(tool_list) + system = builder.telemetry(config.telemetry).traces(config.telemetry).build() + + # Build TelemetrySession (optional — only if energy monitoring available) + telemetry_session = None + try: + from openjarvis.telemetry.energy_monitor import create_energy_monitor + from openjarvis.telemetry.session import TelemetrySession + + monitor = create_energy_monitor() + if monitor is not None: + telemetry_session = TelemetrySession( + monitor=monitor, interval_ms=100, + ) + except ImportError: + pass + + # Set up run directory + model_slug = config.model.replace("/", "-").replace(":", "-") + if config.output_path: + run_dir = _Path(config.output_path).parent + else: + run_dir = _Path("results") + run_dir = run_dir / f"agentic_{config.benchmark}_{model_slug}" + run_dir.mkdir(parents=True, exist_ok=True) + + # Build runner + event_recorder = EventRecorder() + runner = AgenticRunner( + agent=system, + dataset=dataset, + telemetry_session=telemetry_session, + config={ + "model": config.model, + "benchmark": config.benchmark, + "agent": agent_name, + "tools": tool_list, + "temperature": config.temperature, + "max_tokens": config.max_tokens, + }, + event_recorder=event_recorder, + run_dir=run_dir, + concurrency=concurrency, + query_timeout=query_timeout, + ) + + # Execute with telemetry session context + try: + ctx = telemetry_session if telemetry_session is not None else _nullctx() + with ctx: + with console.status("Running agentic evaluation..."): + traces = asyncio.run(runner.run(max_queries=config.max_samples)) + finally: + system.close() + if telemetry_session is not None and hasattr(telemetry_session, "stop"): + try: + telemetry_session.stop() + except Exception: + pass + + # Export results + jsonl_path = run_dir / "traces.jsonl" + export_jsonl(traces, jsonl_path) + console.print(f" [green]Traces:[/green] {jsonl_path}") + + summary_path = run_dir / "summary.json" + export_summary_json( + traces, + config={ + "model": config.model, + "benchmark": config.benchmark, + "agent": agent_name, + "concurrency": concurrency, + "query_timeout": query_timeout, + }, + path=summary_path, + ) + console.print(f" [green]Summary:[/green] {summary_path}") + + manifest = export_artifacts_manifest(run_dir) + if manifest: + console.print(f" [green]Manifest:[/green] {manifest}") + + # Try HF dataset export (optional) + try: + from openjarvis.evals.core.export import export_hf_dataset + hf_path = run_dir / "hf_dataset" + export_hf_dataset(traces, hf_path) + console.print(f" [green]HF Arrow:[/green] {hf_path}") + except ImportError: + pass + + # Print summary table + print_section(console, "Agentic Results") + _print_agentic_summary(console, traces, config) + + +def _nullctx(): + """Return a no-op context manager.""" + from contextlib import nullcontext + return nullcontext() + + +def _print_agentic_summary(console: Console, traces, config) -> None: + """Print a rich summary of agentic run results.""" + from rich.table import Table + + completed = sum(1 for t in traces if t.completed) + resolved = sum(1 for t in traces if t.is_resolved is True) + timed_out = sum(1 for t in traces if t.timed_out) + total_turns = sum(t.num_turns for t in traces) + total_tool_calls = sum(t.total_tool_calls for t in traces) + total_in_tok = sum(t.total_input_tokens for t in traces) + total_out_tok = sum(t.total_output_tokens for t in traces) + total_wall = sum(t.total_wall_clock_s for t in traces) + + gpu_energies = [ + t.total_gpu_energy_joules for t in traces + if t.total_gpu_energy_joules is not None + ] + total_gpu_energy = sum(gpu_energies) if gpu_energies else None + + costs = [t.total_cost_usd for t in traces if t.total_cost_usd is not None] + total_cost = sum(costs) if costs else None + + table = Table( + title=f"[bold]{config.benchmark} / {config.model}[/bold]", + border_style="bright_blue", + ) + table.add_column("Metric", style="cyan", no_wrap=True) + table.add_column("Value", style="white") + + table.add_row("Queries", str(len(traces))) + table.add_row("Completed", f"{completed}/{len(traces)}") + if any(t.is_resolved is not None for t in traces): + table.add_row("Resolved", f"{resolved}/{len(traces)}") + table.add_row("Timed out", str(timed_out)) + table.add_row("Total turns", str(total_turns)) + avg_t = f"{total_turns / len(traces):.1f}" if traces else "0" + table.add_row("Avg turns/query", avg_t) + table.add_row("Total tool calls", str(total_tool_calls)) + table.add_row("Input tokens", f"{total_in_tok:,}") + table.add_row("Output tokens", f"{total_out_tok:,}") + table.add_row("Wall clock", f"{total_wall:.1f}s") + table.add_row( + "Avg query time", + f"{total_wall/len(traces):.1f}s" if traces else "0s", + ) + + if total_gpu_energy is not None: + table.add_row("GPU energy", f"{total_gpu_energy:.2f} J") + if total_cost is not None: + table.add_row("Total cost", f"${total_cost:.4f}") + + # Throughput + if total_out_tok > 0 and total_wall > 0: + table.add_row( + "Throughput", + f"{total_out_tok/total_wall:.1f} tok/s", + ) + + console.print(table) + + +def _run_from_config( + config_path: str, + verbose: bool, + *, + model_filter: str | None = None, +) -> None: """Load a TOML config and run the full models x benchmarks matrix.""" from openjarvis.evals.core.config import expand_suite, load_eval_config @@ -306,6 +569,14 @@ def _run_from_config(config_path: str, verbose: bool) -> None: suite = load_eval_config(config_path) run_configs = expand_suite(suite) + # Filter by model name substring if requested + if model_filter: + run_configs = [rc for rc in run_configs if model_filter in rc.model] + if not run_configs: + raise click.ClickException( + f"No models match filter '{model_filter}'" + ) + suite_name = suite.meta.name or Path(config_path).stem # Banner + configuration @@ -414,6 +685,14 @@ def main(): @click.option("--sheets-creds", "sheets_credentials_path", default="", help="Service account JSON path") +@click.option("--model-filter", default=None, + help="Filter models by name substring (for multi-model configs)") +@click.option("--agentic", is_flag=True, default=False, + help="Use AgenticRunner for multi-turn agent execution") +@click.option("--concurrency", type=int, default=1, + help="Parallel query execution (AgenticRunner only)") +@click.option("--query-timeout", type=float, default=None, + help="Per-query wall-clock timeout in seconds (AgenticRunner only)") @click.option("-v", "--verbose", is_flag=True, help="Verbose logging") @click.pass_context def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, @@ -422,7 +701,7 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, compact, trace_detail, wandb_project, wandb_entity, wandb_tags, wandb_group, sheets_spreadsheet_id, sheets_worksheet, sheets_credentials_path, - verbose): + model_filter, agentic, concurrency, query_timeout, verbose): """Run a single benchmark evaluation, or a full suite from a TOML config.""" _setup_logging(verbose) @@ -430,7 +709,7 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, # Config-driven mode if config_path is not None: - _run_from_config(config_path, verbose) + _run_from_config(config_path, verbose, model_filter=model_filter) return # CLI-driven mode: validate required args @@ -487,6 +766,23 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, workers=max_workers, ) + if agentic: + # --- Agentic runner path --- + print_section(console, "Agentic Evaluation") + console.print( + f" [cyan]Concurrency:[/cyan] {concurrency}" + ) + if query_timeout: + console.print( + f" [cyan]Timeout:[/cyan] {query_timeout}s per query" + ) + _run_agentic( + config, console=console, + concurrency=concurrency, + query_timeout=query_timeout, + ) + return + # Evaluation print_section(console, "Evaluation") summary = _run_single(config, console=console) diff --git a/src/openjarvis/evals/configs/use_case_cloud_comparison.toml b/src/openjarvis/evals/configs/use_case_cloud_comparison.toml new file mode 100644 index 00000000..08ffbbd1 --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_cloud_comparison.toml @@ -0,0 +1,78 @@ +# Local vs Cloud comparison on 5 use-case benchmarks. +# Same agent (native_openhands), same benchmarks, different models. +# Shows cost/accuracy/latency tradeoffs. +# +# Prerequisites: source .env (OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY) +# Local: ollama running with qwen3:8b pulled + +[meta] +name = "use-case-cloud-comparison" +description = "Local (qwen3:8b) vs Cloud (gpt-5-mini, claude-sonnet-4-6, gemini-3-flash) on use-case benchmarks" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-cloud-comparison/" +seed = 42 +telemetry = true + +# --- Models Under Test --- + +[[models]] +name = "qwen3:8b" +engine = "ollama" + +[[models]] +name = "gpt-5-mini" +provider = "openai" + +[[models]] +name = "claude-sonnet-4-6" +provider = "anthropic" + +[[models]] +name = "gemini-3-flash" +provider = "google" + +# --- Benchmarks --- + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "web_search"] +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "web_search"] +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "code_interpreter"] +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_cloud_openhands.toml b/src/openjarvis/evals/configs/use_case_cloud_openhands.toml new file mode 100644 index 00000000..43c4c9e4 --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_cloud_openhands.toml @@ -0,0 +1,87 @@ +# Closed-source cloud models × NativeOpenHands (CodeAct) agent on 5 use-case benchmarks. +# +# Models: Claude Opus 4.6, Claude Sonnet 4.6, GPT 5.4, GPT 5 Mini, +# Gemini 3.1 Pro, Gemini 3.1 Flash +# Agent: NativeOpenHandsAgent (code-execution-centric, Action/Observation loop) +# +# Prerequisites: source .env (OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY) + +[meta] +name = "use-case-cloud-openhands" +description = "Cloud models with NativeOpenHandsAgent (CodeAct) on use-case benchmarks" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-cloud-openhands/" +seed = 42 +telemetry = true + +# --- Models Under Test --- + +[[models]] +name = "claude-opus-4-6" +provider = "anthropic" + +[[models]] +name = "claude-sonnet-4-6" +provider = "anthropic" + +[[models]] +name = "gpt-5.4" +provider = "openai" + +[[models]] +name = "gpt-5-mini" +provider = "openai" + +[[models]] +name = "gemini-3.1-pro-preview" +provider = "google" + +[[models]] +name = "gemini-3-flash-preview" +provider = "google" + +# --- Benchmarks --- + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "web_search"] +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "web_search"] +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "code_interpreter"] +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml b/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml new file mode 100644 index 00000000..81e97695 --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_cloud_orchestrator.toml @@ -0,0 +1,87 @@ +# Closed-source cloud models × Orchestrator agent on 5 use-case benchmarks. +# +# Models: Claude Opus 4.6, Claude Sonnet 4.6, GPT 5.4, GPT 5 Mini, +# Gemini 3.1 Pro, Gemini 3.1 Flash +# Agent: OrchestratorAgent (multi-turn tool-calling loop) +# +# Prerequisites: source .env (OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY) + +[meta] +name = "use-case-cloud-orchestrator" +description = "Cloud models with OrchestratorAgent on use-case benchmarks" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-cloud-orchestrator/" +seed = 42 +telemetry = true + +# --- Models Under Test --- + +[[models]] +name = "claude-opus-4-6" +provider = "anthropic" + +[[models]] +name = "claude-sonnet-4-6" +provider = "anthropic" + +[[models]] +name = "gpt-5.4" +provider = "openai" + +[[models]] +name = "gpt-5-mini" +provider = "openai" + +[[models]] +name = "gemini-3.1-pro-preview" +provider = "google" + +[[models]] +name = "gemini-3-flash-preview" +provider = "google" + +# --- Benchmarks --- + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "web_search"] +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "web_search"] +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "code_interpreter"] +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_direct.toml b/src/openjarvis/evals/configs/use_case_direct.toml new file mode 100644 index 00000000..81d1c248 --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_direct.toml @@ -0,0 +1,47 @@ +# Baseline: direct engine inference (no agent) on 5 use-case benchmarks. +# Compare this against agent-backed configs to measure agent value-add. + +[meta] +name = "use-case-direct-baseline" +description = "Use-case benchmarks — raw engine, no agent (baseline)" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-direct/" +seed = 42 + +[[models]] +name = "qwen3:8b" +engine = "ollama" + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-direct" +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-direct" +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-direct" +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-direct" +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-direct" +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_openhands.toml b/src/openjarvis/evals/configs/use_case_openhands.toml new file mode 100644 index 00000000..b66c507f --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_openhands.toml @@ -0,0 +1,57 @@ +# NativeOpenHandsAgent (CodeAct) on 5 use-case benchmarks. +# Code-execution-centric agent — strongest on coding tasks. + +[meta] +name = "use-case-openhands" +description = "Use-case benchmarks — NativeOpenHandsAgent (CodeAct)" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-openhands/" +seed = 42 + +[[models]] +name = "qwen3:8b" +engine = "ollama" + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "web_search"] +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "web_search"] +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "code_interpreter"] +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_opensource_openhands.toml b/src/openjarvis/evals/configs/use_case_opensource_openhands.toml new file mode 100644 index 00000000..1969ed31 --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_opensource_openhands.toml @@ -0,0 +1,101 @@ +# Open-source models × NativeOpenHands (CodeAct) agent on 5 use-case benchmarks. +# +# Models: Qwen3.5-397B-A17B, Qwen3.5-122B-A10B, GLM-5, gpt-oss-120b, GLM-4.7-Flash +# Agent: NativeOpenHandsAgent (code-execution-centric, Action/Observation loop) +# Engine: vLLM (large MoE models need tensor parallelism) +# +# Setup: +# - Start vLLM with each model, e.g.: +# vllm serve unsloth/Qwen3.5-397B-A17B-GGUF --tensor-parallel-size 8 +# - Or use Ollama for smaller models: +# ollama pull glm4.7-flash + +[meta] +name = "use-case-opensource-openhands" +description = "Open-source models with NativeOpenHandsAgent (CodeAct) on use-case benchmarks" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-opensource-openhands/" +seed = 42 +telemetry = true +gpu_metrics = true + +# --- Models Under Test --- + +[[models]] +name = "unsloth/Qwen3.5-397B-A17B-GGUF" +engine = "vllm" +param_count_b = 397.0 +active_params_b = 17.0 +num_gpus = 8 + +[[models]] +name = "unsloth/Qwen3.5-122B-A10B-GGUF" +engine = "vllm" +param_count_b = 122.0 +active_params_b = 10.0 +num_gpus = 4 + +[[models]] +name = "unsloth/GLM-5-GGUF" +engine = "vllm" +param_count_b = 100.0 +num_gpus = 4 + +[[models]] +name = "unsloth/gpt-oss-120b-GGUF" +engine = "vllm" +param_count_b = 120.0 +num_gpus = 4 + +[[models]] +name = "unsloth/GLM-4.7-Flash-GGUF" +engine = "vllm" +param_count_b = 30.0 +active_params_b = 3.0 +num_gpus = 2 + +# --- Benchmarks --- + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "web_search"] +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "web_search"] +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["think", "code_interpreter"] +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml b/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml new file mode 100644 index 00000000..c40d60e1 --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_opensource_orchestrator.toml @@ -0,0 +1,101 @@ +# Open-source models × Orchestrator agent on 5 use-case benchmarks. +# +# Models: Qwen3.5-397B-A17B, Qwen3.5-122B-A10B, GLM-5, gpt-oss-120b, GLM-4.7-Flash +# Agent: OrchestratorAgent (multi-turn tool-calling loop) +# Engine: vLLM (large MoE models need tensor parallelism) +# +# Setup: +# - Start vLLM with each model, e.g.: +# vllm serve unsloth/Qwen3.5-397B-A17B-GGUF --tensor-parallel-size 8 +# - Or use Ollama for smaller models: +# ollama pull glm4.7-flash + +[meta] +name = "use-case-opensource-orchestrator" +description = "Open-source models with OrchestratorAgent on use-case benchmarks" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-opensource-orchestrator/" +seed = 42 +telemetry = true +gpu_metrics = true + +# --- Models Under Test --- + +[[models]] +name = "unsloth/Qwen3.5-397B-A17B-GGUF" +engine = "vllm" +param_count_b = 397.0 +active_params_b = 17.0 +num_gpus = 8 + +[[models]] +name = "unsloth/Qwen3.5-122B-A10B-GGUF" +engine = "vllm" +param_count_b = 122.0 +active_params_b = 10.0 +num_gpus = 4 + +[[models]] +name = "unsloth/GLM-5-GGUF" +engine = "vllm" +param_count_b = 100.0 +num_gpus = 4 + +[[models]] +name = "unsloth/gpt-oss-120b-GGUF" +engine = "vllm" +param_count_b = 120.0 +num_gpus = 4 + +[[models]] +name = "unsloth/GLM-4.7-Flash-GGUF" +engine = "vllm" +param_count_b = 30.0 +active_params_b = 3.0 +num_gpus = 2 + +# --- Benchmarks --- + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "web_search"] +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "web_search"] +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "code_interpreter"] +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_orchestrator.toml b/src/openjarvis/evals/configs/use_case_orchestrator.toml new file mode 100644 index 00000000..40257d4d --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_orchestrator.toml @@ -0,0 +1,57 @@ +# Orchestrator agent on 5 use-case benchmarks. +# Multi-turn tool-calling loop — OpenJarvis's flagship agent. + +[meta] +name = "use-case-orchestrator" +description = "Use-case benchmarks — OrchestratorAgent with tools" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-orchestrator/" +seed = 42 + +[[models]] +name = "qwen3:8b" +engine = "ollama" + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "web_search"] +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "web_search"] +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["think", "code_interpreter"] +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_react.toml b/src/openjarvis/evals/configs/use_case_react.toml new file mode 100644 index 00000000..08741fbc --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_react.toml @@ -0,0 +1,57 @@ +# NativeReActAgent on 5 use-case benchmarks. +# Thought-Action-Observation loop — classic ReAct paradigm. + +[meta] +name = "use-case-react" +description = "Use-case benchmarks — NativeReActAgent (Thought-Action-Observation)" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases-react/" +seed = 42 + +[[models]] +name = "qwen3:8b" +engine = "ollama" + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-agent" +agent = "native_react" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-agent" +agent = "native_react" +tools = ["think", "web_search"] +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-agent" +agent = "native_react" +tools = ["think", "web_search"] +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-agent" +agent = "native_react" +tools = ["think"] +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-agent" +agent = "native_react" +tools = ["think", "code_interpreter"] +max_samples = 29 diff --git a/src/openjarvis/evals/configs/use_case_suite.toml b/src/openjarvis/evals/configs/use_case_suite.toml new file mode 100644 index 00000000..a4d2cc7a --- /dev/null +++ b/src/openjarvis/evals/configs/use_case_suite.toml @@ -0,0 +1,48 @@ +# Use-Case Benchmark Suite +# Evaluates OpenJarvis on 5 practical use cases: +# email triage, morning briefs, research, knowledge base QA, coding tasks. + +[meta] +name = "use-case-suite" +description = "Top 5 use-case benchmarks for OpenJarvis product demo" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" + +[run] +max_workers = 4 +output_dir = "results/use-cases/" +seed = 42 + +[[models]] +name = "qwen3:8b" +engine = "ollama" + +[[benchmarks]] +name = "email_triage" +backend = "jarvis-direct" +max_samples = 30 + +[[benchmarks]] +name = "morning_brief" +backend = "jarvis-direct" +max_samples = 15 + +[[benchmarks]] +name = "research_mining" +backend = "jarvis-direct" +max_samples = 30 + +[[benchmarks]] +name = "knowledge_base" +backend = "jarvis-direct" +max_samples = 30 + +[[benchmarks]] +name = "coding_task" +backend = "jarvis-direct" +max_samples = 30 diff --git a/src/openjarvis/evals/core/agentic_runner.py b/src/openjarvis/evals/core/agentic_runner.py new file mode 100644 index 00000000..6b40632e --- /dev/null +++ b/src/openjarvis/evals/core/agentic_runner.py @@ -0,0 +1,616 @@ +"""AgenticRunner — multi-turn agent execution with energy telemetry correlation. + +Orchestrates agentic workloads where a single query may involve multiple +LLM turns and tool calls, capturing per-turn traces with energy attribution. +""" + +from __future__ import annotations + +import asyncio +import copy +import json +import logging +import math +import re +import statistics +import threading +import time +from contextlib import nullcontext +from pathlib import Path +from typing import Any, Callable, Optional + +from openjarvis.evals.core.event_recorder import AgentEvent, EventRecorder, EventType +from openjarvis.evals.core.trace import QueryTrace, TurnTrace + +LOGGER = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Energy computation helpers +# --------------------------------------------------------------------------- + + +def _compute_energy_delta( + readings: list[Any], + gpu_field: str = "gpu_energy_j", +) -> Optional[float]: + """Compute energy delta from first to last reading for a field.""" + values = [ + getattr(s, gpu_field, None) + for s in readings + ] + values = [v for v in values if v is not None and math.isfinite(v)] + if len(values) >= 2: + delta = values[-1] - values[0] + return delta if delta >= 0 else None + return None + + +def _compute_power_avg( + readings: list[Any], + power_field: str = "gpu_power_w", +) -> Optional[float]: + """Compute average power across readings for a field.""" + values = [ + getattr(s, power_field, None) + for s in readings + ] + values = [v for v in values if v is not None and math.isfinite(v)] + return statistics.mean(values) if values else None + + +# --------------------------------------------------------------------------- +# Patch extraction helpers +# --------------------------------------------------------------------------- + +_FENCED_DIFF_RE = re.compile( + r"```(?:diff|patch)\s*\n(.*?)```", re.DOTALL +) +_UNIFIED_DIFF_MARKERS = ("diff --git", "--- a/", "+++ b/", "@@ ") + + +def _extract_patch(text: str) -> Optional[str]: + """Extract a unified-diff patch from agent response text.""" + fenced = _FENCED_DIFF_RE.findall(text) + if fenced: + return "\n\n".join(block.strip() for block in fenced) + + lines = text.splitlines() + patch_lines: list[str] = [] + in_diff = False + for line in lines: + if any(line.startswith(m) for m in _UNIFIED_DIFF_MARKERS): + in_diff = True + if in_diff: + patch_lines.append(line) + + if patch_lines: + return "\n".join(patch_lines) + return None + + +class AgenticRunner: + """Orchestrate multi-turn agent runs with energy telemetry correlation. + + Designed for agentic workloads where a single query may involve multiple + LLM turns and tool calls. Captures per-turn ``TurnTrace`` objects with + energy attribution and builds ``QueryTrace`` aggregates. + """ + + _FLUSH_INTERVAL = 50 + + def __init__( + self, + agent: Any, + dataset: Any, + telemetry_session: Any = None, + config: Optional[dict[str, Any]] = None, + event_recorder: Optional[EventRecorder] = None, + run_dir: Optional[Path] = None, + concurrency: int = 1, + agent_factory: Optional[Callable[[], Any]] = None, + query_timeout: Optional[float] = None, + ) -> None: + self._agent = agent + self._dataset = dataset + self._telemetry = telemetry_session + self._config = config or {} + self._event_recorder = ( + event_recorder if event_recorder is not None else EventRecorder() + ) + self._run_dir = run_dir + self._traces: list[QueryTrace] = [] + self._concurrency = max(1, concurrency) + self._agent_factory = agent_factory + self._query_timeout = query_timeout + self._results_lock = threading.Lock() + + async def run(self, max_queries: Optional[int] = None) -> list[QueryTrace]: + """Run the agent over the dataset, collecting traces and telemetry. + + Args: + max_queries: Maximum number of queries to process. None means all. + + Returns: + List of ``QueryTrace`` objects with energy-correlated telemetry. + """ + records = list(self._dataset.iter_records()) + total = max_queries if max_queries is not None else len(records) + records = records[:total] + model = self._config.get("model", "unknown") + + work_items = list(enumerate(records)) + + if self._concurrency <= 1: + return await self._run_sequential(work_items, model) + return await self._run_concurrent(work_items, model) + + async def _run_sequential( + self, + work_items: list[tuple[int, Any]], + model: str, + ) -> list[QueryTrace]: + """Sequential execution path.""" + for index, record in work_items: + query_id = f"q{index:04d}" + start_time = time.time() + try: + fut = self._run_single_query( + index, record, model, self._agent, self._event_recorder + ) + if self._query_timeout: + trace = await asyncio.wait_for( + fut, timeout=self._query_timeout + ) + else: + trace = await fut + except asyncio.TimeoutError: + elapsed = time.time() - start_time + LOGGER.warning( + "Query %s timed out after %.0fs (limit=%ss)", + query_id, elapsed, self._query_timeout, + ) + workload_type = getattr(record, "category", "agentic") + trace = QueryTrace( + query_id=query_id, + workload_type=str(workload_type), + query_text=record.problem, + response_text=f"Query timed out after {elapsed:.0f}s", + total_wall_clock_s=elapsed, + completed=False, + timed_out=True, + is_resolved=record.metadata.get("is_resolved"), + ) + self._traces.append(trace) + + status = ( + "TIMEOUT" if trace.timed_out + else ("OK" if trace.completed else "FAIL") + ) + LOGGER.info( + "Task %s: %s in %.1fs", + query_id, status, trace.total_wall_clock_s, + ) + + if self._run_dir: + self._save_query_artifacts(index, record, trace) + + if len(self._traces) % self._FLUSH_INTERVAL == 0: + LOGGER.debug( + "Processed %d/%d queries", + len(self._traces), len(work_items), + ) + + return self._traces + + async def _run_concurrent( + self, + work_items: list[tuple[int, Any]], + model: str, + ) -> list[QueryTrace]: + """Concurrent execution via asyncio.Semaphore + thread pool.""" + total = len(work_items) + LOGGER.info( + "Running %d queries with concurrency=%d", + total, self._concurrency, + ) + + result_slots: list[Optional[QueryTrace]] = [None] * total + semaphore = asyncio.Semaphore(self._concurrency) + loop = asyncio.get_event_loop() + + async def _process(slot: int, index: int, record: Any) -> None: + async with semaphore: + if self._agent_factory is not None: + agent = self._agent_factory() + else: + agent = copy.deepcopy(self._agent) + recorder = EventRecorder() + + query_id = f"q{index:04d}" + start_time = time.time() + + try: + fut = loop.run_in_executor( + None, + self._run_single_query_sync, + index, record, model, agent, recorder, + ) + if self._query_timeout: + trace = await asyncio.wait_for( + fut, timeout=self._query_timeout + ) + else: + trace = await fut + except asyncio.TimeoutError: + elapsed = time.time() - start_time + LOGGER.warning( + "Query %s timed out after %.0fs (limit=%ss)", + query_id, elapsed, self._query_timeout, + ) + workload_type = getattr(record, "category", "agentic") + trace = QueryTrace( + query_id=query_id, + workload_type=str(workload_type), + query_text=record.problem, + response_text=f"Query timed out after {elapsed:.0f}s", + total_wall_clock_s=elapsed, + completed=False, + timed_out=True, + is_resolved=record.metadata.get("is_resolved"), + ) + + status = ( + "TIMEOUT" if trace.timed_out + else ("OK" if trace.completed else "FAIL") + ) + LOGGER.info( + "Task %s: %s in %.1fs", + query_id, status, trace.total_wall_clock_s, + ) + + if self._run_dir: + self._save_query_artifacts(index, record, trace) + + with self._results_lock: + result_slots[slot] = trace + + tasks = [ + _process(slot, index, record) + for slot, (index, record) in enumerate(work_items) + ] + await asyncio.gather(*tasks) + + for slot_result in result_slots: + if slot_result is not None: + self._traces.append(slot_result) + + return self._traces + + def _run_single_query_sync( + self, + index: int, + record: Any, + model: str, + agent: Any, + event_recorder: EventRecorder, + ) -> QueryTrace: + """Synchronous wrapper for ``_run_single_query``.""" + return asyncio.run( + self._run_single_query(index, record, model, agent, event_recorder) + ) + + async def _run_single_query( + self, + index: int, + record: Any, + model: str, + agent: Optional[Any] = None, + event_recorder: Optional[EventRecorder] = None, + ) -> QueryTrace: + """Run a single query through the agent with telemetry capture.""" + agent = agent or self._agent + event_recorder = event_recorder or self._event_recorder + + query_id = f"q{index:04d}" + workload_type = getattr(record, "category", "agentic") + + start_time = time.time() + start_ns = time.monotonic_ns() + + event_recorder.clear() + + # Set up per-query workspace + if self._run_dir and hasattr(agent, "set_workspace"): + instance_id = record.metadata.get("instance_id", record.record_id) + slug = re.sub(r"[^a-zA-Z0-9_-]", "_", str(instance_id))[:80] + workspace = ( + self._run_dir / "artifacts" / f"q{index:04d}_{slug}" / "workspace" + ) + workspace.mkdir(parents=True, exist_ok=True) + agent.set_workspace(str(workspace)) + + # Create per-task execution environment (e.g. Docker for TerminalBench) + task_env = None + if hasattr(self._dataset, "create_task_env"): + task_env = self._dataset.create_task_env(record) + ctx = task_env if task_env is not None else nullcontext() + + response_text = "" + result_tokens: dict[str, int] = {} + + try: + with ctx: + # Forward task metadata to agent + if task_env is not None and hasattr(agent, "set_task_metadata"): + agent.set_task_metadata(record.metadata) + + # Run the agent + if hasattr(agent, "ask"): + # SystemBuilder-based agent (JarvisSystem) + result = agent.ask(record.problem) + if isinstance(result, dict): + response_text = result.get("content", "") + usage = result.get("usage", {}) + result_tokens = { + "input_tokens": usage.get("prompt_tokens", 0), + "output_tokens": usage.get("completion_tokens", 0), + "cost_usd": result.get("cost_usd", 0.0), + } + else: + response_text = str(result) + elif hasattr(agent, "run"): + # BaseAgent-style + result = agent.run(record.problem) + response_text = getattr(result, "content", str(result)) + result_tokens = { + "input_tokens": getattr(result, "input_tokens", 0) + or 0, + "output_tokens": getattr(result, "output_tokens", 0) + or 0, + "cost_usd": getattr(result, "cost_usd", 0.0) or 0.0, + } + else: + response_text = str(agent(record.problem)) + + # Run tests if task env supports it + if task_env is not None and hasattr(task_env, "run_tests"): + task_env.run_tests() + + except Exception as exc: + LOGGER.warning("Agent failed on query %s: %s", query_id, exc) + end_time = time.time() + return QueryTrace( + query_id=query_id, + workload_type=str(workload_type), + query_text=record.problem, + response_text=str(exc), + total_wall_clock_s=end_time - start_time, + completed=False, + is_resolved=record.metadata.get("is_resolved"), + ) + + end_time = time.time() + end_ns = time.monotonic_ns() + + # Collect telemetry samples for this query window + readings: list[Any] = [] + if self._telemetry is not None: + readings = self._telemetry.window(start_ns, end_ns) + + # Build turn traces from event recorder + events = event_recorder.get_events() + turns = self._build_turn_traces(events, readings) + + # Synthetic turn when EventRecorder captured nothing + in_tok = result_tokens.get("input_tokens", 0) + out_tok = result_tokens.get("output_tokens", 0) + cost = result_tokens.get("cost_usd", 0.0) + if not turns and (in_tok > 0 or out_tok > 0): + turns = [TurnTrace( + turn_index=0, + input_tokens=in_tok, + output_tokens=out_tok, + wall_clock_s=end_time - start_time, + cost_usd=cost if cost else None, + )] + + # Backfill tokens from result when turns have zero tokens + if turns and in_tok > 0 and out_tok > 0: + total_turn_in = sum(t.input_tokens for t in turns) + total_turn_out = sum(t.output_tokens for t in turns) + if total_turn_in == 0 and total_turn_out == 0: + turns[0].input_tokens = in_tok + turns[0].output_tokens = out_tok + turns[0].wall_clock_s = ( + turns[0].wall_clock_s or (end_time - start_time) + ) + if cost and turns[0].cost_usd is None: + turns[0].cost_usd = cost + + # Compute cost for turns missing it + for turn in turns: + if turn.cost_usd is None and ( + turn.input_tokens > 0 or turn.output_tokens > 0 + ): + from openjarvis.evals.core.pricing import compute_turn_cost + turn.cost_usd = compute_turn_cost( + model, turn.input_tokens, turn.output_tokens + ) + + # Query-level energy from telemetry window + query_gpu_energy = _compute_energy_delta(readings, "gpu_energy_j") + query_cpu_energy = _compute_energy_delta(readings, "cpu_energy_j") + query_gpu_power_avg = _compute_power_avg(readings, "gpu_power_w") + query_cpu_power_avg = _compute_power_avg(readings, "cpu_power_w") + + trace = QueryTrace( + query_id=query_id, + workload_type=str(workload_type), + query_text=record.problem, + response_text=response_text, + turns=turns, + total_wall_clock_s=end_time - start_time, + completed=True, + query_gpu_energy_joules=query_gpu_energy, + query_cpu_energy_joules=query_cpu_energy, + query_gpu_power_avg_watts=query_gpu_power_avg, + query_cpu_power_avg_watts=query_cpu_power_avg, + is_resolved=record.metadata.get("is_resolved"), + ) + + # Correlate energy with trace + trace = self._correlate_energy(trace, readings) + + return trace + + def _build_turn_traces( + self, + events: list[AgentEvent], + readings: list[Any], + ) -> list[TurnTrace]: + """Build TurnTrace objects from recorded events.""" + turns: list[TurnTrace] = [] + current_turn_index = 0 + current_turn_start: Optional[float] = None + current_tools: list[str] = [] + current_tool_latencies: dict[str, float] = {} + tool_start_times: dict[str, float] = {} + input_tokens = 0 + output_tokens = 0 + + for event in events: + etype = event.event_type + + if etype == EventType.LM_INFERENCE_START: + current_turn_start = event.timestamp + + elif etype == EventType.LM_INFERENCE_END: + wall_clock = 0.0 + if current_turn_start is not None: + wall_clock = event.timestamp - current_turn_start + + input_tokens = event.metadata.get("prompt_tokens", 0) + output_tokens = event.metadata.get("completion_tokens", 0) + + turn = TurnTrace( + turn_index=current_turn_index, + input_tokens=input_tokens, + output_tokens=output_tokens, + tools_called=list(current_tools), + tool_latencies_s=dict(current_tool_latencies), + wall_clock_s=wall_clock, + ) + turns.append(turn) + + current_turn_index += 1 + current_turn_start = None + current_tools = [] + current_tool_latencies = {} + input_tokens = 0 + output_tokens = 0 + + elif etype == EventType.TOOL_CALL_START: + tool_name = event.metadata.get("tool", "unknown") + tool_start_times[tool_name] = event.timestamp + + elif etype == EventType.TOOL_CALL_END: + tool_name = event.metadata.get("tool", "unknown") + current_tools.append(tool_name) + start_ts = tool_start_times.pop(tool_name, None) + if start_ts is not None: + current_tool_latencies[tool_name] = ( + event.timestamp - start_ts + ) + + # Synthetic turn if events but no complete LM_START/END pair + if not turns and events: + turns.append( + TurnTrace( + turn_index=0, + tools_called=current_tools, + tool_latencies_s=current_tool_latencies, + ) + ) + + return turns + + def _correlate_energy( + self, + trace: QueryTrace, + readings: list[Any], + ) -> QueryTrace: + """Distribute energy across turns proportionally by wall clock time. + + Only runs when per-turn energy was not already populated from events. + """ + if not readings or not trace.turns: + return trace + + has_turn_energy = any( + t.gpu_energy_joules is not None for t in trace.turns + ) + if has_turn_energy: + return trace + + total_gpu_energy = _compute_energy_delta(readings, "gpu_energy_j") + total_cpu_energy = _compute_energy_delta(readings, "cpu_energy_j") + + total_wall = sum(t.wall_clock_s for t in trace.turns) + if total_wall > 0: + for turn in trace.turns: + fraction = turn.wall_clock_s / total_wall + if total_gpu_energy is not None: + turn.gpu_energy_joules = total_gpu_energy * fraction + if total_cpu_energy is not None: + turn.cpu_energy_joules = total_cpu_energy * fraction + + return trace + + def _save_query_artifacts( + self, + index: int, + record: Any, + trace: QueryTrace, + ) -> None: + """Save per-query artifacts to structured subdirectories.""" + assert self._run_dir is not None + instance_id = record.metadata.get("instance_id", record.record_id) + slug = re.sub(r"[^a-zA-Z0-9_-]", "_", str(instance_id))[:80] + query_dir = self._run_dir / "artifacts" / f"q{index:04d}_{slug}" + query_dir.mkdir(parents=True, exist_ok=True) + + (query_dir / "response.txt").write_text( + trace.response_text or "", encoding="utf-8" + ) + + meta: dict[str, object] = { + "query_id": trace.query_id, + "instance_id": str(instance_id), + "completed": trace.completed, + "timed_out": trace.timed_out, + "wall_clock_s": trace.total_wall_clock_s, + "num_turns": trace.num_turns, + } + for key in ( + "repo", "base_commit", "dataset_name", + "is_resolved", "test_results", + ): + val = record.metadata.get(key) + if val is not None: + meta[key] = val + (query_dir / "metadata.json").write_text( + json.dumps(meta, indent=2, default=str), encoding="utf-8" + ) + + patch = _extract_patch(trace.response_text or "") + if patch: + (query_dir / "patch.diff").write_text(patch, encoding="utf-8") + + @property + def traces(self) -> list[QueryTrace]: + """Return collected traces.""" + return list(self._traces) + + +__all__ = ["AgenticRunner"] diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index 3c1329ba..962cb280 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -39,6 +39,8 @@ KNOWN_BENCHMARKS = { "simpleqa", "wildchat", "ipw", "gaia", "frames", "swebench", "swefficiency", "terminalbench", "terminalbench-native", + "email_triage", "morning_brief", "research_mining", + "knowledge_base", "coding_task", } diff --git a/src/openjarvis/evals/core/dataset.py b/src/openjarvis/evals/core/dataset.py index c3a31887..883fe113 100644 --- a/src/openjarvis/evals/core/dataset.py +++ b/src/openjarvis/evals/core/dataset.py @@ -3,7 +3,8 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Iterable, Optional +from contextlib import AbstractContextManager +from typing import Iterable, List, Optional from openjarvis.evals.core.types import EvalRecord @@ -32,5 +33,15 @@ class DatasetProvider(ABC): def size(self) -> int: """Return the number of loaded records.""" + def create_task_env( + self, record: EvalRecord, + ) -> Optional[AbstractContextManager]: + """Return a task environment context manager, or None.""" + return None + + def verify_requirements(self) -> List[str]: + """Return list of unsatisfied requirements, or empty list.""" + return [] + __all__ = ["DatasetProvider"] diff --git a/src/openjarvis/evals/core/event_recorder.py b/src/openjarvis/evals/core/event_recorder.py new file mode 100644 index 00000000..aaf169ad --- /dev/null +++ b/src/openjarvis/evals/core/event_recorder.py @@ -0,0 +1,98 @@ +"""Event recording system for agent execution telemetry.""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List + + +class EventType(str, Enum): + """Supported event types for agent telemetry.""" + + LM_INFERENCE_START = "lm_inference_start" + LM_INFERENCE_END = "lm_inference_end" + TOOL_CALL_START = "tool_call_start" + TOOL_CALL_END = "tool_call_end" + PREFILL_START = "prefill_start" + PREFILL_END = "prefill_end" + DECODE_START = "decode_start" + DECODE_END = "decode_end" + # Submodel call events (for MCP tools that call inference servers) + SUBMODEL_CALL_START = "submodel_call_start" + SUBMODEL_CALL_END = "submodel_call_end" + + +@dataclass +class AgentEvent: + """Single event recorded during agent execution.""" + + event_type: str + timestamp: float # Unix timestamp from time.time() + metadata: Dict[str, Any] = field(default_factory=dict) + + def __repr__(self) -> str: + return ( + f"AgentEvent({self.event_type!r}, " + f"ts={self.timestamp:.3f}, meta={self.metadata})" + ) + + +class EventRecorder: + """Thread-safe recorder for agent execution events. + + Records events with timestamps for later correlation with energy telemetry. + All operations are thread-safe for use in concurrent agent execution. + + Example: + >>> recorder = EventRecorder() + >>> recorder.record('tool_call_start', tool='calculator') + >>> recorder.record('tool_call_end', tool='calculator') + >>> events = recorder.get_events() + >>> len(events) + 2 + """ + + def __init__(self) -> None: + """Initialize the event recorder.""" + self._events: List[AgentEvent] = [] + self._lock = threading.Lock() + + def record(self, event_type: str, **metadata: Any) -> None: + """Record an event with current timestamp. + + Args: + event_type: Type of event (e.g., 'tool_call_start', 'lm_inference_end') + **metadata: Additional metadata to attach to the event + """ + event = AgentEvent( + event_type=event_type, + timestamp=time.time(), + metadata=metadata, + ) + with self._lock: + self._events.append(event) + + def get_events(self) -> List[AgentEvent]: + """Return a copy of all recorded events. + + Returns: + List of all recorded events in chronological order. + """ + with self._lock: + return list(self._events) + + def clear(self) -> None: + """Clear all recorded events.""" + with self._lock: + self._events.clear() + + def __len__(self) -> int: + """Return the number of recorded events.""" + with self._lock: + return len(self._events) + + +__all__ = ["AgentEvent", "EventRecorder", "EventType"] diff --git a/src/openjarvis/evals/core/export.py b/src/openjarvis/evals/core/export.py new file mode 100644 index 00000000..dd22ac1f --- /dev/null +++ b/src/openjarvis/evals/core/export.py @@ -0,0 +1,223 @@ +"""Export functions for agentic run traces and profiling records.""" + +from __future__ import annotations + +import json +import statistics +import time +from pathlib import Path +from typing import Any, Optional, Sequence + +from openjarvis.evals.core.trace import QueryTrace + + +def _agg_stats(values: Sequence[Optional[float]]) -> dict[str, Optional[float]]: + """Return {avg, median, min, max, std} filtering None values.""" + clean = [v for v in values if v is not None] + if not clean: + return {"avg": None, "median": None, "min": None, "max": None, "std": None} + return { + "avg": statistics.mean(clean), + "median": statistics.median(clean), + "min": min(clean), + "max": max(clean), + "std": statistics.stdev(clean) if len(clean) > 1 else 0.0, + } + + +def export_jsonl(traces: list[QueryTrace], path: Path) -> Path: + """Export traces as JSONL (one JSON object per line). + + Args: + traces: List of QueryTrace objects to export. + path: Output file path. Parent directories are created if needed. + + Returns: + The path to the written file. + """ + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for trace in traces: + f.write(json.dumps(trace.to_dict()) + "\n") + return path + + +def export_hf_dataset(traces: list[QueryTrace], path: Path) -> Path: + """Export traces as a HuggingFace Arrow dataset. + + Args: + traces: List of QueryTrace objects to export. + path: Output directory for the Arrow dataset. + + Returns: + The path to the saved dataset directory. + + Raises: + ImportError: If the ``datasets`` package is not installed. + """ + ds = QueryTrace.to_hf_dataset(traces) + path.parent.mkdir(parents=True, exist_ok=True) + ds.save_to_disk(str(path)) + return path + + +def export_summary_json( + traces: list[QueryTrace], + config: dict[str, Any], + path: Path, +) -> Path: + """Export aggregate summary as JSON. + + Args: + traces: List of QueryTrace objects. + config: Run configuration dictionary. + path: Output file path. + + Returns: + The path to the written file. + """ + total_queries = len(traces) + completed = sum(1 for t in traces if t.completed) + total_turns = sum(t.num_turns for t in traces) + total_tool_calls = sum(t.total_tool_calls for t in traces) + + total_input_tokens = sum(t.total_input_tokens for t in traces) + total_output_tokens = sum(t.total_output_tokens for t in traces) + total_wall_clock_s = sum(t.total_wall_clock_s for t in traces) + + gpu_energy_values = [ + t.total_gpu_energy_joules for t in traces + if t.total_gpu_energy_joules is not None + ] + total_gpu_energy = sum(gpu_energy_values) if gpu_energy_values else None + + cpu_energy_values: list[float] = [] + for trace in traces: + cpu_vals = [ + turn.cpu_energy_joules for turn in trace.turns + if turn.cpu_energy_joules is not None + ] + if cpu_vals: + cpu_energy_values.append(sum(cpu_vals)) + total_cpu_energy = sum(cpu_energy_values) if cpu_energy_values else None + + resolved = sum(1 for t in traces if t.is_resolved is True) + unresolved = sum(1 for t in traces if t.is_resolved is False) + + cost_values = [ + t.total_cost_usd for t in traces + if t.total_cost_usd is not None + ] + total_cost = sum(cost_values) if cost_values else None + + avg_turns = total_turns / total_queries if total_queries > 0 else 0 + avg_wall_clock = total_wall_clock_s / total_queries if total_queries > 0 else 0 + avg_gpu_energy = ( + total_gpu_energy / total_queries + if total_gpu_energy is not None and total_queries > 0 + else None + ) + + stats = { + "wall_clock_s": _agg_stats([t.total_wall_clock_s for t in traces]), + "gpu_energy_joules": _agg_stats( + [t.total_gpu_energy_joules for t in traces], + ), + "cpu_energy_joules": _agg_stats( + [t.total_cpu_energy_joules for t in traces], + ), + "gpu_power_watts": _agg_stats( + [t.avg_gpu_power_watts for t in traces], + ), + "cpu_power_watts": _agg_stats( + [t.avg_cpu_power_watts for t in traces], + ), + "input_tokens": _agg_stats( + [float(t.total_input_tokens) for t in traces], + ), + "output_tokens": _agg_stats( + [float(t.total_output_tokens) for t in traces], + ), + "total_tokens": _agg_stats( + [float(t.total_tokens) for t in traces], + ), + "throughput_tokens_per_sec": _agg_stats( + [t.throughput_tokens_per_sec for t in traces], + ), + "energy_per_token_joules": _agg_stats( + [t.energy_per_token_joules for t in traces], + ), + "cost_usd": _agg_stats([t.total_cost_usd for t in traces]), + "turns": _agg_stats([float(t.num_turns) for t in traces]), + "tool_calls": _agg_stats( + [float(t.total_tool_calls) for t in traces], + ), + } + + summary = { + "generated_at": time.time(), + "config": config, + "totals": { + "queries": total_queries, + "completed": completed, + "resolved": resolved, + "unresolved": unresolved, + "turns": total_turns, + "tool_calls": total_tool_calls, + "input_tokens": total_input_tokens, + "output_tokens": total_output_tokens, + "total_tokens": total_input_tokens + total_output_tokens, + "wall_clock_s": total_wall_clock_s, + "gpu_energy_joules": total_gpu_energy, + "cpu_energy_joules": total_cpu_energy, + "cost_usd": total_cost, + }, + "averages": { + "turns_per_query": avg_turns, + "wall_clock_per_query_s": avg_wall_clock, + "gpu_energy_per_query_joules": avg_gpu_energy, + }, + "statistics": stats, + } + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(summary, indent=2, default=str)) + return path + + +def export_artifacts_manifest(run_dir: Path) -> Optional[Path]: + """Scan ``{run_dir}/artifacts/`` and write ``artifacts_manifest.json``. + + The manifest lists every per-query artifact directory together with + the files it contains, making it easy for downstream tools to discover + what was produced without walking the directory tree themselves. + + Returns: + The manifest path, or ``None`` if there is no artifacts directory. + """ + artifacts_root = run_dir / "artifacts" + if not artifacts_root.is_dir(): + return None + + entries: list[dict[str, object]] = [] + for query_dir in sorted(artifacts_root.iterdir()): + if not query_dir.is_dir(): + continue + files = sorted( + str(p.relative_to(artifacts_root)) + for p in query_dir.rglob("*") + if p.is_file() + ) + entries.append({"query_dir": query_dir.name, "files": files}) + + manifest_path = run_dir / "artifacts_manifest.json" + manifest_path.write_text(json.dumps(entries, indent=2), encoding="utf-8") + return manifest_path + + +__all__ = [ + "export_jsonl", + "export_hf_dataset", + "export_summary_json", + "export_artifacts_manifest", +] diff --git a/src/openjarvis/evals/core/pricing.py b/src/openjarvis/evals/core/pricing.py new file mode 100644 index 00000000..ce4199b4 --- /dev/null +++ b/src/openjarvis/evals/core/pricing.py @@ -0,0 +1,17 @@ +"""Cost computation for agentic eval runs — wraps engine/cloud.py pricing.""" + +from __future__ import annotations + +from openjarvis.engine.cloud import PRICING, estimate_cost + + +def compute_turn_cost(model: str, input_tokens: int, output_tokens: int) -> float: + """Compute USD cost for a single agent turn. + + Delegates to the canonical ``estimate_cost()`` from ``engine/cloud.py``. + Returns 0.0 for models not in the pricing table (e.g. local models). + """ + return estimate_cost(model, input_tokens, output_tokens) + + +__all__ = ["PRICING", "compute_turn_cost", "estimate_cost"] diff --git a/src/openjarvis/evals/core/trace.py b/src/openjarvis/evals/core/trace.py new file mode 100644 index 00000000..9377c51e --- /dev/null +++ b/src/openjarvis/evals/core/trace.py @@ -0,0 +1,261 @@ +"""Trace data model for agentic eval runs.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + + +@dataclass +class TurnTrace: + """Per-turn telemetry data.""" + + turn_index: int + input_tokens: int = 0 + output_tokens: int = 0 + tool_result_tokens: int = 0 + tools_called: List[str] = field(default_factory=list) + tool_latencies_s: Dict[str, float] = field(default_factory=dict) + wall_clock_s: float = 0.0 + error: Optional[str] = None + # Energy and cost fields + gpu_energy_joules: Optional[float] = None + cpu_energy_joules: Optional[float] = None + gpu_power_avg_watts: Optional[float] = None + cpu_power_avg_watts: Optional[float] = None + cost_usd: Optional[float] = None + + def to_dict(self) -> Dict[str, Any]: + return { + "turn_index": self.turn_index, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "tool_result_tokens": self.tool_result_tokens, + "tools_called": list(self.tools_called), + "tool_latencies_s": dict(self.tool_latencies_s), + "wall_clock_s": self.wall_clock_s, + "error": self.error, + "gpu_energy_joules": self.gpu_energy_joules, + "cpu_energy_joules": self.cpu_energy_joules, + "gpu_power_avg_watts": self.gpu_power_avg_watts, + "cpu_power_avg_watts": self.cpu_power_avg_watts, + "cost_usd": self.cost_usd, + } + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> TurnTrace: + return cls( + turn_index=d["turn_index"], + input_tokens=d.get("input_tokens", 0), + output_tokens=d.get("output_tokens", 0), + tool_result_tokens=d.get("tool_result_tokens", 0), + tools_called=d.get("tools_called", []), + tool_latencies_s=d.get("tool_latencies_s", {}), + wall_clock_s=d.get("wall_clock_s", 0.0), + error=d.get("error"), + gpu_energy_joules=d.get("gpu_energy_joules"), + cpu_energy_joules=d.get("cpu_energy_joules"), + gpu_power_avg_watts=d.get("gpu_power_avg_watts"), + cpu_power_avg_watts=d.get("cpu_power_avg_watts"), + cost_usd=d.get("cost_usd"), + ) + + +@dataclass +class QueryTrace: + """Per-query aggregate telemetry.""" + + query_id: str + workload_type: str + query_text: str = "" + response_text: str = "" + turns: List[TurnTrace] = field(default_factory=list) + total_wall_clock_s: float = 0.0 + completed: bool = False + timed_out: bool = False + # Query-level energy fields (populated even when turns are empty) + query_gpu_energy_joules: Optional[float] = None + query_cpu_energy_joules: Optional[float] = None + query_gpu_power_avg_watts: Optional[float] = None + query_cpu_power_avg_watts: Optional[float] = None + is_resolved: Optional[bool] = None + + @property + def num_turns(self) -> int: + return len(self.turns) + + @property + def total_input_tokens(self) -> int: + return sum(t.input_tokens for t in self.turns) + + @property + def total_output_tokens(self) -> int: + return sum(t.output_tokens for t in self.turns) + + @property + def tool_call_count(self) -> int: + return sum(len(t.tools_called) for t in self.turns) + + @property + def total_tool_calls(self) -> int: + return self.tool_call_count + + @property + def total_gpu_energy_joules(self) -> Optional[float]: + values = [ + t.gpu_energy_joules for t in self.turns + if t.gpu_energy_joules is not None + ] + if values: + return sum(values) + return self.query_gpu_energy_joules + + @property + def total_cpu_energy_joules(self) -> Optional[float]: + values = [ + t.cpu_energy_joules for t in self.turns + if t.cpu_energy_joules is not None + ] + if values: + return sum(values) + return self.query_cpu_energy_joules + + @property + def total_cost_usd(self) -> Optional[float]: + values = [t.cost_usd for t in self.turns if t.cost_usd is not None] + return sum(values) if values else None + + @property + def total_tokens(self) -> int: + """Total tokens (input + output) across all turns.""" + return self.total_input_tokens + self.total_output_tokens + + @property + def avg_gpu_power_watts(self) -> Optional[float]: + """Mean GPU power across turns; falls back to query-level power.""" + values = [ + t.gpu_power_avg_watts for t in self.turns + if t.gpu_power_avg_watts is not None + ] + if values: + return sum(values) / len(values) + return self.query_gpu_power_avg_watts + + @property + def avg_cpu_power_watts(self) -> Optional[float]: + """Mean CPU power across turns; falls back to query-level power.""" + values = [ + t.cpu_power_avg_watts for t in self.turns + if t.cpu_power_avg_watts is not None + ] + if values: + return sum(values) / len(values) + return self.query_cpu_power_avg_watts + + @property + def throughput_tokens_per_sec(self) -> Optional[float]: + """Output tokens per second; None if zero tokens or zero time.""" + if self.total_output_tokens > 0 and self.total_wall_clock_s > 0: + return self.total_output_tokens / self.total_wall_clock_s + return None + + @property + def energy_per_token_joules(self) -> Optional[float]: + """GPU energy per output token; None if no energy data or zero tokens.""" + gpu_energy = self.total_gpu_energy_joules + if gpu_energy is not None and gpu_energy > 0 and self.total_output_tokens > 0: + return gpu_energy / self.total_output_tokens + return None + + def to_dict(self) -> Dict[str, Any]: + return { + "query_id": self.query_id, + "workload_type": self.workload_type, + "query_text": self.query_text, + "response_text": self.response_text, + "turns": [t.to_dict() for t in self.turns], + "total_wall_clock_s": self.total_wall_clock_s, + "completed": self.completed, + "timed_out": self.timed_out, + "query_gpu_energy_joules": self.query_gpu_energy_joules, + "query_cpu_energy_joules": self.query_cpu_energy_joules, + "query_gpu_power_avg_watts": self.query_gpu_power_avg_watts, + "query_cpu_power_avg_watts": self.query_cpu_power_avg_watts, + "is_resolved": self.is_resolved, + } + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> QueryTrace: + return cls( + query_id=d["query_id"], + workload_type=d["workload_type"], + query_text=d.get("query_text", ""), + response_text=d.get("response_text", ""), + turns=[TurnTrace.from_dict(t) for t in d.get("turns", [])], + total_wall_clock_s=d.get("total_wall_clock_s", 0.0), + completed=d.get("completed", False), + timed_out=d.get("timed_out", False), + query_gpu_energy_joules=d.get("query_gpu_energy_joules"), + query_cpu_energy_joules=d.get("query_cpu_energy_joules"), + query_gpu_power_avg_watts=d.get("query_gpu_power_avg_watts"), + query_cpu_power_avg_watts=d.get("query_cpu_power_avg_watts"), + is_resolved=d.get("is_resolved"), + ) + + def save_jsonl(self, path: Path) -> None: + """Append this trace as a JSONL line.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "a") as f: + f.write(json.dumps(self.to_dict()) + "\n") + + @classmethod + def load_jsonl(cls, path: Path) -> List[QueryTrace]: + """Load traces from a JSONL file.""" + traces = [] + with open(path) as f: + for line in f: + line = line.strip() + if line: + traces.append(cls.from_dict(json.loads(line))) + return traces + + @staticmethod + def to_hf_dataset(traces: List[QueryTrace]) -> Any: + """Convert a list of QueryTrace objects to a HuggingFace Dataset. + + Returns: + A datasets.Dataset with one row per trace. + """ + from datasets import Dataset + + rows = [] + for trace in traces: + rows.append({ + "query_id": trace.query_id, + "workload_type": trace.workload_type, + "query_text": trace.query_text, + "response_text": trace.response_text, + "num_turns": trace.num_turns, + "total_input_tokens": trace.total_input_tokens, + "total_output_tokens": trace.total_output_tokens, + "total_tool_calls": trace.total_tool_calls, + "total_wall_clock_s": trace.total_wall_clock_s, + "total_gpu_energy_joules": trace.total_gpu_energy_joules, + "total_cpu_energy_joules": trace.total_cpu_energy_joules, + "total_tokens": trace.total_tokens, + "total_cost_usd": trace.total_cost_usd, + "avg_gpu_power_watts": trace.avg_gpu_power_watts, + "avg_cpu_power_watts": trace.avg_cpu_power_watts, + "throughput_tokens_per_sec": trace.throughput_tokens_per_sec, + "energy_per_token_joules": trace.energy_per_token_joules, + "completed": trace.completed, + "timed_out": trace.timed_out, + "is_resolved": trace.is_resolved, + "trace_json": json.dumps(trace.to_dict()), + }) + return Dataset.from_list(rows) + + +__all__ = ["TurnTrace", "QueryTrace"] diff --git a/src/openjarvis/evals/datasets/coding_task.py b/src/openjarvis/evals/datasets/coding_task.py new file mode 100644 index 00000000..3679e5d1 --- /dev/null +++ b/src/openjarvis/evals/datasets/coding_task.py @@ -0,0 +1,282 @@ +"""Coding task benchmark dataset. + +Standalone function-level coding problems with test cases for evaluating +code generation accuracy. +""" + +from __future__ import annotations + +import random +from typing import Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +_PROMPT_TEMPLATE = """Write a Python function that solves the following problem. Return ONLY the function definition, no explanations. + +{spec} + +Function signature: {signature} + +Examples: +{examples}""" + +_TASKS = [ + { + "spec": "Given a list of integers, return a new list containing only the elements that appear exactly once, in their original order.", + "signature": "def unique_elements(lst: list[int]) -> list[int]", + "examples": "unique_elements([1, 2, 3, 2, 1, 4]) -> [3, 4]\nunique_elements([1, 1, 1]) -> []\nunique_elements([5]) -> [5]", + "test_cases": "assert unique_elements([1, 2, 3, 2, 1, 4]) == [3, 4]\nassert unique_elements([1, 1, 1]) == []\nassert unique_elements([5]) == [5]\nassert unique_elements([]) == []\nassert unique_elements([1, 2, 3]) == [1, 2, 3]", + "reference": "def unique_elements(lst):\n from collections import Counter\n counts = Counter(lst)\n return [x for x in lst if counts[x] == 1]", + }, + { + "spec": "Flatten a nested list of arbitrary depth into a single flat list.", + "signature": "def flatten(nested: list) -> list", + "examples": "flatten([1, [2, [3, 4], 5]]) -> [1, 2, 3, 4, 5]\nflatten([[1, 2], [3, [4]]]) -> [1, 2, 3, 4]\nflatten([]) -> []", + "test_cases": "assert flatten([1, [2, [3, 4], 5]]) == [1, 2, 3, 4, 5]\nassert flatten([[1, 2], [3, [4]]]) == [1, 2, 3, 4]\nassert flatten([]) == []\nassert flatten([1, 2, 3]) == [1, 2, 3]\nassert flatten([[[1]]]) == [1]", + "reference": "def flatten(nested):\n result = []\n for item in nested:\n if isinstance(item, list):\n result.extend(flatten(item))\n else:\n result.append(item)\n return result", + }, + { + "spec": "Given a string, return the longest substring without repeating characters.", + "signature": "def longest_unique_substring(s: str) -> str", + "examples": 'longest_unique_substring("abcabcbb") -> "abc"\nlongest_unique_substring("bbbbb") -> "b"\nlongest_unique_substring("pwwkew") -> "wke"', + "test_cases": 'assert longest_unique_substring("abcabcbb") == "abc"\nassert longest_unique_substring("bbbbb") == "b"\nassert longest_unique_substring("pwwkew") == "wke"\nassert longest_unique_substring("") == ""\nassert longest_unique_substring("abcdef") == "abcdef"', + "reference": "def longest_unique_substring(s):\n start = 0\n best = ''\n seen = {}\n for i, c in enumerate(s):\n if c in seen and seen[c] >= start:\n start = seen[c] + 1\n seen[c] = i\n if i - start + 1 > len(best):\n best = s[start:i+1]\n return best", + }, + { + "spec": "Implement a function that converts a Roman numeral string to an integer.", + "signature": "def roman_to_int(s: str) -> int", + "examples": 'roman_to_int("III") -> 3\nroman_to_int("IV") -> 4\nroman_to_int("MCMXCIV") -> 1994', + "test_cases": 'assert roman_to_int("III") == 3\nassert roman_to_int("IV") == 4\nassert roman_to_int("IX") == 9\nassert roman_to_int("MCMXCIV") == 1994\nassert roman_to_int("LVIII") == 58', + "reference": "def roman_to_int(s):\n vals = {'I':1,'V':5,'X':10,'L':50,'C':100,'D':500,'M':1000}\n total = 0\n for i in range(len(s)):\n if i+1 < len(s) and vals[s[i]] < vals[s[i+1]]:\n total -= vals[s[i]]\n else:\n total += vals[s[i]]\n return total", + }, + { + "spec": "Given a list of intervals [start, end], merge all overlapping intervals and return the result sorted by start time.", + "signature": "def merge_intervals(intervals: list[list[int]]) -> list[list[int]]", + "examples": "merge_intervals([[1,3],[2,6],[8,10],[15,18]]) -> [[1,6],[8,10],[15,18]]\nmerge_intervals([[1,4],[4,5]]) -> [[1,5]]", + "test_cases": "assert merge_intervals([[1,3],[2,6],[8,10],[15,18]]) == [[1,6],[8,10],[15,18]]\nassert merge_intervals([[1,4],[4,5]]) == [[1,5]]\nassert merge_intervals([[1,4],[0,4]]) == [[0,4]]\nassert merge_intervals([]) == []\nassert merge_intervals([[1,2]]) == [[1,2]]", + "reference": "def merge_intervals(intervals):\n if not intervals:\n return []\n intervals.sort()\n merged = [intervals[0]]\n for s, e in intervals[1:]:\n if s <= merged[-1][1]:\n merged[-1][1] = max(merged[-1][1], e)\n else:\n merged.append([s, e])\n return merged", + }, + { + "spec": "Implement a function that checks if a string of brackets is balanced. Valid brackets are (), [], {}.", + "signature": "def is_balanced(s: str) -> bool", + "examples": 'is_balanced("()[]{}") -> True\nis_balanced("([)]") -> False\nis_balanced("{[]}") -> True', + "test_cases": 'assert is_balanced("()[]{}") == True\nassert is_balanced("([)]") == False\nassert is_balanced("{[]}") == True\nassert is_balanced("") == True\nassert is_balanced("(") == False\nassert is_balanced("([{}])") == True', + "reference": "def is_balanced(s):\n stack = []\n pairs = {')':'(', ']':'[', '}':'{'}\n for c in s:\n if c in '([{':\n stack.append(c)\n elif c in pairs:\n if not stack or stack[-1] != pairs[c]:\n return False\n stack.pop()\n return len(stack) == 0", + }, + { + "spec": "Given a matrix (list of lists), rotate it 90 degrees clockwise in-place and return it.", + "signature": "def rotate_matrix(matrix: list[list[int]]) -> list[list[int]]", + "examples": "rotate_matrix([[1,2,3],[4,5,6],[7,8,9]]) -> [[7,4,1],[8,5,2],[9,6,3]]", + "test_cases": "assert rotate_matrix([[1,2,3],[4,5,6],[7,8,9]]) == [[7,4,1],[8,5,2],[9,6,3]]\nassert rotate_matrix([[1]]) == [[1]]\nassert rotate_matrix([[1,2],[3,4]]) == [[3,1],[4,2]]", + "reference": "def rotate_matrix(matrix):\n n = len(matrix)\n for i in range(n):\n for j in range(i, n):\n matrix[i][j], matrix[j][i] = matrix[j][i], matrix[i][j]\n for row in matrix:\n row.reverse()\n return matrix", + }, + { + "spec": "Implement a function that computes the nth Fibonacci number using memoization. F(0)=0, F(1)=1.", + "signature": "def fibonacci(n: int) -> int", + "examples": "fibonacci(0) -> 0\nfibonacci(1) -> 1\nfibonacci(10) -> 55\nfibonacci(20) -> 6765", + "test_cases": "assert fibonacci(0) == 0\nassert fibonacci(1) == 1\nassert fibonacci(10) == 55\nassert fibonacci(20) == 6765\nassert fibonacci(30) == 832040", + "reference": "def fibonacci(n, memo={}):\n if n in memo:\n return memo[n]\n if n <= 1:\n return n\n memo[n] = fibonacci(n-1, memo) + fibonacci(n-2, memo)\n return memo[n]", + }, + { + "spec": "Given a list of words, group them by anagrams. Return a list of groups, where each group is a sorted list of words.", + "signature": "def group_anagrams(words: list[str]) -> list[list[str]]", + "examples": 'group_anagrams(["eat","tea","tan","ate","nat","bat"]) -> [["ate","eat","tea"],["nat","tan"],["bat"]]', + "test_cases": 'result = group_anagrams(["eat","tea","tan","ate","nat","bat"])\nresult = [sorted(g) for g in result]\nresult.sort()\nassert result == [["bat"], ["ate","eat","tea"], ["nat","tan"]] or result == [["ate","eat","tea"], ["bat"], ["nat","tan"]]\nassert group_anagrams([""]) == [[""]]\nassert group_anagrams(["a"]) == [["a"]]', + "reference": "def group_anagrams(words):\n from collections import defaultdict\n groups = defaultdict(list)\n for w in words:\n key = ''.join(sorted(w))\n groups[key].append(w)\n return [sorted(g) for g in groups.values()]", + }, + { + "spec": "Implement binary search on a sorted list. Return the index of the target if found, otherwise return -1.", + "signature": "def binary_search(arr: list[int], target: int) -> int", + "examples": "binary_search([1, 3, 5, 7, 9], 5) -> 2\nbinary_search([1, 3, 5, 7, 9], 4) -> -1\nbinary_search([], 1) -> -1", + "test_cases": "assert binary_search([1, 3, 5, 7, 9], 5) == 2\nassert binary_search([1, 3, 5, 7, 9], 4) == -1\nassert binary_search([], 1) == -1\nassert binary_search([1], 1) == 0\nassert binary_search([1, 3, 5, 7, 9], 1) == 0\nassert binary_search([1, 3, 5, 7, 9], 9) == 4", + "reference": "def binary_search(arr, target):\n lo, hi = 0, len(arr) - 1\n while lo <= hi:\n mid = (lo + hi) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n lo = mid + 1\n else:\n hi = mid - 1\n return -1", + }, + { + "spec": "Given a string containing digits, return all possible valid IP addresses that can be formed by inserting dots.", + "signature": "def restore_ip_addresses(s: str) -> list[str]", + "examples": 'restore_ip_addresses("25525511135") -> ["255.255.11.135","255.255.111.35"]\nrestore_ip_addresses("0000") -> ["0.0.0.0"]', + "test_cases": 'assert sorted(restore_ip_addresses("25525511135")) == ["255.255.11.135","255.255.111.35"]\nassert restore_ip_addresses("0000") == ["0.0.0.0"]\nassert restore_ip_addresses("1111") == ["1.1.1.1"]\nassert restore_ip_addresses("010010") == ["0.10.0.10","0.100.1.0"]', + "reference": "def restore_ip_addresses(s):\n result = []\n def bt(start, parts):\n if len(parts) == 4:\n if start == len(s):\n result.append('.'.join(parts))\n return\n for end in range(start+1, min(start+4, len(s)+1)):\n seg = s[start:end]\n if (seg[0] == '0' and len(seg) > 1) or int(seg) > 255:\n continue\n bt(end, parts + [seg])\n bt(0, [])\n return result", + }, + { + "spec": "Implement a function that computes the longest common subsequence of two strings.", + "signature": "def lcs(s1: str, s2: str) -> str", + "examples": 'lcs("abcde", "ace") -> "ace"\nlcs("abc", "abc") -> "abc"\nlcs("abc", "def") -> ""', + "test_cases": 'assert lcs("abcde", "ace") == "ace"\nassert lcs("abc", "abc") == "abc"\nassert lcs("abc", "def") == ""\nassert lcs("", "abc") == ""\nassert lcs("abcd", "abdc") in ("abc", "abd")', + "reference": "def lcs(s1, s2):\n m, n = len(s1), len(s2)\n dp = [[''] * (n+1) for _ in range(m+1)]\n for i in range(1, m+1):\n for j in range(1, n+1):\n if s1[i-1] == s2[j-1]:\n dp[i][j] = dp[i-1][j-1] + s1[i-1]\n else:\n dp[i][j] = max(dp[i-1][j], dp[i][j-1], key=len)\n return dp[m][n]", + }, + { + "spec": "Given a list of integers and a target sum, return all unique pairs that sum to the target. Each pair should be sorted, and the result should contain no duplicate pairs.", + "signature": "def two_sum_pairs(nums: list[int], target: int) -> list[list[int]]", + "examples": "two_sum_pairs([1, 2, 3, 4, 5], 6) -> [[1, 5], [2, 4]]\ntwo_sum_pairs([1, 1, 2, 3], 4) -> [[1, 3]]", + "test_cases": "assert two_sum_pairs([1, 2, 3, 4, 5], 6) == [[1, 5], [2, 4]]\nassert two_sum_pairs([1, 1, 2, 3], 4) == [[1, 3]]\nassert two_sum_pairs([], 5) == []\nassert two_sum_pairs([3, 3], 6) == [[3, 3]]", + "reference": "def two_sum_pairs(nums, target):\n nums.sort()\n result, seen = [], set()\n lo, hi = 0, len(nums)-1\n while lo < hi:\n s = nums[lo] + nums[hi]\n if s == target:\n pair = (nums[lo], nums[hi])\n if pair not in seen:\n result.append(list(pair))\n seen.add(pair)\n lo += 1; hi -= 1\n elif s < target:\n lo += 1\n else:\n hi -= 1\n return result", + }, + { + "spec": "Implement a function that converts an integer to its English words representation.", + "signature": "def int_to_words(num: int) -> str", + "examples": 'int_to_words(123) -> "One Hundred Twenty Three"\nint_to_words(0) -> "Zero"\nint_to_words(1000000) -> "One Million"', + "test_cases": 'assert int_to_words(0) == "Zero"\nassert int_to_words(123) == "One Hundred Twenty Three"\nassert int_to_words(1000) == "One Thousand"\nassert int_to_words(1000000) == "One Million"\nassert int_to_words(15) == "Fifteen"', + "reference": "def int_to_words(num):\n if num == 0:\n return 'Zero'\n ones = ['','One','Two','Three','Four','Five','Six','Seven','Eight','Nine',\n 'Ten','Eleven','Twelve','Thirteen','Fourteen','Fifteen','Sixteen',\n 'Seventeen','Eighteen','Nineteen']\n tens = ['','','Twenty','Thirty','Forty','Fifty','Sixty','Seventy','Eighty','Ninety']\n scales = ['','Thousand','Million','Billion']\n def helper(n):\n if n == 0: return []\n if n < 20: return [ones[n]]\n if n < 100: return [tens[n//10]] + helper(n%10)\n return [ones[n//100], 'Hundred'] + helper(n%100)\n parts, i = [], 0\n while num > 0:\n if num % 1000 != 0:\n chunk = helper(num % 1000)\n if scales[i]: chunk.append(scales[i])\n parts = chunk + parts\n num //= 1000; i += 1\n return ' '.join(parts)", + }, + { + "spec": "Implement a function that finds the kth largest element in an unsorted list without fully sorting it.", + "signature": "def kth_largest(nums: list[int], k: int) -> int", + "examples": "kth_largest([3, 2, 1, 5, 6, 4], 2) -> 5\nkth_largest([3, 2, 3, 1, 2, 4, 5, 5, 6], 4) -> 4", + "test_cases": "assert kth_largest([3, 2, 1, 5, 6, 4], 2) == 5\nassert kth_largest([3, 2, 3, 1, 2, 4, 5, 5, 6], 4) == 4\nassert kth_largest([1], 1) == 1\nassert kth_largest([7, 7, 7], 1) == 7", + "reference": "def kth_largest(nums, k):\n import heapq\n return heapq.nlargest(k, nums)[-1]", + }, + { + "spec": "Given a string, determine if it is a valid palindrome considering only alphanumeric characters and ignoring case.", + "signature": "def is_palindrome(s: str) -> bool", + "examples": 'is_palindrome("A man, a plan, a canal: Panama") -> True\nis_palindrome("race a car") -> False\nis_palindrome("") -> True', + "test_cases": 'assert is_palindrome("A man, a plan, a canal: Panama") == True\nassert is_palindrome("race a car") == False\nassert is_palindrome("") == True\nassert is_palindrome(" ") == True\nassert is_palindrome("ab") == False', + "reference": "def is_palindrome(s):\n cleaned = ''.join(c.lower() for c in s if c.isalnum())\n return cleaned == cleaned[::-1]", + }, + { + "spec": "Implement a simple LRU cache with get and put operations. The cache has a fixed capacity.", + "signature": "class LRUCache:\n def __init__(self, capacity: int): ...\n def get(self, key: int) -> int: ...\n def put(self, key: int, value: int) -> None: ...", + "examples": "cache = LRUCache(2)\ncache.put(1, 1)\ncache.put(2, 2)\ncache.get(1) -> 1\ncache.put(3, 3) # evicts key 2\ncache.get(2) -> -1", + "test_cases": "cache = LRUCache(2)\ncache.put(1, 1)\ncache.put(2, 2)\nassert cache.get(1) == 1\ncache.put(3, 3)\nassert cache.get(2) == -1\ncache.put(4, 4)\nassert cache.get(1) == -1\nassert cache.get(3) == 3\nassert cache.get(4) == 4", + "reference": "from collections import OrderedDict\nclass LRUCache:\n def __init__(self, capacity):\n self.cap = capacity\n self.cache = OrderedDict()\n def get(self, key):\n if key not in self.cache: return -1\n self.cache.move_to_end(key)\n return self.cache[key]\n def put(self, key, value):\n if key in self.cache:\n self.cache.move_to_end(key)\n self.cache[key] = value\n if len(self.cache) > self.cap:\n self.cache.popitem(last=False)", + }, + { + "spec": "Given a non-negative integer represented as a list of digits, add one to the number.", + "signature": "def plus_one(digits: list[int]) -> list[int]", + "examples": "plus_one([1, 2, 3]) -> [1, 2, 4]\nplus_one([9, 9, 9]) -> [1, 0, 0, 0]\nplus_one([0]) -> [1]", + "test_cases": "assert plus_one([1, 2, 3]) == [1, 2, 4]\nassert plus_one([9, 9, 9]) == [1, 0, 0, 0]\nassert plus_one([0]) == [1]\nassert plus_one([9]) == [1, 0]\nassert plus_one([1, 0, 0]) == [1, 0, 1]", + "reference": "def plus_one(digits):\n for i in range(len(digits)-1, -1, -1):\n if digits[i] < 9:\n digits[i] += 1\n return digits\n digits[i] = 0\n return [1] + digits", + }, + { + "spec": "Implement a function that finds the maximum profit from buying and selling a stock once. You can only buy before you sell.", + "signature": "def max_profit(prices: list[int]) -> int", + "examples": "max_profit([7, 1, 5, 3, 6, 4]) -> 5\nmax_profit([7, 6, 4, 3, 1]) -> 0", + "test_cases": "assert max_profit([7, 1, 5, 3, 6, 4]) == 5\nassert max_profit([7, 6, 4, 3, 1]) == 0\nassert max_profit([1, 2]) == 1\nassert max_profit([2, 1]) == 0\nassert max_profit([]) == 0", + "reference": "def max_profit(prices):\n if not prices: return 0\n min_price = prices[0]\n profit = 0\n for p in prices[1:]:\n profit = max(profit, p - min_price)\n min_price = min(min_price, p)\n return profit", + }, + { + "spec": "Given a string, find all starting indices of substrings that are anagrams of a given pattern.", + "signature": "def find_anagrams(s: str, p: str) -> list[int]", + "examples": 'find_anagrams("cbaebabacd", "abc") -> [0, 6]\nfind_anagrams("abab", "ab") -> [0, 1, 2]', + "test_cases": 'assert find_anagrams("cbaebabacd", "abc") == [0, 6]\nassert find_anagrams("abab", "ab") == [0, 1, 2]\nassert find_anagrams("", "a") == []\nassert find_anagrams("a", "ab") == []', + "reference": "def find_anagrams(s, p):\n from collections import Counter\n if len(p) > len(s): return []\n pc = Counter(p)\n wc = Counter(s[:len(p)])\n result = []\n if wc == pc: result.append(0)\n for i in range(len(p), len(s)):\n wc[s[i]] += 1\n old = s[i - len(p)]\n wc[old] -= 1\n if wc[old] == 0: del wc[old]\n if wc == pc: result.append(i - len(p) + 1)\n return result", + }, + { + "spec": "Implement a function that evaluates a mathematical expression given as a string containing +, -, *, / and parentheses. Assume valid input with integer operands.", + "signature": "def evaluate(expression: str) -> float", + "examples": 'evaluate("3+2*2") -> 7.0\nevaluate("(1+2)*3") -> 9.0\nevaluate("10/3") -> 3.333...', + "test_cases": 'assert evaluate("3+2*2") == 7.0\nassert evaluate("(1+2)*3") == 9.0\nassert abs(evaluate("10/3") - 3.333333) < 0.01\nassert evaluate("2*(3+4)") == 14.0', + "reference": "def evaluate(expression):\n def helper(tokens, pos):\n def parse_num():\n nonlocal pos\n if tokens[pos] == '(':\n pos += 1\n val = parse_expr()\n pos += 1\n return val\n num = 0\n while pos < len(tokens) and tokens[pos].isdigit():\n num = num * 10 + int(tokens[pos])\n pos += 1\n return float(num)\n def parse_term():\n val = parse_num()\n while pos < len(tokens) and tokens[pos] in '*/':\n op = tokens[pos]\n nonlocal pos\n pos += 1\n r = parse_num()\n val = val * r if op == '*' else val / r\n return val\n def parse_expr():\n val = parse_term()\n while pos < len(tokens) and tokens[pos] in '+-':\n op = tokens[pos]\n nonlocal pos\n pos += 1\n r = parse_term()\n val = val + r if op == '+' else val - r\n return val\n return parse_expr()\n tokens = list(expression.replace(' ', ''))\n return helper(tokens, 0)", + }, + { + "spec": "Implement a trie (prefix tree) with insert, search, and startsWith methods.", + "signature": "class Trie:\n def __init__(self): ...\n def insert(self, word: str) -> None: ...\n def search(self, word: str) -> bool: ...\n def starts_with(self, prefix: str) -> bool: ...", + "examples": 'trie = Trie()\ntrie.insert("apple")\ntrie.search("apple") -> True\ntrie.search("app") -> False\ntrie.starts_with("app") -> True', + "test_cases": 'trie = Trie()\ntrie.insert("apple")\nassert trie.search("apple") == True\nassert trie.search("app") == False\nassert trie.starts_with("app") == True\ntrie.insert("app")\nassert trie.search("app") == True', + "reference": "class Trie:\n def __init__(self):\n self.children = {}\n self.is_end = False\n def insert(self, word):\n node = self\n for c in word:\n if c not in node.children:\n node.children[c] = Trie()\n node = node.children[c]\n node.is_end = True\n def search(self, word):\n node = self\n for c in word:\n if c not in node.children: return False\n node = node.children[c]\n return node.is_end\n def starts_with(self, prefix):\n node = self\n for c in prefix:\n if c not in node.children: return False\n node = node.children[c]\n return True", + }, + { + "spec": "Given a 2D grid of '1's (land) and '0's (water), count the number of islands. An island is surrounded by water and formed by connecting adjacent lands horizontally or vertically.", + "signature": "def count_islands(grid: list[list[str]]) -> int", + "examples": 'count_islands([["1","1","0"],["1","1","0"],["0","0","1"]]) -> 2', + "test_cases": 'assert count_islands([["1","1","0"],["1","1","0"],["0","0","1"]]) == 2\nassert count_islands([["0","0"],["0","0"]]) == 0\nassert count_islands([["1"]]) == 1\nassert count_islands([["1","0","1"],["0","1","0"],["1","0","1"]]) == 5', + "reference": "def count_islands(grid):\n if not grid: return 0\n rows, cols = len(grid), len(grid[0])\n count = 0\n def dfs(r, c):\n if r < 0 or r >= rows or c < 0 or c >= cols or grid[r][c] != '1': return\n grid[r][c] = '#'\n dfs(r+1,c); dfs(r-1,c); dfs(r,c+1); dfs(r,c-1)\n for r in range(rows):\n for c in range(cols):\n if grid[r][c] == '1':\n count += 1\n dfs(r, c)\n return count", + }, + { + "spec": "Implement a function that generates all valid combinations of n pairs of parentheses.", + "signature": "def generate_parentheses(n: int) -> list[str]", + "examples": 'generate_parentheses(1) -> ["()"]\ngenerate_parentheses(2) -> ["(())", "()()"]\ngenerate_parentheses(3) -> ["((()))", "(()())", "(())()", "()(())", "()()()"]', + "test_cases": 'assert generate_parentheses(1) == ["()"]\nassert sorted(generate_parentheses(2)) == ["(())", "()()"]\nassert len(generate_parentheses(3)) == 5\nassert len(generate_parentheses(4)) == 14', + "reference": "def generate_parentheses(n):\n result = []\n def bt(s, o, c):\n if len(s) == 2*n:\n result.append(s)\n return\n if o < n: bt(s+'(', o+1, c)\n if c < o: bt(s+')', o, c+1)\n bt('', 0, 0)\n return result", + }, + { + "spec": "Given a list of non-negative integers representing elevation heights, compute how much water can be trapped after raining.", + "signature": "def trap_water(heights: list[int]) -> int", + "examples": "trap_water([0,1,0,2,1,0,1,3,2,1,2,1]) -> 6\ntrap_water([4,2,0,3,2,5]) -> 9", + "test_cases": "assert trap_water([0,1,0,2,1,0,1,3,2,1,2,1]) == 6\nassert trap_water([4,2,0,3,2,5]) == 9\nassert trap_water([]) == 0\nassert trap_water([3]) == 0\nassert trap_water([3, 0, 3]) == 3", + "reference": "def trap_water(heights):\n if not heights: return 0\n l, r = 0, len(heights)-1\n lmax = rmax = water = 0\n while l < r:\n if heights[l] < heights[r]:\n if heights[l] >= lmax: lmax = heights[l]\n else: water += lmax - heights[l]\n l += 1\n else:\n if heights[r] >= rmax: rmax = heights[r]\n else: water += rmax - heights[r]\n r -= 1\n return water", + }, + { + "spec": "Implement a function that serializes and deserializes a binary tree. The tree node has val, left, right attributes.", + "signature": "def serialize(root) -> str\ndef deserialize(data: str) -> TreeNode", + "examples": 'serialize a tree [1,2,3,null,null,4,5] -> some string\ndeserialize that string -> same tree', + "test_cases": "class TreeNode:\n def __init__(self, val=0, left=None, right=None):\n self.val, self.left, self.right = val, left, right\nroot = TreeNode(1, TreeNode(2), TreeNode(3, TreeNode(4), TreeNode(5)))\ns = serialize(root)\nnew_root = deserialize(s)\nassert new_root.val == 1\nassert new_root.left.val == 2\nassert new_root.right.val == 3\nassert new_root.right.left.val == 4", + "reference": "class TreeNode:\n def __init__(self, val=0, left=None, right=None):\n self.val, self.left, self.right = val, left, right\ndef serialize(root):\n if not root: return 'null'\n return f'{root.val},{serialize(root.left)},{serialize(root.right)}'\ndef deserialize(data):\n vals = iter(data.split(','))\n def build():\n v = next(vals)\n if v == 'null': return None\n node = TreeNode(int(v))\n node.left = build()\n node.right = build()\n return node\n return build()", + }, + { + "spec": "Implement a function that converts a number from base 10 to any base (2-36). Use lowercase letters for digits > 9.", + "signature": "def to_base(num: int, base: int) -> str", + "examples": 'to_base(255, 16) -> "ff"\nto_base(10, 2) -> "1010"\nto_base(0, 5) -> "0"', + "test_cases": 'assert to_base(255, 16) == "ff"\nassert to_base(10, 2) == "1010"\nassert to_base(0, 5) == "0"\nassert to_base(100, 10) == "100"\nassert to_base(35, 36) == "z"', + "reference": "def to_base(num, base):\n if num == 0: return '0'\n digits = '0123456789abcdefghijklmnopqrstuvwxyz'\n result = []\n neg = num < 0\n num = abs(num)\n while num:\n result.append(digits[num % base])\n num //= base\n if neg: result.append('-')\n return ''.join(reversed(result))", + }, + { + "spec": "Given a string of words, reverse the order of words while preserving whitespace normalization (single spaces, no leading/trailing).", + "signature": "def reverse_words(s: str) -> str", + "examples": 'reverse_words("the sky is blue") -> "blue is sky the"\nreverse_words(" hello world ") -> "world hello"', + "test_cases": 'assert reverse_words("the sky is blue") == "blue is sky the"\nassert reverse_words(" hello world ") == "world hello"\nassert reverse_words("a") == "a"\nassert reverse_words(" spaces between ") == "between spaces"', + "reference": "def reverse_words(s):\n return ' '.join(s.split()[::-1])", + }, + { + "spec": "Implement a function that computes the power set (all subsets) of a list of unique integers.", + "signature": "def power_set(nums: list[int]) -> list[list[int]]", + "examples": "power_set([1, 2, 3]) -> [[], [1], [2], [1,2], [3], [1,3], [2,3], [1,2,3]]", + "test_cases": "result = power_set([1, 2, 3])\nassert len(result) == 8\nassert [] in result\nassert [1, 2, 3] in result\nassert power_set([]) == [[]]", + "reference": "def power_set(nums):\n result = [[]]\n for num in nums:\n result += [subset + [num] for subset in result]\n return result", + }, +] + + +class CodingTaskDataset(DatasetProvider): + """Coding task benchmark: function-level code generation with test cases.""" + + dataset_id = "coding_task" + dataset_name = "Coding Task" + + def __init__(self) -> None: + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + rows = list(_TASKS) + + if seed is not None: + rng = random.Random(seed) + rng.shuffle(rows) + + if max_samples is not None: + rows = rows[:max_samples] + + self._records = [] + for idx, task in enumerate(rows): + prompt = _PROMPT_TEMPLATE.format( + spec=task["spec"], + signature=task["signature"], + examples=task["examples"], + ) + self._records.append(EvalRecord( + record_id=f"coding-task-{idx}", + problem=prompt, + reference=task["reference"], + category="use-case", + subject="coding_task", + metadata={ + "test_cases": task["test_cases"], + "signature": task["signature"], + }, + )) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + +__all__ = ["CodingTaskDataset"] diff --git a/src/openjarvis/evals/datasets/email_triage.py b/src/openjarvis/evals/datasets/email_triage.py new file mode 100644 index 00000000..366da967 --- /dev/null +++ b/src/openjarvis/evals/datasets/email_triage.py @@ -0,0 +1,335 @@ +"""Email triage benchmark dataset. + +Synthetic email threads for evaluating urgency classification, +category assignment, and draft response generation. +""" + +from __future__ import annotations + +import random +from typing import Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +_PROMPT_TEMPLATE = """You are an AI email assistant. Analyze the following email and provide: +1. Urgency level: critical, high, medium, or low +2. Category: action, decision, info, or social +3. A brief draft reply (2-3 sentences) + +Format your response exactly as: +urgency: +category: +draft: + +Email: +From: {sender} +Subject: {subject} +Date: {date} + +{body}""" + +# Synthetic email records covering a range of urgency/category combinations +_EMAILS = [ + { + "sender": "cto@company.com", + "subject": "URGENT: Production database is down", + "date": "2025-12-01 03:14", + "body": "Our primary Postgres cluster is unreachable. Customers are seeing 500 errors. I need the on-call team assembled immediately. Please acknowledge and start the incident runbook.", + "urgency": "critical", + "category": "action", + }, + { + "sender": "alice@partner.org", + "subject": "Contract renewal deadline tomorrow", + "date": "2025-12-01 09:00", + "body": "Hi, just a reminder that our service contract expires tomorrow at midnight. We need your signed renewal by EOD or we'll need to pause service. The updated terms are attached. Please review and sign at your earliest convenience.", + "urgency": "critical", + "category": "decision", + }, + { + "sender": "security@company.com", + "subject": "Security patch required: CVE-2025-4321", + "date": "2025-12-01 10:30", + "body": "A critical vulnerability has been disclosed in our authentication library. All production services must be patched within 24 hours. The fix is available in version 3.2.1. Please coordinate with your team to schedule the update.", + "urgency": "critical", + "category": "action", + }, + { + "sender": "boss@company.com", + "subject": "Q4 budget approval needed", + "date": "2025-12-02 08:00", + "body": "I need your approval on the Q4 infrastructure budget by Friday. The proposal includes $45K for GPU servers and $12K for monitoring tools. Let me know if you have concerns or want to discuss before I submit to finance.", + "urgency": "high", + "category": "decision", + }, + { + "sender": "hr@company.com", + "subject": "New hire starting Monday — setup needed", + "date": "2025-12-02 11:00", + "body": "Sarah Chen is joining the ML team on Monday. Please ensure her dev environment, Slack access, and GitHub permissions are set up before she arrives. She'll need access to the model-training and inference repos.", + "urgency": "high", + "category": "action", + }, + { + "sender": "client@bigcorp.com", + "subject": "API integration failing in staging", + "date": "2025-12-02 14:00", + "body": "We're getting 401 errors when calling your /v2/inference endpoint from our staging environment. Our API key was rotated last week. Could you check if the new key is properly allowlisted? We need this resolved before our Thursday demo.", + "urgency": "high", + "category": "action", + }, + { + "sender": "pm@company.com", + "subject": "Feature spec review: voice commands", + "date": "2025-12-02 15:30", + "body": "I've drafted the PRD for voice command support. It covers wake word detection, streaming transcription, and intent parsing. Can you review the technical feasibility section and leave comments by next Wednesday?", + "urgency": "high", + "category": "decision", + }, + { + "sender": "devops@company.com", + "subject": "Kubernetes cluster upgrade scheduled", + "date": "2025-12-03 09:00", + "body": "We'll be upgrading the production k8s cluster from 1.28 to 1.30 this Saturday at 2am UTC. Expected downtime is 15 minutes. Please ensure your services are compatible with the new version. No action needed if you're using the standard Helm charts.", + "urgency": "medium", + "category": "info", + }, + { + "sender": "data-team@company.com", + "subject": "Weekly model performance report", + "date": "2025-12-03 10:00", + "body": "Here's this week's performance summary:\n- Inference latency: P50=42ms, P99=180ms (stable)\n- Accuracy on validation set: 94.2% (up 0.3%)\n- Daily active users: 12,400 (up 8%)\n- Error rate: 0.02% (down from 0.05%)\nAll metrics within SLA. No action needed.", + "urgency": "low", + "category": "info", + }, + { + "sender": "intern@company.com", + "subject": "Question about coding style", + "date": "2025-12-03 11:00", + "body": "Hi! I'm working on the data pipeline refactor. Should I use dataclasses or Pydantic models for the new config objects? I saw both patterns in the codebase and want to stay consistent. Thanks!", + "urgency": "low", + "category": "decision", + }, + { + "sender": "marketing@company.com", + "subject": "Blog post draft for review", + "date": "2025-12-03 13:00", + "body": "I've written a blog post about our new on-device inference feature. Could you review the technical accuracy of the benchmarks section? No rush — we're planning to publish next week. Draft is in the shared Google Doc.", + "urgency": "medium", + "category": "action", + }, + { + "sender": "colleague@company.com", + "subject": "Lunch tomorrow?", + "date": "2025-12-03 16:00", + "body": "Hey, want to grab lunch tomorrow? There's a new ramen place that opened on 5th street. I heard it's really good. Let me know if you're free around noon.", + "urgency": "low", + "category": "social", + }, + { + "sender": "legal@company.com", + "subject": "Updated data processing agreement", + "date": "2025-12-04 09:00", + "body": "Please review the updated DPA for our EU customers. The main changes are around data retention periods and right-to-erasure timelines. We need engineering sign-off by end of next week to stay compliant with the new regulations.", + "urgency": "medium", + "category": "decision", + }, + { + "sender": "support@vendor.io", + "subject": "Your support ticket #8842 has been resolved", + "date": "2025-12-04 10:00", + "body": "Hi, we've resolved the memory leak issue you reported in v4.1.2. The fix is included in v4.1.3 which was released today. Please upgrade at your convenience and let us know if the issue persists.", + "urgency": "medium", + "category": "action", + }, + { + "sender": "recruiter@external.com", + "subject": "Exciting ML opportunity", + "date": "2025-12-04 11:00", + "body": "I came across your profile and think you'd be a great fit for our Senior ML Engineer role. We're building cutting-edge on-device AI systems. Would you be open to a brief chat next week?", + "urgency": "low", + "category": "social", + }, + { + "sender": "cfo@company.com", + "subject": "Cost optimization meeting — Thursday 2pm", + "date": "2025-12-04 14:00", + "body": "I'm setting up a meeting to discuss cloud cost optimization. Our GPU compute spend increased 40% last quarter. Please come prepared with data on your team's usage and any ideas for reducing costs without impacting performance.", + "urgency": "high", + "category": "action", + }, + { + "sender": "ops@company.com", + "subject": "SSL certificate expiring in 7 days", + "date": "2025-12-05 08:00", + "body": "The SSL certificate for api.openjarvis.dev expires on December 12. Auto-renewal is configured but failed last time due to DNS validation issues. Please verify the CNAME record is correct and trigger a manual renewal if needed.", + "urgency": "high", + "category": "action", + }, + { + "sender": "team@company.com", + "subject": "Retrospective notes from sprint 47", + "date": "2025-12-05 09:00", + "body": "Here are the key takeaways from yesterday's retro:\n- What went well: shipped voice feature on time, test coverage improved to 92%\n- What to improve: PR review turnaround is still slow (avg 2 days)\n- Action items: implement PR review SLA, set up automated test reports\nFull notes in Confluence.", + "urgency": "low", + "category": "info", + }, + { + "sender": "partner@aicloud.io", + "subject": "Partnership proposal — joint webinar", + "date": "2025-12-05 10:00", + "body": "We'd love to co-host a webinar on local vs cloud AI inference with your team. We think a balanced comparison would be valuable for both our audiences. Proposed date: January 15. Would you be interested?", + "urgency": "medium", + "category": "decision", + }, + { + "sender": "monitoring@company.com", + "subject": "Alert: Memory usage at 85% on prod-gpu-03", + "date": "2025-12-05 22:00", + "body": "Memory usage on prod-gpu-03 has been above 85% for the past 30 minutes. Current usage: 85.2%. Threshold: 90%. The inference service is still healthy but may degrade if usage increases. Consider restarting the service or scaling out.", + "urgency": "medium", + "category": "action", + }, + { + "sender": "design@company.com", + "subject": "New dashboard mockups ready", + "date": "2025-12-06 09:00", + "body": "The updated mockups for the savings dashboard are ready in Figma. Key changes: added monthly projection cards, cloud agent comparison section, and a cost calculator widget. Please take a look when you have a chance.", + "urgency": "low", + "category": "info", + }, + { + "sender": "board@company.com", + "subject": "Board meeting prep — AI strategy deck", + "date": "2025-12-06 10:00", + "body": "The board meeting is next Thursday. I need 3-4 slides on our AI cost savings strategy — specifically how on-device inference reduces our operational costs. Can you prepare the slides with actual numbers by Tuesday?", + "urgency": "high", + "category": "action", + }, + { + "sender": "opensource@contributor.dev", + "subject": "PR submitted: improved memory backend", + "date": "2025-12-06 11:00", + "body": "Hi, I've submitted PR #234 with a hybrid memory backend that uses RRF fusion for better retrieval. Tests are passing. Would appreciate a review when you get a chance. Happy to make any changes.", + "urgency": "medium", + "category": "action", + }, + { + "sender": "training@company.com", + "subject": "Mandatory security training due Dec 15", + "date": "2025-12-06 13:00", + "body": "This is a reminder that all engineering staff must complete the annual security awareness training by December 15. The course takes approximately 30 minutes. Access it through the Learning Portal. Please complete it at your earliest convenience.", + "urgency": "medium", + "category": "action", + }, + { + "sender": "friend@personal.com", + "subject": "Happy holidays!", + "date": "2025-12-06 17:00", + "body": "Hey! Just wanted to wish you happy holidays. Hope you're doing well. Let's catch up over coffee sometime in January. Miss our chats!", + "urgency": "low", + "category": "social", + }, + { + "sender": "ceo@company.com", + "subject": "Company all-hands: cost savings milestone", + "date": "2025-12-07 08:00", + "body": "Great news — we've saved over $500K this year by running AI inference locally. I want to highlight this at the all-hands on Friday. Can someone from the infra team prepare a 5-minute demo of the savings dashboard?", + "urgency": "high", + "category": "action", + }, + { + "sender": "vendor@cloudprovider.com", + "subject": "Your monthly invoice is ready", + "date": "2025-12-07 09:00", + "body": "Your December invoice for cloud compute services is now available. Total: $3,247.88. This represents a 60% reduction from your peak monthly spend in June ($8,120.00). View details in your account dashboard.", + "urgency": "low", + "category": "info", + }, + { + "sender": "qa@company.com", + "subject": "Regression found in v2.8.1", + "date": "2025-12-07 10:00", + "body": "We found a regression in the latest release: the memory search endpoint returns empty results when the query contains special characters (e.g., '@', '#'). This affects the email triage workflow. Bisected to commit abc123. Please fix before the next release.", + "urgency": "high", + "category": "action", + }, + { + "sender": "newsletter@techdigest.com", + "subject": "This week in AI: local inference trends", + "date": "2025-12-07 12:00", + "body": "Top stories this week:\n1. On-device AI adoption up 200% year-over-year\n2. New quantization techniques enable 70B models on consumer GPUs\n3. Cloud AI costs continue to rise, driving interest in local alternatives\n4. Open-source AI frameworks see record contributions", + "urgency": "low", + "category": "info", + }, + { + "sender": "compliance@company.com", + "subject": "GDPR audit scheduled for January", + "date": "2025-12-07 14:00", + "body": "Our annual GDPR compliance audit is scheduled for January 20-22. The auditors will review our data processing pipelines, consent mechanisms, and retention policies. Please ensure your team's documentation is up to date. I'll send a detailed checklist next week.", + "urgency": "medium", + "category": "info", + }, +] + + +class EmailTriageDataset(DatasetProvider): + """Email triage benchmark: urgency classification + category + draft reply.""" + + dataset_id = "email_triage" + dataset_name = "Email Triage" + + def __init__(self) -> None: + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + rows = list(_EMAILS) + + if seed is not None: + rng = random.Random(seed) + rng.shuffle(rows) + + if max_samples is not None: + rows = rows[:max_samples] + + self._records = [] + for idx, email in enumerate(rows): + prompt = _PROMPT_TEMPLATE.format( + sender=email["sender"], + subject=email["subject"], + date=email["date"], + body=email["body"], + ) + reference = ( + f"urgency: {email['urgency']}\n" + f"category: {email['category']}" + ) + self._records.append(EvalRecord( + record_id=f"email-triage-{idx}", + problem=prompt, + reference=reference, + category="use-case", + subject="email_triage", + metadata={ + "urgency": email["urgency"], + "category": email["category"], + "sender": email["sender"], + "subject_line": email["subject"], + }, + )) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + +__all__ = ["EmailTriageDataset"] diff --git a/src/openjarvis/evals/datasets/knowledge_base.py b/src/openjarvis/evals/datasets/knowledge_base.py new file mode 100644 index 00000000..4c2bb24e --- /dev/null +++ b/src/openjarvis/evals/datasets/knowledge_base.py @@ -0,0 +1,225 @@ +"""Knowledge base benchmark dataset. + +Document-grounded retrieval questions for evaluating retrieval accuracy +and answer correctness from a knowledge corpus. +""" + +from __future__ import annotations + +import random +from typing import Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +_PROMPT_TEMPLATE = """You are a knowledge base assistant. Answer the following question using only the information provided in the document excerpts below. If the answer cannot be determined from the documents, say "Cannot be determined from the provided documents." + +Documents: +{documents} + +Question: {question} + +Provide a concise, accurate answer based solely on the documents above.""" + +_RECORDS = [ + { + "documents": "OpenJarvis Architecture Guide (v2.9):\nOpenJarvis is organized around five composable pillars: Intelligence, Engine, Agents, Tools, and Learning. The Intelligence pillar handles model definition and catalog management via ModelRegistry. The Engine pillar provides inference through backends including Ollama, vLLM, SGLang, llama.cpp, MLX, and LM Studio. All engines implement the InferenceEngine ABC with generate(), stream(), list_models(), and health() methods.", + "question": "What are the five pillars of OpenJarvis and what does the Engine pillar provide?", + "answer": "The five pillars are Intelligence, Engine, Agents, Tools, and Learning. The Engine pillar provides inference through backends including Ollama, vLLM, SGLang, llama.cpp, MLX, and LM Studio.", + }, + { + "documents": "Memory Backend Comparison (Internal Doc):\nSQLite/FTS5 is the default memory backend, providing full-text search with minimal dependencies. FAISS offers dense vector similarity search suitable for semantic retrieval. ColBERTv2 provides late-interaction retrieval with higher accuracy but requires more compute. BM25 is a traditional sparse retrieval method. The Hybrid backend uses Reciprocal Rank Fusion (RRF) to combine results from multiple backends.", + "question": "What is the default memory backend and how does the Hybrid backend combine results?", + "answer": "SQLite/FTS5 is the default memory backend. The Hybrid backend uses Reciprocal Rank Fusion (RRF) to combine results from multiple backends.", + }, + { + "documents": "Security Architecture Document:\nOpenJarvis implements a multi-layer security approach. SecretScanner and PIIScanner implement the BaseScanner ABC. GuardrailsEngine wraps inference engines with input/output scanning supporting WARN, REDACT, and BLOCK modes. The AuditLogger uses a Merkle hash chain with SHA-256 for tamper evidence. CapabilityPolicy provides RBAC with 10 capabilities and glob matching, enforced in ToolExecutor.", + "question": "How does the AuditLogger ensure tamper evidence?", + "answer": "The AuditLogger uses a Merkle hash chain with SHA-256 for tamper evidence.", + }, + { + "documents": "Agent System Documentation:\nOpenJarvis agents follow a hierarchy: BaseAgent ABC provides helpers for turn management and message building. ToolUsingAgent extends BaseAgent with tool dispatch via ToolExecutor. SimpleAgent handles single-turn queries. OrchestratorAgent manages multi-turn tool loops. NativeReActAgent implements Thought-Action-Observation patterns. The LoopGuard system prevents infinite loops via SHA-256 hash tracking and ping-pong detection.", + "question": "What is the agent hierarchy and how are infinite loops prevented?", + "answer": "The hierarchy is BaseAgent -> ToolUsingAgent -> specific agents (SimpleAgent, OrchestratorAgent, NativeReActAgent, etc.). Infinite loops are prevented by the LoopGuard system which uses SHA-256 hash tracking and ping-pong detection.", + }, + { + "documents": "Telemetry System Guide:\nInstrumentedEngine wraps any inference engine to collect telemetry transparently. TelemetryStore persists records to SQLite. TelemetryAggregator provides read-only queries for statistics. EnergyMonitor ABC has vendor-specific implementations: NvidiaEnergyMonitor (hardware counters/polling), AmdEnergyMonitor (amdsmi), AppleEnergyMonitor (zeus-ml), and RaplEnergyMonitor (sysfs). EnergyBatch tracks batch-level energy-per-token metrics.", + "question": "What energy monitoring backends are available and what library does the Apple backend use?", + "answer": "The available energy monitoring backends are NvidiaEnergyMonitor (hardware counters/polling), AmdEnergyMonitor (amdsmi), AppleEnergyMonitor (zeus-ml), and RaplEnergyMonitor (sysfs). The Apple backend uses the zeus-ml library.", + }, + { + "documents": "Configuration Reference:\nOpenJarvis uses TOML configuration with pillar-aligned sections. The config file is located at ~/.openjarvis/config.toml. Key sections include [engine] with nested per-backend configs (e.g., [engine.ollama], [engine.vllm]), [intelligence] for model defaults, [agent] for agent configuration, [tools.storage] for memory backend settings, [learning] with nested routing/intelligence/agent/metrics sub-policies, and [security] with capabilities and rate limiting.", + "question": "Where is the config file located and what format does it use?", + "answer": "The config file is located at ~/.openjarvis/config.toml and uses TOML format.", + }, + { + "documents": "Learning Subsystem Overview:\nThe Learning pillar supports multiple routing policies. HeuristicRouter uses rule-based routing. SFTRouterPolicy learns query-to-model mapping from traces. GRPORouterPolicy uses softmax sampling with group relative advantage and per-query-class weights. BanditRouterPolicy implements Thompson Sampling and UCB1 with per-arm statistics. SkillDiscovery mines tool subsequences from traces to auto-generate skill manifests.", + "question": "What routing policies are available and what algorithm does BanditRouterPolicy implement?", + "answer": "Available routing policies are HeuristicRouter, SFTRouterPolicy, GRPORouterPolicy, and BanditRouterPolicy. BanditRouterPolicy implements Thompson Sampling and UCB1 algorithms.", + }, + { + "documents": "MCP Integration Guide:\nThe Model Context Protocol (MCP) is used for all tool management. MCPToolAdapter wraps external MCP tools as native BaseTool instances. MCPToolProvider discovers tools from MCP servers. The built-in MCP server exposes all tools via JSON-RPC tools/list and tools/call endpoints following the MCP spec 2025-11-25. Tool templates in tools/templates/ allow dynamic tool construction from TOML specifications with 10 builtin templates.", + "question": "How does OpenJarvis integrate with external MCP tools?", + "answer": "MCPToolAdapter wraps external MCP tools as native BaseTool instances, and MCPToolProvider discovers tools from MCP servers.", + }, + { + "documents": "Workflow Engine Documentation:\nThe workflow engine uses DAG-based WorkflowGraph with cycle detection and topological sort. Parallel stages execute via ThreadPoolExecutor. WorkflowBuilder provides a fluent API for graph construction. WorkflowEngine executes workflows against a JarvisSystem instance. Workflows can be defined in TOML files. Supported node types: agent, tool, condition, parallel, loop, and transform.", + "question": "What node types does the workflow engine support and how are parallel stages executed?", + "answer": "The workflow engine supports agent, tool, condition, parallel, loop, and transform node types. Parallel stages are executed via ThreadPoolExecutor.", + }, + { + "documents": "Speech Subsystem Technical Reference:\nOpenJarvis supports speech-to-text with pluggable backends. SpeechBackend ABC defines transcribe(), health(), and supported_formats() methods. Three backends are available: FasterWhisperBackend (local, CTranslate2, key 'faster-whisper'), OpenAIWhisperBackend (cloud, whisper-1, key 'openai'), and DeepgramSpeechBackend (cloud, nova-2, key 'deepgram'). Auto-discovery prioritizes local backends.", + "question": "What speech backends are available and which models do the cloud backends use?", + "answer": "Three backends are available: FasterWhisperBackend (local), OpenAIWhisperBackend (cloud, using whisper-1), and DeepgramSpeechBackend (cloud, using nova-2). Auto-discovery prioritizes local backends.", + }, + { + "documents": "Sandbox System Guide:\nOpenJarvis provides two sandbox options. ContainerRunner manages Docker/Podman container lifecycle with mount validation. WasmRunner uses wasmtime-py with fuel and memory limits. SandboxedAgent is a transparent wrapper that routes agent execution through either sandbox. MountAllowlist prevents path traversal attacks. The create_sandbox_runner() factory selects the appropriate runner.", + "question": "What are the two sandbox options and what prevents path traversal attacks?", + "answer": "The two sandbox options are ContainerRunner (Docker/Podman) and WasmRunner (wasmtime-py). MountAllowlist prevents path traversal attacks.", + }, + { + "documents": "Channel System Architecture:\nOpenJarvis supports multi-platform messaging via the BaseChannel ABC. Available channels include WhatsAppBaileysChannel, LINEChannel, ViberChannel, MessengerChannel, RedditChannel, MastodonChannel, XMPPChannel, RocketChatChannel, ZulipChannel, TwitchChannel, and NostrChannel. All channels use @ChannelRegistry.register() for discovery and integrate with the EventBus.", + "question": "How many messaging channels does OpenJarvis support and how are they discovered?", + "answer": "OpenJarvis supports 11 messaging channels. They are discovered via the @ChannelRegistry.register() decorator.", + }, + { + "documents": "Eval Framework Guide:\nThe evaluation framework includes 15 real benchmark datasets from IPW: SuperGPQA, GPQA, MMLU-Pro, MATH-500, Natural Reasoning, HLE, SimpleQA, WildChat, IPW, GAIA, FRAMES, SWE-bench, SWEfficiency, TerminalBench, and TerminalBench Native. Scorer types include MCQ letter extraction, LLM-judge, exact match, and structural validation. EvalRunner supports parallel execution.", + "question": "How many benchmark datasets are available and what scorer types exist?", + "answer": "There are 15 benchmark datasets available. Scorer types include MCQ letter extraction, LLM-judge, exact match, and structural validation.", + }, + { + "documents": "Session Management Documentation:\nSessionStore uses SQLite for cross-channel persistent sessions. SessionIdentity provides canonical user identification across channels. The consolidate() method summarizes old messages to reduce storage. The decay() method removes expired sessions based on configurable retention policies.", + "question": "How are sessions managed across channels and how is storage reduced?", + "answer": "Sessions are managed via SessionStore using SQLite with SessionIdentity for canonical user identification across channels. Storage is reduced through the consolidate() method which summarizes old messages.", + }, + { + "documents": "Desktop Application Guide:\nThe OpenJarvis desktop app uses Tauri 2.0. It includes 5 dashboard panels: EnergyDashboard (real-time power monitoring with recharts), TraceDebugger (timeline inspection with step-type color coding), LearningCurve (policy visualization for GRPO/bandit stats), MemoryBrowser (search and stats), and AdminPanel (health, agents, server control). Tauri commands proxy to the OpenJarvis REST API.", + "question": "What framework does the desktop app use and what are the 5 dashboard panels?", + "answer": "The desktop app uses Tauri 2.0. The 5 panels are EnergyDashboard, TraceDebugger, LearningCurve, MemoryBrowser, and AdminPanel.", + }, + { + "documents": "API Server Reference:\nThe OpenAI-compatible server is started via 'jarvis serve'. Core endpoints: POST /v1/chat/completions, GET /v1/models, GET /health. Extended endpoints cover agents, memory, traces, telemetry, learning, skills, sessions, budget, and metrics. SSE streaming is supported on /v1/chat/completions with stream=true. WebSocket streaming is available at WS /v1/chat/stream.", + "question": "How is the API server started and what streaming options are available?", + "answer": "The API server is started via 'jarvis serve'. SSE streaming is available on /v1/chat/completions with stream=true, and WebSocket streaming is available at WS /v1/chat/stream.", + }, + { + "documents": "Trace System Documentation:\nTraces capture full interaction records via TraceStep objects. Step types include route, retrieve, generate, tool_call, and respond, each with timing information. TraceStore persists traces to SQLite. TraceCollector auto-wraps agents to capture traces. TraceAnalyzer generates statistics used by the learning subsystem.", + "question": "What step types can a trace contain and how are traces used for learning?", + "answer": "Traces can contain route, retrieve, generate, tool_call, and respond steps. TraceAnalyzer generates statistics from traces that are used by the learning subsystem.", + }, + { + "documents": "Skill System Reference:\nSkills are defined as TOML manifests with sequential tool steps and template rendering. SkillExecutor runs the steps in order. Ed25519 signature verification ensures skill integrity. SkillTool adapter wraps skills as invocable tools. There are 20 bundled skills covering file management, research, code quality, productivity, and document processing.", + "question": "How many bundled skills are there and how is their integrity verified?", + "answer": "There are 20 bundled skills. Their integrity is verified through Ed25519 signature verification.", + }, + { + "documents": "Vault System Guide:\nThe vault provides encrypted credential storage at ~/.openjarvis/vault.enc using Fernet encryption. The encryption key is auto-generated with 0o600 file permissions for security. CLI commands: 'jarvis vault set KEY' to store, 'jarvis vault get KEY' to retrieve, and 'jarvis vault list' to list stored keys.", + "question": "What encryption does the vault use and where is it stored?", + "answer": "The vault uses Fernet encryption and is stored at ~/.openjarvis/vault.enc.", + }, + { + "documents": "Recipe System Documentation:\nRecipes are composable TOML configs that wire all 5 pillars. Each Recipe dataclass provides to_builder_kwargs() for SystemBuilder integration. Three built-in recipes exist: coding_assistant, research_assistant, and general_assistant. Operator recipes include researcher (4h cycle), correspondent (5min interval), and sentinel (2h cycle). Recipes are discovered via discover_recipes() and resolved via resolve_recipe().", + "question": "What built-in recipes exist and what are the operator recipe cycle times?", + "answer": "Built-in recipes are coding_assistant, research_assistant, and general_assistant. Operator recipe cycle times: researcher (4h), correspondent (5min), sentinel (2h).", + }, + { + "documents": "A2A Protocol Implementation:\nOpenJarvis implements the Google Agent-to-Agent spec using JSON-RPC 2.0. A2AServer supports tasks/send, tasks/get, and tasks/cancel operations. Agent discovery is available at /.well-known/agent.json. A2AClient enables calling external A2A agents. A2AAgentTool adapts A2A agents as tool-callable resources.", + "question": "What protocol does A2A use and how are agents discovered?", + "answer": "A2A uses JSON-RPC 2.0. Agents are discovered via /.well-known/agent.json.", + }, + { + "documents": "Deployment Guide:\nOpenJarvis provides three Docker variants: Dockerfile (Python 3.12-slim for CPU), Dockerfile.gpu (NVIDIA CUDA 12.4), and Dockerfile.gpu.rocm (AMD ROCm 6.2). The docker-compose.yml runs two services: jarvis on port 8000 and ollama on port 11434. SystemD and launchd service files are also provided for system-level deployment.", + "question": "What Docker variants are available and what ports do the services use?", + "answer": "Three Docker variants: CPU (Python 3.12-slim), NVIDIA (CUDA 12.4), and AMD (ROCm 6.2). Jarvis runs on port 8000 and Ollama on port 11434.", + }, + { + "documents": "Event System Reference:\nThe EventBus provides synchronous pub/sub event dispatch. Approximately 30 EventType values cover inference, tools, memory, agents, telemetry, traces, channels, security, scheduler, workflow, skills, sessions, and A2A. Subscribers register handlers for specific event types. Events are dispatched synchronously within the publishing thread.", + "question": "How many EventType values does the system define and how are events dispatched?", + "answer": "The system defines approximately 30 EventType values. Events are dispatched synchronously within the publishing thread.", + }, + { + "documents": "Cost Savings Analysis (Q4 2025):\nBy running inference locally using Ollama with quantized models, the team eliminated all API costs. Previous cloud spending: $8,120/month on GPT-4 API calls for email triage (every 5 min), research queries (20/day), and monitoring (continuous). After migration to local inference: $0/month in API fees, plus approximately $15/month in electricity costs for a consumer GPU running 24/7.", + "question": "What was the monthly cloud spending before migration and what are the costs after?", + "answer": "Cloud spending was $8,120/month. After migration to local inference: $0/month in API fees plus approximately $15/month in electricity for a consumer GPU.", + }, + { + "documents": "Hardware Detection System:\nOpenJarvis auto-detects GPU vendor, model, and VRAM using platform-specific tools: nvidia-smi for NVIDIA GPUs, rocm-smi for AMD GPUs, system_profiler for Apple Silicon, and /proc/cpuinfo for CPU features. Based on detection, it recommends the appropriate inference engine (vLLM for NVIDIA, MLX for Apple, llama.cpp for CPU-only).", + "question": "How does OpenJarvis detect hardware and what engine is recommended for Apple Silicon?", + "answer": "OpenJarvis detects hardware using nvidia-smi, rocm-smi, system_profiler, and /proc/cpuinfo. MLX is recommended for Apple Silicon.", + }, + { + "documents": "Template System Documentation:\nAgent templates are pre-configured TOML manifests with system prompts, tool sets, and behavioral parameters. 15 built-in templates exist including code-reviewer, debugger, architect, deep-researcher, fact-checker, and summarizer. Templates are loaded via load_template() and discovered via discover_templates(). Each template specifies the agent type, tools, and generation parameters.", + "question": "How many built-in agent templates exist and name at least four of them.", + "answer": "There are 15 built-in templates. Examples include code-reviewer, debugger, architect, deep-researcher, fact-checker, and summarizer.", + }, + { + "documents": "Scheduler System Guide:\nTaskScheduler supports three scheduling types: cron (cron expressions), interval (fixed time intervals), and once (one-time execution). Tasks are persisted to SQLite. Five MCP tools are available for task management. The scheduler integrates with the EventBus for notifications. CLI commands: create, list, pause, resume, cancel, logs, and start.", + "question": "What scheduling types does TaskScheduler support?", + "answer": "TaskScheduler supports three scheduling types: cron (cron expressions), interval (fixed time intervals), and once (one-time execution).", + }, + { + "documents": "TUI Dashboard Reference:\nThe terminal-based dashboard uses the textual library and is available with the [dashboard] optional dependency. It provides panels for system status, event stream, telemetry visualization, agent activity monitoring, and session management. The dashboard runs in the terminal and provides a real-time view of system operations.", + "question": "What library powers the TUI dashboard and what panels does it include?", + "answer": "The TUI dashboard is powered by the textual library. It includes panels for system status, event stream, telemetry, agent activity, and sessions.", + }, + { + "documents": "MCP Quick-Add Feature:\nThe 'jarvis add' command provides quick MCP server setup with 8 built-in templates: github, filesystem, slack, postgres, brave-search, memory, puppeteer, and google-maps. Configuration is saved as JSON to ~/.openjarvis/mcp/. Each template includes the server command, arguments, and required environment variables.", + "question": "How many MCP server templates are available via 'jarvis add' and where is configuration saved?", + "answer": "8 MCP server templates are available. Configuration is saved to ~/.openjarvis/mcp/.", + }, + { + "documents": "Composition Layer Documentation:\nSystemBuilder provides a fluent builder pattern that produces JarvisSystem instances. The builder wires together engine, model, agent, tools, telemetry, traces, workflow, sessions, and capability policy. JarvisSystem exposes ask() for queries and close() for resource cleanup. The SDK (Jarvis class) wraps SystemBuilder with a simplified sync API.", + "question": "What is the relationship between SystemBuilder, JarvisSystem, and the Jarvis SDK class?", + "answer": "SystemBuilder is a fluent builder that produces JarvisSystem instances. The Jarvis SDK class wraps SystemBuilder with a simplified synchronous API.", + }, +] + + +class KnowledgeBaseDataset(DatasetProvider): + """Knowledge base benchmark: document-grounded retrieval and QA.""" + + dataset_id = "knowledge_base" + dataset_name = "Knowledge Base" + + def __init__(self) -> None: + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + rows = list(_RECORDS) + + if seed is not None: + rng = random.Random(seed) + rng.shuffle(rows) + + if max_samples is not None: + rows = rows[:max_samples] + + self._records = [] + for idx, rec in enumerate(rows): + prompt = _PROMPT_TEMPLATE.format( + documents=rec["documents"], + question=rec["question"], + ) + self._records.append(EvalRecord( + record_id=f"knowledge-base-{idx}", + problem=prompt, + reference=rec["answer"], + category="use-case", + subject="knowledge_base", + metadata={}, + )) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + +__all__ = ["KnowledgeBaseDataset"] diff --git a/src/openjarvis/evals/datasets/morning_brief.py b/src/openjarvis/evals/datasets/morning_brief.py new file mode 100644 index 00000000..1fa79d3c --- /dev/null +++ b/src/openjarvis/evals/datasets/morning_brief.py @@ -0,0 +1,215 @@ +"""Morning brief benchmark dataset. + +Synthetic user-context records for evaluating daily briefing generation: +calendar events, todo items, news topics, and pending messages. +""" + +from __future__ import annotations + +import random +from typing import Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +_PROMPT_TEMPLATE = """You are a personal AI assistant preparing a morning briefing. Based on the following context, generate a concise, prioritized morning brief. + +The brief should: +1. Highlight the most important/urgent items first +2. Summarize calendar events for today +3. List top priorities from the todo list +4. Include relevant news highlights +5. Note any pending messages that need attention + +Format your response as a structured briefing with clear sections. + +--- User Context --- +Date: {date} + +Calendar: +{calendar} + +Todo List: +{todos} + +News Topics of Interest: +{news_topics} + +Pending Messages: +{messages}""" + +_CONTEXTS = [ + { + "date": "Monday, December 1, 2025", + "calendar": "- 9:00 AM: Team standup (15 min)\n- 10:30 AM: 1:1 with Sarah (engineering lead)\n- 2:00 PM: Product review meeting\n- 4:00 PM: Candidate interview — senior ML engineer", + "todos": "- Review PR #234 (memory backend optimization)\n- Submit Q4 budget proposal (due today)\n- Update deployment runbook\n- Prepare interview questions for 4pm candidate", + "news_topics": "AI, machine learning, local inference, open source", + "messages": "- Slack from CTO: 'Can we discuss cloud cost reduction at standup?'\n- Email from legal: contract renewal deadline is Wednesday\n- GitHub notification: CI failing on main branch", + "key_priorities": "Q4 budget due today, CI failure on main, CTO wants cost discussion", + }, + { + "date": "Tuesday, December 2, 2025", + "calendar": "- 8:30 AM: All-hands meeting (company update)\n- 11:00 AM: Architecture review — new caching layer\n- 1:00 PM: Lunch with visiting researcher\n- 3:30 PM: Sprint planning", + "todos": "- Fix regression in search endpoint (high priority)\n- Write design doc for voice command feature\n- Review 3 pending PRs\n- Order new GPU server (approved in budget)", + "news_topics": "semiconductor industry, GPU availability, NVIDIA earnings", + "messages": "- Email from vendor: GPU server shipping delayed 2 weeks\n- Slack from QA: 'Found another edge case in triage workflow'\n- PR comment: need to address review feedback on #230", + "key_priorities": "Search regression fix, GPU delay impacts timeline, QA edge case", + }, + { + "date": "Wednesday, December 3, 2025", + "calendar": "- 9:00 AM: Team standup\n- 10:00 AM: Security review (quarterly)\n- 2:00 PM: Demo to potential partner\n- 5:00 PM: Yoga class", + "todos": "- Complete security training (due Dec 15)\n- Prepare demo slides for partner meeting\n- Merge approved PRs\n- Update CLAUDE.md with Phase 25 changes", + "news_topics": "cybersecurity, data privacy, GDPR compliance", + "messages": "- Email from partner: 'Looking forward to the demo, can you show the savings dashboard?'\n- Slack from intern: question about coding conventions\n- Calendar reminder: contract renewal deadline is today", + "key_priorities": "Contract renewal deadline today, partner demo at 2pm, security review", + }, + { + "date": "Thursday, December 4, 2025", + "calendar": "- 9:00 AM: Team standup\n- 11:00 AM: Deep work block (no meetings)\n- 2:00 PM: Cost optimization meeting with CFO\n- 4:30 PM: Mentoring session with junior developer", + "todos": "- Prepare cost analysis slides for CFO meeting\n- Implement rate limiting for API endpoints\n- Code review: agent loop guard improvements\n- Write blog post draft on local inference benefits", + "news_topics": "cloud computing costs, serverless trends, edge AI", + "messages": "- Email from CFO: 'Please bring specific numbers on GPU vs cloud costs'\n- Slack from devops: 'k8s upgrade went smoothly, all services healthy'\n- GitHub: 2 new issues filed by community users", + "key_priorities": "CFO meeting needs cost data, rate limiting implementation, community issues", + }, + { + "date": "Friday, December 5, 2025", + "calendar": "- 9:00 AM: Team standup\n- 10:00 AM: Retrospective\n- 12:00 PM: Team lunch\n- 3:00 PM: Release planning for v3.0", + "todos": "- Tag v2.9.1 release\n- Update changelog\n- Clear inbox (20+ unread)\n- Submit expense report\n- Plan next week's priorities", + "news_topics": "open source funding, AI regulation, developer tools", + "messages": "- Email from CEO: 'Great work on the cost savings! Can we present at all-hands?'\n- Slack from PM: 'v3.0 feature list needs final sign-off'\n- PR merged: speech backend improvements", + "key_priorities": "Release v2.9.1, CEO presentation request, v3.0 planning", + }, + { + "date": "Monday, December 8, 2025", + "calendar": "- 9:00 AM: Team standup\n- 10:00 AM: Board meeting prep\n- 1:00 PM: Interview panel — engineering manager\n- 3:00 PM: Tech debt review", + "todos": "- Prepare 3-4 slides on AI cost savings for board\n- Review candidate resume\n- Fix flaky test in CI (test_memory_search)\n- Update oncall rotation for holidays", + "news_topics": "AI startups, venture capital, talent market", + "messages": "- Email from board member: 'Send me the cost savings data before Thursday'\n- Slack from team: 'Who's covering oncall over the holidays?'\n- GitHub: security advisory for dependency", + "key_priorities": "Board slides due, security advisory, holiday oncall coverage", + }, + { + "date": "Tuesday, December 9, 2025", + "calendar": "- 8:00 AM: Breakfast meeting with advisor\n- 10:00 AM: Pair programming session\n- 2:00 PM: Customer feedback review\n- 4:00 PM: Design review — mobile app", + "todos": "- Finish board presentation slides\n- Review customer feedback summary\n- Benchmark new quantization approach\n- Update documentation for new API endpoints", + "news_topics": "mobile AI, quantization techniques, model compression", + "messages": "- Email from customer: 'Love the product, but need better docs for the SDK'\n- Slack from design: 'Mobile mockups ready for review'\n- PR: community contribution for AMD GPU support", + "key_priorities": "Board slides must be finished, customer docs feedback, mobile review", + }, + { + "date": "Wednesday, December 10, 2025", + "calendar": "- 9:00 AM: Team standup\n- 11:00 AM: Cross-team sync (ML + Platform)\n- 1:30 PM: Vendor call — monitoring tools\n- 3:00 PM: Office hours (open door)", + "todos": "- Deploy rate limiter to staging\n- Test SSL certificate auto-renewal\n- Write incident postmortem from last week\n- Prepare for GDPR audit (January)", + "news_topics": "observability, monitoring, SRE practices", + "messages": "- Email from ops: 'SSL cert renewed successfully'\n- Slack from ML team: 'New model achieves 96% accuracy on benchmark'\n- Calendar reminder: GDPR audit docs due in 2 weeks", + "key_priorities": "Rate limiter deployment, incident postmortem overdue, GDPR prep", + }, + { + "date": "Thursday, December 11, 2025", + "calendar": "- 9:00 AM: Board meeting (all day)\n- 12:00 PM: Working lunch during board meeting\n- 5:00 PM: Team happy hour", + "todos": "- Present AI cost savings slides at board meeting\n- Answer board questions on AI strategy\n- Send follow-up notes after board meeting\n- Approve holiday PTO requests", + "news_topics": "corporate AI strategy, ROI of AI investments", + "messages": "- Email from CEO: 'Board is excited about the cost savings numbers'\n- Slack from team: '3 PTO requests pending your approval'\n- Email from recruiter: candidate accepted the offer!", + "key_priorities": "Board presentation today, PTO approvals needed, new hire accepted", + }, + { + "date": "Friday, December 12, 2025", + "calendar": "- 9:00 AM: Team standup\n- 10:00 AM: Post-board debrief with CEO\n- 11:30 AM: Release review\n- 2:00 PM: End of year planning", + "todos": "- Send board meeting follow-up notes\n- Plan Q1 OKRs\n- Review and approve Q4 bonuses\n- Archive completed project boards\n- Buy holiday gifts for team", + "news_topics": "year-end reviews, tech trends 2026, planning", + "messages": "- Email from CEO: 'Board approved the 2026 AI budget, well done!'\n- Slack from HR: 'Q4 bonus recommendations due Monday'\n- Team Slack: 'Can we do Secret Santa?'", + "key_priorities": "Board follow-up notes, Q1 OKR planning, bonus recommendations due Monday", + }, + { + "date": "Monday, December 15, 2025", + "calendar": "- 9:00 AM: Team standup\n- 10:00 AM: Q1 planning kickoff\n- 2:00 PM: Onboarding new hire — Sarah Chen\n- 4:00 PM: Security training deadline", + "todos": "- Complete security training (deadline today!)\n- Prepare onboarding materials for Sarah\n- Submit Q4 bonus recommendations\n- Draft Q1 OKR proposals\n- Review year-end performance self-assessments", + "news_topics": "hiring trends, employee onboarding best practices", + "messages": "- Email from HR: 'Security training deadline is today'\n- Slack from Sarah: 'Excited to start! Any prep I should do?'\n- Email from finance: 'Bonus recommendations received, thank you'", + "key_priorities": "Security training deadline TODAY, new hire onboarding, Q1 OKR drafts", + }, + { + "date": "Tuesday, December 16, 2025", + "calendar": "- 9:00 AM: Team standup\n- 10:00 AM: Sarah's first day — team intro\n- 1:00 PM: Architecture deep dive with Sarah\n- 3:00 PM: Vendor evaluation — new CI tool", + "todos": "- Verify Sarah's access to all repos and tools\n- Review CI vendor proposals\n- Update team wiki with 2025 retrospective\n- Close out stale GitHub issues", + "news_topics": "CI/CD trends, developer experience, DevOps", + "messages": "- Slack from Sarah: 'All access is working, thank you!'\n- Email from CI vendor: 'Demo scheduled for 3pm today'\n- GitHub: 5 issues marked as stale, need triage", + "key_priorities": "Sarah's onboarding going well, CI vendor evaluation, stale issue triage", + }, + { + "date": "Wednesday, December 17, 2025", + "calendar": "- 9:00 AM: Team standup\n- 10:30 AM: Performance review calibration\n- 2:00 PM: External talk prep — local AI meetup\n- 4:00 PM: 1:1 with Sarah (first week check-in)", + "todos": "- Prepare performance review comments\n- Polish meetup talk slides\n- Review Sarah's first PR\n- Plan team holiday event", + "news_topics": "AI community events, developer conferences, tech talks", + "messages": "- Email from meetup organizer: 'Your talk is confirmed for Jan 8'\n- Slack from Sarah: 'Submitted my first PR!'\n- Email from HR: 'Performance reviews due Dec 22'", + "key_priorities": "Performance reviews due in 5 days, Sarah's first PR, meetup talk prep", + }, + { + "date": "Thursday, December 18, 2025", + "calendar": "- 9:00 AM: Team standup\n- 11:00 AM: Holiday party planning committee\n- 2:00 PM: Final architecture review for v3.0\n- 4:30 PM: Holiday card signing", + "todos": "- Write performance reviews (3 direct reports)\n- Finalize v3.0 architecture decisions\n- Order catering for holiday party\n- Backup critical data before holiday freeze", + "news_topics": "work-life balance, remote work, team culture", + "messages": "- Slack from team: 'Can we do a white elephant gift exchange?'\n- Email from IT: 'Code freeze starts Dec 23'\n- GitHub: v3.0 milestone at 85% completion", + "key_priorities": "Performance reviews writing, v3.0 architecture decisions, code freeze approaching", + }, + { + "date": "Friday, December 19, 2025", + "calendar": "- 9:00 AM: Team standup (last of the year)\n- 10:00 AM: Year-end retrospective\n- 12:00 PM: Holiday lunch\n- 2:00 PM: Wrap-up and handoffs", + "todos": "- Submit performance reviews (due Mon)\n- Document handoff for holiday oncall\n- Send team thank-you notes\n- Set out-of-office auto-reply\n- Final check on monitoring alerts", + "news_topics": "year in review, tech predictions 2026", + "messages": "- Email from CEO: 'Thank you for an amazing year!'\n- Slack from oncall volunteer: 'I have the runbook, all set'\n- Team Slack: 'Happy holidays everyone!'", + "key_priorities": "Performance reviews due Monday, holiday handoffs, monitoring verification", + }, +] + + +class MorningBriefDataset(DatasetProvider): + """Morning brief benchmark: generate prioritized daily briefing from context.""" + + dataset_id = "morning_brief" + dataset_name = "Morning Brief" + + def __init__(self) -> None: + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + rows = list(_CONTEXTS) + + if seed is not None: + rng = random.Random(seed) + rng.shuffle(rows) + + if max_samples is not None: + rows = rows[:max_samples] + + self._records = [] + for idx, ctx in enumerate(rows): + prompt = _PROMPT_TEMPLATE.format( + date=ctx["date"], + calendar=ctx["calendar"], + todos=ctx["todos"], + news_topics=ctx["news_topics"], + messages=ctx["messages"], + ) + self._records.append(EvalRecord( + record_id=f"morning-brief-{idx}", + problem=prompt, + reference=ctx["key_priorities"], + category="use-case", + subject="morning_brief", + metadata={"date": ctx["date"]}, + )) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + +__all__ = ["MorningBriefDataset"] diff --git a/src/openjarvis/evals/datasets/research_mining.py b/src/openjarvis/evals/datasets/research_mining.py new file mode 100644 index 00000000..499ddd8c --- /dev/null +++ b/src/openjarvis/evals/datasets/research_mining.py @@ -0,0 +1,264 @@ +"""Research mining benchmark dataset. + +Research questions spanning multiple domains for evaluating synthesis, +source quality, and accuracy of AI research assistants. +""" + +from __future__ import annotations + +import random +from typing import Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +_PROMPT_TEMPLATE = """You are a research assistant. Please research the following question and provide a comprehensive answer. + +Your response should include: +1. Key findings (3-5 main points) +2. Supporting evidence or data +3. A synthesis paragraph connecting the findings + +Research question: {question} +Domain: {domain} +Scope: {scope}""" + +_QUESTIONS = [ + { + "question": "What are the main approaches to running large language models on consumer hardware, and what trade-offs does each approach involve?", + "domain": "AI/ML", + "scope": "Technical survey", + "key_facts": "quantization (GPTQ, AWQ, GGUF), distillation, pruning, speculative decoding, offloading; trade-offs include accuracy loss vs speed gain vs memory reduction", + }, + { + "question": "How does the energy consumption of cloud-based AI inference compare to on-device inference for typical consumer workloads?", + "domain": "Energy/Sustainability", + "scope": "Comparative analysis", + "key_facts": "cloud inference involves network transfer, datacenter cooling, shared GPU utilization; on-device avoids network overhead but may use less efficient hardware; total energy depends on workload volume and hardware", + }, + { + "question": "What are the privacy implications of sending personal data to cloud AI services versus processing it locally?", + "domain": "Privacy/Security", + "scope": "Policy analysis", + "key_facts": "data exposure in transit and at rest, third-party access, GDPR/CCPA compliance, data retention policies, local processing eliminates transmission risk", + }, + { + "question": "What mechanisms exist for retrieval-augmented generation (RAG) and how do they compare in accuracy and latency?", + "domain": "AI/ML", + "scope": "Technical comparison", + "key_facts": "dense retrieval (DPR, ColBERT), sparse retrieval (BM25), hybrid approaches (RRF fusion), chunk size effects, re-ranking strategies, context window limits", + }, + { + "question": "What is the current state of hardware acceleration for AI inference on consumer devices?", + "domain": "Hardware", + "scope": "Market survey", + "key_facts": "NVIDIA consumer GPUs (RTX series), Apple Silicon (M-series Neural Engine), AMD GPUs (ROCm), Intel Arc, Qualcomm NPUs, VRAM as primary bottleneck", + }, + { + "question": "How do different model quantization methods affect inference quality for code generation tasks?", + "domain": "AI/ML", + "scope": "Empirical analysis", + "key_facts": "INT8, INT4, FP8, GPTQ vs AWQ vs GGUF; code generation sensitive to quantization; larger models tolerate more aggressive quantization; benchmark results vary by task complexity", + }, + { + "question": "What are the economic factors driving the shift from cloud AI services to on-premises or local AI deployments?", + "domain": "Economics", + "scope": "Market analysis", + "key_facts": "API cost escalation, GPU cost amortization, regulatory compliance costs, data sovereignty requirements, latency-sensitive applications, predictable vs variable costs", + }, + { + "question": "How effective are agent-based AI systems at automating software engineering tasks?", + "domain": "Software Engineering", + "scope": "Research survey", + "key_facts": "SWE-bench results, agentic loop patterns (ReAct, CodeAct), tool use, multi-step reasoning, error recovery, cost per task solved, current accuracy limitations", + }, + { + "question": "What role does knowledge graph technology play in improving AI system accuracy and explainability?", + "domain": "AI/Knowledge Management", + "scope": "Technical survey", + "key_facts": "entity-relation storage, graph-augmented retrieval, fact verification, reasoning chains, hybrid approaches with vector databases, explainable AI connections", + }, + { + "question": "What are the best practices for evaluating LLM performance across diverse task categories?", + "domain": "AI Evaluation", + "scope": "Methodological survey", + "key_facts": "benchmark suites (MMLU, GPQA, HLE), LLM-as-judge, human evaluation, contamination concerns, multi-dimensional scoring, statistical significance testing", + }, + { + "question": "How can scheduled AI agents reduce operational costs for routine business tasks?", + "domain": "Business Automation", + "scope": "Practical analysis", + "key_facts": "cron-based scheduling, email triage automation, report generation, monitoring/alerting, cost comparison vs human labor vs cloud API calls", + }, + { + "question": "What is the environmental impact of training and deploying large language models?", + "domain": "Sustainability", + "scope": "Impact assessment", + "key_facts": "training carbon footprint, inference energy per query, datacenter water usage, hardware lifecycle, renewable energy offsets, efficiency improvements over time", + }, + { + "question": "How do different memory architectures in AI assistants affect long-term user experience?", + "domain": "Human-Computer Interaction", + "scope": "Design analysis", + "key_facts": "session-based vs persistent memory, vector similarity search, knowledge graph memory, decay and consolidation, privacy controls, user trust", + }, + { + "question": "What security vulnerabilities are unique to AI-powered applications?", + "domain": "Cybersecurity", + "scope": "Threat analysis", + "key_facts": "prompt injection, jailbreaking, data exfiltration via LLM, model inversion, supply chain attacks on model weights, PII leakage, SSRF via tool calls", + }, + { + "question": "How do open-source and proprietary AI models compare for enterprise deployment?", + "domain": "Enterprise AI", + "scope": "Comparative analysis", + "key_facts": "licensing terms, customization flexibility, support and SLA, total cost of ownership, data privacy, model quality trends, community ecosystem", + }, + { + "question": "What approaches exist for making AI systems more energy-efficient during inference?", + "domain": "Green AI", + "scope": "Technical survey", + "key_facts": "model pruning, quantization, speculative decoding, dynamic batching, hardware-software co-design, early exit strategies, mixture of experts", + }, + { + "question": "How can AI assistants effectively manage multi-channel communication across platforms like Slack, email, and messaging apps?", + "domain": "Communication Systems", + "scope": "Architecture analysis", + "key_facts": "channel abstraction, unified session identity, message routing, webhook vs polling, rate limiting, cross-channel context, user preference management", + }, + { + "question": "What are the current limitations and future directions of on-device speech recognition?", + "domain": "Speech Processing", + "scope": "Technology assessment", + "key_facts": "Whisper variants, CTranslate2 optimization, streaming vs batch, vocabulary and language coverage, noise robustness, privacy benefits, latency requirements", + }, + { + "question": "How do workflow engines and DAG-based execution improve AI agent reliability?", + "domain": "AI Systems", + "scope": "Architecture survey", + "key_facts": "directed acyclic graphs, topological sort, parallel execution, error recovery, conditional branching, checkpointing, loop detection, human-in-the-loop gates", + }, + { + "question": "What is the role of telemetry and observability in production AI systems?", + "domain": "MLOps", + "scope": "Best practices survey", + "key_facts": "token usage tracking, latency monitoring, cost attribution, energy measurement, model drift detection, alert thresholds, privacy-preserving telemetry", + }, + { + "question": "How can reinforcement learning improve AI agent decision-making in real-time applications?", + "domain": "AI/ML", + "scope": "Research survey", + "key_facts": "GRPO, bandit algorithms (Thompson Sampling, UCB1), reward shaping, sample efficiency, exploration vs exploitation, online vs offline RL, safety constraints", + }, + { + "question": "What are the emerging standards and protocols for AI agent interoperability?", + "domain": "AI Standards", + "scope": "Standards review", + "key_facts": "MCP (Model Context Protocol), A2A (Agent-to-Agent), OpenAI function calling, tool use formats, JSON-RPC, agent card discovery, capability negotiation", + }, + { + "question": "How does the Model Context Protocol (MCP) enable extensible tool integration in AI systems?", + "domain": "AI Infrastructure", + "scope": "Technical analysis", + "key_facts": "JSON-RPC 2.0, tools/list and tools/call, server discovery, template-based tool generation, security considerations, adapter patterns, community ecosystem", + }, + { + "question": "What strategies exist for preventing and detecting AI hallucinations in production systems?", + "domain": "AI Safety", + "scope": "Technical survey", + "key_facts": "retrieval augmentation, chain-of-thought verification, self-consistency checking, confidence calibration, knowledge grounding, citation generation, human review loops", + }, + { + "question": "How do edge computing and on-device AI complement each other for IoT applications?", + "domain": "Edge Computing", + "scope": "Architecture analysis", + "key_facts": "latency requirements, bandwidth constraints, intermittent connectivity, model size limits, federated learning, privacy preservation, hardware heterogeneity", + }, + { + "question": "What approaches exist for cost-effective fine-tuning of language models on domain-specific data?", + "domain": "AI/ML", + "scope": "Practical guide", + "key_facts": "LoRA, QLoRA, prefix tuning, adapter methods, data quality over quantity, synthetic data generation, evaluation methodology, compute budgeting", + }, + { + "question": "How can AI systems maintain consistency and accuracy across multi-turn conversations?", + "domain": "Conversational AI", + "scope": "Technical analysis", + "key_facts": "context window management, session persistence, fact tracking, contradiction detection, memory consolidation, user modeling, conversation summarization", + }, + { + "question": "What are the trade-offs between different vector database implementations for AI memory systems?", + "domain": "Data Infrastructure", + "scope": "Comparative analysis", + "key_facts": "FAISS vs Chroma vs Weaviate vs Qdrant, indexing speed, query latency, memory footprint, scalability, filtering, hybrid search, SQLite/FTS5 as lightweight alternative", + }, + { + "question": "How do capability-based access control and taint tracking improve AI system security?", + "domain": "AI Security", + "scope": "Architecture analysis", + "key_facts": "RBAC vs capability model, principle of least privilege, taint propagation, sink policies, audit logging, Merkle chains for tamper evidence, tool-level enforcement", + }, + { + "question": "What metrics best capture the efficiency of AI inference in terms of intelligence per resource consumed?", + "domain": "AI Evaluation", + "scope": "Metrics design", + "key_facts": "Intelligence Per Watt (IPW), Intelligence Per Joule (IPJ), tokens per second per watt, accuracy per dollar, MFU/MBU, energy per output token, steady-state detection", + }, + { + "question": "How can desktop AI applications provide a seamless user experience while managing local compute resources?", + "domain": "Desktop Applications", + "scope": "UX/Architecture analysis", + "key_facts": "Tauri/Electron frameworks, background service management, resource monitoring, model switching, system tray integration, auto-start, update mechanisms", + }, +] + + +class ResearchMiningDataset(DatasetProvider): + """Research mining benchmark: evaluate research synthesis and accuracy.""" + + dataset_id = "research_mining" + dataset_name = "Research Mining" + + def __init__(self) -> None: + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + rows = list(_QUESTIONS) + + if seed is not None: + rng = random.Random(seed) + rng.shuffle(rows) + + if max_samples is not None: + rows = rows[:max_samples] + + self._records = [] + for idx, q in enumerate(rows): + prompt = _PROMPT_TEMPLATE.format( + question=q["question"], + domain=q["domain"], + scope=q["scope"], + ) + self._records.append(EvalRecord( + record_id=f"research-mining-{idx}", + problem=prompt, + reference=q["key_facts"], + category="use-case", + subject="research_mining", + metadata={"domain": q["domain"], "scope": q["scope"]}, + )) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + +__all__ = ["ResearchMiningDataset"] diff --git a/src/openjarvis/evals/datasets/terminalbench_native.py b/src/openjarvis/evals/datasets/terminalbench_native.py index 97fc709e..eca582b7 100644 --- a/src/openjarvis/evals/datasets/terminalbench_native.py +++ b/src/openjarvis/evals/datasets/terminalbench_native.py @@ -112,6 +112,8 @@ class TerminalBenchNativeDataset(DatasetProvider): "tags": getattr(task, "tags", None), "difficulty": getattr(task, "difficulty", None), "timeout": getattr(task, "timeout", None), + "task": task, + "task_paths": task_paths, } return EvalRecord( @@ -124,4 +126,25 @@ class TerminalBenchNativeDataset(DatasetProvider): ) + def create_task_env(self, record): + """Return a TerminalBenchTaskEnv for the given record.""" + try: + from openjarvis.evals.execution.terminalbench_env import ( + TerminalBenchTaskEnv, + ) + return TerminalBenchTaskEnv(record.metadata) + except ImportError: + return None + + def verify_requirements(self): + """Check that terminal-bench and docker are available.""" + issues = [] + if not _HAS_TERMINALBENCH: + issues.append("terminal-bench package not installed") + import shutil + if not shutil.which("docker"): + issues.append("docker not found in PATH") + return issues + + __all__ = ["TerminalBenchNativeDataset"] diff --git a/src/openjarvis/evals/execution/__init__.py b/src/openjarvis/evals/execution/__init__.py new file mode 100644 index 00000000..1ae9fafa --- /dev/null +++ b/src/openjarvis/evals/execution/__init__.py @@ -0,0 +1 @@ +"""Execution environments for agentic eval workloads.""" diff --git a/src/openjarvis/evals/execution/terminalbench_env.py b/src/openjarvis/evals/execution/terminalbench_env.py new file mode 100644 index 00000000..07f9dd98 --- /dev/null +++ b/src/openjarvis/evals/execution/terminalbench_env.py @@ -0,0 +1,187 @@ +"""TerminalBench task environment — per-task Docker lifecycle + test execution.""" + +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path +from types import TracebackType +from typing import Any, MutableMapping, Optional, Type + +LOGGER = logging.getLogger(__name__) + + +class TerminalBenchTaskEnv: + """Per-task Docker environment for TerminalBench. + + Context manager that spins up a Docker container, creates a tmux session, + and runs test scripts after the agent finishes. + """ + + def __init__(self, metadata: MutableMapping[str, Any]) -> None: + self._metadata = metadata + self._terminal: Any = None + self._terminal_cm: Any = None + self._logs_tmpdir: Any = None + + # ------------------------------------------------------------------ + # Context manager + # ------------------------------------------------------------------ + + def __enter__(self) -> TerminalBenchTaskEnv: + from terminal_bench.terminal.terminal import spin_up_terminal + + task = self._metadata.get("task") + task_paths = self._metadata.get("task_paths") + task_id = self._metadata.get("task_id", "unknown") + + if task is None or task_paths is None: + raise ValueError( + "Task metadata missing 'task' or 'task_paths'. " + "Use the 'terminalbench-native' dataset." + ) + + docker_image_prefix = f"tb__{task_id}".replace(".", "-") + client_image_name = f"{docker_image_prefix}__client" + client_container_name = f"oj-{task_id}".replace(".", "-") + + self._logs_tmpdir = tempfile.TemporaryDirectory(prefix="oj_tb_logs_") + logs_path = Path(self._logs_tmpdir.name) + + self._terminal_cm = spin_up_terminal( + client_container_name=client_container_name, + client_image_name=client_image_name, + docker_compose_path=task_paths.docker_compose_path, + docker_image_name_prefix=docker_image_prefix, + sessions_logs_path=logs_path, + disable_recording=task.disable_asciinema, + ) + self._terminal = self._terminal_cm.__enter__() + + session = self._terminal.create_session( + "agent", is_active_stream=False, as_configured_user=True + ) + + self._metadata["terminal"] = self._terminal + self._metadata["session"] = session + self._metadata["container"] = client_container_name + + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self._metadata.pop("terminal", None) + self._metadata.pop("session", None) + self._metadata.pop("container", None) + + if self._terminal_cm is not None: + self._terminal_cm.__exit__(exc_type, exc_val, exc_tb) + self._terminal_cm = None + self._terminal = None + + if self._logs_tmpdir is not None: + self._logs_tmpdir.cleanup() + self._logs_tmpdir = None + + # ------------------------------------------------------------------ + # Test execution + # ------------------------------------------------------------------ + + def run_tests(self) -> tuple[bool, dict[str, Any]]: + """Copy test scripts into container, execute, parse results.""" + from terminal_bench.parsers.base_parser import UnitTestStatus + from terminal_bench.parsers.parser_factory import ParserFactory + from terminal_bench.terminal.docker_compose_manager import ( + DockerComposeManager, + ) + + task = self._metadata["task"] + task_paths = self._metadata["task_paths"] + terminal = self._terminal + results: dict[str, Any] = {} + + if terminal is None: + results["error"] = "terminal_not_running" + self._metadata["is_resolved"] = False + self._metadata["test_results"] = results + return False, results + + try: + paths_to_copy = [task_paths.run_tests_path] + if task_paths.test_dir.exists(): + paths_to_copy.append(task_paths.test_dir) + + terminal.copy_to_container( + paths=paths_to_copy, + container_dir=str(DockerComposeManager.CONTAINER_TEST_DIR), + ) + + if not task.run_tests_in_same_shell: + test_session = terminal.create_session( + "tests", is_active_stream=False, as_configured_user=False + ) + else: + test_session = terminal.create_session( + "agent-tests", + is_active_stream=False, + as_configured_user=True, + ) + + test_timeout = task.max_test_timeout_sec + test_script_path = ( + DockerComposeManager.CONTAINER_TEST_DIR + / task_paths.run_tests_path.name + ) + + try: + test_session.send_keys( + ["bash ", str(test_script_path), "Enter"], + block=True, + max_timeout_sec=test_timeout, + ) + except TimeoutError: + LOGGER.warning( + "Test command timed out after %.0fs", test_timeout + ) + results["error"] = "test_timeout" + self._metadata["is_resolved"] = False + self._metadata["test_results"] = results + return False, results + + post_test_pane = test_session.capture_pane(capture_entire=True) + results["test_output"] = post_test_pane[:10000] + + parser = ParserFactory.get_parser(task.parser_name) + try: + parser_results = parser.parse(post_test_pane) + results["parser_results"] = { + name: status.value + for name, status in parser_results.items() + } + is_resolved = all( + status == UnitTestStatus.PASSED + for status in parser_results.values() + ) + except Exception as exc: + LOGGER.warning("Parser failed: %s", exc) + results["parse_error"] = str(exc) + is_resolved = False + + results["is_resolved"] = is_resolved + self._metadata["is_resolved"] = is_resolved + self._metadata["test_results"] = results + return is_resolved, results + + except Exception as exc: + LOGGER.exception("Test execution failed") + results["error"] = str(exc) + self._metadata["is_resolved"] = False + self._metadata["test_results"] = results + return False, results + + +__all__ = ["TerminalBenchTaskEnv"] diff --git a/src/openjarvis/evals/scorers/coding_task.py b/src/openjarvis/evals/scorers/coding_task.py new file mode 100644 index 00000000..7329096d --- /dev/null +++ b/src/openjarvis/evals/scorers/coding_task.py @@ -0,0 +1,142 @@ +"""Coding task scorer — test pass rate + structural validation. + +Extracts the function/class from model output and runs test cases +to determine correctness. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import Scorer +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + + +def _extract_code(answer: str) -> str: + """Extract Python code from model answer, handling markdown fences.""" + # Try markdown code fence first + fence_match = re.search( + r"```(?:python)?\s*\n(.*?)```", + answer, + re.DOTALL, + ) + if fence_match: + return fence_match.group(1).strip() + + # Look for function/class definitions + lines = answer.strip().split("\n") + code_lines = [] + in_code = False + for line in lines: + stripped = line.lstrip() + if stripped.startswith(("def ", "class ", "from ", "import ")): + in_code = True + if in_code: + code_lines.append(line) + + if code_lines: + return "\n".join(code_lines) + + # Last resort: return the whole answer + return answer.strip() + + +def _run_tests(code: str, test_cases: str) -> Tuple[int, int, str]: + """Execute code + tests in a restricted namespace. Returns (passed, total, error).""" + namespace: Dict[str, Any] = {} + + try: + exec(code, namespace) # noqa: S102 + except Exception as exc: + return 0, 0, f"Code execution error: {exc}" + + # Parse individual test assertions + test_lines = [ + line.strip() + for line in test_cases.strip().split("\n") + if line.strip() + ] + + passed = 0 + total = 0 + + # Some tests span multiple lines (setup + assert), so we run + # the whole block together but count assertions + try: + exec(test_cases, namespace) # noqa: S102 + # Count assertions in the test code + total = sum(1 for line in test_lines if "assert " in line) + passed = total + return passed, total, "" + except AssertionError as exc: + # Count how many assertions were in the code + total = sum(1 for line in test_lines if "assert " in line) + # Run line by line to count individual passes + passed = 0 + for line in test_lines: + if "assert " not in line: + try: + exec(line, namespace) # noqa: S102 + except Exception: + pass + continue + try: + exec(line, namespace) # noqa: S102 + passed += 1 + except (AssertionError, Exception): + pass + return passed, total, str(exc) + except Exception as exc: + total = sum(1 for line in test_lines if "assert " in line) + return 0, max(total, 1), f"Test execution error: {exc}" + + +class CodingTaskScorer(Scorer): + """Score coding tasks by running test cases against model output.""" + + scorer_id = "coding_task" + + def __init__(self, judge_backend=None, judge_model: str = "") -> None: + # Accept same constructor args as LLMJudgeScorer for compatibility + # but don't need the judge for this scorer + pass + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + test_cases = record.metadata.get("test_cases", "") + if not test_cases: + return None, {"reason": "no_test_cases"} + + code = _extract_code(model_answer) + if not code: + return False, {"reason": "no_code_extracted"} + + passed, total, error = _run_tests(code, test_cases) + + if total == 0: + return None, {"reason": "no_assertions_found"} + + pass_rate = passed / total + is_correct = pass_rate == 1.0 + + meta: Dict[str, Any] = { + "match_type": "test_execution", + "tests_passed": passed, + "tests_total": total, + "pass_rate": pass_rate, + } + if error: + meta["error"] = error + + return is_correct, meta + + +__all__ = ["CodingTaskScorer"] diff --git a/src/openjarvis/evals/scorers/email_triage.py b/src/openjarvis/evals/scorers/email_triage.py new file mode 100644 index 00000000..0023d74a --- /dev/null +++ b/src/openjarvis/evals/scorers/email_triage.py @@ -0,0 +1,134 @@ +"""Email triage scorer — classification accuracy + draft quality. + +Scores urgency and category via exact match, with LLM fallback +for draft reply quality assessment. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + +_DRAFT_JUDGE_PROMPT = """You are evaluating an AI-generated email triage response. + +The AI was asked to classify an email and draft a reply. + +Reference classification: +{reference} + +AI response: +{response} + +Evaluate: +1. Did the AI correctly identify the urgency level? (critical/high/medium/low) +2. Did the AI correctly identify the category? (action/decision/info/social) +3. Is the draft reply appropriate, professional, and helpful? + +Your response MUST use exactly this format: +urgency_correct: +category_correct: +draft_quality: +reasoning: +overall_correct: """ + + +def _extract_field(text: str, field: str) -> str: + """Extract a field value from structured model output.""" + match = re.search( + rf"^{field}\s*:\s*(.+)", + text, + re.MULTILINE | re.IGNORECASE, + ) + return match.group(1).strip().lower() if match else "" + + +class EmailTriageScorer(LLMJudgeScorer): + """Score email triage: classification accuracy + draft quality.""" + + scorer_id = "email_triage" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + ref_urgency = record.metadata.get("urgency", "") + ref_category = record.metadata.get("category", "") + + # Try exact field extraction first + pred_urgency = _extract_field(model_answer, "urgency") + pred_category = _extract_field(model_answer, "category") + + urgency_match = pred_urgency == ref_urgency + category_match = pred_category == ref_category + + # If both match exactly, score as correct without LLM call + if urgency_match and category_match: + return True, { + "match_type": "exact", + "urgency_correct": True, + "category_correct": True, + "pred_urgency": pred_urgency, + "pred_category": pred_category, + } + + # Fall back to LLM judge for ambiguous cases + try: + prompt = _DRAFT_JUDGE_PROMPT.format( + reference=record.reference, + response=model_answer, + ) + raw = self._ask_judge(prompt, temperature=0.0, max_tokens=512) + + overall_match = re.search( + r"^overall_correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + is_correct = ( + overall_match.group(1).lower() == "yes" + if overall_match + else (urgency_match and category_match) + ) + + urg_match = re.search( + r"^urgency_correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + cat_match = re.search( + r"^category_correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + + return is_correct, { + "match_type": "llm_judge", + "urgency_correct": ( + urg_match.group(1).lower() == "yes" if urg_match else urgency_match + ), + "category_correct": ( + cat_match.group(1).lower() == "yes" if cat_match else category_match + ), + "pred_urgency": pred_urgency, + "pred_category": pred_category, + "raw_judge_output": raw, + } + except Exception as exc: + LOGGER.error("Email triage LLM judge failed: %s", exc) + return urgency_match and category_match, { + "match_type": "exact_fallback", + "urgency_correct": urgency_match, + "category_correct": category_match, + "error": str(exc), + } + + +__all__ = ["EmailTriageScorer"] diff --git a/src/openjarvis/evals/scorers/knowledge_base.py b/src/openjarvis/evals/scorers/knowledge_base.py new file mode 100644 index 00000000..10e0a3e6 --- /dev/null +++ b/src/openjarvis/evals/scorers/knowledge_base.py @@ -0,0 +1,120 @@ +"""Knowledge base scorer — answer correctness via normalized match + LLM fallback. + +Evaluates document-grounded QA by checking if the model answer +matches the reference answer using exact match with normalization, +falling back to LLM judge for semantic comparison. +""" + +from __future__ import annotations + +import logging +import re +import string +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + + +def _normalize(text: str) -> str: + """Lowercase, remove punctuation, collapse whitespace.""" + text = text.lower() + text = text.translate(str.maketrans("", "", string.punctuation)) + return " ".join(text.split()) + + +def _contains_key_phrases(answer: str, reference: str) -> bool: + """Check if answer contains the key phrases from the reference.""" + ref_norm = _normalize(reference) + ans_norm = _normalize(answer) + + # Split reference into key phrases (sentences or comma-separated items) + phrases = [p.strip() for p in re.split(r"[.,;]", ref_norm) if p.strip()] + if not phrases: + return False + + matched = sum(1 for p in phrases if p in ans_norm) + return matched / len(phrases) >= 0.5 + + +_LLM_JUDGE_PROMPT = """You are evaluating a knowledge base QA response. + +The AI was given document excerpts and asked a question. It should answer based solely on the provided documents. + +Reference answer: {reference} +AI answer: {answer} + +Evaluate whether the AI's answer is correct and consistent with the reference. Minor wording differences are acceptable as long as the meaning is preserved. + +Your response MUST use exactly this format: +factually_correct: +completeness: +reasoning: +overall_correct: """ + + +class KnowledgeBaseScorer(LLMJudgeScorer): + """Score knowledge base QA: answer correctness.""" + + scorer_id = "knowledge_base" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + reference = record.reference + if not reference or not reference.strip(): + return None, {"reason": "no_ground_truth"} + + # Try phrase containment check first + if _contains_key_phrases(model_answer, reference): + return True, {"match_type": "phrase_match"} + + # Fall back to LLM judge + try: + prompt = _LLM_JUDGE_PROMPT.format( + reference=reference, + answer=model_answer, + ) + raw = self._ask_judge(prompt, temperature=0.0, max_tokens=512) + + overall_match = re.search( + r"^overall_correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + is_correct = ( + overall_match.group(1).lower() == "yes" + if overall_match + else False + ) + + completeness_match = re.search( + r"^completeness:\s*(complete|partial|missing)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + completeness = ( + completeness_match.group(1).lower() + if completeness_match + else "unknown" + ) + + return is_correct, { + "match_type": "llm_judge", + "completeness": completeness, + "raw_judge_output": raw, + } + except Exception as exc: + LOGGER.error("Knowledge base LLM judge failed: %s", exc) + return False, { + "match_type": "llm_judge_error", + "error": str(exc), + } + + +__all__ = ["KnowledgeBaseScorer"] diff --git a/src/openjarvis/evals/scorers/morning_brief.py b/src/openjarvis/evals/scorers/morning_brief.py new file mode 100644 index 00000000..b5bf8a38 --- /dev/null +++ b/src/openjarvis/evals/scorers/morning_brief.py @@ -0,0 +1,117 @@ +"""Morning brief scorer — LLM judge for briefing quality. + +Evaluates completeness, prioritization, conciseness, and actionability +of generated morning briefings. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + +_JUDGE_PROMPT = """You are evaluating an AI-generated morning briefing. + +The AI was given calendar events, todos, news topics, and pending messages, and asked to produce a prioritized morning brief. + +Key priorities that should be highlighted: +{key_priorities} + +Original user context (summary): +{problem_excerpt} + +AI-generated briefing: +{response} + +Evaluate the briefing on these criteria: +1. Completeness: Does it cover calendar, todos, messages, and news? +2. Prioritization: Are the most urgent/important items highlighted first? +3. Conciseness: Is it well-organized and scannable, not too verbose? +4. Actionability: Does it clearly indicate what needs attention today? +5. Key priorities: Does it mention the key priorities listed above? + +Your response MUST use exactly this format: +completeness: <1-5> +prioritization: <1-5> +conciseness: <1-5> +actionability: <1-5> +key_priorities_covered: +reasoning: +overall_correct: """ + + +class MorningBriefScorer(LLMJudgeScorer): + """Score morning briefings on quality dimensions.""" + + scorer_id = "morning_brief" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + # Truncate problem for the judge prompt to save tokens + problem_excerpt = record.problem[:500] + "..." if len(record.problem) > 500 else record.problem + + try: + prompt = _JUDGE_PROMPT.format( + key_priorities=record.reference, + problem_excerpt=problem_excerpt, + response=model_answer, + ) + raw = self._ask_judge(prompt, temperature=0.0, max_tokens=512) + + # Extract scores + scores = {} + for dim in ("completeness", "prioritization", "conciseness", "actionability"): + match = re.search( + rf"^{dim}:\s*(\d)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + scores[dim] = int(match.group(1)) if match else 3 + + priorities_match = re.search( + r"^key_priorities_covered:\s*(yes|partial|no)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + priorities_covered = ( + priorities_match.group(1).lower() if priorities_match else "partial" + ) + + overall_match = re.search( + r"^overall_correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + + # Consider correct if average score >= 3.5 and priorities at least partial + avg_score = sum(scores.values()) / len(scores) + if overall_match: + is_correct = overall_match.group(1).lower() == "yes" + else: + is_correct = avg_score >= 3.5 and priorities_covered != "no" + + return is_correct, { + "match_type": "llm_judge", + "scores": scores, + "avg_score": avg_score, + "key_priorities_covered": priorities_covered, + "raw_judge_output": raw, + } + except Exception as exc: + LOGGER.error("Morning brief LLM judge failed: %s", exc) + return None, { + "match_type": "llm_judge_error", + "error": str(exc), + } + + +__all__ = ["MorningBriefScorer"] diff --git a/src/openjarvis/evals/scorers/research_mining.py b/src/openjarvis/evals/scorers/research_mining.py new file mode 100644 index 00000000..2682df38 --- /dev/null +++ b/src/openjarvis/evals/scorers/research_mining.py @@ -0,0 +1,105 @@ +"""Research mining scorer — LLM judge for research synthesis quality. + +Evaluates accuracy, depth, source quality, and synthesis of +AI-generated research responses. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + +_JUDGE_PROMPT = """You are evaluating an AI-generated research response. + +The AI was asked to research a topic and provide key findings, supporting evidence, and a synthesis. + +Expected key facts/topics that should be covered: +{key_facts} + +Research question: +{question} + +AI response: +{response} + +Evaluate the response on these criteria: +1. Accuracy: Are the stated facts correct and not hallucinated? +2. Coverage: Does it address the key facts/topics listed above? +3. Depth: Does it go beyond surface-level observations? +4. Structure: Is it well-organized with clear findings and synthesis? +5. Relevance: Does it stay focused on the research question? + +Your response MUST use exactly this format: +accuracy: <1-5> +coverage: <1-5> +depth: <1-5> +structure: <1-5> +relevance: <1-5> +reasoning: +overall_correct: """ + + +class ResearchMiningScorer(LLMJudgeScorer): + """Score research mining responses on quality dimensions.""" + + scorer_id = "research_mining" + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + if not model_answer or not model_answer.strip(): + return False, {"reason": "empty_response"} + + # Extract original question from the prompt + question = record.problem.split("Research question: ")[-1].split("\n")[0].strip() + + try: + prompt = _JUDGE_PROMPT.format( + key_facts=record.reference, + question=question, + response=model_answer, + ) + raw = self._ask_judge(prompt, temperature=0.0, max_tokens=512) + + scores = {} + for dim in ("accuracy", "coverage", "depth", "structure", "relevance"): + match = re.search( + rf"^{dim}:\s*(\d)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + scores[dim] = int(match.group(1)) if match else 3 + + overall_match = re.search( + r"^overall_correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, + ) + + avg_score = sum(scores.values()) / len(scores) + if overall_match: + is_correct = overall_match.group(1).lower() == "yes" + else: + is_correct = avg_score >= 3.5 + + return is_correct, { + "match_type": "llm_judge", + "scores": scores, + "avg_score": avg_score, + "raw_judge_output": raw, + } + except Exception as exc: + LOGGER.error("Research mining LLM judge failed: %s", exc) + return None, { + "match_type": "llm_judge_error", + "error": str(exc), + } + + +__all__ = ["ResearchMiningScorer"] diff --git a/src/openjarvis/server/app.py b/src/openjarvis/server/app.py index 806f06f6..ca0f5e10 100644 --- a/src/openjarvis/server/app.py +++ b/src/openjarvis/server/app.py @@ -10,6 +10,7 @@ from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles from openjarvis.server.api_routes import include_all_routes +from openjarvis.server.comparison import comparison_router from openjarvis.server.dashboard import dashboard_router from openjarvis.server.routes import router @@ -122,6 +123,7 @@ def create_app( app.include_router(router) app.include_router(dashboard_router) + app.include_router(comparison_router) include_all_routes(app) # Add security headers middleware diff --git a/src/openjarvis/server/comparison.py b/src/openjarvis/server/comparison.py new file mode 100644 index 00000000..89f58436 --- /dev/null +++ b/src/openjarvis/server/comparison.py @@ -0,0 +1,448 @@ +"""Comparison page -- static cost comparison of local vs cloud inference.""" + +from __future__ import annotations + +from fastapi import APIRouter +from fastapi.responses import HTMLResponse + +comparison_router = APIRouter() + +COMPARISON_HTML = """\ + + + + + +OpenJarvis — Cost Comparison + + + + +
+

OpenJarvis Cost Comparison

+
+ Dashboard + Comparison +
+
+ +
+ + +
+

Run AI Locally — $0 API Costs

+
$0
+
estimated annual savings vs cloud APIs
+
+ OpenJarvis runs models on your own hardware. Every inference call that would + cost money on a cloud API is completely free when running locally. +
+
+ + +
Choose a Use Case
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
OpenJarvis (Local)GPT-5.3Claude Opus 4.6Gemini 3.1 Pro
Monthly Cost$0.00------
Annual Cost$0.00------
+
+ + +
+

Interactive Cost Calculator

+
+
+ +
50
+ +
+
+ +
1000
+ +
+
+
+
+
OpenJarvis
+
$0.00/mo
+
+
+
GPT-5.3
+
--
+
+
+
Claude Opus 4.6
+
--
+
+
+
Gemini 3.1 Pro
+
--
+
+
+
+ + +
+

Start Saving Today

+
Install OpenJarvis and run AI locally + with zero API costs.
+
+ pip install openjarvis + +
+
+ +
+ + + + +""" + + +@comparison_router.get("/comparison", response_class=HTMLResponse) +async def comparison(): + """Serve the cost comparison page.""" + return HTMLResponse(content=COMPARISON_HTML) + + +__all__ = ["comparison_router"] diff --git a/src/openjarvis/server/cost_calculator.py b/src/openjarvis/server/cost_calculator.py new file mode 100644 index 00000000..a79459cc --- /dev/null +++ b/src/openjarvis/server/cost_calculator.py @@ -0,0 +1,135 @@ +"""Cost calculator -- estimate monthly cloud API costs for common use cases.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, List + +from openjarvis.server.savings import CLOUD_PRICING + + +@dataclass(slots=True) +class CostEstimate: + """Estimated cost for a provider given a usage scenario.""" + provider: str + label: str + monthly_cost: float + annual_cost: float + input_cost: float + output_cost: float + total_calls_per_month: int + + +@dataclass(slots=True) +class Scenario: + """A prebuilt usage scenario.""" + name: str + label: str + description: str + calls_per_month: int + avg_input_tokens: int + avg_output_tokens: int + + +SCENARIOS: Dict[str, Scenario] = { + "daily_briefing": Scenario( + name="daily_briefing", + label="Daily Briefing", + description="Morning brief every 5 minutes, 24/7", + calls_per_month=8_640, + avg_input_tokens=500, + avg_output_tokens=200, + ), + "email_triage": Scenario( + name="email_triage", + label="Email Triage", + description="Email classification and drafting every 5 minutes", + calls_per_month=8_640, + avg_input_tokens=800, + avg_output_tokens=300, + ), + "research_assistant": Scenario( + name="research_assistant", + label="Research Assistant", + description="Deep research queries, ~20 per day", + calls_per_month=600, + avg_input_tokens=2_000, + avg_output_tokens=1_500, + ), + "overnight_coder": Scenario( + name="overnight_coder", + label="Overnight Coder", + description="Automated coding tasks, ~100 per night", + calls_per_month=3_000, + avg_input_tokens=3_000, + avg_output_tokens=2_000, + ), + "always_on": Scenario( + name="always_on", + label="Always-On (All Above)", + description="All use cases combined", + calls_per_month=20_880, + avg_input_tokens=1_200, + avg_output_tokens=700, + ), +} + + +def estimate_monthly_cost( + calls_per_month: int, + avg_input_tokens: int, + avg_output_tokens: int, + provider_key: str, +) -> CostEstimate: + """Estimate monthly cost for a provider given usage parameters.""" + pricing = CLOUD_PRICING.get(provider_key) + if pricing is None: + raise ValueError(f"Unknown provider: {provider_key}") + + total_input = calls_per_month * avg_input_tokens + total_output = calls_per_month * avg_output_tokens + + input_cost = (total_input / 1_000_000) * pricing["input_per_1m"] + output_cost = (total_output / 1_000_000) * pricing["output_per_1m"] + monthly = input_cost + output_cost + + return CostEstimate( + provider=provider_key, + label=str(pricing["label"]), + monthly_cost=monthly, + annual_cost=monthly * 12, + input_cost=input_cost, + output_cost=output_cost, + total_calls_per_month=calls_per_month, + ) + + +def estimate_scenario(scenario_name: str) -> List[CostEstimate]: + """Estimate costs for all providers for a named scenario.""" + scenario = SCENARIOS.get(scenario_name) + if scenario is None: + raise ValueError(f"Unknown scenario: {scenario_name}") + return [ + estimate_monthly_cost( + scenario.calls_per_month, + scenario.avg_input_tokens, + scenario.avg_output_tokens, + provider_key, + ) + for provider_key in CLOUD_PRICING + ] + + +def estimate_all_scenarios() -> Dict[str, List[CostEstimate]]: + """Estimate costs for all scenarios and all providers.""" + return {name: estimate_scenario(name) for name in SCENARIOS} + + +__all__ = [ + "SCENARIOS", + "CostEstimate", + "Scenario", + "estimate_all_scenarios", + "estimate_monthly_cost", + "estimate_scenario", +] diff --git a/src/openjarvis/server/dashboard.py b/src/openjarvis/server/dashboard.py index 16398f42..3430dadb 100644 --- a/src/openjarvis/server/dashboard.py +++ b/src/openjarvis/server/dashboard.py @@ -184,7 +184,7 @@ DASHBOARD_HTML = """\
OpenAI
-
GPT 5.2 — $1.75 / $14.00 per 1M tokens
+
GPT-5.3 — $2.00 / $10.00 per 1M tokens
$0.00
@@ -229,16 +229,64 @@ DASHBOARD_HTML = """\
+ +
Monthly Projection
+
+
+
vs OpenAI
+
projected monthly savings
+
$0.00
+
per month at current rate
+
+
+
vs Anthropic
+
projected monthly savings
+
$0.00
+
per month at current rate
+
+
+
vs Google
+
projected monthly savings
+
$0.00
+
per month at current rate
+
+
+ + +
vs Cloud Agent Platforms
+
+
+
Typical Cloud Agent Platform
+
based on published API pricing tiers
+
+
+
MODERATE USE
+
$15–60/mo
+
+
+
HEAVY USE
+
$100–400+/mo
+
+
+
YOUR COST
+
$0.00
+
local inference
+
+
+
+
+
Energy & Compute Avoided
-
Energy Saved (vs GPT 5.2)
+
Energy Saved (vs GPT-5.3)
0 J
0 kWh of cloud datacenter energy avoided
-
FLOPs Avoided (vs GPT 5.2)
+
FLOPs Avoided (vs GPT-5.3)
0 FLOP
cloud compute operations not needed
@@ -306,8 +354,8 @@ async function refresh() { providerMap[p.provider] = p; }); - // OpenAI / GPT 5.2 - const oa = providerMap['gpt-5.2'] || {}; + // OpenAI / GPT-5.3 + const oa = providerMap['gpt-5.3'] || {}; document.getElementById('save-openai') .textContent = fmtDollar(oa.total_cost || 0); document.getElementById('save-openai-in') @@ -333,7 +381,16 @@ async function refresh() { document.getElementById('save-google-out') .textContent = fmtDollar(go.output_cost || 0); - // Energy / FLOPs (use GPT 5.2 as reference) + // Monthly projections + const proj = d.monthly_projection || {}; + document.getElementById('proj-openai') + .textContent = fmtDollar(proj['gpt-5.3'] || 0); + document.getElementById('proj-anthropic') + .textContent = fmtDollar(proj['claude-opus-4.6'] || 0); + document.getElementById('proj-google') + .textContent = fmtDollar(proj['gemini-3.1-pro'] || 0); + + // Energy / FLOPs (use GPT-5.3 as reference) const ej = oa.energy_joules || 0; const eWh = oa.energy_wh || 0; const fl = oa.flops || 0; diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py index a26f2397..9f847919 100644 --- a/src/openjarvis/server/routes.py +++ b/src/openjarvis/server/routes.py @@ -243,6 +243,7 @@ async def savings(request: Request): prompt_tokens=sum(m.prompt_tokens for m in summary.per_model), completion_tokens=sum(m.completion_tokens for m in summary.per_model), total_calls=summary.total_calls, + session_start=session_start if session_start else 0.0, ) return savings_to_dict(result) finally: diff --git a/src/openjarvis/server/savings.py b/src/openjarvis/server/savings.py index af6b95cb..020c23e5 100644 --- a/src/openjarvis/server/savings.py +++ b/src/openjarvis/server/savings.py @@ -2,6 +2,7 @@ from __future__ import annotations +import time from dataclasses import asdict, dataclass, field from typing import Any, Dict, List @@ -61,20 +62,33 @@ class SavingsSummary: total_tokens: int = 0 local_cost: float = 0.0 # always 0 for local inference per_provider: List[ProviderSavings] = field(default_factory=list) + monthly_projection: Dict[str, float] = field(default_factory=dict) + session_start_ts: float = 0.0 + session_duration_hours: float = 0.0 + avg_cost_per_query: Dict[str, float] = field(default_factory=dict) + cloud_agent_equivalent: Dict[str, int] = field(default_factory=dict) def compute_savings( prompt_tokens: int, completion_tokens: int, total_calls: int = 0, + session_start: float = 0.0, ) -> SavingsSummary: """Compute savings vs cloud providers given token counts.""" total_tokens = prompt_tokens + completion_tokens providers: List[ProviderSavings] = [] + now = time.time() + session_duration_hours = (now - session_start) / 3600 if session_start > 0 else 0.0 + + monthly_projection: Dict[str, float] = {} + avg_cost_per_query: Dict[str, float] = {} + for key, pricing in CLOUD_PRICING.items(): input_cost = (prompt_tokens / 1_000_000) * pricing["input_per_1m"] output_cost = (completion_tokens / 1_000_000) * pricing["output_per_1m"] + total_cost = input_cost + output_cost energy_wh = (total_tokens / 1000) * pricing["energy_wh_per_1k_tokens"] flops = total_tokens * pricing["flops_per_token"] @@ -83,12 +97,24 @@ def compute_savings( label=pricing["label"], input_cost=input_cost, output_cost=output_cost, - total_cost=input_cost + output_cost, + total_cost=total_cost, energy_wh=energy_wh, energy_joules=energy_wh * 3600, # 1 Wh = 3600 J flops=flops, )) + # Monthly projection: extrapolate current spend to 720 hours/month + if session_duration_hours > 0: + monthly_projection[key] = (total_cost / session_duration_hours) * 720 + else: + monthly_projection[key] = 0.0 + + # Average cost per query + if total_calls > 0: + avg_cost_per_query[key] = total_cost / total_calls + else: + avg_cost_per_query[key] = 0.0 + return SavingsSummary( total_calls=total_calls, total_prompt_tokens=prompt_tokens, @@ -96,6 +122,16 @@ def compute_savings( total_tokens=total_tokens, local_cost=0.0, per_provider=providers, + monthly_projection=monthly_projection, + session_start_ts=session_start, + session_duration_hours=session_duration_hours, + avg_cost_per_query=avg_cost_per_query, + cloud_agent_equivalent={ + "moderate_low": 15, + "moderate_high": 60, + "heavy_low": 100, + "heavy_high": 400, + }, ) diff --git a/src/openjarvis/telemetry/__init__.py b/src/openjarvis/telemetry/__init__.py index 8c748cd1..f2c68646 100644 --- a/src/openjarvis/telemetry/__init__.py +++ b/src/openjarvis/telemetry/__init__.py @@ -48,6 +48,31 @@ from openjarvis.telemetry.steady_state import ( SteadyStateResult, ) +try: + from openjarvis.telemetry.session import TelemetrySample, TelemetrySession +except ImportError: + pass + +try: + from openjarvis.telemetry.phase_metrics import compute_phase_metrics, split_at_ttft +except ImportError: + pass + +try: + from openjarvis.telemetry.itl import compute_itl_stats +except ImportError: + pass + +try: + from openjarvis.telemetry.flops import ( + GPU_PEAK_TFLOPS_BF16, + MODEL_PARAMS_B, + compute_mfu, + estimate_flops, + ) +except ImportError: + pass + __all__ = [ "AggregatedStats", "BatchMetrics", @@ -69,6 +94,15 @@ __all__ = [ "SteadyStateConfig", "SteadyStateDetector", "SteadyStateResult", + "TelemetrySession", + "TelemetrySample", + "compute_phase_metrics", + "split_at_ttft", + "compute_itl_stats", + "estimate_flops", + "compute_mfu", + "GPU_PEAK_TFLOPS_BF16", + "MODEL_PARAMS_B", "compute_efficiency", "create_energy_monitor", "instrumented_generate", diff --git a/src/openjarvis/telemetry/energy_monitor.py b/src/openjarvis/telemetry/energy_monitor.py index 33ba0d8d..f2b97342 100644 --- a/src/openjarvis/telemetry/energy_monitor.py +++ b/src/openjarvis/telemetry/energy_monitor.py @@ -84,6 +84,14 @@ class EnergyMonitor(ABC): """ yield EnergySample() # pragma: no cover + def snapshot(self) -> EnergySample: + """Return an instantaneous energy reading without start/stop bracket. + + Subclasses should override to provide actual readings. Default returns + an empty sample. + """ + return EnergySample() + @abstractmethod def close(self) -> None: """Release any resources (handles, threads, etc.).""" diff --git a/src/openjarvis/telemetry/flops.py b/src/openjarvis/telemetry/flops.py new file mode 100644 index 00000000..2fd4ce39 --- /dev/null +++ b/src/openjarvis/telemetry/flops.py @@ -0,0 +1,74 @@ +"""FLOPs estimation and Model FLOPs Utilization (MFU) computation.""" + +from __future__ import annotations + +GPU_PEAK_TFLOPS_BF16: dict[str, float] = { + "H100": 989.0, + "H200": 989.0, + "A100": 312.0, + "A10G": 31.2, + "L4": 30.3, + "L40": 181.0, + "L40S": 362.0, + "T4": 65.1, + "V100": 125.0, + "4090": 82.6, + "4080": 48.7, + "3090": 35.6, + "M3 Max": 14.2, + "M3 Ultra": 27.0, + "M4 Max": 18.0, +} + +MODEL_PARAMS_B: dict[str, float] = { + "qwen3:8b": 8.0, + "qwen3:0.6b": 0.6, + "qwen3:4b": 4.0, + "llama-3.1-70b": 70.0, + "llama-3.1-8b": 8.0, + "mistral-7b": 7.0, + "mixtral-8x7b": 47.0, +} + + +def estimate_flops( + model: str, input_tokens: int, output_tokens: int +) -> tuple[float, float]: + """Estimate FLOPs for an inference pass. + + Uses the 2 * P * T approximation where P = params, T = total tokens. + Returns (total_flops, flops_per_token). + """ + params_b = MODEL_PARAMS_B.get(model, 0.0) + if params_b == 0.0: + # Try prefix matching + for key, val in MODEL_PARAMS_B.items(): + if model.startswith(key.split(":")[0]): + params_b = val + break + + total_tokens = input_tokens + output_tokens + params = params_b * 1e9 + total_flops = 2.0 * params * total_tokens + flops_per_token = 2.0 * params if total_tokens > 0 else 0.0 + return (total_flops, flops_per_token) + + +def compute_mfu( + flops: float, duration_s: float, gpu_name: str, num_gpus: int = 1 +) -> float: + """Compute Model FLOPs Utilization. + + MFU = actual_tflops / (peak_tflops * num_gpus) + """ + peak = GPU_PEAK_TFLOPS_BF16.get(gpu_name, 0.0) + if peak == 0.0: + # Try substring matching + for key, val in GPU_PEAK_TFLOPS_BF16.items(): + if key.lower() in gpu_name.lower(): + peak = val + break + if peak <= 0 or duration_s <= 0: + return 0.0 + actual_tflops = flops / (duration_s * 1e12) + return (actual_tflops / (peak * num_gpus)) * 100.0 diff --git a/src/openjarvis/telemetry/itl.py b/src/openjarvis/telemetry/itl.py new file mode 100644 index 00000000..c35fbf84 --- /dev/null +++ b/src/openjarvis/telemetry/itl.py @@ -0,0 +1,47 @@ +"""Inter-token latency percentile computation.""" + +from __future__ import annotations + +import statistics + + +def compute_itl_stats(token_timestamps: list[float]) -> dict: + """Compute ITL statistics from token arrival timestamps (in ms). + + Returns dict with p50_ms, p90_ms, p95_ms, p99_ms, mean_ms, min_ms, max_ms. + """ + if len(token_timestamps) < 2: + return { + "p50_ms": 0, + "p90_ms": 0, + "p95_ms": 0, + "p99_ms": 0, + "mean_ms": 0, + "min_ms": 0, + "max_ms": 0, + } + + # Compute inter-token latencies + itls = [ + token_timestamps[i] - token_timestamps[i - 1] + for i in range(1, len(token_timestamps)) + ] + itls.sort() + + def percentile(data: list[float], p: float) -> float: + k = (len(data) - 1) * p + f = int(k) + c = f + 1 + if c >= len(data): + return data[-1] + return data[f] + (k - f) * (data[c] - data[f]) + + return { + "p50_ms": percentile(itls, 0.50), + "p90_ms": percentile(itls, 0.90), + "p95_ms": percentile(itls, 0.95), + "p99_ms": percentile(itls, 0.99), + "mean_ms": statistics.mean(itls), + "min_ms": min(itls), + "max_ms": max(itls), + } diff --git a/src/openjarvis/telemetry/phase_metrics.py b/src/openjarvis/telemetry/phase_metrics.py new file mode 100644 index 00000000..7f53d499 --- /dev/null +++ b/src/openjarvis/telemetry/phase_metrics.py @@ -0,0 +1,44 @@ +"""Phase metrics computation -- prefill/decode energy separation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openjarvis.telemetry.session import TelemetrySession + + +def compute_phase_metrics( + session: TelemetrySession, + start_ns: int, + end_ns: int, + tokens: int, +) -> dict: + """Compute energy/power metrics for a phase window.""" + gpu_j, cpu_j = session.energy_delta(start_ns, end_ns) + gpu_w, cpu_w = session.avg_power(start_ns, end_ns) + duration_s = (end_ns - start_ns) / 1e9 + energy_per_token = gpu_j / tokens if tokens > 0 else 0.0 + return { + "energy_j": gpu_j, + "cpu_energy_j": cpu_j, + "mean_power_w": gpu_w, + "cpu_mean_power_w": cpu_w, + "duration_s": duration_s, + "energy_per_token_j": energy_per_token, + "tokens": tokens, + } + + +def split_at_ttft( + session: TelemetrySession, + start_ns: int, + ttft_ns: int, + end_ns: int, + input_tokens: int, + output_tokens: int, +) -> tuple[dict, dict]: + """Split energy at TTFT boundary into prefill and decode phases.""" + prefill = compute_phase_metrics(session, start_ns, ttft_ns, input_tokens) + decode = compute_phase_metrics(session, ttft_ns, end_ns, output_tokens) + return (prefill, decode) diff --git a/src/openjarvis/telemetry/session.py b/src/openjarvis/telemetry/session.py new file mode 100644 index 00000000..d75931f9 --- /dev/null +++ b/src/openjarvis/telemetry/session.py @@ -0,0 +1,145 @@ +"""Background-sampling telemetry session. + +Uses Rust ring buffer when available, with Python fallback. +""" + +from __future__ import annotations + +import threading +import time +from collections import deque +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Deque, List, Optional + +if TYPE_CHECKING: + from openjarvis.telemetry.energy_monitor import EnergyMonitor + + +@dataclass +class TelemetrySample: + """Single telemetry sample.""" + + timestamp_ns: int + gpu_power_w: float = 0.0 + cpu_power_w: float = 0.0 + gpu_energy_j: float = 0.0 + cpu_energy_j: float = 0.0 + gpu_util_pct: float = 0.0 + gpu_temp_c: float = 0.0 + gpu_mem_gb: float = 0.0 + + +class _PythonRingBuffer: + """Pure-Python fallback ring buffer.""" + + def __init__(self, capacity: int = 100_000): + self._data: Deque[TelemetrySample] = deque(maxlen=capacity) + + def push(self, sample: TelemetrySample) -> None: + self._data.append(sample) + + def window(self, start_ns: int, end_ns: int) -> List[TelemetrySample]: + return [s for s in self._data if start_ns <= s.timestamp_ns <= end_ns] + + def compute_energy_delta(self, start_ns: int, end_ns: int) -> tuple[float, float]: + samples = self.window(start_ns, end_ns) + if len(samples) < 2: + return (0.0, 0.0) + # Trapezoidal integration over power samples + gpu_j = 0.0 + cpu_j = 0.0 + for i in range(1, len(samples)): + dt = (samples[i].timestamp_ns - samples[i - 1].timestamp_ns) / 1e9 + gpu_j += 0.5 * (samples[i - 1].gpu_power_w + samples[i].gpu_power_w) * dt + cpu_j += 0.5 * (samples[i - 1].cpu_power_w + samples[i].cpu_power_w) * dt + return (gpu_j, cpu_j) + + def compute_avg_power(self, start_ns: int, end_ns: int) -> tuple[float, float]: + samples = self.window(start_ns, end_ns) + if not samples: + return (0.0, 0.0) + gpu_avg = sum(s.gpu_power_w for s in samples) / len(samples) + cpu_avg = sum(s.cpu_power_w for s in samples) / len(samples) + return (gpu_avg, cpu_avg) + + def clear(self) -> None: + self._data.clear() + + def __len__(self) -> int: + return len(self._data) + + +class TelemetrySession: + """Background-sampling telemetry session. + + Spawns a daemon thread that calls monitor.snapshot() at the configured + interval. Stores samples in a ring buffer (Rust-backed if available, + else pure-Python fallback). + """ + + def __init__( + self, + monitor: Optional[EnergyMonitor] = None, + interval_ms: int = 100, + buffer_size: int = 100_000, + ) -> None: + self._monitor = monitor + self._interval_ms = interval_ms + self._buffer = _PythonRingBuffer(buffer_size) + self._thread: Optional[threading.Thread] = None + self._stop_event = threading.Event() + + def _sample_loop(self) -> None: + """Daemon thread loop: poll monitor and push samples.""" + interval_s = self._interval_ms / 1000.0 + while not self._stop_event.is_set(): + try: + if self._monitor is not None: + es = self._monitor.snapshot() + sample = TelemetrySample( + timestamp_ns=time.time_ns(), + gpu_power_w=es.mean_power_watts, + cpu_power_w=0.0, + gpu_energy_j=es.gpu_energy_joules, + cpu_energy_j=es.cpu_energy_joules, + gpu_util_pct=es.mean_utilization_pct, + gpu_temp_c=es.mean_temperature_c, + gpu_mem_gb=es.mean_memory_used_gb, + ) + self._buffer.push(sample) + except Exception: + pass + self._stop_event.wait(interval_s) + + def start(self) -> None: + """Start background sampling thread.""" + if self._monitor is None: + return + if self._thread is not None and self._thread.is_alive(): + return + self._stop_event.clear() + self._thread = threading.Thread(target=self._sample_loop, daemon=True) + self._thread.start() + + def stop(self) -> None: + """Stop sampling thread.""" + self._stop_event.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + def window(self, start_ns: int, end_ns: int) -> List[TelemetrySample]: + return self._buffer.window(start_ns, end_ns) + + def energy_delta(self, start_ns: int, end_ns: int) -> tuple[float, float]: + return self._buffer.compute_energy_delta(start_ns, end_ns) + + def avg_power(self, start_ns: int, end_ns: int) -> tuple[float, float]: + return self._buffer.compute_avg_power(start_ns, end_ns) + + def __enter__(self) -> TelemetrySession: + self.start() + return self + + def __exit__(self, *args: Any) -> None: + self.stop() diff --git a/tests/evals/test_agentic_runner.py b/tests/evals/test_agentic_runner.py new file mode 100644 index 00000000..188b2387 --- /dev/null +++ b/tests/evals/test_agentic_runner.py @@ -0,0 +1,171 @@ +"""Tests for AgenticRunner with mock agent and dataset.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any, Dict, List + +import pytest + +from openjarvis.evals.core.agentic_runner import AgenticRunner, _extract_patch + +# --------------------------------------------------------------------------- +# Mock objects +# --------------------------------------------------------------------------- + + +@dataclass +class MockRecord: + record_id: str + problem: str + expected: str = "" + category: str = "test" + metadata: Dict[str, Any] = field(default_factory=dict) + + +class MockDataset: + def __init__(self, records: List[MockRecord]): + self._records = records + + def iter_records(self): + return iter(self._records) + + +class MockAgent: + """Agent that echoes the query.""" + + def ask(self, query: str) -> dict: + return { + "content": f"Response to: {query}", + "usage": {"prompt_tokens": 50, "completion_tokens": 25}, + "cost_usd": 0.001, + } + + +class MockFailingAgent: + """Agent that always raises.""" + + def ask(self, query: str) -> dict: + raise RuntimeError("Agent error") + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestAgenticRunner: + def _run_async(self, coro): + return asyncio.get_event_loop().run_until_complete(coro) + + @pytest.fixture(autouse=True) + def _setup_loop(self): + try: + asyncio.get_event_loop() + except RuntimeError: + asyncio.set_event_loop(asyncio.new_event_loop()) + + def test_basic_run(self): + records = [ + MockRecord(record_id="r1", problem="What is 2+2?"), + MockRecord(record_id="r2", problem="What is 3+3?"), + ] + dataset = MockDataset(records) + agent = MockAgent() + runner = AgenticRunner(agent=agent, dataset=dataset) + + traces = self._run_async(runner.run()) + assert len(traces) == 2 + assert all(t.completed for t in traces) + assert traces[0].query_id == "q0000" + assert traces[1].query_id == "q0001" + assert "Response to: What is 2+2?" in traces[0].response_text + + def test_max_queries(self): + records = [MockRecord(record_id=f"r{i}", problem=f"Q{i}") for i in range(10)] + dataset = MockDataset(records) + runner = AgenticRunner(agent=MockAgent(), dataset=dataset) + + traces = self._run_async(runner.run(max_queries=3)) + assert len(traces) == 3 + + def test_agent_failure(self): + records = [MockRecord(record_id="r1", problem="test")] + dataset = MockDataset(records) + runner = AgenticRunner(agent=MockFailingAgent(), dataset=dataset) + + traces = self._run_async(runner.run()) + assert len(traces) == 1 + assert not traces[0].completed + assert "Agent error" in traces[0].response_text + + def test_synthetic_turn_created(self): + records = [MockRecord(record_id="r1", problem="test")] + dataset = MockDataset(records) + runner = AgenticRunner(agent=MockAgent(), dataset=dataset) + + traces = self._run_async(runner.run()) + assert traces[0].num_turns == 1 + assert traces[0].turns[0].input_tokens == 50 + assert traces[0].turns[0].output_tokens == 25 + + def test_traces_property(self): + records = [MockRecord(record_id="r1", problem="test")] + dataset = MockDataset(records) + runner = AgenticRunner(agent=MockAgent(), dataset=dataset) + + self._run_async(runner.run()) + assert len(runner.traces) == 1 + + def test_artifacts_saved(self, tmp_path): + records = [MockRecord(record_id="r1", problem="test")] + dataset = MockDataset(records) + runner = AgenticRunner( + agent=MockAgent(), dataset=dataset, run_dir=tmp_path + ) + + self._run_async(runner.run()) + arts = tmp_path / "artifacts" + assert arts.exists() + subdirs = list(arts.iterdir()) + assert len(subdirs) == 1 + assert (subdirs[0] / "response.txt").exists() + assert (subdirs[0] / "metadata.json").exists() + + def test_query_timeout_configured(self): + """Verify timeout is stored and runner accepts the parameter.""" + records = [MockRecord(record_id="r1", problem="test")] + dataset = MockDataset(records) + runner = AgenticRunner( + agent=MockAgent(), dataset=dataset, query_timeout=30.0 + ) + assert runner._query_timeout == 30.0 + + +class TestExtractPatch: + def test_fenced_diff(self): + text = ( + "Here's the fix:\n```diff\n" + "--- a/foo.py\n+++ b/foo.py\n" + "@@ -1 +1 @@\n-old\n+new\n```\n" + ) + patch = _extract_patch(text) + assert patch is not None + assert "--- a/foo.py" in patch + + def test_unfenced_diff(self): + text = ( + "Some explanation\n" + "diff --git a/x.py b/x.py\n" + "--- a/x.py\n+++ b/x.py\n" + "@@ -1 +1 @@\n-old\n+new\n" + ) + patch = _extract_patch(text) + assert patch is not None + assert "diff --git" in patch + + def test_no_patch(self): + text = "This is just a regular response with no code changes." + patch = _extract_patch(text) + assert patch is None diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py index a381a78c..a59db6f9 100644 --- a/tests/evals/test_benchmark_datasets.py +++ b/tests/evals/test_benchmark_datasets.py @@ -266,7 +266,7 @@ class TestConfigBenchmarks: def test_benchmarks_count(self) -> None: from openjarvis.evals.core.config import KNOWN_BENCHMARKS - assert len(KNOWN_BENCHMARKS) == 15 + assert len(KNOWN_BENCHMARKS) == 20 # --------------------------------------------------------------------------- diff --git a/tests/evals/test_eval_trace.py b/tests/evals/test_eval_trace.py new file mode 100644 index 00000000..8d3a73e3 --- /dev/null +++ b/tests/evals/test_eval_trace.py @@ -0,0 +1,151 @@ +"""Tests for QueryTrace and TurnTrace data model.""" + +from __future__ import annotations + +import pytest + +from openjarvis.evals.core.trace import QueryTrace, TurnTrace + + +class TestTurnTrace: + def test_defaults(self): + t = TurnTrace(turn_index=0) + assert t.input_tokens == 0 + assert t.output_tokens == 0 + assert t.tools_called == [] + assert t.gpu_energy_joules is None + assert t.cost_usd is None + + def test_to_dict_roundtrip(self): + t = TurnTrace( + turn_index=1, + input_tokens=100, + output_tokens=50, + tools_called=["calculator"], + tool_latencies_s={"calculator": 0.5}, + wall_clock_s=1.2, + gpu_energy_joules=10.0, + cost_usd=0.001, + ) + d = t.to_dict() + t2 = TurnTrace.from_dict(d) + assert t2.turn_index == 1 + assert t2.input_tokens == 100 + assert t2.output_tokens == 50 + assert t2.tools_called == ["calculator"] + assert t2.tool_latencies_s == {"calculator": 0.5} + assert t2.wall_clock_s == pytest.approx(1.2) + assert t2.gpu_energy_joules == pytest.approx(10.0) + assert t2.cost_usd == pytest.approx(0.001) + + def test_from_dict_missing_keys(self): + d = {"turn_index": 0} + t = TurnTrace.from_dict(d) + assert t.input_tokens == 0 + assert t.tools_called == [] + assert t.error is None + + +class TestQueryTrace: + def _make_trace(self, **kwargs): + defaults = { + "query_id": "q0001", + "workload_type": "coding", + "query_text": "Hello", + "response_text": "World", + "turns": [ + TurnTrace( + turn_index=0, + input_tokens=100, + output_tokens=50, + wall_clock_s=1.0, + gpu_energy_joules=5.0, + cost_usd=0.01, + ), + TurnTrace( + turn_index=1, + input_tokens=150, + output_tokens=75, + wall_clock_s=1.5, + gpu_energy_joules=7.5, + cost_usd=0.015, + ), + ], + "total_wall_clock_s": 2.5, + "completed": True, + } + defaults.update(kwargs) + return QueryTrace(**defaults) + + def test_num_turns(self): + t = self._make_trace() + assert t.num_turns == 2 + + def test_total_tokens(self): + t = self._make_trace() + assert t.total_input_tokens == 250 + assert t.total_output_tokens == 125 + assert t.total_tokens == 375 + + def test_total_gpu_energy(self): + t = self._make_trace() + assert t.total_gpu_energy_joules == pytest.approx(12.5) + + def test_total_gpu_energy_fallback(self): + t = self._make_trace( + turns=[TurnTrace(turn_index=0)], + query_gpu_energy_joules=20.0, + ) + assert t.total_gpu_energy_joules == pytest.approx(20.0) + + def test_total_cost(self): + t = self._make_trace() + assert t.total_cost_usd == pytest.approx(0.025) + + def test_total_cost_none(self): + t = self._make_trace(turns=[TurnTrace(turn_index=0)]) + assert t.total_cost_usd is None + + def test_throughput(self): + t = self._make_trace() + assert t.throughput_tokens_per_sec == pytest.approx(125 / 2.5) + + def test_energy_per_token(self): + t = self._make_trace() + assert t.energy_per_token_joules == pytest.approx(12.5 / 125) + + def test_avg_gpu_power(self): + t = self._make_trace() + # No per-turn power set, so falls back to query-level + assert t.avg_gpu_power_watts is None + + def test_to_dict_roundtrip(self): + t = self._make_trace(is_resolved=True) + d = t.to_dict() + t2 = QueryTrace.from_dict(d) + assert t2.query_id == "q0001" + assert t2.workload_type == "coding" + assert t2.num_turns == 2 + assert t2.completed is True + assert t2.is_resolved is True + assert t2.total_input_tokens == 250 + + def test_save_load_jsonl(self, tmp_path): + t1 = self._make_trace(query_id="q0001") + t2 = self._make_trace(query_id="q0002") + path = tmp_path / "traces.jsonl" + t1.save_jsonl(path) + t2.save_jsonl(path) + loaded = QueryTrace.load_jsonl(path) + assert len(loaded) == 2 + assert loaded[0].query_id == "q0001" + assert loaded[1].query_id == "q0002" + + def test_tool_call_count(self): + t = self._make_trace( + turns=[ + TurnTrace(turn_index=0, tools_called=["calc", "search"]), + TurnTrace(turn_index=1, tools_called=["read"]), + ] + ) + assert t.total_tool_calls == 3 diff --git a/tests/evals/test_event_recorder.py b/tests/evals/test_event_recorder.py new file mode 100644 index 00000000..fab83277 --- /dev/null +++ b/tests/evals/test_event_recorder.py @@ -0,0 +1,89 @@ +"""Tests for EventRecorder thread safety and functionality.""" + +from __future__ import annotations + +import threading + +from openjarvis.evals.core.event_recorder import AgentEvent, EventRecorder, EventType + + +class TestEventType: + def test_enum_values(self): + assert EventType.LM_INFERENCE_START == "lm_inference_start" + assert EventType.TOOL_CALL_END == "tool_call_end" + + def test_all_types_exist(self): + assert len(EventType) == 10 + + +class TestAgentEvent: + def test_creation(self): + e = AgentEvent(event_type="test", timestamp=123.456, metadata={"key": "val"}) + assert e.event_type == "test" + assert e.timestamp == 123.456 + assert e.metadata == {"key": "val"} + + def test_repr(self): + e = AgentEvent(event_type="test", timestamp=1.0) + r = repr(e) + assert "test" in r + assert "1.000" in r + + +class TestEventRecorder: + def test_record_and_get(self): + rec = EventRecorder() + rec.record("tool_call_start", tool="calc") + rec.record("tool_call_end", tool="calc") + events = rec.get_events() + assert len(events) == 2 + assert events[0].event_type == "tool_call_start" + assert events[0].metadata["tool"] == "calc" + assert events[1].event_type == "tool_call_end" + + def test_len(self): + rec = EventRecorder() + assert len(rec) == 0 + rec.record("test") + assert len(rec) == 1 + + def test_clear(self): + rec = EventRecorder() + rec.record("test") + rec.clear() + assert len(rec) == 0 + assert rec.get_events() == [] + + def test_get_events_returns_copy(self): + rec = EventRecorder() + rec.record("test") + events = rec.get_events() + events.clear() + assert len(rec) == 1 + + def test_timestamps_monotonic(self): + rec = EventRecorder() + for _ in range(10): + rec.record("test") + events = rec.get_events() + for i in range(1, len(events)): + assert events[i].timestamp >= events[i - 1].timestamp + + def test_thread_safety(self): + rec = EventRecorder() + barrier = threading.Barrier(4) + + def writer(thread_id): + barrier.wait() + for i in range(100): + rec.record("test", thread=thread_id, index=i) + + threads = [threading.Thread(target=writer, args=(t,)) for t in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(rec) == 400 + events = rec.get_events() + assert len(events) == 400 diff --git a/tests/evals/test_export.py b/tests/evals/test_export.py new file mode 100644 index 00000000..821a5379 --- /dev/null +++ b/tests/evals/test_export.py @@ -0,0 +1,136 @@ +"""Tests for eval export utilities.""" + +from __future__ import annotations + +import json + +from openjarvis.evals.core.export import ( + export_artifacts_manifest, + export_jsonl, + export_summary_json, +) +from openjarvis.evals.core.trace import QueryTrace, TurnTrace + + +def _make_traces(n=3): + traces = [] + for i in range(n): + traces.append(QueryTrace( + query_id=f"q{i:04d}", + workload_type="test", + query_text=f"Question {i}", + response_text=f"Answer {i}", + turns=[ + TurnTrace( + turn_index=0, + input_tokens=100 + i * 10, + output_tokens=50 + i * 5, + wall_clock_s=1.0 + i * 0.5, + gpu_energy_joules=5.0 + i, + cost_usd=0.01, + ), + ], + total_wall_clock_s=1.0 + i * 0.5, + completed=True, + is_resolved=i % 2 == 0, + )) + return traces + + +class TestExportJsonl: + def test_basic_export(self, tmp_path): + traces = _make_traces() + path = tmp_path / "traces.jsonl" + result = export_jsonl(traces, path) + assert result == path + assert path.exists() + + lines = path.read_text().strip().split("\n") + assert len(lines) == 3 + for line in lines: + d = json.loads(line) + assert "query_id" in d + assert "turns" in d + + def test_empty_traces(self, tmp_path): + path = tmp_path / "empty.jsonl" + export_jsonl([], path) + assert path.read_text() == "" + + def test_creates_parent_dirs(self, tmp_path): + path = tmp_path / "a" / "b" / "c" / "traces.jsonl" + export_jsonl(_make_traces(1), path) + assert path.exists() + + +class TestExportSummaryJson: + def test_basic_summary(self, tmp_path): + traces = _make_traces() + path = tmp_path / "summary.json" + result = export_summary_json(traces, {"model": "test"}, path) + assert result == path + assert path.exists() + + summary = json.loads(path.read_text()) + assert summary["totals"]["queries"] == 3 + assert summary["totals"]["completed"] == 3 + assert summary["totals"]["resolved"] == 2 + assert summary["totals"]["input_tokens"] > 0 + assert summary["totals"]["output_tokens"] > 0 + assert summary["config"]["model"] == "test" + assert "statistics" in summary + + def test_statistics_keys(self, tmp_path): + traces = _make_traces() + path = tmp_path / "summary.json" + export_summary_json(traces, {}, path) + summary = json.loads(path.read_text()) + stats = summary["statistics"] + expected_stat_keys = { + "wall_clock_s", "gpu_energy_joules", "cpu_energy_joules", + "gpu_power_watts", "cpu_power_watts", + "input_tokens", "output_tokens", "total_tokens", + "throughput_tokens_per_sec", "energy_per_token_joules", + "cost_usd", "turns", "tool_calls", + } + assert set(stats.keys()) == expected_stat_keys + + def test_agg_stats_fields(self, tmp_path): + traces = _make_traces() + path = tmp_path / "summary.json" + export_summary_json(traces, {}, path) + summary = json.loads(path.read_text()) + wc_stats = summary["statistics"]["wall_clock_s"] + assert "avg" in wc_stats + assert "median" in wc_stats + assert "min" in wc_stats + assert "max" in wc_stats + assert "std" in wc_stats + + def test_empty_traces(self, tmp_path): + path = tmp_path / "summary.json" + export_summary_json([], {}, path) + summary = json.loads(path.read_text()) + assert summary["totals"]["queries"] == 0 + + +class TestExportArtifactsManifest: + def test_no_artifacts_dir(self, tmp_path): + result = export_artifacts_manifest(tmp_path) + assert result is None + + def test_with_artifacts(self, tmp_path): + art_dir = tmp_path / "artifacts" + q_dir = art_dir / "q0001_test" + q_dir.mkdir(parents=True) + (q_dir / "response.txt").write_text("hello") + (q_dir / "metadata.json").write_text("{}") + + result = export_artifacts_manifest(tmp_path) + assert result is not None + assert result.exists() + + manifest = json.loads(result.read_text()) + assert len(manifest) == 1 + assert manifest[0]["query_dir"] == "q0001_test" + assert len(manifest[0]["files"]) == 2 diff --git a/tests/evals/test_pricing.py b/tests/evals/test_pricing.py new file mode 100644 index 00000000..5ed7062a --- /dev/null +++ b/tests/evals/test_pricing.py @@ -0,0 +1,38 @@ +"""Tests for eval pricing module.""" + +from __future__ import annotations + +import pytest + +from openjarvis.evals.core.pricing import PRICING, compute_turn_cost, estimate_cost + + +class TestPricing: + def test_pricing_dict_nonempty(self): + assert isinstance(PRICING, dict) + assert len(PRICING) > 0 + + def test_compute_turn_cost_known_model(self): + # Pick a model that's in PRICING (cloud models) + if not PRICING: + pytest.skip("No models in PRICING dict") + model = next(iter(PRICING)) + cost = compute_turn_cost(model, 1000, 500) + assert isinstance(cost, (int, float)) + assert cost >= 0 + + def test_compute_turn_cost_unknown_model(self): + cost = compute_turn_cost("totally-unknown-local-model", 1000, 500) + assert cost == 0.0 + + def test_compute_turn_cost_zero_tokens(self): + if not PRICING: + pytest.skip("No models in PRICING dict") + model = next(iter(PRICING)) + cost = compute_turn_cost(model, 0, 0) + assert cost == 0.0 + + def test_estimate_cost_alias(self): + # estimate_cost is the same as engine/cloud.py estimate_cost + cost = estimate_cost("unknown-model", 100, 50) + assert cost == 0.0 diff --git a/tests/evals/test_terminalbench_env.py b/tests/evals/test_terminalbench_env.py new file mode 100644 index 00000000..a348d437 --- /dev/null +++ b/tests/evals/test_terminalbench_env.py @@ -0,0 +1,48 @@ +"""Tests for TerminalBenchTaskEnv (mocked terminal_bench dependency).""" + +from __future__ import annotations + +import pytest + +from openjarvis.evals.execution.terminalbench_env import TerminalBenchTaskEnv + +# terminal_bench is an optional dep — skip all tests if unavailable +terminal_bench = pytest.importorskip( + "terminal_bench", reason="terminal_bench not installed" +) + + +class TestTerminalBenchTaskEnv: + def test_init(self): + metadata = {"task_id": "test-1"} + env = TerminalBenchTaskEnv(metadata) + assert env._metadata is metadata + assert env._terminal is None + + def test_enter_without_task_raises(self): + metadata = {"task_id": "test-1"} + env = TerminalBenchTaskEnv(metadata) + with pytest.raises(ValueError, match="Task metadata missing"): + env.__enter__() + + def test_exit_cleans_metadata(self): + metadata = { + "task_id": "test-1", + "terminal": "fake_terminal", + "session": "fake_session", + "container": "fake_container", + } + env = TerminalBenchTaskEnv(metadata) + env.__exit__(None, None, None) + assert "terminal" not in metadata + assert "session" not in metadata + assert "container" not in metadata + + def test_run_tests_without_terminal(self): + metadata = {"task": "mock_task", "task_paths": "mock_paths"} + env = TerminalBenchTaskEnv(metadata) + env._terminal = None + is_resolved, results = env.run_tests() + assert is_resolved is False + assert results["error"] == "terminal_not_running" + assert metadata["is_resolved"] is False diff --git a/tests/evals/test_use_case_benchmarks.py b/tests/evals/test_use_case_benchmarks.py new file mode 100644 index 00000000..f50ed24f --- /dev/null +++ b/tests/evals/test_use_case_benchmarks.py @@ -0,0 +1,391 @@ +"""Tests for the 5 use-case benchmark datasets and scorers. + +Tests verify: +1. Each dataset can be instantiated with correct attributes +2. Each dataset loads synthetic records (no HuggingFace download) +3. Records have expected fields +4. Each scorer can be instantiated +5. Coding task scorer can score a correct answer +6. Email triage scorer handles exact match +7. CLI factory functions work for all 5 benchmarks +8. Cost calculator and savings modules work +""" + +from __future__ import annotations + +from unittest.mock import MagicMock # noqa: I001 + +import pytest + +# --------------------------------------------------------------------------- +# Dataset instantiation and loading +# --------------------------------------------------------------------------- + + +class TestEmailTriageDataset: + def test_instantiation(self) -> None: + from openjarvis.evals.datasets.email_triage import EmailTriageDataset + ds = EmailTriageDataset() + assert ds.dataset_id == "email_triage" + assert ds.dataset_name == "Email Triage" + + def test_load(self) -> None: + from openjarvis.evals.datasets.email_triage import EmailTriageDataset + ds = EmailTriageDataset() + ds.load(max_samples=5, seed=42) + assert ds.size() == 5 + records = list(ds.iter_records()) + assert len(records) == 5 + r = records[0] + assert r.record_id.startswith("email-triage-") + assert r.category == "use-case" + assert "urgency" in r.metadata + assert "category" in r.metadata + + def test_load_all(self) -> None: + from openjarvis.evals.datasets.email_triage import EmailTriageDataset + ds = EmailTriageDataset() + ds.load() + assert ds.size() == 30 + + +class TestMorningBriefDataset: + def test_instantiation(self) -> None: + from openjarvis.evals.datasets.morning_brief import MorningBriefDataset + ds = MorningBriefDataset() + assert ds.dataset_id == "morning_brief" + assert ds.dataset_name == "Morning Brief" + + def test_load(self) -> None: + from openjarvis.evals.datasets.morning_brief import MorningBriefDataset + ds = MorningBriefDataset() + ds.load(max_samples=5, seed=42) + assert ds.size() == 5 + records = list(ds.iter_records()) + r = records[0] + assert r.record_id.startswith("morning-brief-") + assert r.reference # key_priorities not empty + + def test_load_all(self) -> None: + from openjarvis.evals.datasets.morning_brief import MorningBriefDataset + ds = MorningBriefDataset() + ds.load() + assert ds.size() == 15 + + +class TestResearchMiningDataset: + def test_instantiation(self) -> None: + from openjarvis.evals.datasets.research_mining import ResearchMiningDataset + ds = ResearchMiningDataset() + assert ds.dataset_id == "research_mining" + assert ds.dataset_name == "Research Mining" + + def test_load(self) -> None: + from openjarvis.evals.datasets.research_mining import ResearchMiningDataset + ds = ResearchMiningDataset() + ds.load(max_samples=5, seed=42) + assert ds.size() == 5 + records = list(ds.iter_records()) + r = records[0] + assert r.record_id.startswith("research-mining-") + assert "domain" in r.metadata + + def test_load_all(self) -> None: + from openjarvis.evals.datasets.research_mining import ResearchMiningDataset + ds = ResearchMiningDataset() + ds.load() + assert ds.size() == 31 + + +class TestKnowledgeBaseDataset: + def test_instantiation(self) -> None: + from openjarvis.evals.datasets.knowledge_base import KnowledgeBaseDataset + ds = KnowledgeBaseDataset() + assert ds.dataset_id == "knowledge_base" + assert ds.dataset_name == "Knowledge Base" + + def test_load(self) -> None: + from openjarvis.evals.datasets.knowledge_base import KnowledgeBaseDataset + ds = KnowledgeBaseDataset() + ds.load(max_samples=5, seed=42) + assert ds.size() == 5 + records = list(ds.iter_records()) + r = records[0] + assert r.record_id.startswith("knowledge-base-") + assert r.reference # answer not empty + + def test_load_all(self) -> None: + from openjarvis.evals.datasets.knowledge_base import KnowledgeBaseDataset + ds = KnowledgeBaseDataset() + ds.load() + assert ds.size() == 30 + + +class TestCodingTaskDataset: + def test_instantiation(self) -> None: + from openjarvis.evals.datasets.coding_task import CodingTaskDataset + ds = CodingTaskDataset() + assert ds.dataset_id == "coding_task" + assert ds.dataset_name == "Coding Task" + + def test_load(self) -> None: + from openjarvis.evals.datasets.coding_task import CodingTaskDataset + ds = CodingTaskDataset() + ds.load(max_samples=5, seed=42) + assert ds.size() == 5 + records = list(ds.iter_records()) + r = records[0] + assert r.record_id.startswith("coding-task-") + assert "test_cases" in r.metadata + assert "signature" in r.metadata + + def test_load_all(self) -> None: + from openjarvis.evals.datasets.coding_task import CodingTaskDataset + ds = CodingTaskDataset() + ds.load() + assert ds.size() == 29 + + +# --------------------------------------------------------------------------- +# Scorer instantiation +# --------------------------------------------------------------------------- + + +class TestScorerInstantiation: + def _mock_backend(self): + return MagicMock() + + def test_email_triage_scorer(self) -> None: + from openjarvis.evals.scorers.email_triage import EmailTriageScorer + scorer = EmailTriageScorer(self._mock_backend(), "gpt-5-mini") + assert scorer.scorer_id == "email_triage" + + def test_morning_brief_scorer(self) -> None: + from openjarvis.evals.scorers.morning_brief import MorningBriefScorer + scorer = MorningBriefScorer(self._mock_backend(), "gpt-5-mini") + assert scorer.scorer_id == "morning_brief" + + def test_research_mining_scorer(self) -> None: + from openjarvis.evals.scorers.research_mining import ResearchMiningScorer + scorer = ResearchMiningScorer(self._mock_backend(), "gpt-5-mini") + assert scorer.scorer_id == "research_mining" + + def test_knowledge_base_scorer(self) -> None: + from openjarvis.evals.scorers.knowledge_base import KnowledgeBaseScorer + scorer = KnowledgeBaseScorer(self._mock_backend(), "gpt-5-mini") + assert scorer.scorer_id == "knowledge_base" + + def test_coding_task_scorer(self) -> None: + from openjarvis.evals.scorers.coding_task import CodingTaskScorer + scorer = CodingTaskScorer(self._mock_backend(), "gpt-5-mini") + assert scorer.scorer_id == "coding_task" + + +# --------------------------------------------------------------------------- +# Scorer functional tests +# --------------------------------------------------------------------------- + + +class TestCodingTaskScoring: + """Test the coding task scorer with actual code execution.""" + + def test_correct_answer(self) -> None: + from openjarvis.evals.core.types import EvalRecord + from openjarvis.evals.scorers.coding_task import CodingTaskScorer + + scorer = CodingTaskScorer() + record = EvalRecord( + record_id="test-1", + problem="Write is_palindrome", + reference="", + category="use-case", + subject="coding_task", + metadata={ + "test_cases": ( + 'assert is_palindrome("racecar") == True\n' + 'assert is_palindrome("hello") == False\n' + 'assert is_palindrome("") == True' + ), + }, + ) + answer = ( + "def is_palindrome(s):\n" + " return s == s[::-1]" + ) + is_correct, meta = scorer.score(record, answer) + assert is_correct is True + assert meta["tests_passed"] == 3 + assert meta["pass_rate"] == 1.0 + + def test_incorrect_answer(self) -> None: + from openjarvis.evals.core.types import EvalRecord + from openjarvis.evals.scorers.coding_task import CodingTaskScorer + + scorer = CodingTaskScorer() + record = EvalRecord( + record_id="test-2", + problem="Write add", + reference="", + category="use-case", + metadata={ + "test_cases": ( + "assert add(1, 2) == 3\n" + "assert add(0, 0) == 0" + ), + }, + ) + # Wrong implementation + answer = "def add(a, b):\n return a * b" + is_correct, meta = scorer.score(record, answer) + assert is_correct is False + + def test_empty_answer(self) -> None: + from openjarvis.evals.core.types import EvalRecord + from openjarvis.evals.scorers.coding_task import CodingTaskScorer + + scorer = CodingTaskScorer() + record = EvalRecord( + record_id="test-3", + problem="Write something", + reference="", + category="use-case", + metadata={"test_cases": "assert True"}, + ) + is_correct, meta = scorer.score(record, "") + assert is_correct is False + assert meta["reason"] == "empty_response" + + +class TestEmailTriageScoring: + """Test exact-match path of email triage scorer.""" + + def test_exact_match(self) -> None: + from openjarvis.evals.core.types import EvalRecord + from openjarvis.evals.scorers.email_triage import EmailTriageScorer + + scorer = EmailTriageScorer(MagicMock(), "gpt-5-mini") + record = EvalRecord( + record_id="test-1", + problem="...", + reference="urgency: high\ncategory: action", + category="use-case", + metadata={"urgency": "high", "category": "action"}, + ) + answer = "urgency: high\ncategory: action\ndraft: I'll look into this." + is_correct, meta = scorer.score(record, answer) + assert is_correct is True + assert meta["match_type"] == "exact" + + +# --------------------------------------------------------------------------- +# CLI factory tests +# --------------------------------------------------------------------------- + + +class TestCLIFactories: + """Test that _build_dataset and _build_scorer work for new benchmarks.""" + + @pytest.mark.parametrize("benchmark", [ + "email_triage", + "morning_brief", + "research_mining", + "knowledge_base", + "coding_task", + ]) + def test_build_dataset(self, benchmark: str) -> None: + from openjarvis.evals.cli import _build_dataset + ds = _build_dataset(benchmark) + assert ds.dataset_id == benchmark + + @pytest.mark.parametrize("benchmark", [ + "email_triage", + "morning_brief", + "research_mining", + "knowledge_base", + "coding_task", + ]) + def test_build_scorer(self, benchmark: str) -> None: + from openjarvis.evals.cli import _build_scorer + scorer = _build_scorer(benchmark, MagicMock(), "gpt-5-mini") + assert scorer.scorer_id == benchmark + + +# --------------------------------------------------------------------------- +# Cost calculator tests +# --------------------------------------------------------------------------- + + +class TestCostCalculator: + """Test the cost calculator module.""" + + def test_estimate_monthly_cost(self) -> None: + from openjarvis.server.cost_calculator import estimate_monthly_cost + est = estimate_monthly_cost( + calls_per_month=1000, + avg_input_tokens=500, + avg_output_tokens=200, + provider_key="gpt-5.3", + ) + assert est.monthly_cost > 0 + assert est.annual_cost == est.monthly_cost * 12 + assert est.total_calls_per_month == 1000 + + def test_estimate_scenario(self) -> None: + from openjarvis.server.cost_calculator import estimate_scenario + estimates = estimate_scenario("daily_briefing") + assert len(estimates) == 3 # 3 cloud providers + for est in estimates: + assert est.monthly_cost > 0 + + def test_estimate_all_scenarios(self) -> None: + from openjarvis.server.cost_calculator import estimate_all_scenarios + all_est = estimate_all_scenarios() + assert len(all_est) == 5 # 5 scenarios + + def test_unknown_provider(self) -> None: + from openjarvis.server.cost_calculator import estimate_monthly_cost + with pytest.raises(ValueError, match="Unknown provider"): + estimate_monthly_cost(100, 100, 100, "nonexistent") + + def test_unknown_scenario(self) -> None: + from openjarvis.server.cost_calculator import estimate_scenario + with pytest.raises(ValueError, match="Unknown scenario"): + estimate_scenario("nonexistent") + + +# --------------------------------------------------------------------------- +# Savings module tests +# --------------------------------------------------------------------------- + + +class TestSavings: + """Test the savings computation module.""" + + def test_compute_savings_basic(self) -> None: + from openjarvis.server.savings import compute_savings + summary = compute_savings(1000, 500, total_calls=10) + assert summary.total_calls == 10 + assert summary.total_tokens == 1500 + assert summary.local_cost == 0.0 + assert len(summary.per_provider) == 3 + for p in summary.per_provider: + assert p.total_cost > 0 + + def test_compute_savings_with_session(self) -> None: + import time + + from openjarvis.server.savings import compute_savings + start = time.time() - 3600 # 1 hour ago + summary = compute_savings( + 100000, 50000, total_calls=100, session_start=start, + ) + assert summary.session_duration_hours > 0 + assert summary.monthly_projection # not empty + + def test_savings_to_dict(self) -> None: + from openjarvis.server.savings import compute_savings, savings_to_dict + summary = compute_savings(1000, 500, total_calls=5) + d = savings_to_dict(summary) + assert isinstance(d, dict) + assert "per_provider" in d + assert "total_calls" in d diff --git a/tests/telemetry/test_flops_new.py b/tests/telemetry/test_flops_new.py new file mode 100644 index 00000000..e67f9997 --- /dev/null +++ b/tests/telemetry/test_flops_new.py @@ -0,0 +1,74 @@ +"""Tests for FLOPs estimation and MFU computation.""" + +from __future__ import annotations + +import pytest + +from openjarvis.telemetry.flops import ( + GPU_PEAK_TFLOPS_BF16, + MODEL_PARAMS_B, + compute_mfu, + estimate_flops, +) + + +class TestEstimateFlops: + def test_known_model(self): + total, per_tok = estimate_flops("qwen3:8b", 100, 50) + # 2 * 8e9 * 150 = 2.4e12 + assert total == pytest.approx(2.4e12) + # 2 * 8e9 = 16e9 + assert per_tok == pytest.approx(16e9) + + def test_unknown_model_zero(self): + total, per_tok = estimate_flops("totally-unknown-model", 100, 50) + assert total == 0.0 + assert per_tok == 0.0 + + def test_prefix_matching(self): + # "llama-3.1-8b-instruct" should match prefix "llama-3.1-8b" + total, per_tok = estimate_flops("llama-3.1-8b-instruct", 10, 10) + assert total > 0 + # 2 * 8e9 * 20 = 3.2e11 + assert total == pytest.approx(3.2e11) + + def test_zero_tokens(self): + total, per_tok = estimate_flops("qwen3:8b", 0, 0) + assert total == 0.0 + assert per_tok == 0.0 + + def test_flops_proportional_to_tokens(self): + total_100, _ = estimate_flops("qwen3:8b", 50, 50) + total_200, _ = estimate_flops("qwen3:8b", 100, 100) + assert total_200 == pytest.approx(total_100 * 2.0) + + +class TestComputeMfu: + def test_known_gpu(self): + # 100 TFLOPS actual for 1s on H100 → 100 / 989 * 100 ≈ 10.1% + flops = 100e12 + mfu = compute_mfu(flops, 1.0, "H100") + assert mfu == pytest.approx(100.0 / 989.0 * 100.0, rel=1e-3) + + def test_unknown_gpu_zero(self): + mfu = compute_mfu(100e12, 1.0, "QuantumGPU") + assert mfu == 0.0 + + def test_zero_duration(self): + mfu = compute_mfu(100e12, 0.0, "H100") + assert mfu == 0.0 + + def test_multi_gpu(self): + flops = 100e12 + mfu_single = compute_mfu(flops, 1.0, "H100", num_gpus=1) + mfu_dual = compute_mfu(flops, 1.0, "H100", num_gpus=2) + assert mfu_dual == pytest.approx(mfu_single / 2.0) + + def test_substring_matching(self): + # "NVIDIA H100 80GB" should match "H100" + mfu = compute_mfu(100e12, 1.0, "NVIDIA H100 80GB") + assert mfu > 0.0 + + def test_tables_nonempty(self): + assert len(GPU_PEAK_TFLOPS_BF16) > 0 + assert len(MODEL_PARAMS_B) > 0 diff --git a/tests/telemetry/test_itl_new.py b/tests/telemetry/test_itl_new.py new file mode 100644 index 00000000..66f1ec00 --- /dev/null +++ b/tests/telemetry/test_itl_new.py @@ -0,0 +1,63 @@ +"""Tests for inter-token latency percentile computation.""" + +from __future__ import annotations + +import pytest + +from openjarvis.telemetry.itl import compute_itl_stats + + +class TestComputeItlStats: + def test_empty_timestamps(self): + result = compute_itl_stats([]) + assert result["p50_ms"] == 0 + assert result["mean_ms"] == 0 + + def test_single_timestamp(self): + result = compute_itl_stats([100.0]) + assert result["p50_ms"] == 0 + assert result["max_ms"] == 0 + + def test_two_timestamps(self): + result = compute_itl_stats([0.0, 10.0]) + assert result["p50_ms"] == 10.0 + assert result["mean_ms"] == 10.0 + assert result["min_ms"] == 10.0 + assert result["max_ms"] == 10.0 + + def test_uniform_spacing(self): + # 11 timestamps 5ms apart → 10 ITLs all = 5.0 + timestamps = [i * 5.0 for i in range(11)] + result = compute_itl_stats(timestamps) + assert result["p50_ms"] == pytest.approx(5.0) + assert result["p90_ms"] == pytest.approx(5.0) + assert result["p95_ms"] == pytest.approx(5.0) + assert result["p99_ms"] == pytest.approx(5.0) + assert result["mean_ms"] == pytest.approx(5.0) + assert result["min_ms"] == pytest.approx(5.0) + assert result["max_ms"] == pytest.approx(5.0) + + def test_varying_spacing(self): + # [0, 1, 3, 6, 10] → ITLs = [1, 2, 3, 4] + timestamps = [0.0, 1.0, 3.0, 6.0, 10.0] + result = compute_itl_stats(timestamps) + assert result["min_ms"] == pytest.approx(1.0) + assert result["max_ms"] == pytest.approx(4.0) + assert result["mean_ms"] == pytest.approx(2.5) + # Median of sorted [1,2,3,4] → 2.5 + assert result["p50_ms"] == pytest.approx(2.5) + + def test_percentile_ordering(self): + timestamps = [float(i) for i in range(101)] + result = compute_itl_stats(timestamps) + assert result["p50_ms"] <= result["p90_ms"] + assert result["p90_ms"] <= result["p95_ms"] + assert result["p95_ms"] <= result["p99_ms"] + + def test_all_keys_present(self): + result = compute_itl_stats([0.0, 5.0, 10.0]) + expected_keys = { + "p50_ms", "p90_ms", "p95_ms", "p99_ms", + "mean_ms", "min_ms", "max_ms", + } + assert set(result.keys()) == expected_keys diff --git a/tests/telemetry/test_phase_metrics_new.py b/tests/telemetry/test_phase_metrics_new.py new file mode 100644 index 00000000..d4c5f23f --- /dev/null +++ b/tests/telemetry/test_phase_metrics_new.py @@ -0,0 +1,49 @@ +"""Tests for phase metrics computation.""" + +from __future__ import annotations + +import pytest + +from openjarvis.telemetry.phase_metrics import compute_phase_metrics, split_at_ttft +from openjarvis.telemetry.session import TelemetrySample, TelemetrySession + + +class TestComputePhaseMetrics: + def _make_session_with_samples(self): + """Create a session with pre-loaded samples.""" + session = TelemetrySession(monitor=None) + # Manually push samples into the buffer + for i in range(11): + session._buffer.push(TelemetrySample( + timestamp_ns=i * 100_000_000, + gpu_power_w=100.0, + cpu_power_w=50.0, + )) + return session + + def test_basic_metrics(self): + session = self._make_session_with_samples() + result = compute_phase_metrics(session, 0, 1_000_000_000, 100) + assert result["duration_s"] == pytest.approx(1.0, abs=1e-6) + assert result["tokens"] == 100 + assert result["energy_j"] > 0 # trapezoidal integral + + def test_zero_tokens(self): + session = self._make_session_with_samples() + result = compute_phase_metrics(session, 0, 1_000_000_000, 0) + assert result["energy_per_token_j"] == 0.0 + + def test_split_at_ttft(self): + session = self._make_session_with_samples() + prefill, decode = split_at_ttft( + session, + start_ns=0, + ttft_ns=500_000_000, # 500ms + end_ns=1_000_000_000, # 1s + input_tokens=50, + output_tokens=100, + ) + assert prefill["tokens"] == 50 + assert decode["tokens"] == 100 + assert prefill["duration_s"] == pytest.approx(0.5, abs=1e-6) + assert decode["duration_s"] == pytest.approx(0.5, abs=1e-6) diff --git a/tests/telemetry/test_session.py b/tests/telemetry/test_session.py new file mode 100644 index 00000000..7226b27e --- /dev/null +++ b/tests/telemetry/test_session.py @@ -0,0 +1,90 @@ +"""Tests for TelemetrySession and ring buffer.""" + +from __future__ import annotations + +from openjarvis.telemetry.session import ( + TelemetrySample, + TelemetrySession, + _PythonRingBuffer, +) + + +class TestPythonRingBuffer: + def test_push_and_len(self): + buf = _PythonRingBuffer(capacity=10) + assert len(buf) == 0 + buf.push(TelemetrySample(timestamp_ns=100)) + assert len(buf) == 1 + + def test_capacity_overflow(self): + buf = _PythonRingBuffer(capacity=3) + for i in range(5): + buf.push(TelemetrySample(timestamp_ns=i * 100)) + assert len(buf) == 3 + + def test_window(self): + buf = _PythonRingBuffer(capacity=100) + for i in range(10): + buf.push(TelemetrySample(timestamp_ns=i * 1000)) + result = buf.window(2000, 6000) + assert len(result) == 5 # 2000, 3000, 4000, 5000, 6000 + + def test_clear(self): + buf = _PythonRingBuffer(capacity=10) + buf.push(TelemetrySample(timestamp_ns=1)) + buf.clear() + assert len(buf) == 0 + + def test_energy_delta_trapezoidal(self): + """Trapezoidal integration over constant power should yield P*t.""" + buf = _PythonRingBuffer(capacity=100) + # 100W GPU, 50W CPU for 1 second (10 samples, 100ms apart) + for i in range(11): + buf.push(TelemetrySample( + timestamp_ns=i * 100_000_000, # 0ms, 100ms, ..., 1000ms + gpu_power_w=100.0, + cpu_power_w=50.0, + )) + gpu_j, cpu_j = buf.compute_energy_delta(0, 1_000_000_000) + assert abs(gpu_j - 100.0) < 1.0 # 100W * 1s = 100J + assert abs(cpu_j - 50.0) < 1.0 # 50W * 1s = 50J + + def test_energy_delta_insufficient_samples(self): + buf = _PythonRingBuffer(capacity=100) + buf.push(TelemetrySample(timestamp_ns=0, gpu_power_w=100.0)) + gpu_j, cpu_j = buf.compute_energy_delta(0, 1_000_000_000) + assert gpu_j == 0.0 + assert cpu_j == 0.0 + + def test_avg_power(self): + buf = _PythonRingBuffer(capacity=100) + buf.push(TelemetrySample(timestamp_ns=0, gpu_power_w=100.0, cpu_power_w=40.0)) + buf.push(TelemetrySample( + timestamp_ns=500_000_000, gpu_power_w=200.0, + cpu_power_w=60.0, + )) + gpu_w, cpu_w = buf.compute_avg_power(0, 1_000_000_000) + assert gpu_w == 150.0 + assert cpu_w == 50.0 + + +class TestTelemetrySession: + def test_noop_session(self): + """Session with no monitor should be a safe no-op.""" + session = TelemetrySession(monitor=None) + with session: + samples = session.window(0, 1_000_000_000) + assert samples == [] + gpu_j, cpu_j = session.energy_delta(0, 1_000_000_000) + assert gpu_j == 0.0 + assert cpu_j == 0.0 + + def test_context_manager(self): + session = TelemetrySession(monitor=None) + with session as s: + assert s is session + + def test_start_stop(self): + session = TelemetrySession(monitor=None) + session.start() + session.stop()