admin/OpenJarvis
1
0
Fork 0
mirror of https://github.com/open-jarvis/OpenJarvis.git synced 2026-07-30 19:02:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/main into gabebo-ui

Browse Source 
Integrate speech-to-text (Phase 24) and eval trackers features.
Resolve conflicts: add speech Tauri commands to lib.rs, merge
MicButton + useSpeech into InputArea, consolidate speech API
functions into lib/api.ts (removed old api/client.ts).

Made-with: Cursor
This commit is contained in:
Gabriel Bo
2026-03-03 18:44:09 -08:00
parent 62dd0ce6c9 ceb8e6a47d
commit 75dc674e44
49 changed files with 5093 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openjarvis/cli/__init__.py
+2
View File
@@ -14,6 +14,7 @@ from openjarvis.cli.chat_cmd import chat
from openjarvis.cli.daemon_cmd import restart, start, status, stop
from openjarvis.cli.doctor_cmd import doctor
from openjarvis.cli.eval_cmd import eval_group
from openjarvis.cli.host_cmd import host
from openjarvis.cli.init_cmd import init
from openjarvis.cli.memory_cmd import memory
from openjarvis.cli.model import model
@@ -64,6 +65,7 @@ cli.add_command(vault, "vault")
cli.add_command(add, "add")
cli.add_command(operators, "operators")
cli.add_command(eval_group, "eval")
cli.add_command(host, "host")
cli.add_command(quickstart, "quickstart")
src/openjarvis/cli/ask.py
+137 -2
View File
@@ -4,13 +4,15 @@ from __future__ import annotations
import json as json_mod
import sys
import time
import click
from rich.console import Console
from rich.table import Table
from openjarvis.cli.hints import hint_no_engine
from openjarvis.core.config import load_config
from openjarvis.core.events import EventBus
from openjarvis.core.events import EventBus, EventType
from openjarvis.core.types import Message, Role
from openjarvis.engine import (
EngineConnectionError,
@@ -147,6 +149,119 @@ def _run_agent(
return agent.run(query_text, context=ctx)
def _print_profile(
bus: EventBus,
wall_seconds: float,
engine_name: str,
model_name: str,
console: Console,
) -> None:
"""Print an inference telemetry profile table from EventBus history."""
# Collect all INFERENCE_END events (agents may fire multiple)
inf_events = [
e for e in bus.history if e.event_type == EventType.INFERENCE_END
]
if not inf_events:
console.print("[dim]No inference telemetry recorded.[/dim]")
return
total_calls = len(inf_events)
# Aggregate across all inference calls
total_latency = sum(e.data.get("latency", 0.0) for e in inf_events)
total_tokens = sum(
e.data.get("usage", {}).get("completion_tokens", 0)
or e.data.get("completion_tokens", 0)
for e in inf_events
)
total_prompt = sum(
e.data.get("usage", {}).get("prompt_tokens", 0)
for e in inf_events
)
total_energy = sum(e.data.get("energy_joules", 0.0) for e in inf_events)
avg_power = 0.0
power_vals = [e.data.get("power_watts", 0.0) for e in inf_events
if e.data.get("power_watts", 0.0) > 0]
if power_vals:
avg_power = sum(power_vals) / len(power_vals)
throughput = total_tokens / total_latency if total_latency > 0 else 0.0
energy_per_tok = total_energy / total_tokens if total_tokens > 0 else 0.0
tpw = throughput / avg_power if avg_power > 0 else 0.0
tok_per_j = total_tokens / total_energy if total_energy > 0 else 0.0
last = inf_events[-1].data
ttft = last.get("ttft", 0.0)
prefill_lat = last.get("prefill_latency_seconds", 0.0)
decode_lat = last.get("decode_latency_seconds", 0.0)
prefill_e = sum(e.data.get("prefill_energy_joules", 0.0) for e in inf_events)
decode_e = sum(e.data.get("decode_energy_joules", 0.0) for e in inf_events)
gpu_util = last.get("gpu_utilization_pct", 0.0)
gpu_mem = last.get("gpu_memory_used_gb", 0.0)
gpu_temp = last.get("gpu_temperature_c", 0.0)
mean_itl = last.get("mean_itl_ms", 0.0)
e_method = last.get("energy_method", "")
e_vendor = last.get("energy_vendor", "")
# Build the profile table
table = Table(
title=f"Inference Profile ({engine_name} / {model_name})",
show_header=True,
header_style="bold bright_white",
border_style="bright_blue",
title_style="bold cyan",
)
table.add_column("Metric", style="cyan", no_wrap=True)
table.add_column("Value", justify="right", style="green")
def _row(label: str, val: str) -> None:
table.add_row(label, val)
_row("Wall time", f"{wall_seconds:.3f} s")
_row("Inference calls", str(total_calls))
_row("Total latency", f"{total_latency:.3f} s")
if ttft > 0:
_row("TTFT", f"{ttft * 1000:.1f} ms")
if prefill_lat > 0:
_row("Prefill latency", f"{prefill_lat * 1000:.1f} ms")
if decode_lat > 0:
_row("Decode latency", f"{decode_lat:.3f} s")
if mean_itl > 0:
_row("Mean ITL", f"{mean_itl:.2f} ms")
_row("Prompt tokens", str(total_prompt))
_row("Completion tokens", str(total_tokens))
_row("Throughput", f"{throughput:.1f} tok/s")
if total_energy > 0:
_row("", "") # separator
_row("Energy", f"{total_energy:.4f} J")
if prefill_e > 0:
_row(" Prefill energy", f"{prefill_e:.4f} J")
if decode_e > 0:
_row(" Decode energy", f"{decode_e:.4f} J")
_row("Energy / output token", f"{energy_per_tok:.6f} J")
_row("Tokens / joule", f"{tok_per_j:.1f}")
_row("Throughput / watt (IPW)", f"{tpw:.2f} tok/s/W")
if avg_power > 0:
_row("Avg power draw", f"{avg_power:.1f} W")
if e_vendor:
_row("Energy vendor", e_vendor)
if e_method:
_row("Energy method", e_method)
if gpu_util > 0 or gpu_mem > 0 or gpu_temp > 0:
_row("", "") # separator
if gpu_util > 0:
_row("GPU utilization", f"{gpu_util:.1f} %")
if gpu_mem > 0:
_row("GPU memory used", f"{gpu_mem:.2f} GB")
if gpu_temp > 0:
_row("GPU temperature", f"{gpu_temp:.0f} °C")
console.print()
console.print(table)
@click.command()
@click.argument("query", nargs=-1, required=True)
@click.option("-m", "--model", "model_name", default=None, help="Model to use.")
@@ -173,6 +288,10 @@ def _run_agent(
"--tools", "tool_names", default=None,
help="Comma-separated tool names to enable (e.g. calculator,think).",
)
@click.option(
"--profile", "enable_profile", is_flag=True,
help="Print inference telemetry profile (latency, tokens, energy, IPW).",
)
def ask(
query: tuple[str, ...],
model_name: str | None,
@@ -184,11 +303,14 @@ def ask(
no_context: bool,
agent_name: str | None,
tool_names: str | None,
enable_profile: bool,
) -> None:
"""Ask Jarvis a question."""
console = Console(stderr=True)
query_text = " ".join(query)
wall_start = time.monotonic() if enable_profile else None
# Load config
config = load_config()
@@ -228,7 +350,8 @@ def ask(
# Wrap engine with InstrumentedEngine for telemetry (energy + GPU metrics)
energy_monitor = None
if config.telemetry.gpu_metrics:
want_energy = config.telemetry.gpu_metrics or enable_profile
if want_energy:
try:
from openjarvis.telemetry.energy_monitor import create_energy_monitor
@@ -288,6 +411,12 @@ def ask(
else:
click.echo(result.content)
if enable_profile:
_print_profile(
bus, time.monotonic() - wall_start,
engine_name, model_name, console,
)
if telem_store is not None:
try:
telem_store.close()
@@ -341,6 +470,12 @@ def ask(
else:
click.echo(result.get("content", ""))
if enable_profile:
_print_profile(
bus, time.monotonic() - wall_start,
engine_name, model_name, console,
)
# Cleanup
if energy_monitor is not None:
try:
src/openjarvis/cli/eval_cmd.py
+42
View File
@@ -113,6 +113,34 @@ def eval_list() -> None:
"-o", "--output", "output_path", default=None, type=click.Path(),
help="Output JSONL path.",
)
@click.option(
"--wandb-project", "wandb_project", default="",
help="W&B project name (enables W&B tracking).",
)
@click.option(
"--wandb-entity", "wandb_entity", default="",
help="W&B entity (team or user).",
)
@click.option(
"--wandb-tags", "wandb_tags", default="",
help="Comma-separated W&B tags.",
)
@click.option(
"--wandb-group", "wandb_group", default="",
help="W&B run group.",
)
@click.option(
"--sheets-id", "sheets_spreadsheet_id", default="",
help="Google Sheets spreadsheet ID.",
)
@click.option(
"--sheets-worksheet", "sheets_worksheet", default="Results",
help="Google Sheets worksheet name.",
)
@click.option(
"--sheets-creds", "sheets_credentials_path", default="",
help="Path to Google service account JSON.",
)
@click.option(
"-v", "--verbose", "verbose", is_flag=True, default=False,
help="Verbose logging.",
@@ -127,6 +155,13 @@ def eval_run(
tools: str,
telemetry: bool,
output_path: Optional[str],
wandb_project: str,
wandb_entity: str,
wandb_tags: str,
wandb_group: str,
sheets_spreadsheet_id: str,
sheets_worksheet: str,
sheets_credentials_path: str,
verbose: bool,
) -> None:
"""Run evaluation benchmarks."""
@@ -216,6 +251,13 @@ def eval_run(
tools=tool_list,
output_path=output_path,
telemetry=telemetry,
wandb_project=wandb_project,
wandb_entity=wandb_entity,
wandb_tags=wandb_tags,
wandb_group=wandb_group,
sheets_spreadsheet_id=sheets_spreadsheet_id,
sheets_worksheet=sheets_worksheet,
sheets_credentials_path=sheets_credentials_path,
)
try:
src/openjarvis/cli/host_cmd.py
+366
View File
@@ -0,0 +1,366 @@
"""``jarvis host`` — download and serve a model locally with auto backend setup."""
from __future__ import annotations
import os
import shutil
import signal
import subprocess
import sys
from typing import Optional
import click
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
_BACKENDS = {
"mlx": {
"display": "MLX (Apple Silicon)",
"package": "mlx-lm",
"import_check": "mlx_lm",
"pip_spec": "mlx-lm>=0.19",
"uv_extra": "inference-mlx",
"platform": "darwin",
"default_port": 8080,
},
"vllm": {
"display": "vLLM (NVIDIA GPU)",
"package": "vllm",
"import_check": "vllm",
"pip_spec": "vllm",
"uv_extra": None,
"platform": "linux",
"default_port": 8000,
},
"sglang": {
"display": "SGLang (NVIDIA GPU)",
"package": "sglang",
"import_check": "sglang",
"pip_spec": "sglang[all]",
"uv_extra": None,
"platform": None,
"default_port": 30000,
},
"ollama": {
"display": "Ollama",
"package": "ollama",
"import_check": None,
"pip_spec": None,
"uv_extra": None,
"platform": None,
"default_port": 11434,
"binary": "ollama",
},
"llamacpp": {
"display": "llama.cpp",
"package": "llama.cpp",
"import_check": None,
"pip_spec": None,
"uv_extra": None,
"platform": None,
"default_port": 8080,
"binary": "llama-server",
},
}
def _is_package_available(backend: str) -> bool:
"""Check whether the backend's Python package or binary is importable/available."""
info = _BACKENDS[backend]
if info.get("import_check"):
try:
__import__(info["import_check"])
return True
except ImportError:
return False
binary = info.get("binary")
if binary:
return shutil.which(binary) is not None
return False
def _detect_backend() -> str | None:
"""Auto-detect the best backend for the current platform."""
import platform
system = platform.system().lower()
if system == "darwin":
try:
result = subprocess.run(
["sysctl", "-n", "machdep.cpu.brand_string"],
capture_output=True,
text=True,
)
if "Apple" in result.stdout:
return "mlx"
except FileNotFoundError:
pass
return "ollama"
if system == "linux":
if shutil.which("nvidia-smi"):
return "vllm"
return "ollama"
return "ollama"
def _install_backend(backend: str, console: Console) -> bool:
"""Prompt the user and install the backend package. Returns True on success."""
info = _BACKENDS[backend]
display = info["display"]
console.print()
console.print(
f"[yellow]Backend [bold]{display}[/bold] is not installed.[/yellow]"
)
uv_available = shutil.which("uv") is not None
in_uv_project = os.path.exists("pyproject.toml") and os.path.exists("uv.lock")
if info.get("binary") and not info.get("pip_spec"):
return _install_binary_backend(backend, console)
pip_spec = info["pip_spec"]
uv_extra = info.get("uv_extra")
if uv_available and in_uv_project and uv_extra:
install_cmd = ["uv", "pip", "install", pip_spec]
install_label = f"uv pip install {pip_spec}"
elif uv_available:
install_cmd = ["uv", "pip", "install", pip_spec]
install_label = f"uv pip install {pip_spec}"
else:
install_cmd = [sys.executable, "-m", "pip", "install", pip_spec]
install_label = f"pip install {pip_spec}"
console.print(f"\n Install command: [cyan]{install_label}[/cyan]\n")
if not click.confirm("Install now?", default=True):
console.print("[dim]Skipped. Install manually and retry.[/dim]")
return False
console.print(f"[bold]Running:[/bold] {install_label}")
result = subprocess.run(install_cmd)
if result.returncode != 0:
console.print(
f"[red]Installation failed (exit {result.returncode}).[/red]"
)
return False
console.print(f"[green]{display} installed successfully.[/green]\n")
return True
def _install_binary_backend(backend: str, console: Console) -> bool:
"""Guide the user through installing a binary backend (Ollama, llama.cpp)."""
info = _BACKENDS[backend]
binary = info["binary"]
instructions = {
"ollama": (
"Install Ollama:\n"
"\n"
" macOS / Linux:\n"
" curl -fsSL https://ollama.com/install.sh | sh\n"
"\n"
" Or download from: https://ollama.com/download"
),
"llamacpp": (
"Install llama.cpp:\n"
"\n"
" macOS:\n"
" brew install llama.cpp\n"
"\n"
" From source:\n"
" https://github.com/ggerganov/llama.cpp"
),
}
console.print()
console.print(
Panel(
instructions.get(
backend,
f"Install {binary} and ensure it's on PATH.",
),
title=f"{info['display']} Installation",
border_style="yellow",
)
)
if backend == "ollama":
import platform
system = platform.system().lower()
if system in ("linux", "darwin"):
if click.confirm("Run the Ollama install script now?", default=True):
console.print("[bold]Running Ollama installer...[/bold]")
result = subprocess.run(
["sh", "-c", "curl -fsSL https://ollama.com/install.sh | sh"],
)
if result.returncode == 0 and shutil.which("ollama"):
console.print("[green]Ollama installed successfully.[/green]\n")
return True
console.print(
"[red]Installation may have failed. "
"Check above for errors.[/red]"
)
return False
console.print("[dim]Install manually and retry.[/dim]")
return False
def _build_serve_command(backend: str, model: str, port: int) -> list[str]:
"""Build the subprocess command list to start the inference server."""
if backend == "mlx":
return [
sys.executable,
"-m",
"mlx_lm.server",
"--model",
model,
"--port",
str(port),
]
if backend == "vllm":
return ["vllm", "serve", model, "--port", str(port)]
if backend == "sglang":
return [
sys.executable,
"-m",
"sglang.launch_server",
"--model-path",
model,
"--port",
str(port),
]
if backend == "ollama":
return ["ollama", "run", model]
if backend == "llamacpp":
return ["llama-server", "-m", model, "--port", str(port)]
raise ValueError(f"Unknown backend: {backend}")
@click.command()
@click.argument("model")
@click.option(
"-b",
"--backend",
type=click.Choice(list(_BACKENDS.keys()), case_sensitive=False),
default=None,
help="Inference backend to use. Auto-detected if omitted.",
)
@click.option(
"-p",
"--port",
type=int,
default=None,
help="Port to serve on (default depends on backend).",
)
@click.option(
"--trust-remote-code",
is_flag=True,
default=False,
help="Pass --trust-remote-code to the backend.",
)
def host(
model: str,
backend: Optional[str],
port: Optional[int],
trust_remote_code: bool,
) -> None:
"""Download (if needed) and serve a model locally.
Examples:
\b
jarvis host mlx-community/Qwen2.5-7B-4bit --backend mlx
jarvis host Qwen/Qwen3-8B --backend vllm
jarvis host qwen3:8b --backend ollama
jarvis host meta-llama/Llama-3-8B -b sglang
"""
console = Console()
if backend is None:
detected = _detect_backend()
if detected is None:
console.print("[red]Could not auto-detect a suitable backend.[/red]")
console.print("Specify one with [cyan]--backend[/cyan].")
raise SystemExit(1)
backend = detected
name = _BACKENDS[backend]["display"]
console.print(
f"Auto-detected backend: [bold cyan]{name}[/bold cyan]"
)
info = _BACKENDS[backend]
if not _is_package_available(backend):
if not _install_backend(backend, console):
raise SystemExit(1)
if not _is_package_available(backend):
console.print(
f"[red]{info['display']} still not available after install.[/red]"
)
console.print(
"You may need to restart your shell or "
"activate the correct environment."
)
raise SystemExit(1)
serve_port = port or info["default_port"]
cmd = _build_serve_command(backend, model, serve_port)
if trust_remote_code:
if backend in ("vllm", "sglang"):
cmd.append("--trust-remote-code")
elif backend == "mlx":
cmd.extend(["--trust-remote-code", "True"])
host_url = f"http://localhost:{serve_port}"
table = Table.grid(padding=(0, 2))
table.add_row("[bold]Backend:[/bold]", info["display"])
table.add_row("[bold]Model:[/bold]", model)
table.add_row("[bold]Endpoint:[/bold]", host_url)
table.add_row("[bold]Command:[/bold]", " ".join(cmd))
console.print()
console.print(Panel(table, title="Hosting Model", border_style="green"))
console.print()
if backend != "ollama":
console.print(
f"[dim]The model server will be available at {host_url}[/dim]"
)
console.print(
"[dim]OpenJarvis will auto-discover it. "
"Press Ctrl+C to stop.[/dim]\n"
)
try:
proc = subprocess.Popen(cmd)
proc.wait()
except KeyboardInterrupt:
console.print("\n[yellow]Shutting down model server...[/yellow]")
proc.send_signal(signal.SIGTERM)
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
console.print("[green]Server stopped.[/green]")
except FileNotFoundError:
console.print(f"[red]Command not found:[/red] {cmd[0]}")
console.print(f"Make sure {info['display']} is installed and on your PATH.")
raise SystemExit(1)
src/openjarvis/cli/init_cmd.py
+17 -3
View File
@@ -5,6 +5,8 @@ from __future__ import annotations
import click
from rich.console import Console
from rich.panel import Panel
from pathlib import Path
from typing import Optional
from openjarvis.core.config import (
DEFAULT_CONFIG_DIR,
@@ -113,7 +115,12 @@ def _next_steps_text(engine: str) -> str:
@click.option(
"--force", is_flag=True, help="Overwrite existing config without prompting."
)
def init(force: bool) -> None:
@click.option(
"--config",
type=click.Path(exists=True),
help="Path to config file to use.",
)
def init(force: bool, config: Optional[Path]) -> None:
"""Detect hardware and generate ~/.openjarvis/config.toml."""
console = Console()
@@ -137,10 +144,16 @@ def init(force: bool) -> None:
else:
console.print(" GPU : none detected")
toml_content = generate_default_toml(hw)
if config:
toml_content = config.read_text()
else:
toml_content = generate_default_toml(hw)
DEFAULT_CONFIG_DIR.mkdir(parents=True, exist_ok=True)
DEFAULT_CONFIG_PATH.write_text(toml_content)
if config:
config.write_text(toml_content)
else:
DEFAULT_CONFIG_PATH.write_text(toml_content)
console.print()
console.print(
@@ -157,3 +170,4 @@ def init(force: bool) -> None:
border_style="cyan",
)
)
src/openjarvis/cli/serve.py
+11
View File
@@ -185,6 +185,16 @@ def serve(
console.print(f"[yellow]Channel failed to start: {exc}[/yellow]")
channel_bridge = None
# Set up speech backend
speech_backend = None
try:
from openjarvis.speech._discovery import get_speech_backend
speech_backend = get_speech_backend(config)
if speech_backend:
console.print(f" Speech: [cyan]{speech_backend.backend_id}[/cyan]")
except Exception:
pass
# Create app
from openjarvis.server.app import create_app
@@ -192,6 +202,7 @@ def serve(
engine, model_name, agent=agent, bus=bus,
engine_name=engine_name, agent_name=agent_key or "",
channel_bridge=channel_bridge, config=config,
speech_backend=speech_backend,
)
console.print(
src/openjarvis/core/config.py
+14
View File
@@ -830,6 +830,17 @@ class OperatorsConfig:
auto_activate: str = "" # Comma-separated operator IDs
@dataclass(slots=True)
class SpeechConfig:
"""Speech-to-text settings."""
backend: str = "auto" # "auto", "faster-whisper", "openai", "deepgram"
model: str = "base" # Whisper model size: tiny, base, small, medium, large-v3
language: str = "" # Empty = auto-detect
device: str = "auto" # "auto", "cpu", "cuda"
compute_type: str = "float16" # "float16", "int8", "float32"
@dataclass
class JarvisConfig:
"""Top-level configuration for OpenJarvis."""
@@ -851,6 +862,7 @@ class JarvisConfig:
sessions: SessionConfig = field(default_factory=SessionConfig)
a2a: A2AConfig = field(default_factory=A2AConfig)
operators: OperatorsConfig = field(default_factory=OperatorsConfig)
speech: SpeechConfig = field(default_factory=SpeechConfig)
@property
def memory(self) -> StorageConfig:
@@ -945,6 +957,7 @@ def load_config(path: Optional[Path] = None) -> JarvisConfig:
"server", "telemetry", "traces", "security",
"channel", "tools", "sandbox", "scheduler",
"workflow", "sessions", "a2a", "operators",
"speech",
)
for section_name in top_sections:
if section_name in data:
@@ -1196,6 +1209,7 @@ __all__ = [
"SessionConfig",
"SignalChannelConfig",
"SlackChannelConfig",
"SpeechConfig",
"StorageConfig",
"TeamsChannelConfig",
"TelegramChannelConfig",
src/openjarvis/core/registry.py
+5
View File
@@ -137,6 +137,10 @@ class SkillRegistry(RegistryBase[Any]):
"""Registry for skill manifests."""
class SpeechRegistry(RegistryBase[Any]):
"""Registry for speech backend implementations."""
__all__ = [
"AgentRegistry",
"BenchmarkRegistry",
@@ -148,5 +152,6 @@ __all__ = [
"RegistryBase",
"RouterPolicyRegistry",
"SkillRegistry",
"SpeechRegistry",
"ToolRegistry",
]
src/openjarvis/evals/cli.py
+68 -3
View File
@@ -217,6 +217,39 @@ def _print_summary(
print_completion(console, summary, output_path, traces_dir)
def _build_trackers(config) -> list:
"""Build tracker instances from RunConfig fields."""
trackers = []
if getattr(config, "wandb_project", ""):
try:
from openjarvis.evals.trackers.wandb_tracker import WandbTracker
trackers.append(WandbTracker(
project=config.wandb_project,
entity=getattr(config, "wandb_entity", ""),
tags=getattr(config, "wandb_tags", ""),
group=getattr(config, "wandb_group", ""),
))
except ImportError as exc:
raise click.ClickException(
f"wandb not installed: {exc}\n"
"Install with: pip install 'openjarvis[eval-wandb]'"
) from exc
if getattr(config, "sheets_spreadsheet_id", ""):
try:
from openjarvis.evals.trackers.sheets_tracker import SheetsTracker
trackers.append(SheetsTracker(
spreadsheet_id=config.sheets_spreadsheet_id,
worksheet=getattr(config, "sheets_worksheet", "Results"),
credentials_path=getattr(config, "sheets_credentials_path", ""),
))
except ImportError as exc:
raise click.ClickException(
f"gspread not installed: {exc}\n"
"Install with: pip install 'openjarvis[eval-sheets]'"
) from exc
return trackers
def _run_single(config, console: Optional[Console] = None) -> object:
"""Run a single eval from a RunConfig and return the summary."""
from openjarvis.evals.core.runner import EvalRunner
@@ -236,7 +269,8 @@ def _run_single(config, console: Optional[Console] = None) -> object:
judge_backend = _build_judge_backend(config.judge_model)
scorer = _build_scorer(config.benchmark, judge_backend, config.judge_model)
runner = EvalRunner(config, dataset, eval_backend, scorer)
trackers = _build_trackers(config)
runner = EvalRunner(config, dataset, eval_backend, scorer, trackers=trackers)
try:
num_samples = config.max_samples or 0
# Use progress bar if we know the sample count
@@ -292,6 +326,11 @@ def _run_from_config(config_path: str, verbose: bool) -> None:
output_dir = Path(suite.run.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# Auto-set wandb_group to suite name if W&B enabled and no explicit group
for rc in run_configs:
if rc.wandb_project and not rc.wandb_group:
rc.wandb_group = suite_name
summaries = []
for i, rc in enumerate(run_configs, 1):
print_section(
@@ -355,12 +394,30 @@ def main():
help="Enable GPU metrics collection")
@click.option("--compact", is_flag=True, default=False, help="Dense single-table output")
@click.option("--trace-detail", is_flag=True, default=False, help="Full per-step trace listing")
@click.option("--wandb-project", default="",
help="W&B project name (enables tracking)")
@click.option("--wandb-entity", default="",
help="W&B entity (team or user)")
@click.option("--wandb-tags", default="",
help="Comma-separated W&B tags")
@click.option("--wandb-group", default="",
help="W&B run group")
@click.option("--sheets-id", "sheets_spreadsheet_id", default="",
help="Google Sheets spreadsheet ID")
@click.option("--sheets-worksheet", default="Results",
help="Google Sheets worksheet name")
@click.option("--sheets-creds", "sheets_credentials_path",
default="",
help="Service account JSON path")
@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
@click.pass_context
def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name,
tools, max_samples, max_workers, judge_model, output_path, seed,
dataset_split, temperature, max_tokens, telemetry, gpu_metrics,
compact, trace_detail, verbose):
compact, trace_detail,
wandb_project, wandb_entity, wandb_tags, wandb_group,
sheets_spreadsheet_id, sheets_worksheet, sheets_credentials_path,
verbose):
"""Run a single benchmark evaluation, or a full suite from a TOML config."""
_setup_logging(verbose)
@@ -404,6 +461,13 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name,
dataset_split=dataset_split,
telemetry=telemetry,
gpu_metrics=gpu_metrics,
wandb_project=wandb_project,
wandb_entity=wandb_entity,
wandb_tags=wandb_tags,
wandb_group=wandb_group,
sheets_spreadsheet_id=sheets_spreadsheet_id,
sheets_worksheet=sheets_worksheet,
sheets_credentials_path=sheets_credentials_path,
)
# Banner + config
@@ -493,7 +557,8 @@ def run_all(model, engine_key, max_samples, max_workers, judge_model,
judge_backend = _build_judge_backend(judge_model)
scorer = _build_scorer(bench_name, judge_backend, judge_model)
runner = EvalRunner(config, dataset, eval_backend, scorer)
trackers = _build_trackers(config)
runner = EvalRunner(config, dataset, eval_backend, scorer, trackers=trackers)
try:
if max_samples and max_samples > 0:
with Progress(
src/openjarvis/evals/core/config.py
+14
View File
@@ -99,6 +99,13 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig:
gpu_metrics=bool(run_raw.get("gpu_metrics", False)),
warmup_samples=int(run_raw.get("warmup_samples", 0)),
energy_vendor=run_raw.get("energy_vendor", ""),
wandb_project=run_raw.get("wandb_project", ""),
wandb_entity=run_raw.get("wandb_entity", ""),
wandb_tags=run_raw.get("wandb_tags", ""),
wandb_group=run_raw.get("wandb_group", ""),
sheets_spreadsheet_id=run_raw.get("sheets_spreadsheet_id", ""),
sheets_worksheet=run_raw.get("sheets_worksheet", "Results"),
sheets_credentials_path=run_raw.get("sheets_credentials_path", ""),
)
# Parse [[models]]
@@ -243,6 +250,13 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]:
gpu_metrics=suite.run.gpu_metrics,
metadata=model_meta,
warmup_samples=suite.run.warmup_samples,
wandb_project=suite.run.wandb_project,
wandb_entity=suite.run.wandb_entity,
wandb_tags=suite.run.wandb_tags,
wandb_group=suite.run.wandb_group,
sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id,
sheets_worksheet=suite.run.sheets_worksheet,
sheets_credentials_path=suite.run.sheets_credentials_path,
))
return configs
src/openjarvis/evals/core/runner.py
+47 -1
View File
@@ -14,7 +14,14 @@ from typing import Any, Callable, Dict, List, Optional
from openjarvis.evals.core.backend import InferenceBackend
from openjarvis.evals.core.dataset import DatasetProvider
from openjarvis.evals.core.scorer import Scorer
from openjarvis.evals.core.types import EvalRecord, EvalResult, MetricStats, RunConfig, RunSummary
from openjarvis.evals.core.tracker import ResultTracker
from openjarvis.evals.core.types import (
EvalRecord,
EvalResult,
MetricStats,
RunConfig,
RunSummary,
)
try:
from openjarvis.telemetry.efficiency import compute_efficiency
@@ -33,11 +40,13 @@ class EvalRunner:
dataset: DatasetProvider,
backend: InferenceBackend,
scorer: Scorer,
trackers: Optional[List[ResultTracker]] = None,
) -> None:
self._config = config
self._dataset = dataset
self._backend = backend
self._scorer = scorer
self._trackers: List[ResultTracker] = trackers or []
self._results: List[EvalResult] = []
self._output_file: Optional[Any] = None
@@ -79,6 +88,16 @@ class EvalRunner:
output_path.parent.mkdir(parents=True, exist_ok=True)
self._output_file = open(output_path, "w")
# Notify trackers of run start
for tracker in self._trackers:
try:
tracker.on_run_start(cfg)
except Exception as exc:
LOGGER.warning(
"Tracker %s.on_run_start failed: %s",
type(tracker).__name__, exc,
)
total = len(records)
try:
with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool:
@@ -99,6 +118,23 @@ class EvalRunner:
ended_at = time.time()
summary = self._compute_summary(records, started_at, ended_at)
# Notify trackers of summary and run end
for tracker in self._trackers:
try:
tracker.on_summary(summary)
except Exception as exc:
LOGGER.warning(
"Tracker %s.on_summary failed: %s",
type(tracker).__name__, exc,
)
try:
tracker.on_run_end()
except Exception as exc:
LOGGER.warning(
"Tracker %s.on_run_end failed: %s",
type(tracker).__name__, exc,
)
# Write summary JSON alongside JSONL
traces_dir: Optional[Path] = None
if output_path:
@@ -255,6 +291,16 @@ class EvalRunner:
self._output_file.write(json.dumps(record_dict) + "\n")
self._output_file.flush()
# Notify trackers of each result
for tracker in self._trackers:
try:
tracker.on_result(result, self._config)
except Exception as exc:
LOGGER.warning(
"Tracker %s.on_result failed: %s",
type(tracker).__name__, exc,
)
def _resolve_output_path(self) -> Optional[Path]:
"""Determine the output file path."""
if self._config.output_path:
src/openjarvis/evals/core/tracker.py
+34
View File
@@ -0,0 +1,34 @@
"""ResultTracker ABC for external experiment tracking."""
from __future__ import annotations
from abc import ABC, abstractmethod
from openjarvis.evals.core.types import EvalResult, RunConfig, RunSummary
class ResultTracker(ABC):
"""Abstract base class for experiment result trackers.
Lifecycle: on_run_start -> on_result (per sample)
-> on_summary -> on_run_end.
"""
@abstractmethod
def on_run_start(self, config: RunConfig) -> None:
"""Called once before evaluation begins."""
@abstractmethod
def on_result(self, result: EvalResult, config: RunConfig) -> None:
"""Called after each sample is evaluated."""
@abstractmethod
def on_summary(self, summary: RunSummary) -> None:
"""Called after all samples are evaluated with aggregate stats."""
@abstractmethod
def on_run_end(self) -> None:
"""Called at the very end of a run for cleanup."""
__all__ = ["ResultTracker"]
src/openjarvis/evals/core/types.py
+14
View File
@@ -71,6 +71,13 @@ class RunConfig:
gpu_metrics: bool = False
metadata: Dict[str, Any] = field(default_factory=dict)
warmup_samples: int = 0
wandb_project: str = ""
wandb_entity: str = ""
wandb_tags: str = ""
wandb_group: str = ""
sheets_spreadsheet_id: str = ""
sheets_worksheet: str = "Results"
sheets_credentials_path: str = ""
@dataclass(slots=True)
@@ -176,6 +183,13 @@ class ExecutionConfig:
gpu_metrics: bool = False
warmup_samples: int = 0
energy_vendor: str = ""
wandb_project: str = ""
wandb_entity: str = ""
wandb_tags: str = ""
wandb_group: str = ""
sheets_spreadsheet_id: str = ""
sheets_worksheet: str = "Results"
sheets_credentials_path: str = ""
@dataclass(slots=True)
src/openjarvis/evals/trackers/__init__.py
+23
View File
@@ -0,0 +1,23 @@
"""External experiment trackers for the eval framework.
Trackers are lazily imported to avoid mandatory dependencies on wandb/gspread.
"""
from __future__ import annotations
def WandbTracker(*args, **kwargs): # noqa: N802
"""Lazy constructor — imports the real class on first use."""
from openjarvis.evals.trackers.wandb_tracker import WandbTracker as _Cls
return _Cls(*args, **kwargs)
def SheetsTracker(*args, **kwargs): # noqa: N802
"""Lazy constructor — imports the real class on first use."""
from openjarvis.evals.trackers.sheets_tracker import SheetsTracker as _Cls
return _Cls(*args, **kwargs)
__all__ = ["WandbTracker", "SheetsTracker"]
src/openjarvis/evals/trackers/sheets_tracker.py
+162
View File
@@ -0,0 +1,162 @@
"""Google Sheets experiment tracker for the eval framework."""
from __future__ import annotations
import logging
import time
from typing import Any, List, Optional
from openjarvis.evals.core.tracker import ResultTracker
from openjarvis.evals.core.types import EvalResult, MetricStats, RunConfig, RunSummary
try:
import gspread
from google.oauth2.service_account import Credentials
except ImportError:
gspread = None # type: ignore[assignment]
Credentials = None # type: ignore[assignment,misc]
LOGGER = logging.getLogger(__name__)
# Canonical column order for the summary row.
SHEET_COLUMNS: List[str] = [
"timestamp",
"benchmark",
"model",
"backend",
"total_samples",
"scored_samples",
"correct",
"accuracy",
"errors",
"mean_latency_seconds",
"total_cost_usd",
"total_energy_joules",
"avg_power_watts",
"total_input_tokens",
"total_output_tokens",
"latency_mean",
"latency_p90",
"latency_p95",
"energy_mean",
"energy_p90",
"throughput_mean",
"throughput_p90",
"ipw_mean",
"ipj_mean",
"mfu_mean",
"mbu_mean",
"ttft_mean",
"ttft_p90",
"gpu_utilization_mean",
]
def _stat_val(ms: Optional[MetricStats], attr: str) -> Any:
"""Safely extract a stat value from a MetricStats, returning '' if None."""
if ms is None:
return ""
return getattr(ms, attr, "")
class SheetsTracker(ResultTracker):
"""Appends a summary row to a Google Sheet after each eval run."""
def __init__(
self,
spreadsheet_id: str,
worksheet: str = "Results",
credentials_path: str = "",
) -> None:
if gspread is None:
raise ImportError(
"gspread is not installed. "
"Install it with: pip install 'openjarvis[eval-sheets]'"
)
self._spreadsheet_id = spreadsheet_id
self._worksheet_name = worksheet
self._credentials_path = credentials_path
def on_run_start(self, config: RunConfig) -> None:
pass
def on_result(self, result: EvalResult, config: RunConfig) -> None:
# No-op: summary-only to avoid excessive API calls.
pass
def on_summary(self, summary: RunSummary) -> None:
row = self._build_row(summary)
try:
gc = self._authorize()
spreadsheet = gc.open_by_key(self._spreadsheet_id)
try:
ws = spreadsheet.worksheet(self._worksheet_name)
except gspread.exceptions.WorksheetNotFound:
ws = spreadsheet.add_worksheet(
title=self._worksheet_name, rows=1000, cols=len(SHEET_COLUMNS),
)
# Ensure header row exists (idempotent)
existing = ws.row_values(1)
if not existing or existing[0] != SHEET_COLUMNS[0]:
ws.update(range_name="A1", values=[SHEET_COLUMNS])
ws.append_row(row, value_input_option="RAW")
LOGGER.info("Appended summary row to Google Sheet")
except Exception as exc:
LOGGER.warning("SheetsTracker.on_summary failed: %s", exc)
def on_run_end(self) -> None:
pass
def _authorize(self):
"""Authenticate with Google Sheets API."""
scopes = [
"https://www.googleapis.com/auth/spreadsheets",
"https://www.googleapis.com/auth/drive",
]
if self._credentials_path:
creds = Credentials.from_service_account_file(
self._credentials_path, scopes=scopes,
)
else:
# Fall back to Application Default Credentials
import google.auth
creds, _ = google.auth.default(scopes=scopes)
return gspread.authorize(creds)
def _build_row(self, s: RunSummary) -> List[Any]:
"""Build a flat row matching SHEET_COLUMNS order."""
return [
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
s.benchmark,
s.model,
s.backend,
s.total_samples,
s.scored_samples,
s.correct,
s.accuracy,
s.errors,
s.mean_latency_seconds,
s.total_cost_usd,
s.total_energy_joules,
s.avg_power_watts,
s.total_input_tokens,
s.total_output_tokens,
_stat_val(s.latency_stats, "mean"),
_stat_val(s.latency_stats, "p90"),
_stat_val(s.latency_stats, "p95"),
_stat_val(s.energy_stats, "mean"),
_stat_val(s.energy_stats, "p90"),
_stat_val(s.throughput_stats, "mean"),
_stat_val(s.throughput_stats, "p90"),
_stat_val(s.ipw_stats, "mean"),
_stat_val(s.ipj_stats, "mean"),
_stat_val(s.mfu_stats, "mean"),
_stat_val(s.mbu_stats, "mean"),
_stat_val(s.ttft_stats, "mean"),
_stat_val(s.ttft_stats, "p90"),
_stat_val(s.gpu_utilization_stats, "mean"),
]
__all__ = ["SheetsTracker", "SHEET_COLUMNS"]
src/openjarvis/evals/trackers/wandb_tracker.py
+154
View File
@@ -0,0 +1,154 @@
"""W&B experiment tracker for the eval framework."""
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
from openjarvis.evals.core.tracker import ResultTracker
from openjarvis.evals.core.types import EvalResult, MetricStats, RunConfig, RunSummary
try:
import wandb
except ImportError:
wandb = None # type: ignore[assignment]
LOGGER = logging.getLogger(__name__)
def _flatten_metric_stats(prefix: str, ms: Optional[MetricStats]) -> Dict[str, float]:
"""Flatten a MetricStats into a dict with prefixed keys."""
if ms is None:
return {}
return {
f"{prefix}_mean": ms.mean,
f"{prefix}_median": ms.median,
f"{prefix}_min": ms.min,
f"{prefix}_max": ms.max,
f"{prefix}_std": ms.std,
f"{prefix}_p90": ms.p90,
f"{prefix}_p95": ms.p95,
f"{prefix}_p99": ms.p99,
}
class WandbTracker(ResultTracker):
"""Streams per-sample metrics to Weights & Biases."""
def __init__(
self,
project: str,
entity: str = "",
tags: str = "",
group: str = "",
) -> None:
if wandb is None:
raise ImportError(
"wandb is not installed. "
"Install it with: pip install 'openjarvis[eval-wandb]'"
)
self._project = project
self._entity = entity or None
self._tags: List[str] = [
t.strip() for t in tags.split(",") if t.strip()
] if tags else []
self._group = group or None
self._run: Any = None
self._step = 0
def on_run_start(self, config: RunConfig) -> None:
run_config = {
"benchmark": config.benchmark,
"model": config.model,
"backend": config.backend,
"max_samples": config.max_samples,
"max_workers": config.max_workers,
"temperature": config.temperature,
"max_tokens": config.max_tokens,
"seed": config.seed,
}
if config.agent_name:
run_config["agent_name"] = config.agent_name
if config.tools:
run_config["tools"] = ",".join(config.tools)
if config.engine_key:
run_config["engine_key"] = config.engine_key
self._run = wandb.init(
project=self._project,
entity=self._entity,
tags=self._tags or None,
group=self._group,
config=run_config,
reinit=True,
)
self._step = 0
def on_result(self, result: EvalResult, config: RunConfig) -> None:
if self._run is None:
return
self._step += 1
log_data: Dict[str, Any] = {
"sample/is_correct": 1.0 if result.is_correct else 0.0,
"sample/latency_seconds": result.latency_seconds,
"sample/prompt_tokens": result.prompt_tokens,
"sample/completion_tokens": result.completion_tokens,
"sample/cost_usd": result.cost_usd,
"sample/ttft": result.ttft,
"sample/energy_joules": result.energy_joules,
"sample/power_watts": result.power_watts,
"sample/throughput_tok_per_sec": result.throughput_tok_per_sec,
"sample/ipw": result.ipw,
"sample/ipj": result.ipj,
}
if result.error:
log_data["sample/has_error"] = 1.0
wandb.log(log_data, step=self._step)
def on_summary(self, summary: RunSummary) -> None:
if self._run is None:
return
flat: Dict[str, Any] = {
"accuracy": summary.accuracy,
"total_samples": summary.total_samples,
"scored_samples": summary.scored_samples,
"correct": summary.correct,
"errors": summary.errors,
"mean_latency_seconds": summary.mean_latency_seconds,
"total_cost_usd": summary.total_cost_usd,
"total_energy_joules": summary.total_energy_joules,
"avg_power_watts": summary.avg_power_watts,
"total_input_tokens": summary.total_input_tokens,
"total_output_tokens": summary.total_output_tokens,
}
flat.update(_flatten_metric_stats("accuracy", summary.accuracy_stats))
flat.update(_flatten_metric_stats("latency", summary.latency_stats))
flat.update(_flatten_metric_stats("ttft", summary.ttft_stats))
flat.update(_flatten_metric_stats("energy", summary.energy_stats))
flat.update(_flatten_metric_stats("power", summary.power_stats))
flat.update(
_flatten_metric_stats("gpu_utilization", summary.gpu_utilization_stats)
)
flat.update(_flatten_metric_stats("throughput", summary.throughput_stats))
flat.update(_flatten_metric_stats("mfu", summary.mfu_stats))
flat.update(_flatten_metric_stats("mbu", summary.mbu_stats))
flat.update(_flatten_metric_stats("ipw", summary.ipw_stats))
flat.update(_flatten_metric_stats("ipj", summary.ipj_stats))
flat.update(_flatten_metric_stats(
"energy_per_output_token",
summary.energy_per_output_token_stats,
))
flat.update(_flatten_metric_stats(
"throughput_per_watt",
summary.throughput_per_watt_stats,
))
flat.update(_flatten_metric_stats("itl", summary.itl_stats))
wandb.run.summary.update(flat)
def on_run_end(self) -> None:
if self._run is not None:
self._run.finish()
self._run = None
__all__ = ["WandbTracker"]
src/openjarvis/sdk.py
+18 -2
View File
@@ -108,6 +108,12 @@ class MemoryHandle:
self._backend.close()
self._backend = None
def __enter__(self) -> MemoryHandle:
return self
def __exit__(self, *exc: Any) -> None:
self.close()
class Jarvis:
"""High-level OpenJarvis SDK.
@@ -116,9 +122,13 @@ class Jarvis:
from openjarvis import Jarvis
with Jarvis() as j:
response = j.ask("Hello, what can you do?")
print(response)
# Or without context manager:
j = Jarvis()
response = j.ask("Hello, what can you do?")
print(response)
response = j.ask("Hello")
j.close()
"""
@@ -479,5 +489,11 @@ class Jarvis:
self._audit_logger = None
self._engine = None
def __enter__(self) -> Jarvis:
return self
def __exit__(self, *exc: Any) -> None:
self.close()
__all__ = ["Jarvis", "JarvisSystem", "MemoryHandle", "SystemBuilder"]
src/openjarvis/server/api_routes.py
+47
View File
@@ -618,6 +618,51 @@ async def learning_policy(request: Request):
return result
# ---- Speech routes ----
speech_router = APIRouter(prefix="/v1/speech", tags=["speech"])
@speech_router.post("/transcribe")
async def transcribe_speech(request: Request):
"""Transcribe uploaded audio to text."""
backend = getattr(request.app.state, "speech_backend", None)
if backend is None:
raise HTTPException(status_code=501, detail="Speech backend not configured")
form = await request.form()
audio_file = form.get("file")
if audio_file is None:
raise HTTPException(status_code=400, detail="Missing 'file' field")
audio_bytes = await audio_file.read()
language = form.get("language")
# Detect format from filename
filename = getattr(audio_file, "filename", "audio.wav")
ext = filename.rsplit(".", 1)[-1] if "." in filename else "wav"
result = backend.transcribe(audio_bytes, format=ext, language=language or None)
return {
"text": result.text,
"language": result.language,
"confidence": result.confidence,
"duration_seconds": result.duration_seconds,
}
@speech_router.get("/health")
async def speech_health(request: Request):
"""Check if a speech backend is available."""
backend = getattr(request.app.state, "speech_backend", None)
if backend is None:
return {"available": False, "reason": "No speech backend configured"}
return {
"available": backend.health(),
"backend": backend.backend_id,
}
def include_all_routes(app) -> None:
"""Include all extended API routers in a FastAPI app."""
app.include_router(agents_router)
@@ -630,6 +675,7 @@ def include_all_routes(app) -> None:
app.include_router(metrics_router)
app.include_router(websocket_router)
app.include_router(learning_router)
app.include_router(speech_router)
__all__ = [
@@ -644,4 +690,5 @@ __all__ = [
"metrics_router",
"websocket_router",
"learning_router",
"speech_router",
]
src/openjarvis/server/app.py
+2
View File
@@ -52,6 +52,7 @@ def create_app(
agent_name: str = "",
channel_bridge=None,
config=None,
speech_backend=None,
) -> FastAPI:
"""Create and configure the FastAPI application.
@@ -116,6 +117,7 @@ def create_app(
getattr(agent, "agent_id", None) if agent else None
)
app.state.channel_bridge = channel_bridge
app.state.speech_backend = speech_backend
app.state.session_start = time.time()
app.include_router(router)
src/openjarvis/speech/__init__.py
+10
View File
@@ -0,0 +1,10 @@
"""Speech subsystem — speech-to-text backends."""
import importlib
# Optional backends — each registers itself via @SpeechRegistry.register()
for _mod in ("faster_whisper", "openai_whisper", "deepgram"):
try:
importlib.import_module(f".{_mod}", __name__)
except ImportError:
pass
src/openjarvis/speech/_discovery.py
+75
View File
@@ -0,0 +1,75 @@
"""Auto-discover available speech-to-text backends."""
from __future__ import annotations
import os
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from openjarvis.core.config import JarvisConfig
from openjarvis.speech._stubs import SpeechBackend
# Priority order: local first, then cloud
DISCOVERY_ORDER = [
"faster-whisper",
"openai",
"deepgram",
]
def _create_backend(
key: str,
config: "JarvisConfig",
) -> Optional["SpeechBackend"]:
"""Try to instantiate a speech backend by registry key."""
from openjarvis.core.registry import SpeechRegistry
if not SpeechRegistry.contains(key):
return None
try:
backend_cls = SpeechRegistry.get(key)
if key == "faster-whisper":
return backend_cls(
model_size=config.speech.model,
device=config.speech.device,
compute_type=config.speech.compute_type,
)
elif key == "openai":
api_key = os.environ.get("OPENAI_API_KEY", "")
if not api_key:
return None
return backend_cls(api_key=api_key)
elif key == "deepgram":
api_key = os.environ.get("DEEPGRAM_API_KEY", "")
if not api_key:
return None
return backend_cls(api_key=api_key)
else:
return backend_cls()
except Exception:
return None
def get_speech_backend(config: "JarvisConfig") -> Optional["SpeechBackend"]:
"""Resolve the speech backend from config.
If ``config.speech.backend`` is ``"auto"``, tries backends in
priority order and returns the first healthy one.
"""
# Trigger registration of built-in backends
import openjarvis.speech # noqa: F401
backend_key = config.speech.backend
if backend_key != "auto":
return _create_backend(backend_key, config)
# Auto-discovery: try each in priority order
for key in DISCOVERY_ORDER:
backend = _create_backend(key, config)
if backend is not None:
return backend
return None
src/openjarvis/speech/_stubs.py
+55
View File
@@ -0,0 +1,55 @@
"""Abstract base classes and data types for the speech subsystem."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import List, Optional
@dataclass
class Segment:
"""A timed segment of transcribed text."""
text: str
start: float # Start time in seconds
end: float # End time in seconds
confidence: Optional[float] = None
@dataclass
class TranscriptionResult:
"""Result of a speech-to-text transcription."""
text: str
language: Optional[str] = None
confidence: Optional[float] = None
duration_seconds: float = 0.0
segments: List[Segment] = field(default_factory=list)
class SpeechBackend(ABC):
"""Abstract base class for speech-to-text backends."""
backend_id: str = ""
@abstractmethod
def transcribe(
self,
audio: bytes,
*,
format: str = "wav",
language: Optional[str] = None,
) -> TranscriptionResult:
"""Transcribe audio bytes to text."""
@abstractmethod
def health(self) -> bool:
"""Check if the backend is ready."""
@abstractmethod
def supported_formats(self) -> List[str]:
"""Return list of supported audio formats."""
__all__ = ["Segment", "SpeechBackend", "TranscriptionResult"]
src/openjarvis/speech/deepgram.py
+96
View File
@@ -0,0 +1,96 @@
"""Deepgram speech-to-text backend (cloud)."""
from __future__ import annotations
import os
from typing import List, Optional
from openjarvis.core.registry import SpeechRegistry
from openjarvis.speech._stubs import SpeechBackend, TranscriptionResult
try:
from deepgram import DeepgramClient, PrerecordedOptions
except ImportError:
DeepgramClient = None # type: ignore[assignment, misc]
PrerecordedOptions = None # type: ignore[assignment, misc]
@SpeechRegistry.register("deepgram")
class DeepgramSpeechBackend(SpeechBackend):
"""Cloud speech-to-text using Deepgram API."""
backend_id = "deepgram"
def __init__(self, api_key: Optional[str] = None) -> None:
self._api_key = api_key or os.environ.get("DEEPGRAM_API_KEY", "")
self._client = None
if self._api_key and DeepgramClient is not None:
self._client = DeepgramClient(self._api_key)
def transcribe(
self,
audio: bytes,
*,
format: str = "wav",
language: Optional[str] = None,
) -> TranscriptionResult:
"""Transcribe audio using Deepgram's API."""
if self._client is None:
raise RuntimeError("Deepgram client not initialized (missing API key?)")
mime_map = {
"wav": "audio/wav",
"mp3": "audio/mpeg",
"ogg": "audio/ogg",
"flac": "audio/flac",
"webm": "audio/webm",
"m4a": "audio/mp4",
}
mime_type = mime_map.get(format, "audio/wav")
options_kwargs: dict = {"model": "nova-2", "smart_format": True}
if language:
options_kwargs["language"] = language
else:
options_kwargs["detect_language"] = True
payload = {"buffer": audio, "mimetype": mime_type}
if PrerecordedOptions is not None:
options = PrerecordedOptions(**options_kwargs)
else:
options = options_kwargs
response = self._client.listen.rest.v("1").transcribe_file(
payload, options,
)
# Extract transcript from response
channels = response.results.channels
if channels and channels[0].alternatives:
alt = channels[0].alternatives[0]
text = alt.transcript
confidence = getattr(alt, "confidence", None)
else:
text = ""
confidence = None
detected_lang = None
if channels:
detected_lang = getattr(channels[0], "detected_language", None)
duration = getattr(response.metadata, "duration", 0.0)
return TranscriptionResult(
text=text,
language=detected_lang,
confidence=confidence,
duration_seconds=duration,
segments=[],
)
def health(self) -> bool:
return self._client is not None and bool(self._api_key)
def supported_formats(self) -> List[str]:
return ["wav", "mp3", "ogg", "flac", "webm", "m4a"]
src/openjarvis/speech/faster_whisper.py
+100
View File
@@ -0,0 +1,100 @@
"""Faster-Whisper speech-to-text backend (local, CTranslate2-based)."""
from __future__ import annotations
import tempfile
from typing import List, Optional
from openjarvis.core.registry import SpeechRegistry
from openjarvis.speech._stubs import Segment, SpeechBackend, TranscriptionResult
try:
from faster_whisper import WhisperModel
except ImportError:
WhisperModel = None # type: ignore[assignment, misc]
@SpeechRegistry.register("faster-whisper")
class FasterWhisperBackend(SpeechBackend):
"""Local speech-to-text using Faster-Whisper (CTranslate2)."""
backend_id = "faster-whisper"
def __init__(
self,
model_size: str = "base",
device: str = "auto",
compute_type: str = "float16",
) -> None:
self._model_size = model_size
self._device = device
self._compute_type = compute_type
self._model: Optional[WhisperModel] = None
def _ensure_model(self) -> WhisperModel:
"""Lazy-load the Whisper model on first use."""
if self._model is None:
if WhisperModel is None:
raise ImportError(
"faster-whisper is not installed. "
"Install with: pip install 'openjarvis[speech]'"
)
self._model = WhisperModel(
self._model_size,
device=self._device,
compute_type=self._compute_type,
)
return self._model
def transcribe(
self,
audio: bytes,
*,
format: str = "wav",
language: Optional[str] = None,
) -> TranscriptionResult:
"""Transcribe audio bytes using Faster-Whisper."""
model = self._ensure_model()
# Write audio to a temp file (faster-whisper needs a file path)
suffix = f".{format}" if not format.startswith(".") else format
with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as tmp:
tmp.write(audio)
tmp.flush()
kwargs = {}
if language:
kwargs["language"] = language
segments_iter, info = model.transcribe(tmp.name, **kwargs)
segments_list = list(segments_iter)
# Build result
text = "".join(seg.text for seg in segments_list).strip()
segments = [
Segment(
text=seg.text.strip(),
start=seg.start,
end=seg.end,
confidence=None,
)
for seg in segments_list
]
return TranscriptionResult(
text=text,
language=getattr(info, "language", None),
confidence=getattr(info, "language_probability", None),
duration_seconds=getattr(info, "duration", 0.0),
segments=segments,
)
def health(self) -> bool:
"""Check if model is loaded or loadable."""
if self._model is not None:
return True
return WhisperModel is not None
def supported_formats(self) -> List[str]:
"""Supported audio formats (same as ffmpeg/Whisper)."""
return ["wav", "mp3", "m4a", "ogg", "flac", "webm"]
src/openjarvis/speech/openai_whisper.py
+64
View File
@@ -0,0 +1,64 @@
"""OpenAI Whisper API speech-to-text backend (cloud)."""
from __future__ import annotations
import io
import os
from typing import List, Optional
from openjarvis.core.registry import SpeechRegistry
from openjarvis.speech._stubs import SpeechBackend, TranscriptionResult
try:
from openai import OpenAI
except ImportError:
OpenAI = None # type: ignore[assignment, misc]
@SpeechRegistry.register("openai")
class OpenAIWhisperBackend(SpeechBackend):
"""Cloud speech-to-text using OpenAI Whisper API."""
backend_id = "openai"
def __init__(self, api_key: Optional[str] = None) -> None:
self._api_key = api_key or os.environ.get("OPENAI_API_KEY", "")
self._client: Optional[OpenAI] = None
if self._api_key and OpenAI is not None:
self._client = OpenAI(api_key=self._api_key)
def transcribe(
self,
audio: bytes,
*,
format: str = "wav",
language: Optional[str] = None,
) -> TranscriptionResult:
"""Transcribe audio using OpenAI's Whisper API."""
if self._client is None:
raise RuntimeError("OpenAI client not initialized (missing API key?)")
ext = format if not format.startswith(".") else format[1:]
audio_file = io.BytesIO(audio)
audio_file.name = f"audio.{ext}"
kwargs: dict = {"model": "whisper-1", "file": audio_file}
if language:
kwargs["language"] = language
kwargs["response_format"] = "verbose_json"
response = self._client.audio.transcriptions.create(**kwargs)
return TranscriptionResult(
text=getattr(response, "text", str(response)),
language=getattr(response, "language", None),
confidence=None,
duration_seconds=getattr(response, "duration", 0.0),
segments=[],
)
def health(self) -> bool:
return self._client is not None and bool(self._api_key)
def supported_formats(self) -> List[str]:
return ["mp3", "mp4", "mpeg", "mpga", "m4a", "wav", "webm"]
src/openjarvis/system.py
+23
View File
@@ -40,6 +40,7 @@ class JarvisSystem:
session_store: Optional[Any] = None # SessionStore
capability_policy: Optional[Any] = None # CapabilityPolicy
operator_manager: Optional[Any] = None # OperatorManager
speech_backend: Optional[Any] = None # SpeechBackend
_learning_orchestrator: Optional[Any] = None # LearningOrchestrator
def ask(
@@ -274,6 +275,12 @@ class JarvisSystem:
if self.trace_store and hasattr(self.trace_store, "close"):
self.trace_store.close()
def __enter__(self) -> JarvisSystem:
return self
def __exit__(self, *exc: Any) -> None:
self.close()
class SystemBuilder:
"""Config-driven fluent builder for JarvisSystem."""
@@ -304,6 +311,7 @@ class SystemBuilder:
self._scheduler: Optional[bool] = None
self._workflow: Optional[bool] = None
self._sessions: Optional[bool] = None
self._speech: Optional[bool] = None
def engine(self, key: str) -> SystemBuilder:
self._engine_key = key
@@ -345,6 +353,10 @@ class SystemBuilder:
self._sessions = enabled
return self
def speech(self, enabled: bool) -> SystemBuilder:
self._speech = enabled
return self
def event_bus(self, bus: EventBus) -> SystemBuilder:
self._bus = bus
return self
@@ -447,6 +459,16 @@ class SystemBuilder:
# Set up learning orchestrator (when training is enabled)
learning_orchestrator = self._setup_learning_orchestrator(config)
# Set up speech backend
speech_backend = None
speech_enabled = self._speech if self._speech is not None else True
if speech_enabled:
try:
from openjarvis.speech._discovery import get_speech_backend
speech_backend = get_speech_backend(config)
except Exception:
pass
system = JarvisSystem(
config=config,
bus=bus,
@@ -466,6 +488,7 @@ class SystemBuilder:
workflow_engine=workflow_engine,
session_store=session_store,
capability_policy=capability_policy,
speech_backend=speech_backend,
)
system._learning_orchestrator = learning_orchestrator
return system