mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 10:52:15 +00:00
chore: remove extraneous files from repo root
Remove internal docs (CLAUDE.md, README.md, ROADMAP.md, VISION.md, NOTES.md), testing scripts (run_dry_run.py, run_full_optimization.py), and update .gitignore to prevent re-accumulation of logs/, traces/, coding_task_*, *.jsonl, *.npz, and NFS lock files. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jon Saad-Falcon
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
2ed56fb0b3
commit
f19970dece
+6
-1
@@ -42,7 +42,12 @@ Thumbs.db
|
||||
# Project
|
||||
*.sqlite
|
||||
*.db
|
||||
*.jsonl
|
||||
*.npz
|
||||
results/
|
||||
logs/
|
||||
traces/
|
||||
coding_task_*
|
||||
get-pip.py
|
||||
|
||||
# MkDocs build output
|
||||
@@ -71,4 +76,4 @@ docs/plans/
|
||||
**/src-tauri/gen/schemas/
|
||||
|
||||
# Rust build artifacts
|
||||
rust/target/
|
||||
rust/target/.claude/.nfs*
|
||||
|
||||
@@ -1,247 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Status
|
||||
|
||||
OpenJarvis is a research framework for studying on-device AI systems. Phase 25 complete. Five composable pillars: Intelligence, Engine, Agents, Tools (with storage + MCP), and Learning — with trace-driven learning as a cross-cutting concern. Speech subsystem (STT) with pluggable backends. ~3405 tests pass (~44 skipped for optional deps). Python SDK (`Jarvis` class), composition layer (`SystemBuilder`/`JarvisSystem`), eval framework (20 real benchmarks), composable recipes, agent templates, bundled skills, operator recipes, trace-driven learning pipeline, Docker deployment, Tauri desktop app, 41+ tools, 20+ CLI commands, 40+ API endpoints all ready.
|
||||
|
||||
## Build & Development Commands
|
||||
|
||||
```bash
|
||||
uv sync --extra dev # Install deps + dev tools
|
||||
uv run pytest tests/ -v # Run ~3295 tests (~44 skipped if optional deps missing)
|
||||
uv run ruff check src/ tests/ # Lint
|
||||
uv run jarvis --version # 1.0.0
|
||||
uv run jarvis ask "Hello" # Query via discovered engine (direct mode)
|
||||
uv run jarvis ask --agent simple "Hello" # SimpleAgent route
|
||||
uv run jarvis ask --agent orchestrator "Hello" # OrchestratorAgent route
|
||||
uv run jarvis ask --agent orchestrator --tools calculator,think "What is 2+2?"
|
||||
uv run jarvis ask --agent native_react --tools calculator "What is 2+2?" # NativeReActAgent
|
||||
uv run jarvis ask --agent react "Hello" # Alias for native_react
|
||||
uv run jarvis ask --agent native_openhands "Hello" # NativeOpenHandsAgent (CodeAct)
|
||||
uv run jarvis ask --agent openhands "Hello" # Real OpenHands SDK (requires openhands-sdk)
|
||||
uv run jarvis ask --router heuristic "Hello" # Explicit heuristic policy
|
||||
uv run jarvis ask --no-context "Hello" # Query without memory context injection
|
||||
uv run jarvis model list # List models from running engines
|
||||
uv run jarvis model info qwen3:8b # Show model details
|
||||
uv run jarvis memory index ./docs/ # Index documents into memory
|
||||
uv run jarvis memory search "topic" # Search memory for relevant chunks
|
||||
uv run jarvis memory stats # Show memory backend statistics
|
||||
uv run jarvis telemetry stats # Show aggregated telemetry stats
|
||||
uv run jarvis telemetry export --format json # Export records as JSON
|
||||
uv run jarvis telemetry export --format csv # Export records as CSV
|
||||
uv run jarvis telemetry clear --yes # Delete all telemetry records
|
||||
uv run jarvis channel list # List available messaging channels
|
||||
uv run jarvis channel send slack "Hello" # Send a message to a channel
|
||||
uv run jarvis channel status # Show channel bridge connection status
|
||||
uv run jarvis scheduler create "Check weather" --type cron --value "0 9 * * *"
|
||||
uv run jarvis scheduler list # List scheduled tasks
|
||||
uv run jarvis scheduler start # Start scheduler daemon (foreground)
|
||||
uv run jarvis bench run # Run all benchmarks against engine
|
||||
uv run jarvis bench run -b energy -w 5 -n 20 --json # Energy benchmark with warmup
|
||||
uv run jarvis serve --port 8000 # OpenAI-compatible API server (requires openjarvis[server])
|
||||
uv run jarvis doctor # Run diagnostic checks (config, engines, models, deps)
|
||||
uv run jarvis doctor --json # Machine-readable diagnostics
|
||||
uv run jarvis start # Start server as background daemon
|
||||
uv run jarvis stop # Stop background daemon
|
||||
uv run jarvis restart # Restart background daemon
|
||||
uv run jarvis status # Show daemon status (PID, uptime)
|
||||
uv run jarvis chat # Interactive REPL (/quit, /clear, /model, /help, /history)
|
||||
uv run jarvis chat --agent orchestrator --tools calculator # REPL with agent
|
||||
uv run jarvis agent list # List registered agents
|
||||
uv run jarvis agent info native_react # Show agent details
|
||||
uv run jarvis workflow list # List available workflows
|
||||
uv run jarvis workflow run my_workflow # Execute a workflow
|
||||
uv run jarvis skill list # List installed skills
|
||||
uv run jarvis skill install path/to/skill.toml # Install a skill
|
||||
uv run jarvis vault set MY_KEY # Store encrypted credential
|
||||
uv run jarvis vault get MY_KEY # Retrieve credential
|
||||
uv run jarvis vault list # List stored keys
|
||||
uv run jarvis add github # Quick-add MCP server (github, slack, postgres, etc.)
|
||||
uv run jarvis eval list # List 15 benchmarks and backends
|
||||
uv run jarvis eval run -b supergpqa -m "qwen3:8b" # Single benchmark run
|
||||
uv run jarvis eval run -c src/openjarvis/evals/configs/suite.toml # Suite mode (models x benchmarks)
|
||||
uv run jarvis eval compare result1.jsonl result2.jsonl # Compare runs side-by-side
|
||||
uv run jarvis eval report result.jsonl # Detailed report with per-subject breakdown
|
||||
uv run jarvis --help # Show all subcommands
|
||||
uv run jarvis init --force # Detect hardware, write ~/.openjarvis/config.toml
|
||||
# Eval framework (direct module invocation)
|
||||
source .env # Load API keys before running evals
|
||||
uv run python -m openjarvis.evals run -c src/openjarvis/evals/configs/glm-4.7-flash-openhands.toml -v # Run eval suite from TOML config
|
||||
uv run python -m openjarvis.evals run -b supergpqa -m "qwen3:8b" -n 50 # Run single benchmark
|
||||
uv run python -m openjarvis.evals summarize results/supergpqa_qwen3-8b.jsonl # Summarize results
|
||||
```
|
||||
|
||||
### Config File Conventions
|
||||
|
||||
- **Runtime config (source of truth):** `configs/openjarvis/config.toml` — Pillar-aligned OpenJarvis config. Copied to `~/.openjarvis/config.toml` at runtime (which is where `load_config()` reads from).
|
||||
- **Eval suite configs:** `src/openjarvis/evals/configs/*.toml` — TOML configs defining models x benchmarks matrices.
|
||||
- **API keys:** `.env` file in project root (gitignored). Source with `source .env` before running evals or cloud operations.
|
||||
- **Never save configs to `~/.openjarvis/` directly** — always maintain the canonical copy in `configs/openjarvis/` and copy/symlink to `~/.openjarvis/`.
|
||||
|
||||
### Python SDK
|
||||
|
||||
```python
|
||||
from openjarvis import Jarvis
|
||||
|
||||
j = Jarvis() # Uses default config + auto-detected engine
|
||||
j = Jarvis(model="qwen3:8b") # Override model
|
||||
j = Jarvis(engine_key="ollama") # Override engine
|
||||
|
||||
response = j.ask("Hello") # Returns string
|
||||
full = j.ask_full("Hello") # Returns dict with content, usage, model, engine
|
||||
response = j.ask("Hello", agent="orchestrator", tools=["calculator"])
|
||||
|
||||
j.memory.index("./docs/") # Index documents
|
||||
results = j.memory.search("topic") # Search memory
|
||||
j.memory.stats() # Backend stats
|
||||
|
||||
j.list_models() # Available models
|
||||
j.list_engines() # Registered engines
|
||||
j.close() # Release resources
|
||||
```
|
||||
|
||||
- **Package manager:** `uv` with `hatchling` build backend
|
||||
- **Config:** `pyproject.toml` with extras for optional backends (e.g., `openjarvis[inference-vllm]`, `openjarvis[inference-mlx]`, `openjarvis[memory-colbert]`, `openjarvis[server]`, `openjarvis[openclaw]`, `openjarvis[energy-amd]`, `openjarvis[energy-apple]`, `openjarvis[energy-all]`, `openjarvis[security-signing]`, `openjarvis[sandbox-wasm]`, `openjarvis[dashboard]`, `openjarvis[browser]`, `openjarvis[media]`, `openjarvis[pdf]`, `openjarvis[channel-line]`, `openjarvis[channel-viber]`, `openjarvis[channel-reddit]`, `openjarvis[channel-mastodon]`, `openjarvis[channel-xmpp]`, `openjarvis[channel-rocketchat]`, `openjarvis[channel-zulip]`, `openjarvis[channel-twitch]`, `openjarvis[channel-nostr]`, `openjarvis[speech]`, `openjarvis[speech-deepgram]`)
|
||||
- **CLI entry point:** `jarvis` (Click-based) — subcommands: `init`, `ask`, `serve`, `start`, `stop`, `restart`, `status`, `chat`, `model`, `memory`, `telemetry`, `bench`, `eval`, `channel`, `scheduler`, `doctor`, `agent`, `workflow`, `skill`, `vault`, `add`
|
||||
- **Python:** 3.10+ required
|
||||
- **Node.js:** 22+ required only for OpenClaw agent
|
||||
|
||||
## Architecture
|
||||
|
||||
OpenJarvis is a research framework for on-device AI organized around **five composable pillars**, each with a clear ABC interface and a decorator-based registry for runtime discovery.
|
||||
|
||||
### Five Pillars
|
||||
|
||||
1. **Intelligence** (`src/openjarvis/intelligence/`) — Model definition, catalog, and generation defaults. `ModelRegistry` maps model keys to `ModelSpec`. `IntelligenceConfig` holds model identity (default/fallback model, model_path, checkpoint_path, quantization, preferred_engine, provider) and generation defaults (temperature, max_tokens, top_p, top_k, repetition_penalty, stop_sequences). Model catalog maintains `BUILTIN_MODELS` with auto-discovery via `merge_discovered_models()`.
|
||||
2. **Engine** (`src/openjarvis/engine/`) — The inference runtime. Backends: vLLM, SGLang, Ollama, llama.cpp, MLX, LM Studio. All implement `InferenceEngine` ABC with `generate()`, `stream()`, `list_models()`, `health()`. Engines extract and pass through `tool_calls` in OpenAI format.
|
||||
3. **Agents** (`src/openjarvis/agents/`) — Pluggable logic for queries, tool/API calls, memory. Hierarchy: `BaseAgent` ABC (helpers: `_emit_turn_start/end`, `_build_messages`, `_generate`, `_max_turns_result`, `_strip_think_tags`, `_check_continuation`) → `ToolUsingAgent` (adds `tools`, `ToolExecutor`, `max_turns`). Agents: `SimpleAgent` (single-turn), `OrchestratorAgent` (multi-turn tool loop), `NativeReActAgent` (Thought-Action-Observation, key `"native_react"`, alias `"react"`), `NativeOpenHandsAgent` (CodeAct, key `"native_openhands"`), `RLMAgent` (recursive LM), `OpenHandsAgent` (real `openhands-sdk`, key `"openhands"`, requires Python 3.12+), `OpenClawAgent` (HTTP/subprocess transport), `ClaudeCodeAgent` (Claude Agent SDK via Node.js, key `"claude_code"`), `SandboxedAgent` (Docker wrapper, key `"sandboxed"`), `MonitorOperativeAgent` (long-horizon monitoring with 4 configurable strategies: memory extraction, observation compression, retrieval, task decomposition, key `"monitor_operative"`). `accepts_tools` class attribute for CLI/SDK auto-detection. Agents call `engine.generate()` directly — telemetry handled by `InstrumentedEngine` wrapper.
|
||||
4. **Tools** (`src/openjarvis/tools/`) — All tools managed via MCP (Model Context Protocol).
|
||||
- **API tools**: `CalculatorTool`, `ThinkTool`, `FileReadTool`, `FileWriteTool`, `WebSearchTool`, `CodeInterpreterTool`, `LLMTool`, `ShellExecTool`, `ApplyPatchTool`, `HttpRequestTool`, `DatabaseQueryTool`, `PDFExtractTool`, `ImageGenerateTool`, `AudioTranscribeTool` — all implement `BaseTool` ABC
|
||||
- **Git tools** (`git_tool.py`): `GitStatusTool`, `GitDiffTool`, `GitCommitTool`, `GitLogTool`
|
||||
- **Browser tools** (`browser.py`): `BrowserNavigateTool`, `BrowserClickTool`, `BrowserTypeTool`, `BrowserScreenshotTool`, `BrowserExtractTool` (Playwright, optional `[browser]`)
|
||||
- **Agent tools** (`agent_tools.py`): `AgentSpawnTool`, `AgentSendTool`, `AgentListTool`, `AgentKillTool`
|
||||
- **Storage tools** (`storage_tools.py`): `MemoryStoreTool`, `MemoryRetrieveTool`, `MemorySearchTool`, `MemoryIndexTool`
|
||||
- **Storage backends** (`tools/storage/`): SQLite/FTS5 (default), FAISS, ColBERTv2, BM25, Hybrid (RRF fusion), KnowledgeGraph. All implement `MemoryBackend` ABC. Canonical import: `from openjarvis.tools.storage.sqlite import SQLiteMemory`.
|
||||
- **Scheduler tools** (`scheduler/tools.py`): 5 MCP tools for task scheduling
|
||||
- **Knowledge graph tools** (`knowledge_tools.py`): `KGAddEntityTool`, `KGAddRelationTool`, `KGQueryTool`, `KGNeighborsTool`
|
||||
- **MCP adapter** (`mcp_adapter.py`): `MCPToolAdapter` wraps external MCP tools as native `BaseTool`; `MCPToolProvider` discovers from server
|
||||
- **MCP server** (`mcp/server.py`): Exposes all built-in tools via JSON-RPC `tools/list` + `tools/call` (MCP spec 2025-11-25)
|
||||
- **MCP templates** (`tools/templates/`): `ToolTemplate` dynamically constructs tools from TOML specs. 10 builtin templates. `discover_templates()` auto-discovers.
|
||||
- **`ToolExecutor`**: dispatch with RBAC check + taint check, `timeout_seconds` on `ToolSpec` (default 30s via `ThreadPoolExecutor`), event bus integration
|
||||
- All registered via `@ToolRegistry.register("name")` decorator
|
||||
5. **Learning** (`src/openjarvis/learning/`) — Structured learning with nested per-pillar sub-policies. `LearningConfig` sections: `routing` (heuristic/learned/grpo/bandit), `intelligence` (none/sft), `agent` (none/agent_advisor/icl_updater), `metrics` (accuracy/latency/cost/efficiency weights). Policies: `SFTRouterPolicy` (query→model from traces), `AgentAdvisorPolicy` (LM-guided), `ICLUpdaterPolicy` (in-context with example DB, versioning, rollback, quality gates), `GRPORouterPolicy` (softmax sampling, group relative advantage, per-query-class weights), `BanditRouterPolicy` (Thompson Sampling / UCB1, per-arm stats). `SkillDiscovery` mines tool subsequences from traces to auto-generate skill manifests. Router policies: `HeuristicRouter`, `TraceDrivenPolicy`. Orchestrator training subpackage provides SFT and GRPO pipelines. **Trace-driven learning pipeline**: `TrainingDataMiner` (extracts SFT pairs from traces with quality filters), `LoRATrainer` (LoRA fine-tuning with configurable rank/alpha, requires torch), `AgentConfigEvolver` (LM-guided agent config recommendations from trace patterns), `LearningOrchestrator` (wired into `SystemBuilder`, orchestrates mine→train→evolve cycle on schedule).
|
||||
|
||||
### Speech Subsystem
|
||||
|
||||
- **Speech** (`src/openjarvis/speech/`) — Speech-to-text with pluggable backends. `SpeechBackend` ABC (`transcribe()`, `health()`, `supported_formats()`). `TranscriptionResult` + `Segment` dataclasses. Backends: `FasterWhisperBackend` (local, CTranslate2, key `"faster-whisper"`), `OpenAIWhisperBackend` (cloud, `whisper-1`, key `"openai"`), `DeepgramSpeechBackend` (cloud, `nova-2`, key `"deepgram"`). Auto-discovery with local-first priority. `SpeechConfig` in `JarvisConfig`. `SpeechRegistry` for backend registration. Wired into `SystemBuilder.speech()`, `create_app()`, `jarvis serve`. API: `POST /v1/speech/transcribe` (multipart), `GET /v1/speech/health`. Frontend: `useSpeech` hook + `MicButton` component. Tauri: `transcribe_audio` + `speech_health` commands. Optional deps: `openjarvis[speech]` (faster-whisper), `openjarvis[speech-deepgram]` (deepgram-sdk).
|
||||
|
||||
### Cross-cutting Systems
|
||||
|
||||
- **Traces** (`src/openjarvis/traces/`) — Full interaction recording. `Trace` captures `TraceStep`s (route, retrieve, generate, tool_call, respond) with timing. `TraceStore` (SQLite), `TraceCollector` (auto-wraps agents), `TraceAnalyzer` (stats for learning).
|
||||
- **Telemetry** (`src/openjarvis/telemetry/`) — `InstrumentedEngine` wraps any engine, publishing events to SQLite via `TelemetryStore`. `TelemetryAggregator` for read-only queries. `EnergyMonitor` ABC with vendor-specific implementations: `NvidiaEnergyMonitor` (hw counters/polling), `AmdEnergyMonitor` (amdsmi), `AppleEnergyMonitor` (zeus-ml), `RaplEnergyMonitor` (sysfs). `EnergyBatch` for batch-level energy-per-token. `SteadyStateDetector` for thermal equilibrium (CV-based).
|
||||
- **Security** (`src/openjarvis/security/`) — `SecretScanner` + `PIIScanner` (implement `BaseScanner` ABC). `GuardrailsEngine` wraps engines with input/output scanning (WARN/REDACT/BLOCK modes). `AuditLogger` with Merkle hash chain (SHA-256 tamper-evidence). `CapabilityPolicy` RBAC (10 capabilities with glob matching, enforced in `ToolExecutor`). `TaintLabel`/`TaintSet` information flow control with `SINK_POLICY`. Ed25519 signing via `cryptography` (optional `[security-signing]`). `file_policy.py` for sensitive file detection. `InjectionScanner` (11 regex patterns: prompt override, identity override, code/shell injection, exfiltration, jailbreak, delimiter injection). `check_ssrf()` SSRF protection (RFC 1918, loopback, link-local, cloud metadata blocking). `RateLimiter` with `TokenBucket` (thread-safe, per-key). `run_sandboxed()` subprocess isolation (`os.setsid`, process group kill, env clearing). Security HTTP middleware (7 headers: CSP, HSTS, X-Frame-Options, etc.).
|
||||
|
||||
### Composition & Infrastructure
|
||||
|
||||
- **Composition Layer** (`system.py`) — `SystemBuilder` fluent builder → `JarvisSystem` with `ask()`, `close()`. Wires engine, model, agent, tools, telemetry, traces, workflow, sessions, capability policy.
|
||||
- **SDK** (`sdk.py`) — `Jarvis` class: high-level sync API with `ask()`/`ask_full()`, `MemoryHandle`, lazy init, telemetry. Also exports `JarvisSystem`/`SystemBuilder`.
|
||||
- **Benchmarks** (`bench/`) — `LatencyBenchmark`, `ThroughputBenchmark`, `EnergyBenchmark`. All registered via `BenchmarkRegistry`. CLI: `jarvis bench run`.
|
||||
- **Eval Framework** (`src/openjarvis/evals/`) — 20 real benchmark datasets: SuperGPQA, GPQA, MMLU-Pro, MATH-500, Natural Reasoning, HLE, SimpleQA, WildChat, IPW, GAIA, FRAMES, SWE-bench, SWEfficiency, TerminalBench, TerminalBench Native, LogHub, AMA-Bench, LifelongAgentBench, WebChoreArena, WorkArena++. Scorer types: MCQ letter extraction, LLM-judge, exact match, structural validation. `EvalRunner` with parallel execution and episode mode (sequential processing with shared agent state). `EnvironmentProvider` ABC for Docker/ServiceNow environments. CLI: `jarvis eval list|run|compare|report`.
|
||||
- **Recipes** (`src/openjarvis/recipes/`) — Composable TOML configs that wire all 5 pillars. `Recipe` dataclass with `to_builder_kwargs()`. `load_recipe()`, `discover_recipes()`, `resolve_recipe()`. 3 built-in recipes in `data/`: coding_assistant, research_assistant, general_assistant. Operator recipes in `data/operators/`: researcher (4h cycle), correspondent (5min interval), sentinel (2h cycle), monitor (2h cycle, causality graph + hybrid retrieval).
|
||||
- **Agent Templates** (`src/openjarvis/templates/`) — Pre-configured TOML manifests with system prompts, tool sets, behavioral parameters. `AgentTemplate` dataclass, `load_template()`, `discover_templates()`. 15 built-in templates in `data/` (code-reviewer, debugger, architect, deep-researcher, fact-checker, summarizer, etc.).
|
||||
- **Bundled Skills** (`src/openjarvis/skills/data/`) — 20 ready-to-use TOML skill manifests. Categories: file management (organizer, deduplicator, backup), research (web-summarize, topic-research, knowledge-extract), code quality (lint, test-gen, security-scan, dependency-audit), productivity (email-draft, meeting-notes, daily-digest), document processing (compare, translate, data-analyze).
|
||||
- **OpenClaw** (`agents/openclaw*.py`) — `OpenClawAgent` with `HttpTransport`/`SubprocessTransport`, JSON-line protocol, `ProviderPlugin`, `MemorySearchManager`.
|
||||
- **API Server** (`server/`) — OpenAI-compatible via `jarvis serve` (FastAPI + uvicorn). Endpoints: `POST /v1/chat/completions`, `GET /v1/models`, `GET /health`, channel endpoints. SSE streaming.
|
||||
- **Channels** (`channels/`) — `BaseChannel` ABC. `OpenClawChannelBridge` (WebSocket/HTTP to OpenClaw gateway). `WhatsAppBaileysChannel` (Baileys protocol, Node.js bridge, QR auth). Phase 21 channels: `LINEChannel`, `ViberChannel`, `MessengerChannel`, `RedditChannel`, `MastodonChannel`, `XMPPChannel`, `RocketChatChannel`, `ZulipChannel`, `TwitchChannel`, `NostrChannel`. All follow `BaseChannel` ABC with env var fallbacks, `@ChannelRegistry.register()`, `EventBus` integration.
|
||||
- **Sandbox** (`sandbox/`) — `ContainerRunner` (Docker/Podman lifecycle, mount validation). `WasmRunner` (wasmtime-py, fuel/memory limits, optional `[sandbox-wasm]`). `SandboxedAgent` transparent wrapper. `create_sandbox_runner()` factory. `MountAllowlist` with path traversal prevention.
|
||||
- **Scheduler** (`scheduler/`) — `TaskScheduler` with cron/interval/once scheduling, SQLite persistence, 5 MCP tools, event bus. CLI: `jarvis scheduler create|list|pause|resume|cancel|logs|start`.
|
||||
- **Agent Hardening** (`agents/loop_guard.py`) — `LoopGuard`: SHA-256 hash tracking (identical calls), ping-pong detection (A-B-A-B patterns), poll-tool budget, context overflow recovery. `BaseAgent._check_continuation()` auto-resumes on `finish_reason=length`.
|
||||
- **Workflow Engine** (`workflow/`) — DAG-based `WorkflowGraph` (cycle detection, topological sort, parallel stages via `ThreadPoolExecutor`). `WorkflowBuilder` fluent API. `WorkflowEngine` executes against `JarvisSystem`. TOML loader. Node types: agent, tool, condition, parallel, loop, transform.
|
||||
- **Skills** (`skills/`) — `SkillManifest`/`SkillExecutor` (sequential tool steps with template rendering). Ed25519 signature verification. `SkillTool` adapter wraps skills as invocable tools. TOML loader.
|
||||
- **Knowledge Graph** (`tools/storage/knowledge_graph.py`) — `KnowledgeGraphMemory(MemoryBackend)`: SQLite entity-relation store. `add_entity()`, `add_relation()`, `neighbors()`, `query_pattern()`. Registered as `"knowledge_graph"`.
|
||||
- **Sessions** (`sessions/`) — `SessionStore` (SQLite): cross-channel persistent sessions. `SessionIdentity` canonical user across channels. `consolidate()` summarizes old messages, `decay()` removes expired.
|
||||
- **A2A Protocol** (`a2a/`) — Google Agent-to-Agent spec (JSON-RPC 2.0). `A2AServer` (tasks/send, tasks/get, tasks/cancel, `/.well-known/agent.json`). `A2AClient`. `A2AAgentTool` adapter.
|
||||
- **TUI Dashboard** (`cli/dashboard.py`) — `textual`-based terminal dashboard (optional `[dashboard]`). Panels: system status, event stream, telemetry, agent activity, sessions.
|
||||
- **Desktop App** (`desktop/`) — Tauri 2.0 native desktop application. 5 dashboard panels: EnergyDashboard (real-time power monitoring with recharts), TraceDebugger (timeline inspection with step-type color coding), LearningCurve (policy visualization, GRPO/bandit stats), MemoryBrowser (search + stats), AdminPanel (health, agents, server control). Tauri commands proxy to OpenJarvis REST API. Plugins: notification, shell, global-shortcut, autostart, updater, single-instance. CI: `.github/workflows/desktop.yml` (Linux/macOS/Windows).
|
||||
- **Vault** (`cli/vault_cmd.py`) — Fernet-encrypted credential store at `~/.openjarvis/vault.enc` with auto-generated key (`0o600` permissions).
|
||||
- **MCP Quick-Add** (`cli/add_cmd.py`) — `jarvis add <server>` with 8 templates (github, filesystem, slack, postgres, brave-search, memory, puppeteer, google-maps). Saves JSON config to `~/.openjarvis/mcp/`.
|
||||
|
||||
### Core Module (`src/openjarvis/core/`)
|
||||
|
||||
- `registry.py` — `RegistryBase[T]` generic base. Subclasses: `ModelRegistry`, `EngineRegistry`, `MemoryRegistry`, `AgentRegistry`, `ToolRegistry`, `RouterPolicyRegistry`, `BenchmarkRegistry`, `ChannelRegistry`, `LearningRegistry`, `SkillRegistry`.
|
||||
- `types.py` — `Message`, `Conversation`, `ModelSpec`, `ToolResult`, `TelemetryRecord`, `StepType`, `TraceStep`, `Trace`, `RoutingContext`.
|
||||
- `config.py` — `JarvisConfig` dataclass hierarchy with TOML loader. Config classes for each pillar/subsystem. TOML sections: `[engine]` (+ nested `[engine.ollama]`, `[engine.vllm]`, `[engine.sglang]`, `[engine.llamacpp]`, `[engine.mlx]`, `[engine.lmstudio]`), `[intelligence]`, `[agent]`, `[tools.storage]`, `[tools.mcp]`, `[tools.browser]`, `[learning]` (+ nested routing/intelligence/agent/metrics), `[server]`, `[telemetry]`, `[traces]`, `[channel]`, `[security]` (+ `[security.capabilities]`, `ssrf_protection`, `rate_limit_*`), `[sandbox]`, `[scheduler]`, `[workflow]`, `[sessions]`, `[a2a]`. Backward-compat: `engine.ollama_host` → `engine.ollama.host`, `agent.default_tools` → `agent.tools`, TOML migration for cross-section moves.
|
||||
- `events.py` — Pub/sub event bus (synchronous dispatch). ~30 EventType values covering inference, tools, memory, agents, telemetry, traces, channels, security, scheduler, workflow, skills, sessions, A2A.
|
||||
|
||||
### Docker & Deployment
|
||||
|
||||
- `deploy/docker/Dockerfile` — Multi-stage: Python 3.12-slim, `.[server]`, entrypoint `jarvis serve`
|
||||
- `deploy/docker/Dockerfile.gpu` — NVIDIA CUDA 12.4 variant
|
||||
- `deploy/docker/Dockerfile.gpu.rocm` — AMD ROCm 6.2 variant
|
||||
- `deploy/docker/docker-compose.yml` — `jarvis` (8000) + `ollama` (11434). ROCm override: `docker-compose.gpu.rocm.yml`
|
||||
- `deploy/systemd/openjarvis.service`, `deploy/launchd/com.openjarvis.plist`
|
||||
|
||||
### Query Flow
|
||||
|
||||
User query → Security scanning (input) → Intelligence resolves model → Agentic Logic (tools/memory) → Memory retrieval → Context injection → Engine generates → Security scanning (output) → Trace recorded → Telemetry recorded → Learning policies update.
|
||||
|
||||
### API Surface
|
||||
|
||||
OpenAI-compatible server via `jarvis serve`:
|
||||
- **Core**: `POST /v1/chat/completions`, `GET /v1/models`, `GET /health`
|
||||
- **Channels**: `GET /v1/channels`, `POST /v1/channels/send`, `GET /v1/channels/status`
|
||||
- **Agents**: `GET /v1/agents`, `POST /v1/agents`, `DELETE /v1/agents/{id}`, `POST /v1/agents/{id}/message`
|
||||
- **Memory**: `POST /v1/memory/store`, `POST /v1/memory/search`, `GET /v1/memory/stats`
|
||||
- **Traces**: `GET /v1/traces`, `GET /v1/traces/{id}`
|
||||
- **Telemetry**: `GET /v1/telemetry/stats`, `GET /v1/telemetry/energy`
|
||||
- **Learning**: `GET /v1/learning/stats`, `GET /v1/learning/policy`
|
||||
- **Skills**: `GET /v1/skills`, `POST /v1/skills`, `DELETE /v1/skills/{name}`
|
||||
- **Speech**: `POST /v1/speech/transcribe`, `GET /v1/speech/health`
|
||||
- **Sessions**: `GET /v1/sessions`, `GET /v1/sessions/{id}`
|
||||
- **Budget**: `GET /v1/budget`, `PUT /v1/budget/limits`
|
||||
- **Metrics**: `GET /metrics` (Prometheus-compatible)
|
||||
- **WebSocket**: `WS /v1/chat/stream` (JSON chunked streaming)
|
||||
- SSE streaming on `/v1/chat/completions` with `stream=true`
|
||||
|
||||
## Key Design Patterns
|
||||
|
||||
- **Registry pattern:** All extensible components use `@XRegistry.register("name")` decorator for registration and runtime discovery.
|
||||
- **ABC interfaces:** Each pillar defines an ABC. Implement the ABC + register via decorator to add a new backend.
|
||||
- **Offline-first:** Cloud APIs are optional. All core functionality works without network.
|
||||
- **Hardware-aware:** Auto-detect GPU vendor/model/VRAM via `nvidia-smi`, `rocm-smi`, `system_profiler`, `/proc/cpuinfo`. Recommend engine accordingly.
|
||||
- **Telemetry opt-in:** `InstrumentedEngine` wraps inference transparently. Agents unaware of telemetry.
|
||||
- **Backward-compat shims:** `intelligence/` re-exports from `learning/`, `agents/react.py` re-exports as `ReActAgent`, registry alias `"react"` → `NativeReActAgent`. Old config keys continue to work.
|
||||
- **`ensure_registered()` pattern:** Benchmark and learning modules use lazy registration to survive registry clearing in tests.
|
||||
|
||||
## Development Phases
|
||||
|
||||
| Version | Phase | Delivers |
|
||||
|---------|-------|----------|
|
||||
| v0.1 | 0 | Scaffolding, registries, core types, config, CLI skeleton |
|
||||
| v0.2 | 1 | Intelligence + Inference — `jarvis ask` end-to-end |
|
||||
| v0.3 | 2 | Memory backends, document indexing, context injection |
|
||||
| v0.4 | 3 | Agents, tool system, OpenAI-compatible API server |
|
||||
| v0.5 | 4 | Learning, telemetry aggregation, `--router` CLI |
|
||||
| v1.0 | 5 | SDK, OpenClaw infra, benchmarks, Docker |
|
||||
| v1.1 | 6 | Trace system, trace-driven learning, pluggable agents |
|
||||
| v1.2 | 7 | 5-pillar restructuring, composition layer, MCP, structured learning |
|
||||
| v1.3 | 8 | Intelligence = "The Model", routing → Learning, engine selection |
|
||||
| v1.4 | 9 | Pillar-aligned config, nested configs, TOML migration |
|
||||
| v1.5 | 10 | Agent restructuring, BaseAgent/ToolUsingAgent, `accepts_tools`, OpenHands SDK |
|
||||
| v1.6 | 11 | NanoClaw subsumption: ClaudeCodeAgent, WhatsApp Baileys, Docker sandbox, TaskScheduler |
|
||||
| v1.7 | 12 | EnergyMonitor ABC (NVIDIA/AMD/Apple/RAPL), EnergyBatch, SteadyStateDetector |
|
||||
| v1.8 | 13 | `jarvis doctor`/`init`, MLX engine, AMD multi-GPU, PWA, ROCm Docker |
|
||||
| v1.9 | 14 | Agent hardening: LoopGuard, RBAC CapabilityPolicy, taint tracking, Merkle audit, Ed25519 |
|
||||
| v2.0 | 15 | WorkflowEngine (DAG), SkillSystem, KnowledgeGraphMemory, SessionStore |
|
||||
| v2.1 | 16 | A2A protocol, MCP templates, WasmRunner, TUI dashboard |
|
||||
| v2.2 | 17 | Production tool parity: FileWrite, ApplyPatch, ShellExec, Git, HTTP, DB, Browser, Agent, Media, PDF tools. SSRF protection, injection scanner, rate limiter, subprocess sandbox, security middleware |
|
||||
| v2.3 | 18 | CLI expansion (20 commands): daemon, chat REPL, agent, workflow, skill, vault, add. API expansion (40+ endpoints): agents, memory, traces, telemetry, learning, skills, sessions, budget, metrics, WebSocket streaming |
|
||||
| v2.4 | 19 | Learning productionization: GRPO (softmax/advantage), BanditRouter (Thompson/UCB1), SkillDiscovery (trace mining), ICL updates (versioning/rollback/quality gates) |
|
||||
| v2.5 | 20 | Tauri 2.0 desktop app: energy dashboard, trace debugger, learning curve visualization, memory browser, admin panel. CI for Linux/macOS/Windows |
|
||||
| v2.6 | 21 | 10 new channels: LINE, Viber, Messenger, Reddit, Mastodon, XMPP, Rocket.Chat, Zulip, Twitch, Nostr |
|
||||
| v2.7 | 22 | Operators: persistent, scheduled autonomous agents with recipe + schedule + channel output |
|
||||
| v2.8 | 23 | Differentiated functionalities: trace-driven learning pipeline (TrainingDataMiner, LoRATrainer, AgentConfigEvolver, LearningOrchestrator), 15 real IPW benchmarks, composable recipes, 15 agent templates, 20 bundled skills, 3 operator recipes |
|
||||
| v2.9 | 24 | Speech subsystem: SpeechBackend ABC, SpeechRegistry, 3 backends (FasterWhisper local, OpenAI cloud, Deepgram cloud), auto-discovery, API endpoints, frontend MicButton + useSpeech hook, Tauri commands, SystemBuilder wiring |
|
||||
| v3.0 | 25 | Operator benchmarks: LogHub, AMA-Bench, LifelongAgentBench, WebChoreArena, WorkArena++. MonitorOperativeAgent with 4 configurable strategies (memory extraction, observation compression, retrieval, task decomposition). EvalRunner episode mode. EnvironmentProvider ABC. browser_axtree tool |
|
||||
@@ -1,492 +0,0 @@
|
||||
# OpenJarvis Development Notes
|
||||
|
||||
Living document tracking implementation progress, testing state, lessons learned, dead ends, and practices for ongoing development. Updated across sessions.
|
||||
|
||||
---
|
||||
|
||||
## Current State (2026-02-21)
|
||||
|
||||
- **Version:** 1.0.0 (trace system added, targeting v1.1)
|
||||
- **All 6 roadmap phases complete** (Phase 0 through Phase 5) + Phase 6 trace system in progress
|
||||
- **Tests:** 576 passed, 8 skipped, 0 failures
|
||||
- **Lint:** ruff clean (`select = ["E", "F", "I", "W"]`)
|
||||
- **Source files:** 76 Python files in `src/openjarvis/`
|
||||
- **Test files:** 78 Python files in `tests/`
|
||||
- **Python:** 3.13 (compatible with 3.10+)
|
||||
- **Package manager:** `uv` with `hatchling` build backend
|
||||
|
||||
### 8 Skipped Tests (Optional Dependencies)
|
||||
|
||||
| Test | Missing Dep | Install Extra |
|
||||
|------|-------------|---------------|
|
||||
| `tests/memory/test_bm25.py` | `rank_bm25` | `openjarvis[memory-bm25]` |
|
||||
| `tests/memory/test_colbert.py` | `colbert` | `openjarvis[memory-colbert]` |
|
||||
| `tests/memory/test_embeddings.py` | `sentence_transformers` | `openjarvis[memory-faiss]` |
|
||||
| `tests/memory/test_faiss.py` | `faiss` | `openjarvis[memory-faiss]` |
|
||||
| `tests/server/test_models_pydantic.py` | `pydantic` | `openjarvis[server]` |
|
||||
| `tests/server/test_routes.py` | `fastapi` | `openjarvis[server]` |
|
||||
| `tests/test_integration.py:165` | `fastapi` | `openjarvis[server]` |
|
||||
| `tests/test_integration.py:190` | `fastapi` | `openjarvis[server]` |
|
||||
|
||||
---
|
||||
|
||||
## Phase Completion Log
|
||||
|
||||
| Phase | Version | Deliverables | Test Count (cumulative) |
|
||||
|-------|---------|-------------|------------------------|
|
||||
| Phase 0 | v0.1 | Scaffolding, registries, core types, config, CLI skeleton, event bus | ~60 |
|
||||
| Phase 1 | v0.2 | Intelligence + Inference — `jarvis ask` end-to-end, heuristic router, engine discovery, basic telemetry | ~160 |
|
||||
| Phase 2 | v0.3 | Memory — SQLite/FAISS/ColBERT/BM25/Hybrid backends, document ingest pipeline, context injection, `jarvis memory` CLI | ~270 |
|
||||
| Phase 3 | v0.4 | Agents (Simple/Orchestrator/Custom/OpenClaw stub), tool system (Calculator/Think/Retrieval/LLM/FileRead), OpenAI-compatible API server, `jarvis serve` | ~360 |
|
||||
| Phase 4 | v0.5 | Learning — HeuristicRouter, HeuristicRewardFunction, GRPORouterPolicy stub, TelemetryAggregator, `jarvis telemetry` CLI, `--router` CLI option | ~432 |
|
||||
| Phase 5 | v1.0 | SDK (`Jarvis` class), OpenClaw infrastructure (protocol/transport/plugin), benchmarks (`jarvis bench`), Docker, docs | ~520 |
|
||||
| Phase 6 | v1.1 | Trace system (TraceStore, TraceCollector, TraceAnalyzer), trace-driven learning (TraceDrivenPolicy) | ~576 |
|
||||
|
||||
---
|
||||
|
||||
## Architecture Quick Reference
|
||||
|
||||
### Directory Layout
|
||||
|
||||
```
|
||||
src/openjarvis/
|
||||
├── __init__.py # __version__ = "1.0.0", exports Jarvis, MemoryHandle
|
||||
├── sdk.py # Python SDK: Jarvis class + MemoryHandle
|
||||
├── core/
|
||||
│ ├── registry.py # RegistryBase[T] + 7 typed registries
|
||||
│ ├── types.py # Message, Conversation, ModelSpec, ToolResult, TelemetryRecord
|
||||
│ ├── config.py # JarvisConfig dataclass hierarchy, TOML loader
|
||||
│ └── events.py # EventBus pub/sub (synchronous)
|
||||
├── intelligence/ # ModelRegistry, HeuristicRouter, model catalog
|
||||
├── traces/ # TraceStore, TraceCollector, TraceAnalyzer
|
||||
├── learning/ # RouterPolicyRegistry, HeuristicRouter, TraceDrivenPolicy, GRPO stub
|
||||
├── memory/ # SQLite/FAISS/ColBERT/BM25/Hybrid backends, chunking, ingest
|
||||
├── agents/ # Simple/Orchestrator/Custom/OpenClaw agents + protocol/transport
|
||||
├── engine/ # Ollama/vLLM/llama.cpp/Cloud engine wrappers
|
||||
├── tools/ # Calculator/Think/Retrieval/LLM/FileRead tools
|
||||
├── bench/ # Latency/Throughput benchmarks, BenchmarkSuite
|
||||
├── telemetry/ # TelemetryStore, TelemetryAggregator, instrumented_generate
|
||||
├── server/ # FastAPI OpenAI-compatible API server
|
||||
└── cli/ # Click CLI: init, ask, serve, model, memory, telemetry, bench
|
||||
```
|
||||
|
||||
### 7 Registries
|
||||
|
||||
All use `RegistryBase[T]` with `@XRegistry.register("name")` or `register_value()`:
|
||||
|
||||
1. `ModelRegistry` — `ModelSpec` objects
|
||||
2. `EngineRegistry` — `InferenceEngine` implementations
|
||||
3. `MemoryRegistry` — `MemoryBackend` implementations
|
||||
4. `AgentRegistry` — `BaseAgent` implementations
|
||||
5. `ToolRegistry` — `BaseTool` implementations
|
||||
6. `RouterPolicyRegistry` — `RouterPolicy` implementations
|
||||
7. `BenchmarkRegistry` — `BaseBenchmark` implementations
|
||||
|
||||
---
|
||||
|
||||
## Patterns and Practices
|
||||
|
||||
### The `ensure_registered()` Pattern
|
||||
|
||||
**Problem:** The `_clean_registries` autouse fixture in `tests/conftest.py` calls `.clear()` on every registry before each test. Module-level `@XRegistry.register("name")` decorators only fire once at import time (Python caches modules in `sys.modules`). After registry clearing, the decorations never re-fire, leaving registries empty for subsequent tests.
|
||||
|
||||
**Solution:** Use lazy registration via `ensure_registered()`:
|
||||
|
||||
```python
|
||||
# src/openjarvis/bench/latency.py
|
||||
_registered = False
|
||||
|
||||
def ensure_registered() -> None:
|
||||
global _registered
|
||||
if _registered:
|
||||
return
|
||||
from openjarvis.core.registry import BenchmarkRegistry
|
||||
if not BenchmarkRegistry.contains("latency"):
|
||||
BenchmarkRegistry.register_value("latency", LatencyBenchmark)
|
||||
_registered = True
|
||||
```
|
||||
|
||||
Then in `__init__.py`:
|
||||
```python
|
||||
def ensure_registered() -> None:
|
||||
from openjarvis.bench.latency import ensure_registered as _reg_latency
|
||||
_reg_latency()
|
||||
```
|
||||
|
||||
And in test files, use an autouse fixture:
|
||||
```python
|
||||
@pytest.fixture(autouse=True)
|
||||
def _register_latency():
|
||||
from openjarvis.bench import ensure_registered
|
||||
ensure_registered()
|
||||
```
|
||||
|
||||
**Where this pattern is used:** `bench/latency.py`, `bench/throughput.py`, `learning/heuristic_policy.py`, `learning/grpo_policy.py`, `learning/heuristic_reward.py`
|
||||
|
||||
**Where this pattern is NOT needed:** Agents, engines, memory backends, and tools use `@register` decorators that work fine because their test files explicitly import and re-register as needed, or the test module import triggers registration.
|
||||
|
||||
### Test Infrastructure
|
||||
|
||||
- **`tests/conftest.py`** — `_clean_registries` autouse fixture clears all 7 registries + clears `EventBus` default listeners before each test. Critical for test isolation.
|
||||
- **Mock engine pattern** — Almost every test that touches the engine layer uses a `MagicMock()` with `.engine_id`, `.health()`, `.list_models()`, `.generate()` stubbed:
|
||||
```python
|
||||
def _make_engine(content="Hello"):
|
||||
engine = MagicMock()
|
||||
engine.engine_id = "mock"
|
||||
engine.health.return_value = True
|
||||
engine.list_models.return_value = ["test-model"]
|
||||
engine.generate.return_value = {
|
||||
"content": content,
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
"model": "test-model",
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
return engine
|
||||
```
|
||||
- **CLI tests** use Click's `CliRunner` with `patch("openjarvis.cli.X.get_engine", ...)` to mock the engine layer.
|
||||
- **Memory tests** use `tmp_path` fixture for SQLite DB paths and test files.
|
||||
- **Optional dep tests** use `pytest.importorskip("module_name")` at module level.
|
||||
|
||||
### Config Defaults
|
||||
|
||||
`JarvisConfig()` with no arguments produces sane defaults:
|
||||
- Engine: auto-discover (Ollama, vLLM, llama.cpp, cloud in priority order)
|
||||
- Memory: `sqlite` backend, `~/.openjarvis/memory.db`
|
||||
- Agent: `simple` (no Node.js dependency)
|
||||
- Intelligence: `qwen3:8b` default, `qwen3:0.6b` fallback
|
||||
- Telemetry: enabled, `~/.openjarvis/telemetry.db`
|
||||
- Learning: `heuristic` default policy
|
||||
|
||||
### File Naming Conventions
|
||||
|
||||
- ABCs and shared dataclasses: `_stubs.py` (e.g., `agents/_stubs.py`, `bench/_stubs.py`, `tools/_stubs.py`)
|
||||
- Internal helpers: `_discovery.py`, `_base.py` (underscore prefix)
|
||||
- CLI commands: `*_cmd.py` (e.g., `bench_cmd.py`, `telemetry_cmd.py`, `memory_cmd.py`)
|
||||
- Test files mirror source: `tests/agents/test_openclaw.py` tests `src/openjarvis/agents/openclaw.py`
|
||||
|
||||
### Import Structure
|
||||
|
||||
- Package `__init__.py` files import submodules to trigger registration
|
||||
- Try/except around optional dependency imports:
|
||||
```python
|
||||
try:
|
||||
from openjarvis.engine.ollama import OllamaEngine # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
```
|
||||
- Top-level `openjarvis/__init__.py` exports: `Jarvis`, `MemoryHandle`, `__version__`
|
||||
|
||||
---
|
||||
|
||||
## Dead Ends and Gotchas
|
||||
|
||||
### 1. `@register` Decorator vs. `ensure_registered()`
|
||||
|
||||
**Dead end:** Initially used `@BenchmarkRegistry.register("latency")` class decorator in `bench/latency.py`. This caused ~10 test failures because:
|
||||
- Registry cleared between tests by `conftest.py`
|
||||
- Module already in `sys.modules`, so `import openjarvis.bench` is a no-op on second import
|
||||
- Registry stays empty after clearing
|
||||
|
||||
**Fix:** Switched to `ensure_registered()` pattern (see above). This is the pattern already used by `learning/` modules.
|
||||
|
||||
**Rule of thumb:** If a module is imported at package init time AND its registry gets cleared in tests, use `ensure_registered()`. If registration only happens in test fixtures or explicit calls, `@register` is fine.
|
||||
|
||||
### 2. Chunk Attribute Names
|
||||
|
||||
`memory/chunking.py` `Chunk` dataclass uses `content` (not `text`). `ChunkConfig` uses `chunk_overlap` (not `overlap`). Easy to get wrong because these aren't obvious from the field names alone. Always read `_stubs.py` or the actual dataclass before using.
|
||||
|
||||
### 3. Test Content Size for Chunking
|
||||
|
||||
`ChunkConfig.min_chunk_size=50` tokens by default. A test string like `"This is test content."` produces 0 chunks. Use at least ~100 words:
|
||||
```python
|
||||
words = " ".join(f"word{i}" for i in range(100))
|
||||
```
|
||||
|
||||
### 4. Version String Locations
|
||||
|
||||
Version is defined in **three places** that must stay in sync:
|
||||
1. `src/openjarvis/__init__.py` — `__version__ = "1.0.0"`
|
||||
2. `pyproject.toml` — `version = "1.0.0"`
|
||||
3. `src/openjarvis/server/app.py` — FastAPI `version="1.0.0"` constructor arg
|
||||
|
||||
Tests that check version: `tests/cli/test_cli.py::test_version_flag`
|
||||
|
||||
### 5. Server Import Guards
|
||||
|
||||
The `server/` module requires `fastapi`, `uvicorn`, `pydantic`. These are behind the `[server]` optional extra. All test files that touch server code use `pytest.importorskip("fastapi")`. The server `__init__.py` wraps imports in try/except.
|
||||
|
||||
### 6. `patch()` Targets for Engine Mocking
|
||||
|
||||
When mocking `get_engine` in CLI tests, the patch target must be the *importing module*, not the source module:
|
||||
```python
|
||||
# CORRECT — patches where it's imported
|
||||
patch("openjarvis.cli.bench_cmd.get_engine", return_value=("mock", engine))
|
||||
|
||||
# WRONG — patches the source, doesn't affect the already-imported reference
|
||||
patch("openjarvis.engine._discovery.get_engine", return_value=("mock", engine))
|
||||
```
|
||||
|
||||
Same for SDK tests: `patch("openjarvis.sdk.get_engine", ...)`.
|
||||
|
||||
### 7. EventBus Clearing
|
||||
|
||||
`EventBus()` creates a new instance each time, but `EventBus._default_listeners` is a class variable. The `conftest.py` fixture resets it. If tests subscribe to events, subscriptions won't persist across tests.
|
||||
|
||||
### 8. Module Shadowing in CLI Package
|
||||
|
||||
In `cli/__init__.py`, `from openjarvis.cli.ask import ask` imports the Click command. This shadows the module name. When you try `mock.patch("openjarvis.cli.ask.get_engine")`, Python resolves `openjarvis.cli.ask` as the Click command (via attribute lookup on the package), not the module.
|
||||
|
||||
**Fix:** Use `importlib.import_module("openjarvis.cli.ask")` to get the actual module object, then `mock.patch.object(module, "get_engine")`.
|
||||
|
||||
---
|
||||
|
||||
## Post-v1.0: Unimplemented Ideas from VISION.md
|
||||
|
||||
These are mentioned in `VISION.md` but not in the roadmap phases. They represent future work:
|
||||
|
||||
### Learning / Router
|
||||
- [ ] Learned router via GRPO (Group Relative Policy Optimization) — `GRPORouterPolicy` is a stub
|
||||
- [ ] Preference learning from user feedback
|
||||
- [ ] Continual fine-tuning on accumulated trajectories
|
||||
- [ ] Multi-objective optimization: quality vs. latency vs. energy vs. cost
|
||||
|
||||
### Memory
|
||||
- [ ] ConversationMemory — sliding window with automatic summarization of older turns
|
||||
- [ ] Personal Notes — user-created persistent notes and preferences
|
||||
- [ ] Episodic Memory — records of past interactions, tool uses, and outcomes
|
||||
- [ ] Vector DB adapters (Qdrant, ChromaDB) for users with existing infrastructure
|
||||
|
||||
### Tools
|
||||
- [ ] WebSearch tool (Tavily, SearXNG, DuckDuckGo)
|
||||
- [ ] CodeInterpreter tool (sandboxed Python execution)
|
||||
- [ ] FileWrite tool (safe file writing with path validation)
|
||||
- [ ] MCP (Model Context Protocol) compatibility
|
||||
|
||||
### Engines
|
||||
- [ ] SGLang engine backend (structured generation, constrained decoding)
|
||||
- [ ] MLX engine backend (Apple Silicon native, Metal acceleration)
|
||||
- [ ] Complete vLLM integration (tensor parallelism config, multi-GPU)
|
||||
|
||||
### OpenClaw
|
||||
- [ ] Full OpenClaw gateway integration (WebSocket, `:18789`)
|
||||
- [ ] OpenClaw skill composition
|
||||
- [ ] Context compaction in OpenClaw agent
|
||||
- [ ] `openjarvis-openclaw` as separate plugin package (currently inline)
|
||||
|
||||
### Infrastructure
|
||||
- [ ] Documentation site (MkDocs or similar)
|
||||
- [ ] Getting started guide
|
||||
- [ ] Plugin development guide
|
||||
- [ ] API reference docs
|
||||
- [ ] CI/CD pipeline
|
||||
- [ ] PyPI publishing
|
||||
|
||||
---
|
||||
|
||||
## Testing Recipes
|
||||
|
||||
### Run all tests
|
||||
```bash
|
||||
uv sync --extra dev
|
||||
uv run pytest tests/ -v --tb=short
|
||||
```
|
||||
|
||||
### Run a specific module's tests
|
||||
```bash
|
||||
uv run pytest tests/bench/ -v
|
||||
uv run pytest tests/sdk/ -v
|
||||
uv run pytest tests/agents/test_openclaw.py -v
|
||||
```
|
||||
|
||||
### Run with optional deps (server)
|
||||
```bash
|
||||
uv sync --extra dev --extra server
|
||||
uv run pytest tests/server/ -v # No longer skipped
|
||||
```
|
||||
|
||||
### Lint
|
||||
```bash
|
||||
uv run ruff check src/ tests/
|
||||
uv run ruff check src/ tests/ --fix # Auto-fix
|
||||
```
|
||||
|
||||
### Quick smoke test
|
||||
```bash
|
||||
uv run jarvis --version # 1.0.0
|
||||
uv run jarvis --help # All subcommands
|
||||
python -c "from openjarvis import Jarvis; print(Jarvis)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Adding New Components
|
||||
|
||||
### New Benchmark
|
||||
|
||||
1. Create `src/openjarvis/bench/my_benchmark.py`:
|
||||
```python
|
||||
from openjarvis.bench._stubs import BaseBenchmark, BenchmarkResult
|
||||
|
||||
class MyBenchmark(BaseBenchmark):
|
||||
@property
|
||||
def name(self) -> str: return "my-bench"
|
||||
@property
|
||||
def description(self) -> str: return "Description"
|
||||
def run(self, engine, model, *, num_samples=10) -> BenchmarkResult: ...
|
||||
|
||||
_registered = False
|
||||
def ensure_registered():
|
||||
global _registered
|
||||
if _registered: return
|
||||
from openjarvis.core.registry import BenchmarkRegistry
|
||||
if not BenchmarkRegistry.contains("my-bench"):
|
||||
BenchmarkRegistry.register_value("my-bench", MyBenchmark)
|
||||
_registered = True
|
||||
```
|
||||
2. Import in `bench/__init__.py` `ensure_registered()`
|
||||
3. Add test file `tests/bench/test_my_benchmark.py` with autouse fixture calling `ensure_registered()`
|
||||
|
||||
### New Tool
|
||||
|
||||
1. Create `src/openjarvis/tools/my_tool.py`:
|
||||
```python
|
||||
from openjarvis.core.registry import ToolRegistry
|
||||
from openjarvis.tools._stubs import BaseTool, ToolSpec
|
||||
|
||||
@ToolRegistry.register("my-tool")
|
||||
class MyTool(BaseTool):
|
||||
@property
|
||||
def spec(self) -> ToolSpec: ...
|
||||
def execute(self, input: str, **params) -> str: ...
|
||||
```
|
||||
2. Import in `tools/__init__.py`
|
||||
3. Add test file `tests/tools/test_my_tool.py`
|
||||
|
||||
### New Memory Backend
|
||||
|
||||
1. Create `src/openjarvis/memory/my_backend.py`:
|
||||
```python
|
||||
from openjarvis.core.registry import MemoryRegistry
|
||||
from openjarvis.memory._stubs import MemoryBackend, RetrievalResult
|
||||
|
||||
@MemoryRegistry.register("my-backend")
|
||||
class MyBackend(MemoryBackend):
|
||||
def store(self, content, *, source="", metadata=None) -> str: ...
|
||||
def retrieve(self, query, top_k=5) -> list[RetrievalResult]: ...
|
||||
def delete(self, doc_id) -> bool: ...
|
||||
def clear(self) -> None: ...
|
||||
```
|
||||
2. Import in `memory/__init__.py` with try/except for optional deps
|
||||
3. Add test file with `pytest.importorskip()` if using optional deps
|
||||
4. Add optional dep group in `pyproject.toml` if needed
|
||||
|
||||
### New Agent
|
||||
|
||||
1. Create `src/openjarvis/agents/my_agent.py`:
|
||||
```python
|
||||
from openjarvis.agents._stubs import AgentResult, BaseAgent
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
|
||||
@AgentRegistry.register("my-agent")
|
||||
class MyAgent(BaseAgent):
|
||||
agent_id = "my-agent"
|
||||
def __init__(self, engine, model, *, bus=None, **kwargs): ...
|
||||
def run(self, input, context=None, **kwargs) -> AgentResult: ...
|
||||
```
|
||||
2. Import in `agents/__init__.py`
|
||||
3. Add test file `tests/agents/test_my_agent.py`
|
||||
|
||||
### New Engine
|
||||
|
||||
1. Create `src/openjarvis/engine/my_engine.py`:
|
||||
```python
|
||||
from openjarvis.core.registry import EngineRegistry
|
||||
from openjarvis.engine._stubs import InferenceEngine
|
||||
|
||||
@EngineRegistry.register("my-engine")
|
||||
class MyEngine(InferenceEngine):
|
||||
engine_id = "my-engine"
|
||||
def generate(self, messages, *, model, **kwargs) -> dict: ...
|
||||
def stream(self, messages, *, model, **kwargs): ...
|
||||
def list_models(self) -> list[str]: ...
|
||||
def health(self) -> bool: ...
|
||||
```
|
||||
2. Import in `engine/__init__.py` with try/except
|
||||
3. Add to `_discovery.py` engine priority list if auto-discoverable
|
||||
|
||||
---
|
||||
|
||||
## Session Log
|
||||
|
||||
### Session 1 (2026-02-16) — Phase 5 Implementation
|
||||
|
||||
**Scope:** Full Phase 5 (v1.0) — SDK, OpenClaw, Benchmarks, Docker, Docs
|
||||
|
||||
**Work completed:**
|
||||
- Step 1: Added `BenchmarkRegistry` to `core/registry.py`, updated `conftest.py`
|
||||
- Step 2: Created `bench/` package — `_stubs.py`, `latency.py`, `throughput.py`, `__init__.py`; CLI `bench_cmd.py`
|
||||
- Step 3: Created `sdk.py` — `Jarvis` class + `MemoryHandle`; updated `__init__.py` exports
|
||||
- Step 4: Created OpenClaw infra — `openclaw_protocol.py`, `openclaw_transport.py`, `openclaw_plugin.py`; rewrote `openclaw.py` from stub
|
||||
- Step 5: Created `Dockerfile`, `Dockerfile.gpu`, `docker-compose.yml`, `deploy/systemd/openjarvis.service`, `deploy/launchd/com.openjarvis.plist`
|
||||
- Step 6: Version bump to 1.0.0, updated `README.md`, `CLAUDE.md`
|
||||
|
||||
**Bugs fixed during implementation:**
|
||||
1. Ruff lint: 17 issues (E501, I001, F401, F841) — all fixed
|
||||
2. Registry clearing broke `@register` decorators — switched to `ensure_registered()` for bench modules
|
||||
3. `ChunkConfig(overlap=...)` should be `ChunkConfig(chunk_overlap=...)` — fixed
|
||||
4. `chunk.text` should be `chunk.content` — fixed
|
||||
5. Test content too short for chunking (0 chunks produced) — used 100 words
|
||||
|
||||
**Final: 520 passed, 8 skipped, 0 failures, ruff clean**
|
||||
|
||||
### Session 2 (2026-02-17) — Test Fixes + Live vLLM Testing
|
||||
|
||||
**Scope:** Fix broken tests, set up live vLLM inference testing
|
||||
|
||||
**Work completed:**
|
||||
- Fixed 6 failed + 13 errored tests in `tests/cli/test_ask_router.py` and `tests/cli/test_ask_agent.py`
|
||||
- **Root cause:** `from openjarvis.cli.ask import ask` in `cli/__init__.py` shadows the `ask` module with the Click command object. When `mock.patch("openjarvis.cli.ask.get_engine")` resolves, it tries to patch an attribute on the Click command, not the module.
|
||||
- **Fix:** Use `importlib.import_module("openjarvis.cli.ask")` + `mock.patch.object(_ask_mod, "get_engine")` instead of string-based patching.
|
||||
- Added tool fallback in `_openai_compat.py`: if server returns 400 when tools are sent (e.g., vLLM without `--enable-auto-tool-choice`), retry without tools.
|
||||
- Verified live vLLM testing: existing vLLM server on port 8003 with `Qwen/Qwen3-8B`
|
||||
- Tested: `jarvis ask`, `jarvis bench run`, `jarvis model list`, `jarvis memory index/search`, `jarvis telemetry stats`, SDK `Jarvis.ask()` and `ask_full()`
|
||||
|
||||
**Gotcha discovered:**
|
||||
8. **Module shadowing with `from X import Y`** — If a package's `__init__.py` does `from openjarvis.cli.ask import ask`, then `openjarvis.cli.ask` in `sys.modules` is the *module*, but accessing it via attribute lookup on `openjarvis.cli` gives the imported *object* (the Click command). Use `importlib.import_module()` for reliable module access when patching.
|
||||
|
||||
**Live vLLM setup notes:**
|
||||
- vLLM 0.15.1 running on Lambda cluster (8x A100-SXM4-80GB)
|
||||
- Config: `~/.openjarvis/config.toml` with `vllm_host = "http://localhost:8003"` and `default_model = "Qwen/Qwen3-8B"`
|
||||
- Tool calling requires `--enable-auto-tool-choice --tool-call-parser hermes` flags on vLLM server
|
||||
- Without tool support, orchestrator falls back to reasoning-only mode
|
||||
|
||||
**Final: 520 passed, 8 skipped, 0 failures, ruff clean**
|
||||
|
||||
### Session 3 (2026-02-21) — Trace System & Research Direction
|
||||
|
||||
**Scope:** Design new research direction (abstractions for local AI), implement trace system
|
||||
|
||||
**Design decisions made:**
|
||||
- OpenJarvis repositioned as a research framework for studying on-device AI
|
||||
- Four core abstractions: Intelligence, Engine, Agentic Logic, Memory
|
||||
- Learning is a cross-cutting concern driven by interaction traces
|
||||
- Agentic Logic should be pluggable — users bring their own architecture (ReAct, OpenHands-style, etc.)
|
||||
- Trace collection is the bridge between static and learned agents
|
||||
- Evolve existing codebase rather than full redesign
|
||||
- Name stays as OpenJarvis
|
||||
- Learning focus: telemetry-driven routing/tool policies (lightweight, always-on)
|
||||
- Agent-model coupling: loose (any agent, any model)
|
||||
|
||||
**Work completed:**
|
||||
- Added `StepType` enum, `TraceStep`, `Trace` dataclasses to `core/types.py`
|
||||
- Added `TRACE_STEP`, `TRACE_COMPLETE` event types to `core/events.py`
|
||||
- Created `traces/` package:
|
||||
- `store.py` — `TraceStore`: SQLite-backed, save/get/list with filters, event bus subscription
|
||||
- `collector.py` — `TraceCollector`: wraps any `BaseAgent`, subscribes to EventBus, records steps automatically
|
||||
- `analyzer.py` — `TraceAnalyzer`: per-route stats, per-tool stats, summaries, export, query-type filtering
|
||||
- Created `learning/trace_policy.py` — `TraceDrivenPolicy`: learns routing from trace outcomes, batch/online updates, registered as `"learned"` policy
|
||||
- Registered `TraceDrivenPolicy` in `learning/__init__.py`
|
||||
- Added 56 new tests across 4 test files in `tests/traces/` and `tests/learning/test_trace_policy.py`
|
||||
- Updated all markdown documentation (README, VISION, ROADMAP, NOTES, CLAUDE)
|
||||
|
||||
**Final: 576 passed, 8 skipped, 0 failures, ruff clean**
|
||||
@@ -1,104 +0,0 @@
|
||||
<div align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="assets/openjarvis-logo-dark.svg">
|
||||
<source media="(prefers-color-scheme: light)" srcset="assets/openjarvis-logo-light.svg">
|
||||
<img alt="OpenJarvis" src="assets/openjarvis-logo-light.svg" width="400">
|
||||
</picture>
|
||||
|
||||
<p><i>Programming abstractions for on-device AI.</i></p>
|
||||
|
||||
<p>
|
||||
<a href="https://www.intelligence-per-watt.ai/"><img src="https://img.shields.io/badge/project-intelligence--per--watt.ai-blue" alt="Project"></a>
|
||||
<a href="https://hazyresearch.stanford.edu/OpenJarvis/"><img src="https://img.shields.io/badge/docs-mkdocs-blue" alt="Docs"></a>
|
||||
<img src="https://img.shields.io/badge/python-%3E%3D3.10-blue" alt="Python">
|
||||
<img src="https://img.shields.io/badge/license-Apache%202.0-green" alt="License">
|
||||
</p>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
> **[Documentation](https://hazyresearch.stanford.edu/OpenJarvis/)**
|
||||
>
|
||||
> **[Project Site](https://www.intelligence-per-watt.ai/)**
|
||||
|
||||
OpenJarvis is a framework for building AI systems that run *entirely on local hardware*. Rather than treating intelligence as a cloud service, OpenJarvis provides composable abstractions for local model selection, inference, agentic reasoning, tool use, and learning — all aware of the hardware they run on.
|
||||
|
||||
```python
|
||||
from openjarvis import Jarvis
|
||||
|
||||
j = Jarvis() # auto-detect hardware + engine
|
||||
response = j.ask("Explain backpropagation") # route to best local model
|
||||
|
||||
j.ask("Solve x^2 - 5x + 6 = 0", # multi-turn agent with tools
|
||||
agent="orchestrator",
|
||||
tools=["calculator", "think"])
|
||||
|
||||
j.memory.index("./papers/") # index documents into local storage
|
||||
results = j.memory.search("attention mechanism") # semantic retrieval
|
||||
j.close()
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install openjarvis # core framework
|
||||
pip install openjarvis[server] # + FastAPI server
|
||||
```
|
||||
|
||||
You also need a local inference backend: [Ollama](https://ollama.com), [vLLM](https://github.com/vllm-project/vllm), [SGLang](https://github.com/sgl-project/sglang), or [llama.cpp](https://github.com/ggerganov/llama.cpp).
|
||||
|
||||
## Quick Start
|
||||
|
||||
The fastest path is Ollama on any machine with Python 3.10+:
|
||||
|
||||
```bash
|
||||
# 1. Install OpenJarvis
|
||||
pip install openjarvis
|
||||
|
||||
# 2. Detect hardware and generate config
|
||||
jarvis init
|
||||
|
||||
# 3. Install and start Ollama (https://ollama.com)
|
||||
curl -fsSL https://ollama.com/install.sh | sh
|
||||
ollama serve # start the Ollama server
|
||||
|
||||
# 4. Pull a model
|
||||
ollama pull qwen3:8b
|
||||
|
||||
# 5. Ask a question
|
||||
jarvis ask "What is the capital of France?"
|
||||
|
||||
# 6. Verify your setup
|
||||
jarvis doctor
|
||||
```
|
||||
|
||||
`jarvis init` auto-detects your hardware and recommends the best engine. After init, it prints engine-specific next steps. Run `jarvis doctor` at any time to diagnose configuration or connectivity issues.
|
||||
|
||||
## The Five Pillars
|
||||
|
||||
| Pillar | What it does | Key abstractions |
|
||||
|--------|-------------|-----------------|
|
||||
| **Intelligence** | Model management and routing | `RouterPolicy`, `QueryAnalyzer`, `ModelCatalog` |
|
||||
| **Engine** | Inference runtime abstraction | `InferenceEngine` ABC — Ollama, vLLM, SGLang, llama.cpp, MLX |
|
||||
| **Agents** | Pluggable reasoning strategies | `BaseAgent` ABC — Simple, Orchestrator, ReAct, OpenHands, OpenClaw |
|
||||
| **Tools** | Capabilities via MCP | `BaseTool` ABC — calculator, code interpreter, web search, memory; external MCP servers auto-discovered |
|
||||
| **Learning** | Trace-driven adaptation | `LearningPolicy` ABC — SFT (model routing), AgentAdvisor (restructuring), ICL (tool usage) |
|
||||
|
||||
Every interaction produces a **Trace** — a structured record of the full reasoning chain. Learning policies consume traces to improve model selection, agent behavior, and tool usage over time.
|
||||
|
||||
## About
|
||||
|
||||
OpenJarvis is part of [Intelligence Per Watt](https://www.intelligence-per-watt.ai/), a research initiative studying the efficiency of on-device AI systems. The project is developed at [Hazy Research](https://hazyresearch.stanford.edu/) and the [Scaling Intelligence Lab](https://scalingintelligence.stanford.edu/) at [Stanford SAIL](https://ai.stanford.edu/).
|
||||
|
||||
## Sponsors
|
||||
|
||||
<p>
|
||||
<a href="https://www.laude.org/">Laude Institute</a> •
|
||||
<a href="https://datascience.stanford.edu/marlowe">Stanford Marlowe</a> •
|
||||
<a href="https://cloud.google.com/">Google Cloud Platform</a> •
|
||||
<a href="https://lambda.ai/">Lambda Labs</a>
|
||||
</p>
|
||||
|
||||
## License
|
||||
|
||||
[Apache 2.0](LICENSE)
|
||||
-374
@@ -1,374 +0,0 @@
|
||||
# OpenJarvis Roadmap
|
||||
|
||||
Phased development plan for OpenJarvis. Phases are ordered to maximize early usability: foundation first, then intelligence + inference (so you can ask questions), then memory (so it remembers), then agents (so it can act), then learning (so it improves).
|
||||
|
||||
---
|
||||
|
||||
## Phase 0 — Foundation (~2-3 weeks)
|
||||
|
||||
**Goal:** Repository scaffolding, core abstractions, and CLI skeleton. Nothing runs yet, but all interfaces are defined.
|
||||
|
||||
**Version milestone:** v0.1
|
||||
|
||||
### Repository structure
|
||||
|
||||
```
|
||||
OpenJarvis/
|
||||
├── pyproject.toml # uv/hatchling, all deps + extras
|
||||
├── src/
|
||||
│ └── openjarvis/
|
||||
│ ├── __init__.py
|
||||
│ ├── core/
|
||||
│ │ ├── registry.py # RegistryBase[T] + typed registries
|
||||
│ │ ├── types.py # Message, Conversation, ModelSpec, ToolResult, TelemetryRecord
|
||||
│ │ ├── config.py # JarvisConfig dataclass hierarchy, TOML loader
|
||||
│ │ └── events.py # Event bus: pub/sub for inter-pillar telemetry
|
||||
│ ├── intelligence/ # Phase 1
|
||||
│ ├── memory/ # Phase 2
|
||||
│ ├── agents/ # Phase 3
|
||||
│ ├── engine/ # Phase 1
|
||||
│ ├── learning/ # Phase 4
|
||||
│ └── cli/ # CLI entry points
|
||||
├── tests/
|
||||
├── VISION.md
|
||||
├── ROADMAP.md
|
||||
└── README.md
|
||||
```
|
||||
|
||||
### Deliverables
|
||||
|
||||
- [ ] **Registry system** — `RegistryBase[T]` adapted from IPW's `registry.py`. Typed subclasses:
|
||||
- `ModelRegistry` — model specs and metadata
|
||||
- `EngineRegistry` — inference engine implementations
|
||||
- `MemoryRegistry` — memory backend implementations
|
||||
- `AgentRegistry` — agent implementations
|
||||
- `ToolRegistry` — tools with `ToolSpec` metadata (category, cost, latency, capabilities)
|
||||
|
||||
- [ ] **Core types** (`core/types.py`):
|
||||
- `Message` — role + content + metadata (tool calls, images, etc.)
|
||||
- `Conversation` — ordered list of messages with sliding window support
|
||||
- `ModelSpec` — model ID, parameter count, quantization, context length, hardware compatibility
|
||||
- `ToolResult` — tool name + output + usage + cost
|
||||
- `TelemetryRecord` — timestamp, model, tokens, latency, energy (optional), cost
|
||||
|
||||
- [ ] **Config system** (`core/config.py`):
|
||||
- `JarvisConfig` dataclass hierarchy: `EngineConfig`, `IntelligenceConfig`, `MemoryConfig`, `AgentConfig`
|
||||
- TOML config file at `~/.openjarvis/config.toml`
|
||||
- Hardware auto-detection: GPU vendor/model/VRAM/platform → populate defaults
|
||||
|
||||
- [ ] **Event bus** (`core/events.py`):
|
||||
- Simple pub/sub for inter-pillar communication
|
||||
- Telemetry events flow without tight coupling between pillars
|
||||
- Synchronous dispatch (async optional later)
|
||||
|
||||
- [ ] **CLI skeleton** (Click-based):
|
||||
- `jarvis init` — create `~/.openjarvis/config.toml` with auto-detected defaults
|
||||
- `jarvis ask` — placeholder (wired in Phase 1)
|
||||
- `jarvis serve` — placeholder (wired in Phase 3)
|
||||
- `jarvis model list|pull|info` — placeholder (wired in Phase 1)
|
||||
- `jarvis memory index|search|stats` — placeholder (wired in Phase 2)
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Intelligence + Inference Engine (~3-4 weeks)
|
||||
|
||||
**Goal:** You can ask OpenJarvis a question and get an answer. Models run locally or via cloud APIs. Basic telemetry records every call.
|
||||
|
||||
**Version milestone:** v0.2 — first usable version
|
||||
|
||||
### Inference Engine
|
||||
|
||||
- [ ] **`InferenceEngine` ABC:**
|
||||
```python
|
||||
class InferenceEngine(ABC):
|
||||
def generate(self, model: str, messages: list[Message], **params) -> Response: ...
|
||||
def stream(self, model: str, messages: list[Message], **params) -> Iterator[ResponseChunk]: ...
|
||||
def list_models(self) -> list[ModelSpec]: ...
|
||||
def health(self) -> bool: ...
|
||||
```
|
||||
|
||||
- [ ] **Engine implementations:**
|
||||
- `OllamaEngine` — wraps Ollama HTTP API (`/api/chat`, `/api/tags`). Apple Silicon + NVIDIA.
|
||||
- `VLLMEngine` — wraps vLLM OpenAI-compatible API. Multi-GPU, tensor parallelism.
|
||||
- `LlamaCppEngine` — wraps `llama-cpp-python` or llama.cpp server. Maximum compatibility.
|
||||
- `CloudEngine` — unified wrapper for OpenAI, Anthropic, and Google APIs. Key-based routing.
|
||||
|
||||
- [ ] **Model management:**
|
||||
- Auto-discovery from running engines (poll `/api/tags`, `/v1/models`)
|
||||
- `ModelSpec` with hardware compatibility matrix (min VRAM, supported engines, quantization options)
|
||||
- `jarvis model list` shows all available models across engines
|
||||
- `jarvis model info <model>` shows spec, hardware requirements, estimated performance
|
||||
|
||||
### Intelligence
|
||||
|
||||
- [ ] **Hardware profiles:**
|
||||
- Auto-detect: `nvidia-smi`, `rocm-smi`, `system_profiler` (macOS), `/proc/cpuinfo`
|
||||
- Map GPU to capabilities: VRAM, compute capability, FP8/FP4 support, unified memory
|
||||
- Recommend engine: Apple Silicon → Ollama/MLX, NVIDIA datacenter → vLLM, AMD → vLLM+ROCm, CPU → llama.cpp
|
||||
|
||||
- [ ] **Heuristic Router V0:**
|
||||
- Rule-based routing: short queries (< 50 tokens) → small model, complex (reasoning keywords, multi-step) → large model, code patterns → code specialist
|
||||
- Fallback chains: if preferred model unavailable, try next in chain
|
||||
- Configurable via `~/.openjarvis/config.toml`
|
||||
|
||||
- [ ] **Basic telemetry:**
|
||||
- Wrap every `generate()` / `stream()` call with timing + token counting
|
||||
- Record to SQLite: model, prompt tokens, completion tokens, latency, cost estimate
|
||||
- `TelemetryRecord` stored via event bus, accumulated for future learning phase
|
||||
|
||||
### Wire-up
|
||||
|
||||
- [ ] **`jarvis ask "What is X?"` works end-to-end:**
|
||||
1. Parse query → detect complexity → route to model
|
||||
2. Generate response via selected engine
|
||||
3. Record telemetry
|
||||
4. Print response (with optional `--json` output)
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Memory / Storage (~3-4 weeks)
|
||||
|
||||
**Goal:** OpenJarvis remembers conversations, can index your documents, and injects relevant context into prompts.
|
||||
|
||||
**Version milestone:** v0.3
|
||||
|
||||
### Memory backends
|
||||
|
||||
- [ ] **`MemoryBackend` ABC:**
|
||||
```python
|
||||
class MemoryBackend(ABC):
|
||||
def store(self, content: str, metadata: dict) -> str: ... # Returns doc ID
|
||||
def retrieve(self, query: str, k: int = 10) -> list[Result]: ...
|
||||
def delete(self, doc_id: str) -> bool: ...
|
||||
def clear(self) -> None: ...
|
||||
```
|
||||
|
||||
- [ ] **Memory subtypes:**
|
||||
- `ConversationMemory` — sliding window (configurable size) + automatic summarization of older turns via LLM call
|
||||
- `KnowledgeBase` — indexed document collection with multi-backend search
|
||||
|
||||
- [ ] **Backend implementations:**
|
||||
|
||||
- **`SQLiteMemory`** — FTS5 full-text search, zero-config default. Always available, no extra dependencies.
|
||||
|
||||
- **`FAISSMemory`** — Dense neural retrieval. Encodes documents with `sentence-transformers`, builds FAISS index (IVF or flat depending on collection size). GPU-accelerated when available.
|
||||
|
||||
- **`ColBERTMemory`** — ColBERTv2 late interaction retrieval. Best retrieval quality.
|
||||
- Package: `colbert-ai[torch,faiss-gpu]`
|
||||
- Indexing: `Indexer(checkpoint="colbertv2.0", config=ColBERTConfig(nbits=2))` → `indexer.index(name, collection)`
|
||||
- Search: `Searcher(index=name)` → `searcher.search(query, k=10)` returns `(passage_ids, ranks, scores)`
|
||||
- Token-level MaxSim matching: each query token attends to each document token, max-pooled per query token, summed
|
||||
- 2-bit residual compression keeps indexes compact (~50x smaller than full embeddings)
|
||||
- Millisecond query latency, substantially better than single-vector methods on complex queries
|
||||
|
||||
- **`BM25Memory`** — Keyword search baseline using `rank-bm25`. No GPU, no embeddings. Fast and effective for keyword-heavy queries.
|
||||
|
||||
- **`HybridMemory`** — Combines BM25 with a dense backend (FAISS or ColBERT) using Reciprocal Rank Fusion (RRF):
|
||||
```
|
||||
RRF_score(d) = sum(1 / (k + rank_i(d))) for each retriever i
|
||||
```
|
||||
Configurable `k` parameter (default 60). Best overall retrieval when you don't know the query type.
|
||||
|
||||
### Document pipeline
|
||||
|
||||
- [ ] **Indexing pipeline:** PDF / Markdown / plain text / code → chunking (configurable size + overlap) → embedding (if using dense/ColBERT backend) → index
|
||||
- [ ] **Context injection:** auto-retrieve top-k relevant chunks before each LLM call, inject into prompt with source attribution (`[Source: filename:line]`)
|
||||
|
||||
### CLI
|
||||
|
||||
- [ ] `jarvis memory index <path>` — index a file or directory
|
||||
- [ ] `jarvis memory search <query>` — search across all memory backends
|
||||
- [ ] `jarvis memory stats` — show index sizes, document counts, backend status
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — Agentic Logic (~3-4 weeks)
|
||||
|
||||
**Goal:** OpenJarvis can use tools, reason over multiple turns, and serve an OpenAI-compatible API. The default agent is OpenClaw's Pi.
|
||||
|
||||
**Version milestone:** v0.4
|
||||
|
||||
### Agent framework
|
||||
|
||||
- [ ] **`BaseAgent` ABC:**
|
||||
```python
|
||||
class BaseAgent(ABC):
|
||||
def run(self, input: str, context: AgentContext) -> AgentResult: ...
|
||||
```
|
||||
`AgentContext` carries: conversation history, memory handle, tool registry, telemetry recorder, model router.
|
||||
`AgentResult` contains: response text, tool calls made, tokens used, telemetry data.
|
||||
|
||||
- [ ] **Agent implementations:**
|
||||
|
||||
- **`OpenClawAgent`** (default) — wraps OpenClaw's Pi agent runtime (`@mariozechner/pi-coding-agent` v0.52.12+). Two modes:
|
||||
1. **HTTP mode:** OpenClaw gateway running locally on `:18789`. Communicate via WebSocket. Best for persistent sessions.
|
||||
2. **Subprocess mode:** invoke `node` with `runEmbeddedPiAgent()` call, JSON over stdin/stdout. No gateway needed.
|
||||
- Capabilities: multi-turn reasoning, tool calling, streaming, skill composition, context compaction
|
||||
- Requires Node.js 22+
|
||||
|
||||
- **`SimpleAgent`** — single-turn: parse query → call model → return response. No tool calling, no multi-turn. Works without Node.js. Good for testing and simple Q&A.
|
||||
|
||||
- **`OrchestratorAgent`** — multi-turn with model selection per step. Adapted from IPW's executor pattern. Each reasoning step can route to a different model (e.g., fast model for planning, large model for synthesis).
|
||||
|
||||
- **`CustomAgent`** — template class for user-defined agents. Subclass `BaseAgent`, implement `run()`, register with `@AgentRegistry.register("my-agent")`.
|
||||
|
||||
### Tool system
|
||||
|
||||
- [ ] **`BaseTool` ABC:**
|
||||
```python
|
||||
class BaseTool(ABC):
|
||||
name: str
|
||||
spec: ToolSpec # category, cost_estimate, latency_estimate, capabilities
|
||||
def execute(self, input: str, **params) -> ToolResult: ...
|
||||
```
|
||||
|
||||
- [ ] **Built-in tools:**
|
||||
- `Calculator` — evaluate math expressions
|
||||
- `WebSearch` — search the web (Tavily, SearXNG, or DuckDuckGo)
|
||||
- `CodeInterpreter` — execute Python in sandboxed environment
|
||||
- `FileRead` / `FileWrite` — local file operations
|
||||
- `Think` — internal reasoning scratchpad (zero-cost tool for chain-of-thought)
|
||||
- `Retrieval` — wired to memory backends, returns relevant documents
|
||||
- `LLMTool` — call another LLM as a tool (for model composition)
|
||||
|
||||
- [ ] **`ToolRegistry`** with discovery:
|
||||
- `ToolSpec` metadata: category, estimated latency, estimated cost, required API keys, capabilities list
|
||||
- Auto-discover available tools based on installed packages and environment
|
||||
|
||||
### API server
|
||||
|
||||
- [ ] **OpenAI-compatible API server:**
|
||||
- `POST /v1/chat/completions` — standard chat completion with tool use
|
||||
- `GET /v1/models` — list available models
|
||||
- Streaming via Server-Sent Events (SSE)
|
||||
- `jarvis serve --port 8000 --agent openclaw`
|
||||
|
||||
### CLI
|
||||
|
||||
- [ ] `jarvis serve --port 8000 --agent <agent>` — start API server
|
||||
- [ ] `jarvis ask` now supports `--agent <agent>` flag
|
||||
|
||||
---
|
||||
|
||||
## Phase 4 — Learning Approach (placeholder)
|
||||
|
||||
**Goal:** Stub interfaces for the learned router. No ML training in this phase — just the contracts and telemetry plumbing so everything is ready when we build it.
|
||||
|
||||
**Version milestone:** v0.5
|
||||
|
||||
### Stubs
|
||||
|
||||
- [ ] **`RouterPolicy` ABC:**
|
||||
```python
|
||||
class RouterPolicy(ABC):
|
||||
def select_model(self, query: str, context: RoutingContext) -> ModelSpec: ...
|
||||
```
|
||||
The heuristic router from Phase 1 implements this as the default.
|
||||
|
||||
- [ ] **`RewardFunction` ABC:**
|
||||
```python
|
||||
class RewardFunction(ABC):
|
||||
def compute(self, trajectory: Trajectory) -> float: ...
|
||||
```
|
||||
Placeholder implementations: `QualityReward` (LLM-judge), `LatencyReward` (inverse latency), `EnergyReward` (inverse energy), `CostReward` (inverse cost), `CompositeReward` (weighted combination).
|
||||
|
||||
- [ ] **`TelemetryAggregator`:**
|
||||
- Reads `TelemetryRecord` entries from SQLite (accumulated since Phase 1)
|
||||
- Computes per-model statistics: average latency, token throughput, cost, quality (when graded)
|
||||
- Exports training-ready datasets for the future GRPO pipeline
|
||||
|
||||
- [ ] **Design document:** `docs/learning-pipeline.md` describing the planned GRPO training pipeline:
|
||||
- Trajectory generation from Phases 1-3 telemetry
|
||||
- Reward model training
|
||||
- Policy optimization with GRPO
|
||||
- Online evaluation and rollout strategy
|
||||
|
||||
---
|
||||
|
||||
## Phase 5 — Integration & Polish (~3-4 weeks)
|
||||
|
||||
**Goal:** Production-ready packaging, OpenClaw integration, benchmarking, SDK, and documentation.
|
||||
|
||||
**Version milestone:** v1.0
|
||||
|
||||
### OpenClaw integration
|
||||
|
||||
- [ ] **`openjarvis-openclaw` plugin package:**
|
||||
- `register()` hook implementing OpenClaw's plugin API
|
||||
- `registerProvider()` — wraps OpenJarvis as an OpenClaw `ProviderPlugin` (routes through OpenJarvis intelligence + engine)
|
||||
- `registerTool()` — exposes OpenJarvis tools to OpenClaw
|
||||
- `MemorySearchManager` — implements OpenClaw's `search()` / `sync()` / `status()` interface, backed by OpenJarvis memory
|
||||
|
||||
### Deployment
|
||||
|
||||
- [ ] **Dockerfile** — multi-stage build with optional GPU support
|
||||
- [ ] **docker-compose.yml** — OpenJarvis + Ollama/vLLM + optional gateway
|
||||
- [ ] **Service files** — systemd (Linux) and launchd (macOS) for running as a system service
|
||||
|
||||
### Python SDK
|
||||
|
||||
- [ ] **Programmatic API:**
|
||||
```python
|
||||
from openjarvis import Jarvis
|
||||
|
||||
j = Jarvis() # Auto-loads config
|
||||
response = await j.ask("Explain transformers") # Uses router + engine
|
||||
await j.memory.index("~/papers/") # Index documents
|
||||
results = await j.memory.search("attention mechanism")
|
||||
```
|
||||
|
||||
### Benchmarking
|
||||
|
||||
- [ ] **`jarvis bench` CLI:**
|
||||
- `BaseBenchmark` / `DatasetBenchmark` ABCs (adapted from IPW's `BenchmarkSuite`)
|
||||
- Run benchmarks across models, measure accuracy + latency + energy
|
||||
- Output JSONL results + summary JSON
|
||||
|
||||
### Documentation
|
||||
|
||||
- [ ] Documentation site (MkDocs or similar)
|
||||
- [ ] Getting started guide
|
||||
- [ ] Plugin development guide
|
||||
- [ ] API reference
|
||||
|
||||
---
|
||||
|
||||
## Phase 6 — Trace System & Learning (~ongoing)
|
||||
|
||||
**Goal:** Full interaction-level trace recording, trace-driven learning, and pluggable agentic architectures. The foundation for studying local AI systems.
|
||||
|
||||
**Version milestone:** v1.1
|
||||
|
||||
### Trace System (complete)
|
||||
|
||||
- [x] **`Trace` and `TraceStep` types** — full interaction recording with step types: route, retrieve, generate, tool_call, respond
|
||||
- [x] **`TraceStore`** — SQLite-backed append-only store with filtering (by agent, model, outcome, time range)
|
||||
- [x] **`TraceCollector`** — wraps any `BaseAgent`, subscribes to EventBus, records steps automatically
|
||||
- [x] **`TraceAnalyzer`** — read-only query layer: per-route stats, per-tool stats, summaries, query-type filtering, export
|
||||
- [x] **`TraceDrivenPolicy`** — learns routing from trace outcomes, batch and online updates, registered as `"learned"` policy
|
||||
- [x] **Event bus integration** — `TRACE_STEP` and `TRACE_COMPLETE` event types
|
||||
|
||||
### Next Steps
|
||||
|
||||
- [ ] Wire `TraceCollector` into SDK/CLI for automatic trace collection
|
||||
- [ ] `jarvis trace` CLI subcommand (list, inspect, export traces)
|
||||
- [ ] User feedback mechanisms (thumbs up/down, quality scores)
|
||||
- [ ] Hierarchical memory (episodic/semantic/procedural layers)
|
||||
- [ ] Pluggable agentic architectures (ReAct, tree-of-thought, custom loops)
|
||||
- [ ] Prompt optimization from traces (DSPy-style compilation for local models)
|
||||
- [ ] Model weight updates from traces (LoRA/QLoRA finetuning)
|
||||
- [ ] GAIA benchmark evaluation with local models
|
||||
|
||||
---
|
||||
|
||||
## Version Summary
|
||||
|
||||
| Version | Phase | What you get |
|
||||
|---------|-------|-------------|
|
||||
| **v0.1** | Phase 0 | Scaffolding, registries, config, CLI skeleton |
|
||||
| **v0.2** | Phase 1 | `jarvis ask` works — local & cloud inference with telemetry |
|
||||
| **v0.3** | Phase 2 | Memory — index docs, conversation history, context injection |
|
||||
| **v0.4** | Phase 3 | Agents + tools + OpenAI-compatible API server |
|
||||
| **v0.5** | Phase 4 | Learning stubs — router policy interface, telemetry aggregation |
|
||||
| **v1.0** | Phase 5 | Production — SDK, OpenClaw plugin, Docker, benchmarks, docs |
|
||||
| **v1.1** | Phase 6 | Trace system, trace-driven learning, pluggable agent architectures |
|
||||
@@ -1,311 +0,0 @@
|
||||
# OpenJarvis
|
||||
|
||||
**Programming abstractions for on-device AI.**
|
||||
|
||||
OpenJarvis defines the abstractions needed to study and build AI systems that run entirely on local hardware. Instead of locking you into one model, one memory system, or one inference engine, OpenJarvis lets you compose your own stack across four core abstractions — then swap any piece without touching the rest.
|
||||
|
||||
Built for researchers studying local AI systems and developers who want full control over their AI stack. Every interaction generates a trace; the system learns from its own usage to improve over time.
|
||||
|
||||
## Why Local AI Needs New Abstractions
|
||||
|
||||
Cloud AI treats intelligence as a **service** — you send a request, get a response, pay per token. Local AI treats intelligence as a **resource** — it lives on your machine, it's always available, it has fixed capabilities, it can be modified, and it accumulates state over time. This inversion changes everything:
|
||||
|
||||
- **Fixed resource budget** — You have 8-24GB VRAM, period. Scheduling and allocation are first-class problems.
|
||||
- **Persistent state is free, compute is expensive** — The opposite of cloud. You can store everything forever but can only run one model at a time.
|
||||
- **You own the weights** — Fine-tuning, RL, prompt compilation are all possible on your data, your hardware, with immediate feedback loops.
|
||||
- **Hardware heterogeneity** — Apple Silicon, NVIDIA consumer GPUs, AMD, CPU-only — each with different optimal strategies.
|
||||
- **Every interaction is a learning signal** — Traces accumulate locally, enabling the system to learn routing, tool selection, and memory strategies from personal usage patterns.
|
||||
|
||||
---
|
||||
|
||||
## The Five Pillars
|
||||
|
||||
OpenJarvis is organized around five composable pillars. Each pillar defines a clear interface; implementations are discovered at runtime via a decorator-based registry system.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ OpenJarvis │
|
||||
│ │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||
│ │ Intelligence │ │ Learning │ │ Memory / │ │
|
||||
│ │ (Models) │ │ Approach │ │ Storage │ │
|
||||
│ │ │ │ (Router) │ │ │ │
|
||||
│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌──────────────────────────────────────────────┐ │
|
||||
│ │ Agentic Logic │ │
|
||||
│ │ (Orchestration, Tools, Reasoning) │ │
|
||||
│ └──────────────────────┬───────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────────────────┐ │
|
||||
│ │ Inference Engine │ │
|
||||
│ │ (vLLM, Ollama, llama.cpp, SGLang, MLX) │ │
|
||||
│ └──────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 1. Intelligence (Model Layer)
|
||||
|
||||
**What it does:** Manages available language models — local and cloud — and routes queries to the best model for the task.
|
||||
|
||||
**What's pluggable:** Models, model providers, routing heuristics.
|
||||
|
||||
**Supported at launch:**
|
||||
|
||||
| Category | Models |
|
||||
|----------|--------|
|
||||
| Open-source (local) | Qwen3 8B, Qwen3 32B, GPT OSS 120B, Kimi-K2.5, MiniMax-M2.5 |
|
||||
| Cloud APIs | Claude (Anthropic), GPT-4o / GPT-5 (OpenAI), Gemini (Google) |
|
||||
|
||||
**Key components:**
|
||||
|
||||
- **`ModelRegistry`** — decorator-based registry mapping model keys to `ModelSpec` objects (parameter count, quantization, hardware compatibility, context length)
|
||||
- **Heuristic Router (V0)** — rule-based routing: short queries → small model, complex reasoning → large model, code → code specialist, fallback chains for unavailable models
|
||||
- **Auto-discovery** — detects models from running inference engines (Ollama, vLLM) and available API keys
|
||||
|
||||
---
|
||||
|
||||
### 2. Learning Approach (Router Policy)
|
||||
|
||||
**What it does:** Determines *which* model handles a given query. Static policies use rules; learned policies update from interaction traces.
|
||||
|
||||
**What's pluggable:** Routing policy, reward functions, training pipeline, trace analyzers.
|
||||
|
||||
**Implemented:**
|
||||
- **Heuristic routing** — rule-based routing based on query characteristics (length, complexity keywords, domain detection), fallback chains
|
||||
- **Trace-driven routing** — learns from accumulated interaction traces which model/agent/tool combinations produce the best outcomes for different query types. Registered as `"learned"` policy.
|
||||
- **Trace system** — every interaction generates a `Trace` recording the full sequence of steps (route, retrieve, generate, tool_call, respond) with timing, inputs, outputs, and outcomes. Stored in SQLite via `TraceStore`.
|
||||
- **Trace analysis** — `TraceAnalyzer` computes per-route stats, per-tool stats, success rates, and query-type distributions from stored traces.
|
||||
|
||||
**Future:**
|
||||
- Learned router via GRPO (Group Relative Policy Optimization)
|
||||
- Preference learning from user feedback
|
||||
- Continual fine-tuning on accumulated trajectories
|
||||
- Multi-objective optimization: quality vs. latency vs. energy vs. cost
|
||||
|
||||
---
|
||||
|
||||
### 3. Memory / Storage
|
||||
|
||||
**What it does:** Provides persistent, searchable memory across conversations, documents, and personal notes. Memory is automatically injected into prompts with source attribution.
|
||||
|
||||
**What's pluggable:** Storage backends, retrieval strategies, embedding models, chunking strategies.
|
||||
|
||||
**Memory types:**
|
||||
- **Conversation Memory** — sliding window with automatic summarization of older turns
|
||||
- **Knowledge Base** — indexed documents (PDF, Markdown, code, text) with multi-backend search
|
||||
- **Personal Notes** — user-created persistent notes and preferences
|
||||
- **Episodic Memory** — records of past interactions, tool uses, and outcomes
|
||||
|
||||
**Backend implementations:**
|
||||
|
||||
| Backend | Type | Description |
|
||||
|---------|------|-------------|
|
||||
| **SQLite** (default) | Keyword + FTS | FTS5 full-text search. Zero dependencies, zero config. Always available. |
|
||||
| **FAISS** | Dense retrieval | Neural semantic search via `sentence-transformers` + FAISS indexes. |
|
||||
| **ColBERTv2** | Late interaction | Token-level MaxSim matching with 2-bit residual compression. Best retrieval quality. Uses `colbert-ai` package with `Indexer` for offline indexing and `Searcher` for millisecond-latency queries. |
|
||||
| **BM25** | Sparse retrieval | Classic keyword search baseline. Fast, no GPU needed. |
|
||||
| **Hybrid** | Fusion | BM25 + dense (or ColBERT) with Reciprocal Rank Fusion (RRF). Best of both worlds. |
|
||||
| **Vector DB adapters** | Dense retrieval | Qdrant, ChromaDB connectors for users with existing vector infrastructure. |
|
||||
|
||||
**ColBERTv2 details:**
|
||||
- Late interaction model: queries and documents are encoded independently, then matched at the token level via MaxSim
|
||||
- 2-bit residual compression keeps indexes compact while preserving quality
|
||||
- Offline indexing via `Indexer(checkpoint="colbertv2.0", config=ColBERTConfig(nbits=2))`
|
||||
- Millisecond query latency via `Searcher(index=name).search(query, k=10)`
|
||||
- Substantially better retrieval quality than single-vector dense methods on complex queries
|
||||
|
||||
---
|
||||
|
||||
### 4. Agentic Logic
|
||||
|
||||
**What it does:** Orchestrates multi-turn reasoning, tool calling, and task execution. The agent layer sits between the user and the model, managing context, tools, and conversation flow.
|
||||
|
||||
**What's pluggable:** Agent implementations, tools, tool registries, execution strategies.
|
||||
|
||||
**Agent implementations:**
|
||||
|
||||
| Agent | Description |
|
||||
|-------|-------------|
|
||||
| **`OpenClawAgent`** (default) | Wraps OpenClaw's Pi agent runtime. Multi-turn reasoning, tool calling, streaming responses, skill composition, context compaction. Two modes: **HTTP** (WebSocket to OpenClaw gateway on `:18789`) or **subprocess** (invoke `node` with `runEmbeddedPiAgent()`, JSON over stdin/stdout). Requires Node.js 22+. |
|
||||
| **`SimpleAgent`** | Single-turn: query → model → response. No tool calling. Works without Node.js. Good for quick answers and testing. |
|
||||
| **`OrchestratorAgent`** | Multi-turn with per-step model selection. Adapted from IPW's executor pattern. Routes each reasoning step to the optimal model. |
|
||||
| **`CustomAgent`** | Template for user-defined agent logic. Subclass `BaseAgent`, implement `run()`, register with `AgentRegistry`. |
|
||||
|
||||
**Tool system:**
|
||||
- `BaseTool` ABC with `ToolSpec` metadata (category, cost estimate, latency estimate, capabilities)
|
||||
- `ToolRegistry` — runtime-discoverable tool catalog
|
||||
- Built-in tools: Calculator, WebSearch, CodeInterpreter, FileRead/Write, Think, Retrieval (wired to memory backends), LLM-as-tool
|
||||
- MCP (Model Context Protocol) compatible
|
||||
|
||||
**API server:**
|
||||
- OpenAI-compatible `/v1/chat/completions` and `/v1/models` endpoints
|
||||
- Streaming via Server-Sent Events (SSE)
|
||||
- Drop-in replacement for any OpenAI-compatible client
|
||||
|
||||
---
|
||||
|
||||
### 5. Inference Engine
|
||||
|
||||
**What it does:** Manages the actual LLM inference runtime — loading models, generating tokens, managing GPU memory.
|
||||
|
||||
**What's pluggable:** Engine backends, hardware profiles, quantization strategies.
|
||||
|
||||
**Supported engines:**
|
||||
|
||||
| Engine | Best for | GPU | CPU |
|
||||
|--------|----------|-----|-----|
|
||||
| **vLLM** | High-throughput server, multi-GPU, production | NVIDIA, AMD | — |
|
||||
| **SGLang** | Structured generation, constrained decoding | NVIDIA, AMD | — |
|
||||
| **Ollama** | Easy setup, Apple Silicon, single-model | NVIDIA, Apple | Yes |
|
||||
| **llama.cpp** | Maximum hardware compatibility, GGUF models | NVIDIA, AMD, Apple | Yes |
|
||||
| **MLX** | Apple Silicon native, Metal acceleration | Apple | Apple |
|
||||
|
||||
**Hardware auto-detection:**
|
||||
- Detects GPU vendor (NVIDIA/AMD/Apple), model, VRAM, compute capability
|
||||
- Recommends the best engine for detected hardware
|
||||
- Apple Silicon → Ollama or MLX; NVIDIA datacenter → vLLM; AMD → vLLM with ROCm; CPU-only → llama.cpp
|
||||
|
||||
---
|
||||
|
||||
## Query Flow
|
||||
|
||||
```
|
||||
User query
|
||||
│
|
||||
▼
|
||||
┌──────────┐ ┌──────────┐ ┌──────────┐
|
||||
│ Agentic │────▶│ Memory │────▶│ Context │
|
||||
│ Logic │ │ Retrieve │ │ Inject │
|
||||
└────┬─────┘ └──────────┘ └────┬─────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────┐ ┌──────────┐ ┌──────────┐
|
||||
│ Learning │────▶│ Model │────▶│ Inference│
|
||||
│ (Router) │ │ Select │ │ Engine │
|
||||
└──────────┘ └──────────┘ └────┬─────┘
|
||||
│
|
||||
▼
|
||||
┌──────────┐
|
||||
│ Response │
|
||||
│ + Telem. │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
1. **Agentic Logic** receives the user query, determines if tools or memory are needed
|
||||
2. **Memory** retrieves relevant context (conversation history, documents, notes)
|
||||
3. **Context Injection** assembles the full prompt with retrieved content and source attribution
|
||||
4. **Learning/Router** selects the best model for this query based on routing policy (heuristic or trace-driven)
|
||||
5. **Inference Engine** runs the selected model and streams the response
|
||||
6. **Trace** records the full interaction sequence: every routing decision, memory retrieval, tool call, and generation step with timing and outcomes
|
||||
7. **Learning** periodically updates routing policies from accumulated traces
|
||||
|
||||
---
|
||||
|
||||
## User Scenarios
|
||||
|
||||
### Developer on M4 Max MacBook Pro (128 GB unified memory)
|
||||
|
||||
```toml
|
||||
# ~/.openjarvis/config.toml
|
||||
[engine]
|
||||
backend = "ollama" # Native Apple Silicon support
|
||||
|
||||
[intelligence]
|
||||
default_model = "qwen3-32b" # Fits in 128 GB unified memory
|
||||
fallback = "qwen3-8b"
|
||||
|
||||
[memory]
|
||||
backend = "sqlite" # Zero-config, always works
|
||||
retrieval = "hybrid" # BM25 + FAISS for local docs
|
||||
|
||||
[agent]
|
||||
type = "openclaw" # Full agent capabilities
|
||||
mode = "subprocess" # No separate gateway needed
|
||||
```
|
||||
|
||||
Day-to-day: codes with `jarvis ask`, indexes project docs with `jarvis memory index`, runs a local OpenAI-compatible server with `jarvis serve` for editor integration.
|
||||
|
||||
### Researcher on DGX Spark (2x B200, 384 GB GPU memory)
|
||||
|
||||
```toml
|
||||
[engine]
|
||||
backend = "vllm"
|
||||
tensor_parallel = 2
|
||||
|
||||
[intelligence]
|
||||
default_model = "qwen3-235b-a22b"
|
||||
router = "heuristic" # Route small queries to 8B, large to 235B
|
||||
|
||||
[memory]
|
||||
backend = "colbert" # Best retrieval quality for papers
|
||||
knowledge_base = "~/papers/"
|
||||
|
||||
[agent]
|
||||
type = "orchestrator" # Multi-model orchestration
|
||||
```
|
||||
|
||||
Running benchmarks with `jarvis bench`, profiling energy per query, comparing model efficiency across hardware configurations.
|
||||
|
||||
### Privacy-Focused Offline Setup
|
||||
|
||||
```toml
|
||||
[engine]
|
||||
backend = "llamacpp" # No server needed
|
||||
network = "offline"
|
||||
|
||||
[intelligence]
|
||||
default_model = "qwen3-8b-q4" # Quantized to fit available RAM
|
||||
|
||||
[memory]
|
||||
backend = "sqlite" # Everything local
|
||||
retrieval = "bm25" # No neural models needed
|
||||
|
||||
[agent]
|
||||
type = "simple" # No external dependencies
|
||||
```
|
||||
|
||||
Fully air-gapped. No cloud APIs, no network calls, no telemetry export. All data stays on the machine.
|
||||
|
||||
---
|
||||
|
||||
## Comparison
|
||||
|
||||
| Feature | OpenJarvis | Ollama | LangChain | OpenClaw | vLLM |
|
||||
|---------|-----------|--------|-----------|----------|------|
|
||||
| **Focus** | Composable AI backend | Model runner | LLM app framework | AI coding assistant | Inference server |
|
||||
| **Model management** | Multi-engine, auto-detect | Single engine | Bring your own | Cloud-first | Single engine |
|
||||
| **Memory** | Multi-backend retrieval | None | Vector store wrappers | Conversation only | None |
|
||||
| **Agents** | Pluggable (Pi, custom) | None | Chain-based | Pi agent (built-in) | None |
|
||||
| **Inference** | vLLM/SGLang/Ollama/llama.cpp/MLX | Ollama only | External | External | vLLM only |
|
||||
| **Hardware-aware** | Auto-detect + recommend | Manual | No | No | Manual |
|
||||
| **Telemetry** | Energy, latency, cost | None | Callbacks | Basic | Metrics |
|
||||
| **Offline** | Full support | Full support | Partial | No | Full support |
|
||||
| **API** | OpenAI-compatible | OpenAI-compatible | Custom | Custom | OpenAI-compatible |
|
||||
| **Language** | Python | Go | Python | TypeScript | Python |
|
||||
|
||||
OpenJarvis is **not** a replacement for these tools — it *composes* them. Ollama and vLLM are inference engine options. OpenClaw's Pi agent is the default agentic logic. LangChain-style chains can be implemented as custom agents.
|
||||
|
||||
---
|
||||
|
||||
## Design Principles
|
||||
|
||||
1. **Pluggable everything** — every component is registered and discoverable at runtime. Swap models, engines, memory backends, and agents without code changes.
|
||||
|
||||
2. **Registry-driven** — `RegistryBase[T]` pattern (adapted from IPW) provides type-safe, decorator-based registration for all extensible components: `ModelRegistry`, `EngineRegistry`, `MemoryRegistry`, `AgentRegistry`, `ToolRegistry`.
|
||||
|
||||
3. **Offline-first** — works without network access. Cloud APIs are optional enhancements, never requirements.
|
||||
|
||||
4. **Telemetry-native** — every inference call records timing, token counts, and (when hardware supports it) energy consumption. Data lands in SQLite for analysis.
|
||||
|
||||
5. **Hardware-aware** — auto-detects GPU vendor, model, VRAM, and platform. Recommends the best engine and model configuration for your hardware.
|
||||
|
||||
6. **Python-first** — core is pure Python (3.10+). Node.js required only for OpenClaw agent integration. No Java, no JVM, no heavy runtimes.
|
||||
|
||||
7. **OpenAI-compatible API** — `jarvis serve` exposes `/v1/chat/completions` and `/v1/models`. Any client that speaks OpenAI protocol works out of the box.
|
||||
|
||||
8. **Standalone** — OpenJarvis is a self-contained backend. OpenClaw is one possible frontend; so is `curl`, a Python SDK call, or any OpenAI-compatible client.
|
||||
-107
@@ -1,107 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the multi-benchmark optimization dry run directly (bypasses CLI)."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
# Load env
|
||||
from pathlib import Path
|
||||
|
||||
env_path = Path(__file__).parent / ".env"
|
||||
if env_path.exists():
|
||||
for line in env_path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
if line.startswith("export "):
|
||||
line = line[7:]
|
||||
key, _, val = line.partition("=")
|
||||
os.environ[key] = val
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(name)s %(levelname)s %(message)s",
|
||||
)
|
||||
log = logging.getLogger("dry_run")
|
||||
|
||||
from openjarvis.optimize.config import ( # noqa: E402
|
||||
load_benchmark_specs,
|
||||
load_objectives,
|
||||
load_optimize_config,
|
||||
)
|
||||
from openjarvis.optimize.llm_optimizer import LLMOptimizer # noqa: E402
|
||||
from openjarvis.optimize.optimizer import OptimizationEngine # noqa: E402
|
||||
from openjarvis.optimize.search_space import build_search_space # noqa: E402
|
||||
from openjarvis.optimize.store import OptimizationStore # noqa: E402
|
||||
from openjarvis.optimize.trial_runner import MultiBenchTrialRunner # noqa: E402
|
||||
|
||||
CONFIG = "src/openjarvis/optimize/configs/qwen3-235b-dry-run.toml"
|
||||
|
||||
log.info("Loading config: %s", CONFIG)
|
||||
data = load_optimize_config(CONFIG)
|
||||
opt = data["optimize"]
|
||||
|
||||
search_space = build_search_space(data)
|
||||
objectives = load_objectives(data)
|
||||
specs = load_benchmark_specs(data)
|
||||
|
||||
log.info("Benchmarks: %s", [(s.benchmark, s.max_samples, s.weight) for s in specs])
|
||||
log.info("Objectives: %s", [(o.metric, o.direction, o.weight) for o in objectives])
|
||||
|
||||
optimizer_model = opt.get("optimizer_model", "claude-opus-4-6")
|
||||
output_dir = opt.get("output_dir", "results/optimize/dry-run/")
|
||||
max_trials = opt.get("max_trials", 2)
|
||||
early_stop = opt.get("early_stop_patience", 5)
|
||||
|
||||
# Build optimizer backend (Anthropic)
|
||||
from openjarvis.evals.cli import _build_judge_backend # noqa: E402
|
||||
|
||||
optimizer_backend = _build_judge_backend(optimizer_model)
|
||||
log.info("Optimizer backend ready: %s", optimizer_model)
|
||||
|
||||
llm_opt = LLMOptimizer(
|
||||
search_space=search_space,
|
||||
optimizer_model=optimizer_model,
|
||||
optimizer_backend=optimizer_backend,
|
||||
)
|
||||
|
||||
runner = MultiBenchTrialRunner(
|
||||
benchmark_specs=specs,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
store = OptimizationStore(Path(output_dir) / "optimize.db")
|
||||
|
||||
engine = OptimizationEngine(
|
||||
search_space=search_space,
|
||||
llm_optimizer=llm_opt,
|
||||
trial_runner=runner,
|
||||
store=store,
|
||||
max_trials=max_trials,
|
||||
early_stop_patience=early_stop,
|
||||
)
|
||||
|
||||
log.info("Starting dry run: max_trials=%d", max_trials)
|
||||
|
||||
run = engine.run(
|
||||
progress_callback=lambda t, m: log.info("Trial %d/%d complete", t, m),
|
||||
)
|
||||
|
||||
store.close()
|
||||
|
||||
log.info("Optimization complete!")
|
||||
log.info(" Run ID: %s", run.run_id)
|
||||
log.info(" Status: %s", run.status)
|
||||
log.info(" Trials: %d", len(run.trials))
|
||||
if run.best_trial:
|
||||
log.info(
|
||||
" Best: %s (accuracy=%.4f)",
|
||||
run.best_trial.trial_id,
|
||||
run.best_trial.accuracy,
|
||||
)
|
||||
|
||||
# Export best recipe
|
||||
if run.best_trial:
|
||||
recipe_path = Path(output_dir) / "best_recipe.toml"
|
||||
engine.export_best_recipe(run, recipe_path)
|
||||
log.info(" Best recipe exported to: %s", recipe_path)
|
||||
@@ -1,126 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the full multi-benchmark optimization (15 trials, 200 samples/benchmark)."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Load env
|
||||
env_path = Path(__file__).parent / ".env"
|
||||
if env_path.exists():
|
||||
for line in env_path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
if line.startswith("export "):
|
||||
line = line[7:]
|
||||
key, _, val = line.partition("=")
|
||||
os.environ[key] = val
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(name)s %(levelname)s %(message)s",
|
||||
)
|
||||
log = logging.getLogger("full_optimization")
|
||||
|
||||
from openjarvis.optimize.config import ( # noqa: E402
|
||||
load_benchmark_specs,
|
||||
load_objectives,
|
||||
load_optimize_config,
|
||||
)
|
||||
from openjarvis.optimize.llm_optimizer import LLMOptimizer # noqa: E402
|
||||
from openjarvis.optimize.optimizer import OptimizationEngine # noqa: E402
|
||||
from openjarvis.optimize.search_space import build_search_space # noqa: E402
|
||||
from openjarvis.optimize.store import OptimizationStore # noqa: E402
|
||||
from openjarvis.optimize.trial_runner import MultiBenchTrialRunner # noqa: E402
|
||||
|
||||
CONFIG = "src/openjarvis/optimize/configs/qwen3.5-joint-agentic.toml"
|
||||
|
||||
# Ensure HF_TOKEN is set for gated datasets (HLE)
|
||||
if not os.environ.get("HF_TOKEN"):
|
||||
log.warning("HF_TOKEN not set — gated datasets like HLE may fail")
|
||||
|
||||
log.info("Loading config: %s", CONFIG)
|
||||
data = load_optimize_config(CONFIG)
|
||||
opt = data["optimize"]
|
||||
|
||||
search_space = build_search_space(data)
|
||||
objectives = load_objectives(data)
|
||||
specs = load_benchmark_specs(data)
|
||||
|
||||
log.info("Benchmarks: %s", [(s.benchmark, s.max_samples, s.weight) for s in specs])
|
||||
log.info("Objectives: %s", [(o.metric, o.direction, o.weight) for o in objectives])
|
||||
|
||||
optimizer_model = opt.get("optimizer_model", "claude-opus-4-6")
|
||||
output_dir = opt.get("output_dir", "results/optimize/qwen3-235b-joint/")
|
||||
max_trials = opt.get("max_trials", 15)
|
||||
early_stop = opt.get("early_stop_patience", 5)
|
||||
|
||||
# Build optimizer backend (Anthropic)
|
||||
from openjarvis.evals.cli import _build_judge_backend # noqa: E402
|
||||
|
||||
optimizer_backend = _build_judge_backend(optimizer_model)
|
||||
log.info("Optimizer backend ready: %s", optimizer_model)
|
||||
|
||||
llm_opt = LLMOptimizer(
|
||||
search_space=search_space,
|
||||
optimizer_model=optimizer_model,
|
||||
optimizer_backend=optimizer_backend,
|
||||
)
|
||||
|
||||
runner = MultiBenchTrialRunner(
|
||||
benchmark_specs=specs,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
store = OptimizationStore(Path(output_dir) / "optimize.db")
|
||||
|
||||
engine = OptimizationEngine(
|
||||
search_space=search_space,
|
||||
llm_optimizer=llm_opt,
|
||||
trial_runner=runner,
|
||||
store=store,
|
||||
max_trials=max_trials,
|
||||
early_stop_patience=early_stop,
|
||||
)
|
||||
|
||||
log.info("Starting full optimization: max_trials=%d, benchmarks=%s",
|
||||
max_trials, [s.benchmark for s in specs])
|
||||
|
||||
run = engine.run(
|
||||
progress_callback=lambda t, m: log.info("=== Trial %d/%d complete ===", t, m),
|
||||
)
|
||||
|
||||
store.close()
|
||||
|
||||
log.info("=" * 60)
|
||||
log.info("OPTIMIZATION COMPLETE")
|
||||
log.info("=" * 60)
|
||||
log.info(" Run ID: %s", run.run_id)
|
||||
log.info(" Status: %s", run.status)
|
||||
log.info(" Trials: %d", len(run.trials))
|
||||
if run.best_trial:
|
||||
log.info(
|
||||
" Best: %s (accuracy=%.4f)",
|
||||
run.best_trial.trial_id,
|
||||
run.best_trial.accuracy,
|
||||
)
|
||||
for t in run.trials:
|
||||
log.info(
|
||||
" Trial %s: acc=%.4f lat=%.2fs",
|
||||
t.trial_id[:8], t.accuracy, t.mean_latency_seconds,
|
||||
)
|
||||
|
||||
# Export best recipe
|
||||
if run.best_trial:
|
||||
recipe_path = Path(output_dir) / "best_recipe.toml"
|
||||
engine.export_best_recipe(run, recipe_path)
|
||||
log.info(" Best recipe exported to: %s", recipe_path)
|
||||
|
||||
# Pareto frontier
|
||||
if run.pareto_frontier:
|
||||
log.info(" Pareto frontier: %d trials", len(run.pareto_frontier))
|
||||
for t in run.pareto_frontier:
|
||||
log.info(" %s: acc=%.4f lat=%.2fs energy=%.1fJ",
|
||||
t.trial_id[:8], t.accuracy, t.mean_latency_seconds,
|
||||
t.total_energy_joules)
|
||||
Reference in New Issue
Block a user