diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05b90a58..d9b4f9cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: run: uv run maturin develop --manifest-path rust/crates/openjarvis-python/Cargo.toml - name: Run tests - run: uv run pytest tests/ -v --tb=short + run: uv run pytest tests/ -v --tb=short -m "not live and not cloud" rust: runs-on: ubuntu-latest diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 14dcf64e..84e2a1e1 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -10,7 +10,7 @@ const OLLAMA_PORT: u16 = 11434; const JARVIS_PORT: u16 = 8222; /// Small, fast model pulled at startup so the app opens quickly. -const STARTUP_MODEL: &str = "qwen3.5:2b"; +const STARTUP_MODEL: &str = "qwen3.5:4b"; /// Tiny fallback model if even the startup model can't be pulled. const FALLBACK_MODEL: &str = "qwen3:0.6b"; @@ -86,12 +86,14 @@ fn models_that_fit() -> Vec<&'static str> { .collect() } -/// Pick the third-largest Qwen3.5 model that fits on this machine. -/// This leaves comfortable headroom for the OS / other apps while -/// still providing a capable model. Falls back gracefully when -/// fewer models fit. +/// Pick the default model — prefers STARTUP_MODEL if it fits, otherwise +/// falls back to the third-largest model that fits on this machine. fn preferred_model() -> &'static str { let fitting = models_that_fit(); + // Prefer STARTUP_MODEL when it fits (fast, good quality) + if fitting.contains(&STARTUP_MODEL) { + return STARTUP_MODEL; + } match fitting.len() { 0 => FALLBACK_MODEL, 1 => fitting[0], @@ -460,17 +462,90 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) { return; } - let project_root = find_project_root(); + let mut project_root = find_project_root(); if project_root.is_none() { - let mut s = status.lock().await; - s.error = Some( - "Could not find the OpenJarvis project directory. \ - Clone it with: git clone https://github.com/open-jarvis/OpenJarvis.git ~/OpenJarvis \ - then relaunch." - .into(), - ); - return; + // Auto-clone on first launch + let git_bin = resolve_bin("git"); + + // Check that git is installed + if !std::path::Path::new(&git_bin).exists() && git_bin == "git" { + let mut s = status.lock().await; + s.error = Some( + "Could not find 'git'. \ + Install it from https://git-scm.com then relaunch." + .into(), + ); + return; + } + + let target_path = std::path::PathBuf::from(home_dir()).join("OpenJarvis"); + let clone_target = target_path.display().to_string(); + + // If the directory exists but is not a valid project, don't overwrite + if target_path.exists() && !target_path.join("pyproject.toml").exists() { + let mut s = status.lock().await; + s.error = Some(format!( + "{} exists but is not a valid OpenJarvis project. \ + Remove it and relaunch, or set OPENJARVIS_ROOT to the correct path.", + clone_target, + )); + return; + } + + { + let mut s = status.lock().await; + s.detail = "Downloading OpenJarvis (first launch)...".into(); + } + + let clone_result = tokio::process::Command::new(&git_bin) + .args([ + "clone", + "--depth", + "1", + "https://github.com/open-jarvis/OpenJarvis.git", + &clone_target, + ]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn(); + + match clone_result { + Ok(child) => match child.wait_with_output().await { + Ok(output) if output.status.success() => { + project_root = Some(target_path); + } + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + let mut s = status.lock().await; + s.error = Some(format!( + "Failed to download OpenJarvis: {}. \ + Clone manually: git clone https://github.com/open-jarvis/OpenJarvis.git {}", + stderr.trim(), + clone_target, + )); + return; + } + Err(e) => { + let mut s = status.lock().await; + s.error = Some(format!( + "Failed to download OpenJarvis: {}. \ + Clone manually: git clone https://github.com/open-jarvis/OpenJarvis.git {}", + e, clone_target, + )); + return; + } + }, + Err(e) => { + let mut s = status.lock().await; + s.error = Some(format!( + "Could not run git: {}. \ + Install git from https://git-scm.com then relaunch.", + e, + )); + return; + } + } } // Kill any leftover server on our port from a previous run diff --git a/docs/getting-started/macos.md b/docs/getting-started/macos.md new file mode 100644 index 00000000..67a6fe1b --- /dev/null +++ b/docs/getting-started/macos.md @@ -0,0 +1,421 @@ +--- +title: macOS Installation Guide +description: Complete step-by-step guide to installing OpenJarvis on macOS with llama.cpp, including common pitfalls and fixes +search: + boost: 2 +--- + +# macOS Installation Guide + +This guide walks through a complete OpenJarvis installation on macOS using **llama.cpp** as +the inference engine. It covers every step from scratch — including pitfalls not documented +elsewhere — and is suitable for both Apple Silicon and Intel Macs. + +!!! tip "Prefer Ollama?" + If you want the fastest possible setup, use [Ollama](installation.md#ollama-recommended) + instead. This guide is for users who want to run GGUF models directly with llama.cpp, + or who want a deeper understanding of the full stack. + +--- + +## What You'll Install + +| Tool | Purpose | +|------|---------| +| Homebrew | macOS package manager — installs everything else | +| uv | Python version and dependency manager | +| Git | Clones the OpenJarvis repo | +| Node.js | Required for the browser UI | +| Rust | Compiles the OpenJarvis security and memory extension | +| llama.cpp | Local inference engine that runs GGUF model files | +| OpenJarvis | The framework itself | +| A GGUF model | The actual AI model (downloaded separately) | + +--- + +## Step-by-Step Installation + +### Step 1 — Install Homebrew + +Homebrew is the standard macOS package manager. Everything else in this guide is installed +through it. + +```bash +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +``` + +If you already have Homebrew, skip this step. + +--- + +### Step 2 — Install uv + +`uv` replaces pip, virtualenv, and pyenv in one tool. OpenJarvis uses it to manage Python +versions, virtual environments, and project dependencies. + +```bash +brew install uv +``` + +--- + +### Step 3 — Install Git + +Git is used to clone the OpenJarvis source code. It may already be present if you have +Xcode Command Line Tools installed. + +```bash +brew install git +``` + +--- + +### Step 4 — Install Node.js + +Node.js is required to build and run the browser frontend. Without it you can still use +the CLI, but not the web UI. + +```bash +brew install node +``` + +--- + +### Step 5 — Install Rust + +OpenJarvis includes a Rust extension that provides security scanning, memory indexing, +rate limiting, and tool execution. It must be compiled from source. + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` + +After the installer finishes, reload your shell so `rustc` is available: + +```bash +source "$HOME/.cargo/env" +``` + +Verify: + +```bash +rustc --version +``` + +--- + +### Step 6 — Install llama.cpp + +llama.cpp is the inference engine that loads and runs GGUF model files. It is not a model +itself — think of it as a media player and the `.gguf` file as the content. + +```bash +brew install llama.cpp +``` + +--- + +### Step 7 — Clone the OpenJarvis repo + +Run this from your home directory or any neutral parent folder. + +```bash +cd ~ +git clone https://github.com/open-jarvis/OpenJarvis.git +cd OpenJarvis +``` + +!!! warning "Do not clone from inside an existing OpenJarvis folder" + A common mistake is running `git clone` while already inside the repo, creating deeply + nested duplicates (`OpenJarvis/OpenJarvis/OpenJarvis`). Always clone from `~` or a + neutral parent directory. + +--- + +### Step 8 — Pin Python to 3.12 + +!!! warning "Critical step — do not skip" + OpenJarvis requires Python 3.10–3.13. Its Rust extension uses PyO3, which does not yet + support Python 3.14. If `uv` has Python 3.14 available, it will use it by default, + causing the Rust extension build to fail silently and resulting in ~250 test failures + with `ModuleNotFoundError: No module named 'openjarvis_rust'`. + +Pin the project to Python 3.12: + +```bash +echo "3.12" > .python-version +uv python install 3.12 +rm -rf .venv +uv venv +``` + +**Restart your terminal**, then verify: + +```bash +uv run python --version +# Must show: Python 3.12.x +``` + +!!! tip "Why restart the terminal?" + Without restarting, the shell may still reference the old virtual environment. This is + the most common reason the version pin appears not to work. + +--- + +### Step 9 — Install Python dependencies + +```bash +uv sync --extra dev --extra server +``` + +The `--extra server` flag adds the FastAPI backend required for the browser UI. + +--- + +### Step 10 — Build the Rust extension + +This compiles the Rust extension and installs it into the virtual environment. It provides +security scanning, memory indexing, MCP tool execution, and rate limiting. This step takes +a few minutes on first run. + +```bash +uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml +``` + +Verify it built correctly: + +```bash +uv run python -c "import openjarvis_rust; print('Rust extension OK')" +``` + +--- + +### Step 11 — Install frontend dependencies + +```bash +cd frontend && npm install && cd .. +``` + +--- + +### Step 12 — Download a model + +OpenJarvis needs a GGUF model file to run inference. First install the Hugging Face CLI, +then download your chosen model. + +```bash +uv tool install huggingface_hub +``` + +!!! note "The CLI command is `hf`, not `huggingface-cli`" + When installed via `uv tool`, the Hugging Face CLI is invoked as `hf`. + +=== "Qwen3 4B (~2.5 GB)" + + Faster, lower RAM requirement. Good for most everyday tasks. + + ```bash + hf download bartowski/Qwen_Qwen3-4B-GGUF \ + --include "Qwen_Qwen3-4B-Q4_K_M.gguf" \ + --local-dir ~/models + ``` + +=== "Qwen3 8B (~4.7 GB)" + + Better reasoning and instruction following. Requires more RAM. + + ```bash + hf download bartowski/Qwen_Qwen3-8B-GGUF \ + --include "Qwen_Qwen3-8B-Q4_K_M.gguf" \ + --local-dir ~/models + ``` + +!!! warning "Use the `Qwen_` prefix" + bartowski's Qwen3 repos use the `Qwen_` prefix (e.g. `Qwen_Qwen3-4B-GGUF`). Using + the shorter name without the prefix returns a "repository not found" error. + +!!! tip "Apple Silicon vs Intel" + On Apple Silicon, both models benefit from Metal GPU acceleration when using the MLX + engine. On Intel, inference runs on CPU — the 4B model is recommended for speed. + +--- + +### Step 13 — Configure OpenJarvis + +Run the init command to detect your hardware and generate a config file: + +```bash +uv run jarvis init +``` + +Then open the config and set the default model to match the filename you downloaded: + +```bash +nano ~/.openjarvis/config.toml +``` + +Find the `default_model` line and update it, for example: + +```toml +default_model = "Qwen_Qwen3-4B-Q4_K_M.gguf" +``` + +--- + +### Step 14 — Verify the installation + +```bash +uv run jarvis doctor +``` + +A healthy setup looks like this: + +``` +✓ Python version 3.12.x +✓ Config file ~/.openjarvis/config.toml +✓ Config parsing Config loaded successfully +✓ Engine: llamacpp Reachable +✓ Models: llamacpp Qwen_Qwen3-4B-Q4_K_M.gguf +✓ Default model Qwen_Qwen3-4B-Q4_K_M.gguf (on llamacpp) +``` + +!!! note "Warnings for other engines are normal" + The `!` warnings for engines like `ollama`, `vllm`, and `lmstudio` simply mean those + backends are not running. You only need `llamacpp` to be reachable. + +--- + +## Running OpenJarvis + +### CLI + +Start llama-server in one terminal, then run queries in another: + +```bash +# Terminal 1 — start the inference engine +llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf -c 4096 -t 8 + +# Terminal 2 — ask a question +cd ~/OpenJarvis +uv run jarvis ask "What is the capital of France?" +``` + +### Browser UI + +```bash +# Terminal 1 — inference engine +llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf -c 4096 -t 8 + +# Terminal 2 — backend +cd ~/OpenJarvis && uv run jarvis serve --port 8000 + +# Terminal 3 — frontend +cd ~/OpenJarvis/frontend && npm run dev +``` + +Then open [http://localhost:5173](http://localhost:5173). + +### Skip typing `uv run` every time + +Activate the virtual environment for your current terminal session: + +```bash +source ~/OpenJarvis/.venv/bin/activate +``` + +Your prompt will show `(openjarvis)` when active, and you can type `jarvis ask "..."` directly. + +--- + +## Performance Tips + +These tips apply when using llama.cpp for CPU inference. + +| Flag | Effect | +|------|--------| +| `-c 4096` | Reduces context window from the 32,768 default, freeing RAM for faster inference | +| `-t 8` | Uses all available CPU threads (default is only 4) — adjust to your machine's thread count | +| `Q4_K_M` quantization | Best balance of size, speed, and quality for CPU inference | + +On Apple Silicon, switching to the [MLX engine](../architecture/engine.md) gives +significantly better performance than llama.cpp for most models. + +--- + +## Common Errors + +### `No such file or directory` when loading model + +The path `path/to/model.gguf` in examples is a placeholder. Replace it with your actual +model path, e.g.: + +```bash +llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf +``` + +--- + +### `No module named 'openjarvis_rust'` + +The Rust extension did not build correctly, or was built against the wrong Python version. + +1. Confirm Python 3.12 is active: `uv run python --version` +2. Rebuild: `uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml` + +If the version shows 3.14, go back to [Step 8](#step-8--pin-python-to-312). + +--- + +### `PyO3 version error — Python 3.14 too new` + +``` +error: the configured Python interpreter version (3.14) is newer than +PyO3's maximum supported version (3.13) +``` + +PyO3 0.23.5 supports Python up to 3.13. Follow [Step 8](#step-8--pin-python-to-312) to +pin to 3.12, then delete `.venv`, recreate it, and restart your terminal before retrying. + +--- + +### `Repository not found` when downloading model + +bartowski's Qwen3 repos use the `Qwen_` prefix. Use: + +``` +bartowski/Qwen_Qwen3-4B-GGUF ✓ +bartowski/Qwen3-4B-GGUF ✗ +``` + +--- + +### `No inference engine available` + +llama-server is not running. Start it in a separate terminal before running any `jarvis` +commands, and wait until you see `model loaded` in the output. + +--- + +### Python version still shows 3.14 after recreating the venv + +Close the terminal completely and reopen it. The old venv path is cached in the shell +environment and persists across commands until the session ends. + +--- + +### `zsh: command not found: huggingface-cli` + +When installed via `uv tool`, the CLI is invoked as `hf`, not `huggingface-cli`: + +```bash +hf download ... # ✓ +huggingface-cli download ... # ✗ +``` + +--- + +## Next Steps + +- [Quick Start](quickstart.md) — Run your first query and explore agents and tools +- [Configuration](configuration.md) — Customize engine hosts, model routing, memory, and more +- [Architecture](../architecture/overview.md) — Understand how OpenJarvis is structured diff --git a/docs/javascripts/leaderboard.js b/docs/javascripts/leaderboard.js index 742b398a..ce22868c 100644 --- a/docs/javascripts/leaderboard.js +++ b/docs/javascripts/leaderboard.js @@ -9,20 +9,24 @@ var allRows = []; var currentPage = 0; - // No-KV-cache formula: FLOPs = params_b * 1e9 * N * (N+1) - // Reference values only (GPT-5.3 constants) — recompute functions below - // are defined but not called; the leaderboard displays database values directly. - var DEFAULT_PARAMS_B = 137; - var ENERGY_WH_PER_FLOP = 0.4 / (1000 * 3e12); + // Outlier detection — hide entries with values that are physically + // implausible relative to their token count. Thresholds are ~1000x + // above legitimate per-token values to avoid false positives. + var MAX_ENERGY_WH_PER_TOKEN = 10; // legit ≈ 0.001 Wh/tok + var MAX_FLOPS_PER_TOKEN = 1e17; // legit ≈ 1e12 /tok + var MAX_DOLLAR_PER_TOKEN = 25.0 / 1e6; // hard ceiling: $25/1M output - function recomputeFlopsNoKvCache(totalTokens) { - var n = Number(totalTokens) || 0; - if (n <= 0) return 0; - return DEFAULT_PARAMS_B * 1e9 * n * (n + 1); - } - - function recomputeEnergyNoKvCache(flops) { - return flops * ENERGY_WH_PER_FLOP; + function isOutlier(row) { + var tokens = Number(row.total_tokens) || 0; + if (tokens <= 0) return false; + var energy = Number(row.energy_wh_saved) || 0; + var flops = Number(row.flops_saved) || 0; + var dollars = Number(row.dollar_savings) || 0; + return ( + energy / tokens > MAX_ENERGY_WH_PER_TOKEN || + flops / tokens > MAX_FLOPS_PER_TOKEN || + dollars / tokens > MAX_DOLLAR_PER_TOKEN + ); } function escapeHtml(s) { @@ -133,17 +137,17 @@ return; } - allRows = rows; + allRows = rows.filter(function (r) { return !isOutlier(r); }); currentPage = 0; - var totalMembers = rows.length; + var totalMembers = allRows.length; var totalDollars = 0; var totalRequests = 0; var totalTokens = 0; - for (var i = 0; i < rows.length; i++) { - totalDollars += Number(rows[i].dollar_savings || 0); - totalRequests += Number(rows[i].total_calls || 0); - totalTokens += Number(rows[i].total_tokens || 0); + for (var i = 0; i < allRows.length; i++) { + totalDollars += Number(allRows[i].dollar_savings || 0); + totalRequests += Number(allRows[i].total_calls || 0); + totalTokens += Number(allRows[i].total_tokens || 0); } var elMembers = document.getElementById("stat-members"); diff --git a/docs/leaderboard.md b/docs/leaderboard.md index f676dfb1..ffc8ef0e 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -16,7 +16,7 @@ See how the OpenJarvis community saves money, energy, and compute by running AI
-
Total Saved
+
Total Saved*
@@ -35,7 +35,7 @@ See how the OpenJarvis community saves money, energy, and compute by running AI # Name - $ Saved + $ Saved* Energy (Wh) FLOPs Requests @@ -55,5 +55,5 @@ See how the OpenJarvis community saves money, energy, and compute by running AI

-*Dollar savings estimates assume local open-source models (e.g. Qwen, Nemotron, Kimi) produce roughly the same number of tokens per request, on average, as closed-source cloud models. +*Dollar savings estimated vs. Claude Opus 4.6 API pricing ($5/1M input, $25/1M output tokens). Assumes local open-source models produce roughly the same number of tokens per request as cloud models.

diff --git a/docs/testing/agent-qa-runbook.md b/docs/testing/agent-qa-runbook.md new file mode 100644 index 00000000..df79fcee --- /dev/null +++ b/docs/testing/agent-qa-runbook.md @@ -0,0 +1,75 @@ +# Agent QA Runbook + +Manual testing scenarios for persistent agents in the CLI and desktop app. + +## Environment Setup + +| Prerequisite | Command / Check | +|---|---| +| Ollama running with model | `ollama list` shows `qwen3:8b` | +| OpenJarvis initialized | `uv run jarvis doctor` all green | +| Rust extension built | `uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml` | +| Desktop app running | `uv run jarvis serve` + `cd frontend && npm run dev` | +| Slack credentials | `SLACK_BOT_TOKEN`, `SLACK_APP_TOKEN` set, bot invited to test channel | +| Gmail credentials | OAuth credentials.json downloaded, token generated | +| Twitter credentials | All 5 env vars set (bearer + OAuth 1.0a) | +| Discord credentials | Bot token set, bot invited to test server | +| Telegram credentials | Bot token from @BotFather, test chat ID known | +| Email credentials | SMTP/IMAP host + credentials for test account | + +## CLI Agent Scenarios + +| # | Scenario | Steps | Expected Result | Pass | +|---|----------|-------|-----------------|------| +| 1 | Template launch | `jarvis agents launch`, pick research_monitor | Agent created, config printed with curated tools | [ ] | +| 2 | Manual run | `jarvis agents run ` | Output shows reasoning + tool calls, status -> idle | [ ] | +| 3 | Immediate ask | `jarvis agents ask "summarize recent AI news"` | Synchronous response in terminal | [ ] | +| 4 | Queued instruct | `jarvis agents instruct "focus on diffusion"`, then `jarvis agents run ` | Queued -> delivered, response in `jarvis agents messages ` | [ ] | +| 5 | Status check | `jarvis agents status` after 3+ runs | total_runs, total_cost, last_run_at populated | [ ] | +| 6 | Pause/resume | `jarvis agents pause `, verify skipped, `jarvis agents resume `, verify fires | Status toggles correctly | [ ] | +| 7 | Daemon scheduling | `jarvis agents daemon` with interval agent (60s) | 3+ ticks fire on schedule, memory accumulates | [ ] | +| 8 | Budget exhaustion | Set max_cost=0.001, run until exceeded | Status becomes budget_exceeded | [ ] | +| 9 | Error recovery | Kill Ollama mid-tick, then `jarvis agents recover ` | Error -> recover -> idle with checkpoint | [ ] | +| 10 | Channel binding | `jarvis agents bind --slack #test`, run tick | Agent sends to Slack | [ ] | +| 11 | Multi-agent | Launch 3 agents, different intervals, run daemon | All fire independently | [ ] | +| 12 | Template tools | Create from each template, `jarvis agents info ` | All curated default tools listed | [ ] | + +## Desktop App Scenarios + +| # | Scenario | Steps | Expected Result | Pass | +|---|----------|-------|-----------------|------| +| 1 | Template wizard | New Agent -> pick each template -> complete wizard | Agent appears in grid with correct config | [ ] | +| 2 | Custom agent | New Agent -> Custom -> manual schedule, pick tools, set credentials | Tools + creds saved, agent created | [ ] | +| 3 | Run Now | Click Run Now on agent card | Status dot: green -> blue -> green, stats increment | [ ] | +| 4 | Immediate chat | Interact tab -> type message -> send (immediate mode) | Response appears in chat UI | [ ] | +| 5 | Queued chat | Interact tab -> send (queued mode) -> click Run Now | Message delivered on tick, response appears | [ ] | +| 6 | Task management | Tasks tab -> create task -> run agent | Task status updates, findings populated | [ ] | +| 7 | Memory inspection | Run 3+ ticks -> Memory tab | Summary memory reflects agent's accumulated knowledge | [ ] | +| 8 | Trace inspection | Run tick -> Logs tab | Trace steps visible with tool calls and results | [ ] | +| 9 | Learning | Enable trace-driven learning -> Learning tab -> trigger | Learning log entries appear | [ ] | +| 10 | Error + recovery | Stop Ollama -> run agent -> verify error badge -> click Recover | Error state shown, recovery resets to idle | [ ] | + +## Channel-Specific QA Matrix + +| Channel | Send Test | Receive Test | Thread/Reply Test | Agent Template | Pass | +|---------|-----------|-------------|-------------------|----------------|------| +| Slack | Post to #test-channel | Socket Mode incoming msg | Reply in thread (thread_ts) | inbox_triager | [ ] | +| Gmail | Send email to test recipient | Poll unread -> handler fires | Reply in thread (threadId) | inbox_triager | [ ] | +| Email (SMTP/IMAP) | Send via SMTP | IMAP poll UNSEEN | In-Reply-To header | inbox_triager | [ ] | +| iMessage (BlueBubbles) | Send to phone number | N/A (send-only) | N/A | research_monitor | [ ] | +| Twitter/X | Post tweet + send DM | Poll mentions | Reply (in_reply_to_tweet_id) | research_monitor | [ ] | +| Discord | Post to #test-channel | Gateway message event | N/A | code_reviewer | [ ] | +| Telegram | Send to test chat | Long-poll update | reply_to_message_id | research_monitor | [ ] | +| WhatsApp (Baileys) | Send to test number | Baileys incoming msg | N/A | inbox_triager | [ ] | + +## Stress & Edge Cases + +| # | Scenario | How to Test | Pass Criteria | Pass | +|---|----------|-------------|---------------|------| +| 1 | Message flood | Queue 50 messages via CLI, run tick | All 50 delivered, response generated | [ ] | +| 2 | Long-running daemon | Run daemon for 1 hour, 60s interval agent | No memory leak, no stall, ~60 ticks | [ ] | +| 3 | Rapid pause/resume | Script: pause -> resume -> pause -> resume during tick | Clean state, no corruption | [ ] | +| 4 | Credential revocation | Revoke Slack token mid-tick | Agent gets tool error, tick completes, status not corrupted | [ ] | +| 5 | Multi-agent load | 10 agents on daemon, mix of intervals | All fire on schedule, no interference | [ ] | +| 6 | Large response handling | Agent produces 10k+ char response | summary_memory truncated to 2000 chars, full response in messages | [ ] | +| 7 | Checkpoint integrity | Kill process mid-tick, restart, recover | Checkpoint restored, agent resumes cleanly | [ ] | diff --git a/docs/user-guide/agents.md b/docs/user-guide/agents.md index 9f21a46f..2a988f54 100644 --- a/docs/user-guide/agents.md +++ b/docs/user-guide/agents.md @@ -626,7 +626,20 @@ These events enable the telemetry and trace systems to record detailed interacti ## Managed Agent Streaming -The Managed Agent API (`/v1/managed-agents/{id}/messages`) supports real-time SSE streaming. Send a message with `stream: true` to receive the agent's response as a Server-Sent Events stream instead of the default asynchronous queue mode. +The Managed Agent API (`/v1/managed-agents/{id}/messages`) supports **real LLM token streaming** via SSE. Send a message with `stream: true` to receive the model's response tokens as they are generated, rather than waiting for the full response. + +### How It Works + +The streaming endpoint calls `engine.stream_full()` directly, which yields `StreamChunk` objects containing content tokens, tool-call fragments, and finish reasons. This provides genuine token-by-token streaming from the LLM -- not a post-hoc word replay. + +For multi-turn tool-calling agents, the streaming loop automatically: + +1. Yields content tokens to the client as they arrive. +2. Accumulates tool-call fragments (OpenAI sends these incrementally). +3. Executes tools when `finish_reason="tool_calls"` is received. +4. Emits tool results as named SSE events (`event: tool_result`). +5. Feeds results back to the LLM for the next turn. +6. Repeats until the model produces a final text response or `max_turns` is reached. ### Streaming Messages @@ -639,19 +652,21 @@ curl -N -X POST http://localhost:8000/v1/managed-agents/{id}/messages \ The response follows the OpenAI SSE format: 1. **Content chunks** -- `data: {"choices": [{"delta": {"content": "token"}}]}` -2. **Tool results** (if the agent used tools) -- `event: tool_results\ndata: {"results": [...]}` -3. **Final chunk** -- `data: {"choices": [{"delta": {}, "finish_reason": "stop"}]}` -4. **Done sentinel** -- `data: [DONE]` +2. **Tool calls** (if the model requests tool use) -- `event: tool_calls\ndata: {"calls": [{"tool_name": "...", "arguments": "..."}]}` +3. **Tool results** -- `event: tool_result\ndata: {"tool_name": "...", "output": "..."}` +4. **Final chunk** -- `data: {"choices": [{"delta": {}, "finish_reason": "stop"}]}` +5. **Done sentinel** -- `data: [DONE]` When `stream: false` (the default), the endpoint behaves exactly as before -- the message is queued and the agent must be triggered separately via `/run`. ### Behavior Details -- The user message is always stored in the database before the agent runs. -- After streaming completes, the full agent response is persisted as an `agent_to_user` message. -- The agent is instantiated from the managed agent's stored `agent_type` and `config`. -- Conversation history from prior messages is automatically loaded as context. +- The user message is always stored in the database before streaming starts. +- After streaming completes, the full collected response is persisted as an `agent_to_user` message. +- Conversation history from prior messages is automatically loaded as LLM context. +- The engine's `stream_full()` method is used for real token streaming. Engines that do not override it fall back to the default implementation which wraps the plain `stream()` method. - If the engine is not available on the server, a `503` error is returned. +- Tool execution during streaming uses the `ToolRegistry` to find and instantiate tools. ### Python Example diff --git a/docs/user-guide/mcp-external-servers.md b/docs/user-guide/mcp-external-servers.md new file mode 100644 index 00000000..7fd316e0 --- /dev/null +++ b/docs/user-guide/mcp-external-servers.md @@ -0,0 +1,158 @@ +# External MCP Server Integration + +OpenJarvis can extend agent capabilities by connecting to external [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) servers. This allows agents to use tools provided by services like Home Assistant, databases, custom APIs, or any MCP-compatible server -- without writing custom tool code. + +## How It Works + +When OpenJarvis starts, it reads the `[tools.mcp]` section in `config.toml`. For each configured server, it: + +1. Opens a connection using the appropriate transport (Streamable HTTP or stdio). +2. Performs the MCP initialize handshake (protocol version negotiation and `initialized` notification). +3. Discovers available tools via `tools/list`. +4. Wraps each discovered tool as a standard `BaseTool` so agents can call them like any built-in tool. + +If a server is unreachable or returns an error, OpenJarvis logs a warning and continues loading the remaining servers. One broken server does not prevent other tools from being available. + +## Configuration + +External MCP servers are configured in `config.toml` under `[tools.mcp]`: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}]' +``` + +The `servers` value is a **JSON-encoded string** containing an array of server objects. Each object defines one external MCP server. + +!!! note + The value must be a JSON string (with single-quote TOML delimiters around it), not a native TOML array. This is because the configuration system passes it through as a single string field. + +## Server Config Schema + +Each server object supports the following fields: + +| Field | Type | Required | Description | +|------------------|----------------|----------|----------------------------------------------------------| +| `name` | string | No | Human-readable name used in log messages. Defaults to ``. | +| `url` | string | No* | URL for Streamable HTTP transport. | +| `command` | string | No* | Command to launch a stdio-based MCP server. | +| `args` | list of strings| No | Arguments passed to the stdio command. | +| `include_tools` | list of strings| No | Whitelist of tool names to import. Only these tools are loaded. | +| `exclude_tools` | list of strings| No | Blacklist of tool names to skip. All other tools are loaded. | + +*Either `url` or `command` must be provided. If neither is set, the server is skipped with a warning. + +When both `include_tools` and `exclude_tools` are specified, the whitelist is applied first, then the blacklist filters the result. + +## Examples + +### Home Assistant via Streamable HTTP + +Connect to the [ha-mcp](https://github.com/tevonsb/ha-mcp) Home Assistant add-on: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}]' +``` + +This discovers all HA tools (entity control, automations, history, etc.) and makes them available to agents. + +### Stdio Server + +Launch a local MCP server as a subprocess: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "myserver", "command": "python", "args": ["-m", "my_mcp_server"]}]' +``` + +OpenJarvis starts the process automatically, communicates via JSON-RPC over stdin/stdout, and terminates it on shutdown. + +### Multiple Servers + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}, {"name": "database", "command": "db-mcp-server", "args": ["--db", "postgres://localhost/mydb"]}]' +``` + +### Tool Filtering + +When a server exposes many tools but you only need a few, use `include_tools` to whitelist: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "ha", "url": "http://172.16.3.1:9583/private_abc123", "include_tools": ["hassTurnOn", "hassTurnOff", "hassGetState"]}]' +``` + +To load everything except specific tools, use `exclude_tools`: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "ha", "url": "http://172.16.3.1:9583/private_abc123", "exclude_tools": ["hassCreateBackup", "hassDeleteBackup"]}]' +``` + +## Transport Types + +### Streamable HTTP + +Used when the `url` field is set. The transport sends JSON-RPC requests as HTTP POST to the given URL using `httpx`. It tracks the `Mcp-Session-Id` header across requests as required by the MCP Streamable HTTP specification. + +**When to use:** Remote MCP servers, services running as HTTP endpoints (e.g., Home Assistant MCP add-on, cloud-hosted MCP servers). + +**Connection parameters:** + +- Connect timeout: 10 seconds +- Request timeout: 60 seconds + +### Stdio + +Used when the `command` field is set. OpenJarvis spawns the command as a subprocess and communicates via JSON-RPC lines on stdin/stdout. + +**When to use:** Local MCP servers distributed as CLI tools, development/testing, servers that require filesystem access on the same machine. + +!!! info "SSETransport alias" + `SSETransport` is provided as a backward-compatible alias for `StreamableHTTPTransport`. Both refer to the same implementation. + +## Error Handling + +OpenJarvis handles MCP server failures gracefully: + +- **Server unreachable:** A warning is logged and the server is skipped. All other servers and built-in tools continue to load normally. +- **Timeout:** HTTP requests time out after 60 seconds. The server is skipped with a warning. +- **Invalid config:** If the `servers` JSON is malformed or a server entry has neither `url` nor `command`, a warning is logged and that entry is skipped. +- **Tool discovery failure:** If `tools/list` fails on a server, the error is caught and the server is skipped. +- **Runtime tool call failure:** If a tool call to an external MCP server fails at runtime, it returns a `ToolResult` with `success=False` and the error message. + +No single server failure causes OpenJarvis to crash or prevents other tools from working. + +## Troubleshooting + +### Server not discovered + +1. Check that `[tools.mcp]` has `enabled = true`. +2. Verify the `servers` JSON is valid. A common mistake is using TOML arrays instead of a JSON string. +3. Check the OpenJarvis logs for warnings like `Failed to discover external MCP tools`. + +### Connection refused / timeout + +1. Verify the server is running and reachable from the OpenJarvis host: `curl -v http://host:port/`. +2. Check firewall rules between the OpenJarvis container and the MCP server. +3. For Docker deployments, ensure both containers are on the same network or use host IPs. + +### Tools not appearing + +1. Run with debug logging to see which tools were discovered. +2. Check if `include_tools` or `exclude_tools` filters are too restrictive. +3. Verify the MCP server actually exposes tools via `tools/list` (some servers only expose resources or prompts). + +### Stdio server crashes immediately + +1. Test the command manually: `python -m my_mcp_server` should start and wait for input on stdin. +2. Check stderr output in the OpenJarvis logs for error messages from the subprocess. +3. Ensure all dependencies for the MCP server are installed in the same environment. diff --git a/docs/user-guide/tools.md b/docs/user-guide/tools.md index 0e3876fd..f22bcd23 100644 --- a/docs/user-guide/tools.md +++ b/docs/user-guide/tools.md @@ -204,7 +204,7 @@ All built-in tools are registered via `@ToolRegistry.register()` and are availab | **Scheduler** | `pause_scheduled_task` | Pause an active scheduled task | | **Scheduler** | `resume_scheduled_task` | Resume a paused scheduled task | | **Scheduler** | `cancel_scheduled_task` | Cancel a scheduled task permanently | -| **Integration** | `mcp_adapter` | Bridge to external MCP tool servers | +| **Integration** | `mcp_adapter` | Bridge to external MCP tool servers (see [External MCP Servers](mcp-external-servers.md)) | --- diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index f619379a..e627931f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -67,10 +67,10 @@ export default function App() { .then((data) => { setSavings(data); if (optInEnabled && optInDisplayName && data) { - const dollarSavings = data.per_provider.reduce( - (sum, p) => sum + p.total_cost, - 0, + const claudeEntry = data.per_provider.find( + (p) => p.provider === 'claude-opus-4.6', ); + const dollarSavings = claudeEntry ? claudeEntry.total_cost : 0; const energySaved = data.per_provider.reduce( (sum, p) => sum + (p.energy_wh || 0), 0, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 201e9c7c..9dc4b238 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -100,6 +100,12 @@ export async function fetchModels(): Promise { return data.data || []; } +export async function fetchRecommendedModel(): Promise<{ model: string; reason: string }> { + const res = await fetch(`${getBase()}/v1/recommended-model`); + if (!res.ok) return { model: '', reason: 'Failed to fetch' }; + return res.json(); +} + export async function pullModel(modelName: string): Promise { // In Tauri, go through the Rust backend directly (avoids CORS / timeout // issues with long model downloads via fetch). @@ -296,6 +302,8 @@ export interface ManagedAgent { total_runs?: number; total_cost?: number; total_tokens?: number; + input_tokens?: number; + output_tokens?: number; last_run_at?: number | null; // Schedule schedule_type?: string; @@ -304,6 +312,8 @@ export interface ManagedAgent { budget?: number; // Learning learning_enabled?: boolean; + // Live progress + current_activity?: string; } export interface AgentTask { diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index 702dee16..0f7288fa 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -20,6 +20,9 @@ import { fetchManagedAgent, fetchAvailableTools, saveToolCredentials, + fetchModels, + updateManagedAgent, + fetchRecommendedModel, } from '../lib/api'; import type { AgentTask, ChannelBinding, AgentTemplate, AgentMessage, ManagedAgent, LearningLogEntry, AgentTrace, ToolInfo } from '../lib/api'; import { @@ -99,8 +102,7 @@ function StatusDot({ status }: { status: string }) { function formatCost(cost?: number): string { if (cost === undefined || cost === null) return '—'; - if (cost < 0.01) return `$${(cost * 100).toFixed(2)}¢`; - return `$${cost.toFixed(3)}`; + return `$${cost.toFixed(4)}`; } function formatRelativeTime(ts?: number | null): string { @@ -204,8 +206,9 @@ function serializeInterval(hours: number, minutes: number, seconds: number): str } interface WizardState { - step: number; + step: 1 | 2; templateId: string; + templateData: AgentTemplate | null; name: string; instruction: string; model: string; @@ -218,8 +221,11 @@ interface WizardState { observationCompression: string; retrievalStrategy: string; taskDecomposition: string; + maxTurns: number; + temperature: number; } + function LaunchWizard({ templates, onClose, @@ -229,9 +235,19 @@ function LaunchWizard({ onClose: () => void; onLaunched: () => void; }) { + const UNIVERSAL_DEFAULTS = { + memoryExtraction: 'structured_json', + observationCompression: 'summarize', + retrievalStrategy: 'sqlite', + taskDecomposition: 'hierarchical', + maxTurns: 25, + temperature: 0.3, + }; + const [wizard, setWizard] = useState({ step: 1, templateId: '', + templateData: null, name: '', instruction: '', model: '', @@ -240,94 +256,60 @@ function LaunchWizard({ selectedTools: [], budget: '', routerPolicy: '', - memoryExtraction: 'causality_graph', - observationCompression: 'summarize', - retrievalStrategy: 'hybrid_with_self_eval', - taskDecomposition: 'phased', + ...UNIVERSAL_DEFAULTS, }); const [launching, setLaunching] = useState(false); + const [recommendedModel, setRecommendedModel] = useState(''); const models = useAppStore((s) => s.models); - const [availableTools, setAvailableTools] = useState([]); - const [expandedCategories, setExpandedCategories] = useState>(new Set()); - const [credentialInputs, setCredentialInputs] = useState>>({}); - const [savingCredentials, setSavingCredentials] = useState(null); useEffect(() => { - fetchAvailableTools().then(setAvailableTools).catch(() => {}); + fetchRecommendedModel().then((r) => { + setRecommendedModel(r.model); + if (!wizard.model) { + setWizard((w) => ({ ...w, model: r.model })); + } + }).catch(() => {}); }, []); - function getToolCategory(tool: ToolInfo): string { - if (tool.category && CATEGORY_MAP[tool.category]) return CATEGORY_MAP[tool.category]; - if (TOOL_NAME_FALLBACK[tool.name]) return TOOL_NAME_FALLBACK[tool.name]; - return 'Reasoning & AI'; - } - - function toggleCategory(cat: string) { - setExpandedCategories((prev) => { - const next = new Set(prev); - if (next.has(cat)) next.delete(cat); - else next.add(cat); - return next; - }); - } - - function handleToggleTool(name: string) { - if (name === 'browser') { - const has = BROWSER_SUB_TOOLS.every((t) => wizard.selectedTools.includes(t)); - if (has) { - update({ selectedTools: wizard.selectedTools.filter((t) => !BROWSER_SUB_TOOLS.includes(t)) }); - } else { - update({ selectedTools: [...new Set([...wizard.selectedTools, ...BROWSER_SUB_TOOLS])] }); - } + function selectTemplate(tpl: AgentTemplate | null) { + if (tpl) { + setWizard((w) => ({ + ...w, + step: 2, + templateId: tpl.id, + templateData: tpl, + name: '', + instruction: '', + model: recommendedModel || w.model, + scheduleType: (tpl as any).schedule_type || 'manual', + scheduleValue: (tpl as any).schedule_value || '', + selectedTools: (tpl as any).tools || [], + memoryExtraction: (tpl as any).memory_extraction || UNIVERSAL_DEFAULTS.memoryExtraction, + observationCompression: (tpl as any).observation_compression || UNIVERSAL_DEFAULTS.observationCompression, + retrievalStrategy: (tpl as any).retrieval_strategy || UNIVERSAL_DEFAULTS.retrievalStrategy, + taskDecomposition: (tpl as any).task_decomposition || UNIVERSAL_DEFAULTS.taskDecomposition, + maxTurns: (tpl as any).max_turns || UNIVERSAL_DEFAULTS.maxTurns, + temperature: (tpl as any).temperature ?? UNIVERSAL_DEFAULTS.temperature, + })); } else { - toggleTool(name); + setWizard((w) => ({ + ...w, + step: 2, + templateId: '', + templateData: null, + name: '', + instruction: '', + model: recommendedModel || w.model, + scheduleType: 'manual', + scheduleValue: '', + selectedTools: [], + ...UNIVERSAL_DEFAULTS, + })); } } - async function handleSaveCredentials(toolName: string) { - const inputs = credentialInputs[toolName]; - if (!inputs) return; - setSavingCredentials(toolName); - try { - await saveToolCredentials(toolName, inputs); - toast.success(`Credentials saved for ${toolName}`); - const updated = await fetchAvailableTools(); - setAvailableTools(updated); - } catch (err: any) { - toast.error(err.message || 'Failed to save credentials'); - } finally { - setSavingCredentials(null); - } - } - - function update(partial: Partial) { - setWizard((prev) => ({ ...prev, ...partial })); - } - - function toggleTool(id: string) { - const next = wizard.selectedTools.includes(id) - ? wizard.selectedTools.filter((t) => t !== id) - : [...wizard.selectedTools, id]; - update({ selectedTools: next }); - } - - function selectTemplate(id: string) { - const tpl = templates.find((t) => t.id === id); - update({ - templateId: id, - name: tpl?.name || wizard.name, - }); - } - async function handleLaunch() { - if ((wizard.scheduleType === 'cron' || wizard.scheduleType === 'interval') && !wizard.instruction.trim()) { - toast.error('Instruction is required for scheduled agents'); - return; - } - if (!wizard.name.trim()) { - toast.error('Agent name is required'); - return; - } + if (!wizard.name.trim()) { toast.error('Name is required'); return; } setLaunching(true); try { const config: Record = { @@ -335,633 +317,259 @@ function LaunchWizard({ schedule_value: wizard.scheduleValue || undefined, tools: wizard.selectedTools, learning_enabled: !!wizard.routerPolicy, + memory_extraction: wizard.memoryExtraction, + observation_compression: wizard.observationCompression, + retrieval_strategy: wizard.retrievalStrategy, + task_decomposition: wizard.taskDecomposition, + max_turns: wizard.maxTurns, + temperature: wizard.temperature, }; if (wizard.budget) config.budget = parseFloat(wizard.budget); if (wizard.instruction.trim()) config.instruction = wizard.instruction.trim(); if (wizard.model) config.model = wizard.model; if (wizard.routerPolicy) config.router_policy = wizard.routerPolicy; - config.memory_extraction = wizard.memoryExtraction; - config.observation_compression = wizard.observationCompression; - config.retrieval_strategy = wizard.retrievalStrategy; - config.task_decomposition = wizard.taskDecomposition; - const created = await createManagedAgent({ - name: wizard.name, + + await createManagedAgent({ + name: wizard.name.trim(), template_id: wizard.templateId || undefined, config, }); - toast.success(`Agent "${wizard.name}" launched`); - // Auto-run first tick for interval agents - if (wizard.scheduleType === 'interval' && created.id) { - runManagedAgent(created.id).catch(() => {}); - } + toast.success(`Agent "${wizard.name}" created`); onLaunched(); - } catch (err) { - toast.error('Could not create agent', { - description: 'Agent manager endpoint not available. Check that agent_manager.enabled = true in your config.', - }); + } catch (err: any) { + toast.error(err.message || 'Failed to create agent'); } finally { setLaunching(false); } } - return ( -
e.target === e.currentTarget && onClose()} - > -
- {/* Header */} -
-
- -

- Launch Agent -

+ const formatScheduleLabel = (type: string, value: string) => { + if (type === 'manual') return 'Manual (run on demand)'; + if (type === 'cron') return `Cron: ${value}`; + if (type === 'interval') { + const secs = parseInt(value, 10); + if (secs >= 3600) return `Every ${secs / 3600}h`; + if (secs >= 60) return `Every ${secs / 60}m`; + return `Every ${secs}s`; + } + return type; + }; + + // ── Step 1: Template Selection ── + if (wizard.step === 1) { + return ( +
+
+
+

New Agent — Choose Template

+
-
- {/* Step indicator */} -
- {([1, 2, 3] as const).map((s) => ( - - s ? 'var(--color-accent)' + '40' : 'var(--color-bg-secondary)', - color: wizard.step >= s ? (wizard.step === s ? '#fff' : 'var(--color-accent)') : 'var(--color-text-tertiary)', - }} - > - {s} - - {s < 3 && } - - ))} -
- + ))} +
+
+ ); + } - {/* Body */} -
- {/* Step 1: Template Picker */} - {wizard.step === 1 && ( + // ── Step 2: Configuration ── + return ( +
+
+
+
+ +

+ {wizard.templateData ? `New ${wizard.templateData.name}` : 'New Custom Agent'} +

+
+ +
+ +
+ {/* Name */} +
+ + setWizard((w) => ({ ...w, name: e.target.value }))} + placeholder="e.g. AI Research Tracker" + className="w-full px-3 py-2 rounded-lg text-sm bg-transparent" + style={{ border: '1px solid var(--color-border)', color: 'var(--color-text)' }} + /> +
+ + {/* Instruction */} +
+ +