mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 05:12:26 +00:00
merge: resolve conflicts with main — keep both DeepResearch tools + MCP functions
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jon Saad-Falcon
@@ -63,7 +63,7 @@ jobs:
|
||||
run: uv run maturin develop --manifest-path rust/crates/openjarvis-python/Cargo.toml
|
||||
|
||||
- name: Run tests
|
||||
run: uv run pytest tests/ -v --tb=short
|
||||
run: uv run pytest tests/ -v --tb=short -m "not live and not cloud"
|
||||
|
||||
rust:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -10,7 +10,7 @@ const OLLAMA_PORT: u16 = 11434;
|
||||
const JARVIS_PORT: u16 = 8222;
|
||||
|
||||
/// Small, fast model pulled at startup so the app opens quickly.
|
||||
const STARTUP_MODEL: &str = "qwen3.5:2b";
|
||||
const STARTUP_MODEL: &str = "qwen3.5:4b";
|
||||
|
||||
/// Tiny fallback model if even the startup model can't be pulled.
|
||||
const FALLBACK_MODEL: &str = "qwen3:0.6b";
|
||||
@@ -86,12 +86,14 @@ fn models_that_fit() -> Vec<&'static str> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Pick the third-largest Qwen3.5 model that fits on this machine.
|
||||
/// This leaves comfortable headroom for the OS / other apps while
|
||||
/// still providing a capable model. Falls back gracefully when
|
||||
/// fewer models fit.
|
||||
/// Pick the default model — prefers STARTUP_MODEL if it fits, otherwise
|
||||
/// falls back to the third-largest model that fits on this machine.
|
||||
fn preferred_model() -> &'static str {
|
||||
let fitting = models_that_fit();
|
||||
// Prefer STARTUP_MODEL when it fits (fast, good quality)
|
||||
if fitting.contains(&STARTUP_MODEL) {
|
||||
return STARTUP_MODEL;
|
||||
}
|
||||
match fitting.len() {
|
||||
0 => FALLBACK_MODEL,
|
||||
1 => fitting[0],
|
||||
@@ -460,17 +462,90 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
|
||||
return;
|
||||
}
|
||||
|
||||
let project_root = find_project_root();
|
||||
let mut project_root = find_project_root();
|
||||
|
||||
if project_root.is_none() {
|
||||
let mut s = status.lock().await;
|
||||
s.error = Some(
|
||||
"Could not find the OpenJarvis project directory. \
|
||||
Clone it with: git clone https://github.com/open-jarvis/OpenJarvis.git ~/OpenJarvis \
|
||||
then relaunch."
|
||||
.into(),
|
||||
);
|
||||
return;
|
||||
// Auto-clone on first launch
|
||||
let git_bin = resolve_bin("git");
|
||||
|
||||
// Check that git is installed
|
||||
if !std::path::Path::new(&git_bin).exists() && git_bin == "git" {
|
||||
let mut s = status.lock().await;
|
||||
s.error = Some(
|
||||
"Could not find 'git'. \
|
||||
Install it from https://git-scm.com then relaunch."
|
||||
.into(),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let target_path = std::path::PathBuf::from(home_dir()).join("OpenJarvis");
|
||||
let clone_target = target_path.display().to_string();
|
||||
|
||||
// If the directory exists but is not a valid project, don't overwrite
|
||||
if target_path.exists() && !target_path.join("pyproject.toml").exists() {
|
||||
let mut s = status.lock().await;
|
||||
s.error = Some(format!(
|
||||
"{} exists but is not a valid OpenJarvis project. \
|
||||
Remove it and relaunch, or set OPENJARVIS_ROOT to the correct path.",
|
||||
clone_target,
|
||||
));
|
||||
return;
|
||||
}
|
||||
|
||||
{
|
||||
let mut s = status.lock().await;
|
||||
s.detail = "Downloading OpenJarvis (first launch)...".into();
|
||||
}
|
||||
|
||||
let clone_result = tokio::process::Command::new(&git_bin)
|
||||
.args([
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"https://github.com/open-jarvis/OpenJarvis.git",
|
||||
&clone_target,
|
||||
])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn();
|
||||
|
||||
match clone_result {
|
||||
Ok(child) => match child.wait_with_output().await {
|
||||
Ok(output) if output.status.success() => {
|
||||
project_root = Some(target_path);
|
||||
}
|
||||
Ok(output) => {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let mut s = status.lock().await;
|
||||
s.error = Some(format!(
|
||||
"Failed to download OpenJarvis: {}. \
|
||||
Clone manually: git clone https://github.com/open-jarvis/OpenJarvis.git {}",
|
||||
stderr.trim(),
|
||||
clone_target,
|
||||
));
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
let mut s = status.lock().await;
|
||||
s.error = Some(format!(
|
||||
"Failed to download OpenJarvis: {}. \
|
||||
Clone manually: git clone https://github.com/open-jarvis/OpenJarvis.git {}",
|
||||
e, clone_target,
|
||||
));
|
||||
return;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
let mut s = status.lock().await;
|
||||
s.error = Some(format!(
|
||||
"Could not run git: {}. \
|
||||
Install git from https://git-scm.com then relaunch.",
|
||||
e,
|
||||
));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Kill any leftover server on our port from a previous run
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
---
|
||||
title: macOS Installation Guide
|
||||
description: Complete step-by-step guide to installing OpenJarvis on macOS with llama.cpp, including common pitfalls and fixes
|
||||
search:
|
||||
boost: 2
|
||||
---
|
||||
|
||||
# macOS Installation Guide
|
||||
|
||||
This guide walks through a complete OpenJarvis installation on macOS using **llama.cpp** as
|
||||
the inference engine. It covers every step from scratch — including pitfalls not documented
|
||||
elsewhere — and is suitable for both Apple Silicon and Intel Macs.
|
||||
|
||||
!!! tip "Prefer Ollama?"
|
||||
If you want the fastest possible setup, use [Ollama](installation.md#ollama-recommended)
|
||||
instead. This guide is for users who want to run GGUF models directly with llama.cpp,
|
||||
or who want a deeper understanding of the full stack.
|
||||
|
||||
---
|
||||
|
||||
## What You'll Install
|
||||
|
||||
| Tool | Purpose |
|
||||
|------|---------|
|
||||
| Homebrew | macOS package manager — installs everything else |
|
||||
| uv | Python version and dependency manager |
|
||||
| Git | Clones the OpenJarvis repo |
|
||||
| Node.js | Required for the browser UI |
|
||||
| Rust | Compiles the OpenJarvis security and memory extension |
|
||||
| llama.cpp | Local inference engine that runs GGUF model files |
|
||||
| OpenJarvis | The framework itself |
|
||||
| A GGUF model | The actual AI model (downloaded separately) |
|
||||
|
||||
---
|
||||
|
||||
## Step-by-Step Installation
|
||||
|
||||
### Step 1 — Install Homebrew
|
||||
|
||||
Homebrew is the standard macOS package manager. Everything else in this guide is installed
|
||||
through it.
|
||||
|
||||
```bash
|
||||
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
|
||||
```
|
||||
|
||||
If you already have Homebrew, skip this step.
|
||||
|
||||
---
|
||||
|
||||
### Step 2 — Install uv
|
||||
|
||||
`uv` replaces pip, virtualenv, and pyenv in one tool. OpenJarvis uses it to manage Python
|
||||
versions, virtual environments, and project dependencies.
|
||||
|
||||
```bash
|
||||
brew install uv
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Step 3 — Install Git
|
||||
|
||||
Git is used to clone the OpenJarvis source code. It may already be present if you have
|
||||
Xcode Command Line Tools installed.
|
||||
|
||||
```bash
|
||||
brew install git
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Step 4 — Install Node.js
|
||||
|
||||
Node.js is required to build and run the browser frontend. Without it you can still use
|
||||
the CLI, but not the web UI.
|
||||
|
||||
```bash
|
||||
brew install node
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Step 5 — Install Rust
|
||||
|
||||
OpenJarvis includes a Rust extension that provides security scanning, memory indexing,
|
||||
rate limiting, and tool execution. It must be compiled from source.
|
||||
|
||||
```bash
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
```
|
||||
|
||||
After the installer finishes, reload your shell so `rustc` is available:
|
||||
|
||||
```bash
|
||||
source "$HOME/.cargo/env"
|
||||
```
|
||||
|
||||
Verify:
|
||||
|
||||
```bash
|
||||
rustc --version
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Step 6 — Install llama.cpp
|
||||
|
||||
llama.cpp is the inference engine that loads and runs GGUF model files. It is not a model
|
||||
itself — think of it as a media player and the `.gguf` file as the content.
|
||||
|
||||
```bash
|
||||
brew install llama.cpp
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Step 7 — Clone the OpenJarvis repo
|
||||
|
||||
Run this from your home directory or any neutral parent folder.
|
||||
|
||||
```bash
|
||||
cd ~
|
||||
git clone https://github.com/open-jarvis/OpenJarvis.git
|
||||
cd OpenJarvis
|
||||
```
|
||||
|
||||
!!! warning "Do not clone from inside an existing OpenJarvis folder"
|
||||
A common mistake is running `git clone` while already inside the repo, creating deeply
|
||||
nested duplicates (`OpenJarvis/OpenJarvis/OpenJarvis`). Always clone from `~` or a
|
||||
neutral parent directory.
|
||||
|
||||
---
|
||||
|
||||
### Step 8 — Pin Python to 3.12
|
||||
|
||||
!!! warning "Critical step — do not skip"
|
||||
OpenJarvis requires Python 3.10–3.13. Its Rust extension uses PyO3, which does not yet
|
||||
support Python 3.14. If `uv` has Python 3.14 available, it will use it by default,
|
||||
causing the Rust extension build to fail silently and resulting in ~250 test failures
|
||||
with `ModuleNotFoundError: No module named 'openjarvis_rust'`.
|
||||
|
||||
Pin the project to Python 3.12:
|
||||
|
||||
```bash
|
||||
echo "3.12" > .python-version
|
||||
uv python install 3.12
|
||||
rm -rf .venv
|
||||
uv venv
|
||||
```
|
||||
|
||||
**Restart your terminal**, then verify:
|
||||
|
||||
```bash
|
||||
uv run python --version
|
||||
# Must show: Python 3.12.x
|
||||
```
|
||||
|
||||
!!! tip "Why restart the terminal?"
|
||||
Without restarting, the shell may still reference the old virtual environment. This is
|
||||
the most common reason the version pin appears not to work.
|
||||
|
||||
---
|
||||
|
||||
### Step 9 — Install Python dependencies
|
||||
|
||||
```bash
|
||||
uv sync --extra dev --extra server
|
||||
```
|
||||
|
||||
The `--extra server` flag adds the FastAPI backend required for the browser UI.
|
||||
|
||||
---
|
||||
|
||||
### Step 10 — Build the Rust extension
|
||||
|
||||
This compiles the Rust extension and installs it into the virtual environment. It provides
|
||||
security scanning, memory indexing, MCP tool execution, and rate limiting. This step takes
|
||||
a few minutes on first run.
|
||||
|
||||
```bash
|
||||
uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml
|
||||
```
|
||||
|
||||
Verify it built correctly:
|
||||
|
||||
```bash
|
||||
uv run python -c "import openjarvis_rust; print('Rust extension OK')"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Step 11 — Install frontend dependencies
|
||||
|
||||
```bash
|
||||
cd frontend && npm install && cd ..
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Step 12 — Download a model
|
||||
|
||||
OpenJarvis needs a GGUF model file to run inference. First install the Hugging Face CLI,
|
||||
then download your chosen model.
|
||||
|
||||
```bash
|
||||
uv tool install huggingface_hub
|
||||
```
|
||||
|
||||
!!! note "The CLI command is `hf`, not `huggingface-cli`"
|
||||
When installed via `uv tool`, the Hugging Face CLI is invoked as `hf`.
|
||||
|
||||
=== "Qwen3 4B (~2.5 GB)"
|
||||
|
||||
Faster, lower RAM requirement. Good for most everyday tasks.
|
||||
|
||||
```bash
|
||||
hf download bartowski/Qwen_Qwen3-4B-GGUF \
|
||||
--include "Qwen_Qwen3-4B-Q4_K_M.gguf" \
|
||||
--local-dir ~/models
|
||||
```
|
||||
|
||||
=== "Qwen3 8B (~4.7 GB)"
|
||||
|
||||
Better reasoning and instruction following. Requires more RAM.
|
||||
|
||||
```bash
|
||||
hf download bartowski/Qwen_Qwen3-8B-GGUF \
|
||||
--include "Qwen_Qwen3-8B-Q4_K_M.gguf" \
|
||||
--local-dir ~/models
|
||||
```
|
||||
|
||||
!!! warning "Use the `Qwen_` prefix"
|
||||
bartowski's Qwen3 repos use the `Qwen_` prefix (e.g. `Qwen_Qwen3-4B-GGUF`). Using
|
||||
the shorter name without the prefix returns a "repository not found" error.
|
||||
|
||||
!!! tip "Apple Silicon vs Intel"
|
||||
On Apple Silicon, both models benefit from Metal GPU acceleration when using the MLX
|
||||
engine. On Intel, inference runs on CPU — the 4B model is recommended for speed.
|
||||
|
||||
---
|
||||
|
||||
### Step 13 — Configure OpenJarvis
|
||||
|
||||
Run the init command to detect your hardware and generate a config file:
|
||||
|
||||
```bash
|
||||
uv run jarvis init
|
||||
```
|
||||
|
||||
Then open the config and set the default model to match the filename you downloaded:
|
||||
|
||||
```bash
|
||||
nano ~/.openjarvis/config.toml
|
||||
```
|
||||
|
||||
Find the `default_model` line and update it, for example:
|
||||
|
||||
```toml
|
||||
default_model = "Qwen_Qwen3-4B-Q4_K_M.gguf"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Step 14 — Verify the installation
|
||||
|
||||
```bash
|
||||
uv run jarvis doctor
|
||||
```
|
||||
|
||||
A healthy setup looks like this:
|
||||
|
||||
```
|
||||
✓ Python version 3.12.x
|
||||
✓ Config file ~/.openjarvis/config.toml
|
||||
✓ Config parsing Config loaded successfully
|
||||
✓ Engine: llamacpp Reachable
|
||||
✓ Models: llamacpp Qwen_Qwen3-4B-Q4_K_M.gguf
|
||||
✓ Default model Qwen_Qwen3-4B-Q4_K_M.gguf (on llamacpp)
|
||||
```
|
||||
|
||||
!!! note "Warnings for other engines are normal"
|
||||
The `!` warnings for engines like `ollama`, `vllm`, and `lmstudio` simply mean those
|
||||
backends are not running. You only need `llamacpp` to be reachable.
|
||||
|
||||
---
|
||||
|
||||
## Running OpenJarvis
|
||||
|
||||
### CLI
|
||||
|
||||
Start llama-server in one terminal, then run queries in another:
|
||||
|
||||
```bash
|
||||
# Terminal 1 — start the inference engine
|
||||
llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf -c 4096 -t 8
|
||||
|
||||
# Terminal 2 — ask a question
|
||||
cd ~/OpenJarvis
|
||||
uv run jarvis ask "What is the capital of France?"
|
||||
```
|
||||
|
||||
### Browser UI
|
||||
|
||||
```bash
|
||||
# Terminal 1 — inference engine
|
||||
llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf -c 4096 -t 8
|
||||
|
||||
# Terminal 2 — backend
|
||||
cd ~/OpenJarvis && uv run jarvis serve --port 8000
|
||||
|
||||
# Terminal 3 — frontend
|
||||
cd ~/OpenJarvis/frontend && npm run dev
|
||||
```
|
||||
|
||||
Then open [http://localhost:5173](http://localhost:5173).
|
||||
|
||||
### Skip typing `uv run` every time
|
||||
|
||||
Activate the virtual environment for your current terminal session:
|
||||
|
||||
```bash
|
||||
source ~/OpenJarvis/.venv/bin/activate
|
||||
```
|
||||
|
||||
Your prompt will show `(openjarvis)` when active, and you can type `jarvis ask "..."` directly.
|
||||
|
||||
---
|
||||
|
||||
## Performance Tips
|
||||
|
||||
These tips apply when using llama.cpp for CPU inference.
|
||||
|
||||
| Flag | Effect |
|
||||
|------|--------|
|
||||
| `-c 4096` | Reduces context window from the 32,768 default, freeing RAM for faster inference |
|
||||
| `-t 8` | Uses all available CPU threads (default is only 4) — adjust to your machine's thread count |
|
||||
| `Q4_K_M` quantization | Best balance of size, speed, and quality for CPU inference |
|
||||
|
||||
On Apple Silicon, switching to the [MLX engine](../architecture/engine.md) gives
|
||||
significantly better performance than llama.cpp for most models.
|
||||
|
||||
---
|
||||
|
||||
## Common Errors
|
||||
|
||||
### `No such file or directory` when loading model
|
||||
|
||||
The path `path/to/model.gguf` in examples is a placeholder. Replace it with your actual
|
||||
model path, e.g.:
|
||||
|
||||
```bash
|
||||
llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `No module named 'openjarvis_rust'`
|
||||
|
||||
The Rust extension did not build correctly, or was built against the wrong Python version.
|
||||
|
||||
1. Confirm Python 3.12 is active: `uv run python --version`
|
||||
2. Rebuild: `uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml`
|
||||
|
||||
If the version shows 3.14, go back to [Step 8](#step-8--pin-python-to-312).
|
||||
|
||||
---
|
||||
|
||||
### `PyO3 version error — Python 3.14 too new`
|
||||
|
||||
```
|
||||
error: the configured Python interpreter version (3.14) is newer than
|
||||
PyO3's maximum supported version (3.13)
|
||||
```
|
||||
|
||||
PyO3 0.23.5 supports Python up to 3.13. Follow [Step 8](#step-8--pin-python-to-312) to
|
||||
pin to 3.12, then delete `.venv`, recreate it, and restart your terminal before retrying.
|
||||
|
||||
---
|
||||
|
||||
### `Repository not found` when downloading model
|
||||
|
||||
bartowski's Qwen3 repos use the `Qwen_` prefix. Use:
|
||||
|
||||
```
|
||||
bartowski/Qwen_Qwen3-4B-GGUF ✓
|
||||
bartowski/Qwen3-4B-GGUF ✗
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `No inference engine available`
|
||||
|
||||
llama-server is not running. Start it in a separate terminal before running any `jarvis`
|
||||
commands, and wait until you see `model loaded` in the output.
|
||||
|
||||
---
|
||||
|
||||
### Python version still shows 3.14 after recreating the venv
|
||||
|
||||
Close the terminal completely and reopen it. The old venv path is cached in the shell
|
||||
environment and persists across commands until the session ends.
|
||||
|
||||
---
|
||||
|
||||
### `zsh: command not found: huggingface-cli`
|
||||
|
||||
When installed via `uv tool`, the CLI is invoked as `hf`, not `huggingface-cli`:
|
||||
|
||||
```bash
|
||||
hf download ... # ✓
|
||||
huggingface-cli download ... # ✗
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Quick Start](quickstart.md) — Run your first query and explore agents and tools
|
||||
- [Configuration](configuration.md) — Customize engine hosts, model routing, memory, and more
|
||||
- [Architecture](../architecture/overview.md) — Understand how OpenJarvis is structured
|
||||
@@ -9,20 +9,24 @@
|
||||
var allRows = [];
|
||||
var currentPage = 0;
|
||||
|
||||
// No-KV-cache formula: FLOPs = params_b * 1e9 * N * (N+1)
|
||||
// Reference values only (GPT-5.3 constants) — recompute functions below
|
||||
// are defined but not called; the leaderboard displays database values directly.
|
||||
var DEFAULT_PARAMS_B = 137;
|
||||
var ENERGY_WH_PER_FLOP = 0.4 / (1000 * 3e12);
|
||||
// Outlier detection — hide entries with values that are physically
|
||||
// implausible relative to their token count. Thresholds are ~1000x
|
||||
// above legitimate per-token values to avoid false positives.
|
||||
var MAX_ENERGY_WH_PER_TOKEN = 10; // legit ≈ 0.001 Wh/tok
|
||||
var MAX_FLOPS_PER_TOKEN = 1e17; // legit ≈ 1e12 /tok
|
||||
var MAX_DOLLAR_PER_TOKEN = 25.0 / 1e6; // hard ceiling: $25/1M output
|
||||
|
||||
function recomputeFlopsNoKvCache(totalTokens) {
|
||||
var n = Number(totalTokens) || 0;
|
||||
if (n <= 0) return 0;
|
||||
return DEFAULT_PARAMS_B * 1e9 * n * (n + 1);
|
||||
}
|
||||
|
||||
function recomputeEnergyNoKvCache(flops) {
|
||||
return flops * ENERGY_WH_PER_FLOP;
|
||||
function isOutlier(row) {
|
||||
var tokens = Number(row.total_tokens) || 0;
|
||||
if (tokens <= 0) return false;
|
||||
var energy = Number(row.energy_wh_saved) || 0;
|
||||
var flops = Number(row.flops_saved) || 0;
|
||||
var dollars = Number(row.dollar_savings) || 0;
|
||||
return (
|
||||
energy / tokens > MAX_ENERGY_WH_PER_TOKEN ||
|
||||
flops / tokens > MAX_FLOPS_PER_TOKEN ||
|
||||
dollars / tokens > MAX_DOLLAR_PER_TOKEN
|
||||
);
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
@@ -133,17 +137,17 @@
|
||||
return;
|
||||
}
|
||||
|
||||
allRows = rows;
|
||||
allRows = rows.filter(function (r) { return !isOutlier(r); });
|
||||
currentPage = 0;
|
||||
|
||||
var totalMembers = rows.length;
|
||||
var totalMembers = allRows.length;
|
||||
var totalDollars = 0;
|
||||
var totalRequests = 0;
|
||||
var totalTokens = 0;
|
||||
for (var i = 0; i < rows.length; i++) {
|
||||
totalDollars += Number(rows[i].dollar_savings || 0);
|
||||
totalRequests += Number(rows[i].total_calls || 0);
|
||||
totalTokens += Number(rows[i].total_tokens || 0);
|
||||
for (var i = 0; i < allRows.length; i++) {
|
||||
totalDollars += Number(allRows[i].dollar_savings || 0);
|
||||
totalRequests += Number(allRows[i].total_calls || 0);
|
||||
totalTokens += Number(allRows[i].total_tokens || 0);
|
||||
}
|
||||
|
||||
var elMembers = document.getElementById("stat-members");
|
||||
|
||||
+3
-3
@@ -16,7 +16,7 @@ See how the OpenJarvis community saves money, energy, and compute by running AI
|
||||
<div class="lb-stat-value" id="stat-members">—</div>
|
||||
</div>
|
||||
<div class="lb-stat-card">
|
||||
<div class="lb-stat-label">Total Saved</div>
|
||||
<div class="lb-stat-label">Total Saved*</div>
|
||||
<div class="lb-stat-value" id="stat-dollars">—</div>
|
||||
</div>
|
||||
<div class="lb-stat-card">
|
||||
@@ -35,7 +35,7 @@ See how the OpenJarvis community saves money, energy, and compute by running AI
|
||||
<tr>
|
||||
<th style="width:50px">#</th>
|
||||
<th>Name</th>
|
||||
<th style="text-align:right">$ Saved</th>
|
||||
<th style="text-align:right">$ Saved*</th>
|
||||
<th style="text-align:right">Energy (Wh)</th>
|
||||
<th style="text-align:right">FLOPs</th>
|
||||
<th style="text-align:right">Requests</th>
|
||||
@@ -55,5 +55,5 @@ See how the OpenJarvis community saves money, energy, and compute by running AI
|
||||
<div id="leaderboard-pagination" class="lb-pagination"></div>
|
||||
|
||||
<p style="font-size:12px;opacity:0.6;margin-top:12px">
|
||||
*Dollar savings estimates assume local open-source models (e.g. Qwen, Nemotron, Kimi) produce roughly the same number of tokens per request, on average, as closed-source cloud models.
|
||||
*Dollar savings estimated vs. Claude Opus 4.6 API pricing ($5/1M input, $25/1M output tokens). Assumes local open-source models produce roughly the same number of tokens per request as cloud models.
|
||||
</p>
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
# Agent QA Runbook
|
||||
|
||||
Manual testing scenarios for persistent agents in the CLI and desktop app.
|
||||
|
||||
## Environment Setup
|
||||
|
||||
| Prerequisite | Command / Check |
|
||||
|---|---|
|
||||
| Ollama running with model | `ollama list` shows `qwen3:8b` |
|
||||
| OpenJarvis initialized | `uv run jarvis doctor` all green |
|
||||
| Rust extension built | `uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml` |
|
||||
| Desktop app running | `uv run jarvis serve` + `cd frontend && npm run dev` |
|
||||
| Slack credentials | `SLACK_BOT_TOKEN`, `SLACK_APP_TOKEN` set, bot invited to test channel |
|
||||
| Gmail credentials | OAuth credentials.json downloaded, token generated |
|
||||
| Twitter credentials | All 5 env vars set (bearer + OAuth 1.0a) |
|
||||
| Discord credentials | Bot token set, bot invited to test server |
|
||||
| Telegram credentials | Bot token from @BotFather, test chat ID known |
|
||||
| Email credentials | SMTP/IMAP host + credentials for test account |
|
||||
|
||||
## CLI Agent Scenarios
|
||||
|
||||
| # | Scenario | Steps | Expected Result | Pass |
|
||||
|---|----------|-------|-----------------|------|
|
||||
| 1 | Template launch | `jarvis agents launch`, pick research_monitor | Agent created, config printed with curated tools | [ ] |
|
||||
| 2 | Manual run | `jarvis agents run <id>` | Output shows reasoning + tool calls, status -> idle | [ ] |
|
||||
| 3 | Immediate ask | `jarvis agents ask <id> "summarize recent AI news"` | Synchronous response in terminal | [ ] |
|
||||
| 4 | Queued instruct | `jarvis agents instruct <id> "focus on diffusion"`, then `jarvis agents run <id>` | Queued -> delivered, response in `jarvis agents messages <id>` | [ ] |
|
||||
| 5 | Status check | `jarvis agents status` after 3+ runs | total_runs, total_cost, last_run_at populated | [ ] |
|
||||
| 6 | Pause/resume | `jarvis agents pause <id>`, verify skipped, `jarvis agents resume <id>`, verify fires | Status toggles correctly | [ ] |
|
||||
| 7 | Daemon scheduling | `jarvis agents daemon` with interval agent (60s) | 3+ ticks fire on schedule, memory accumulates | [ ] |
|
||||
| 8 | Budget exhaustion | Set max_cost=0.001, run until exceeded | Status becomes budget_exceeded | [ ] |
|
||||
| 9 | Error recovery | Kill Ollama mid-tick, then `jarvis agents recover <id>` | Error -> recover -> idle with checkpoint | [ ] |
|
||||
| 10 | Channel binding | `jarvis agents bind <id> --slack #test`, run tick | Agent sends to Slack | [ ] |
|
||||
| 11 | Multi-agent | Launch 3 agents, different intervals, run daemon | All fire independently | [ ] |
|
||||
| 12 | Template tools | Create from each template, `jarvis agents info <id>` | All curated default tools listed | [ ] |
|
||||
|
||||
## Desktop App Scenarios
|
||||
|
||||
| # | Scenario | Steps | Expected Result | Pass |
|
||||
|---|----------|-------|-----------------|------|
|
||||
| 1 | Template wizard | New Agent -> pick each template -> complete wizard | Agent appears in grid with correct config | [ ] |
|
||||
| 2 | Custom agent | New Agent -> Custom -> manual schedule, pick tools, set credentials | Tools + creds saved, agent created | [ ] |
|
||||
| 3 | Run Now | Click Run Now on agent card | Status dot: green -> blue -> green, stats increment | [ ] |
|
||||
| 4 | Immediate chat | Interact tab -> type message -> send (immediate mode) | Response appears in chat UI | [ ] |
|
||||
| 5 | Queued chat | Interact tab -> send (queued mode) -> click Run Now | Message delivered on tick, response appears | [ ] |
|
||||
| 6 | Task management | Tasks tab -> create task -> run agent | Task status updates, findings populated | [ ] |
|
||||
| 7 | Memory inspection | Run 3+ ticks -> Memory tab | Summary memory reflects agent's accumulated knowledge | [ ] |
|
||||
| 8 | Trace inspection | Run tick -> Logs tab | Trace steps visible with tool calls and results | [ ] |
|
||||
| 9 | Learning | Enable trace-driven learning -> Learning tab -> trigger | Learning log entries appear | [ ] |
|
||||
| 10 | Error + recovery | Stop Ollama -> run agent -> verify error badge -> click Recover | Error state shown, recovery resets to idle | [ ] |
|
||||
|
||||
## Channel-Specific QA Matrix
|
||||
|
||||
| Channel | Send Test | Receive Test | Thread/Reply Test | Agent Template | Pass |
|
||||
|---------|-----------|-------------|-------------------|----------------|------|
|
||||
| Slack | Post to #test-channel | Socket Mode incoming msg | Reply in thread (thread_ts) | inbox_triager | [ ] |
|
||||
| Gmail | Send email to test recipient | Poll unread -> handler fires | Reply in thread (threadId) | inbox_triager | [ ] |
|
||||
| Email (SMTP/IMAP) | Send via SMTP | IMAP poll UNSEEN | In-Reply-To header | inbox_triager | [ ] |
|
||||
| iMessage (BlueBubbles) | Send to phone number | N/A (send-only) | N/A | research_monitor | [ ] |
|
||||
| Twitter/X | Post tweet + send DM | Poll mentions | Reply (in_reply_to_tweet_id) | research_monitor | [ ] |
|
||||
| Discord | Post to #test-channel | Gateway message event | N/A | code_reviewer | [ ] |
|
||||
| Telegram | Send to test chat | Long-poll update | reply_to_message_id | research_monitor | [ ] |
|
||||
| WhatsApp (Baileys) | Send to test number | Baileys incoming msg | N/A | inbox_triager | [ ] |
|
||||
|
||||
## Stress & Edge Cases
|
||||
|
||||
| # | Scenario | How to Test | Pass Criteria | Pass |
|
||||
|---|----------|-------------|---------------|------|
|
||||
| 1 | Message flood | Queue 50 messages via CLI, run tick | All 50 delivered, response generated | [ ] |
|
||||
| 2 | Long-running daemon | Run daemon for 1 hour, 60s interval agent | No memory leak, no stall, ~60 ticks | [ ] |
|
||||
| 3 | Rapid pause/resume | Script: pause -> resume -> pause -> resume during tick | Clean state, no corruption | [ ] |
|
||||
| 4 | Credential revocation | Revoke Slack token mid-tick | Agent gets tool error, tick completes, status not corrupted | [ ] |
|
||||
| 5 | Multi-agent load | 10 agents on daemon, mix of intervals | All fire on schedule, no interference | [ ] |
|
||||
| 6 | Large response handling | Agent produces 10k+ char response | summary_memory truncated to 2000 chars, full response in messages | [ ] |
|
||||
| 7 | Checkpoint integrity | Kill process mid-tick, restart, recover | Checkpoint restored, agent resumes cleanly | [ ] |
|
||||
@@ -626,7 +626,20 @@ These events enable the telemetry and trace systems to record detailed interacti
|
||||
|
||||
## Managed Agent Streaming
|
||||
|
||||
The Managed Agent API (`/v1/managed-agents/{id}/messages`) supports real-time SSE streaming. Send a message with `stream: true` to receive the agent's response as a Server-Sent Events stream instead of the default asynchronous queue mode.
|
||||
The Managed Agent API (`/v1/managed-agents/{id}/messages`) supports **real LLM token streaming** via SSE. Send a message with `stream: true` to receive the model's response tokens as they are generated, rather than waiting for the full response.
|
||||
|
||||
### How It Works
|
||||
|
||||
The streaming endpoint calls `engine.stream_full()` directly, which yields `StreamChunk` objects containing content tokens, tool-call fragments, and finish reasons. This provides genuine token-by-token streaming from the LLM -- not a post-hoc word replay.
|
||||
|
||||
For multi-turn tool-calling agents, the streaming loop automatically:
|
||||
|
||||
1. Yields content tokens to the client as they arrive.
|
||||
2. Accumulates tool-call fragments (OpenAI sends these incrementally).
|
||||
3. Executes tools when `finish_reason="tool_calls"` is received.
|
||||
4. Emits tool results as named SSE events (`event: tool_result`).
|
||||
5. Feeds results back to the LLM for the next turn.
|
||||
6. Repeats until the model produces a final text response or `max_turns` is reached.
|
||||
|
||||
### Streaming Messages
|
||||
|
||||
@@ -639,19 +652,21 @@ curl -N -X POST http://localhost:8000/v1/managed-agents/{id}/messages \
|
||||
The response follows the OpenAI SSE format:
|
||||
|
||||
1. **Content chunks** -- `data: {"choices": [{"delta": {"content": "token"}}]}`
|
||||
2. **Tool results** (if the agent used tools) -- `event: tool_results\ndata: {"results": [...]}`
|
||||
3. **Final chunk** -- `data: {"choices": [{"delta": {}, "finish_reason": "stop"}]}`
|
||||
4. **Done sentinel** -- `data: [DONE]`
|
||||
2. **Tool calls** (if the model requests tool use) -- `event: tool_calls\ndata: {"calls": [{"tool_name": "...", "arguments": "..."}]}`
|
||||
3. **Tool results** -- `event: tool_result\ndata: {"tool_name": "...", "output": "..."}`
|
||||
4. **Final chunk** -- `data: {"choices": [{"delta": {}, "finish_reason": "stop"}]}`
|
||||
5. **Done sentinel** -- `data: [DONE]`
|
||||
|
||||
When `stream: false` (the default), the endpoint behaves exactly as before -- the message is queued and the agent must be triggered separately via `/run`.
|
||||
|
||||
### Behavior Details
|
||||
|
||||
- The user message is always stored in the database before the agent runs.
|
||||
- After streaming completes, the full agent response is persisted as an `agent_to_user` message.
|
||||
- The agent is instantiated from the managed agent's stored `agent_type` and `config`.
|
||||
- Conversation history from prior messages is automatically loaded as context.
|
||||
- The user message is always stored in the database before streaming starts.
|
||||
- After streaming completes, the full collected response is persisted as an `agent_to_user` message.
|
||||
- Conversation history from prior messages is automatically loaded as LLM context.
|
||||
- The engine's `stream_full()` method is used for real token streaming. Engines that do not override it fall back to the default implementation which wraps the plain `stream()` method.
|
||||
- If the engine is not available on the server, a `503` error is returned.
|
||||
- Tool execution during streaming uses the `ToolRegistry` to find and instantiate tools.
|
||||
|
||||
### Python Example
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# External MCP Server Integration
|
||||
|
||||
OpenJarvis can extend agent capabilities by connecting to external [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) servers. This allows agents to use tools provided by services like Home Assistant, databases, custom APIs, or any MCP-compatible server -- without writing custom tool code.
|
||||
|
||||
## How It Works
|
||||
|
||||
When OpenJarvis starts, it reads the `[tools.mcp]` section in `config.toml`. For each configured server, it:
|
||||
|
||||
1. Opens a connection using the appropriate transport (Streamable HTTP or stdio).
|
||||
2. Performs the MCP initialize handshake (protocol version negotiation and `initialized` notification).
|
||||
3. Discovers available tools via `tools/list`.
|
||||
4. Wraps each discovered tool as a standard `BaseTool` so agents can call them like any built-in tool.
|
||||
|
||||
If a server is unreachable or returns an error, OpenJarvis logs a warning and continues loading the remaining servers. One broken server does not prevent other tools from being available.
|
||||
|
||||
## Configuration
|
||||
|
||||
External MCP servers are configured in `config.toml` under `[tools.mcp]`:
|
||||
|
||||
```toml
|
||||
[tools.mcp]
|
||||
enabled = true
|
||||
servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}]'
|
||||
```
|
||||
|
||||
The `servers` value is a **JSON-encoded string** containing an array of server objects. Each object defines one external MCP server.
|
||||
|
||||
!!! note
|
||||
The value must be a JSON string (with single-quote TOML delimiters around it), not a native TOML array. This is because the configuration system passes it through as a single string field.
|
||||
|
||||
## Server Config Schema
|
||||
|
||||
Each server object supports the following fields:
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|------------------|----------------|----------|----------------------------------------------------------|
|
||||
| `name` | string | No | Human-readable name used in log messages. Defaults to `<unnamed>`. |
|
||||
| `url` | string | No* | URL for Streamable HTTP transport. |
|
||||
| `command` | string | No* | Command to launch a stdio-based MCP server. |
|
||||
| `args` | list of strings| No | Arguments passed to the stdio command. |
|
||||
| `include_tools` | list of strings| No | Whitelist of tool names to import. Only these tools are loaded. |
|
||||
| `exclude_tools` | list of strings| No | Blacklist of tool names to skip. All other tools are loaded. |
|
||||
|
||||
*Either `url` or `command` must be provided. If neither is set, the server is skipped with a warning.
|
||||
|
||||
When both `include_tools` and `exclude_tools` are specified, the whitelist is applied first, then the blacklist filters the result.
|
||||
|
||||
## Examples
|
||||
|
||||
### Home Assistant via Streamable HTTP
|
||||
|
||||
Connect to the [ha-mcp](https://github.com/tevonsb/ha-mcp) Home Assistant add-on:
|
||||
|
||||
```toml
|
||||
[tools.mcp]
|
||||
enabled = true
|
||||
servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}]'
|
||||
```
|
||||
|
||||
This discovers all HA tools (entity control, automations, history, etc.) and makes them available to agents.
|
||||
|
||||
### Stdio Server
|
||||
|
||||
Launch a local MCP server as a subprocess:
|
||||
|
||||
```toml
|
||||
[tools.mcp]
|
||||
enabled = true
|
||||
servers = '[{"name": "myserver", "command": "python", "args": ["-m", "my_mcp_server"]}]'
|
||||
```
|
||||
|
||||
OpenJarvis starts the process automatically, communicates via JSON-RPC over stdin/stdout, and terminates it on shutdown.
|
||||
|
||||
### Multiple Servers
|
||||
|
||||
```toml
|
||||
[tools.mcp]
|
||||
enabled = true
|
||||
servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}, {"name": "database", "command": "db-mcp-server", "args": ["--db", "postgres://localhost/mydb"]}]'
|
||||
```
|
||||
|
||||
### Tool Filtering
|
||||
|
||||
When a server exposes many tools but you only need a few, use `include_tools` to whitelist:
|
||||
|
||||
```toml
|
||||
[tools.mcp]
|
||||
enabled = true
|
||||
servers = '[{"name": "ha", "url": "http://172.16.3.1:9583/private_abc123", "include_tools": ["hassTurnOn", "hassTurnOff", "hassGetState"]}]'
|
||||
```
|
||||
|
||||
To load everything except specific tools, use `exclude_tools`:
|
||||
|
||||
```toml
|
||||
[tools.mcp]
|
||||
enabled = true
|
||||
servers = '[{"name": "ha", "url": "http://172.16.3.1:9583/private_abc123", "exclude_tools": ["hassCreateBackup", "hassDeleteBackup"]}]'
|
||||
```
|
||||
|
||||
## Transport Types
|
||||
|
||||
### Streamable HTTP
|
||||
|
||||
Used when the `url` field is set. The transport sends JSON-RPC requests as HTTP POST to the given URL using `httpx`. It tracks the `Mcp-Session-Id` header across requests as required by the MCP Streamable HTTP specification.
|
||||
|
||||
**When to use:** Remote MCP servers, services running as HTTP endpoints (e.g., Home Assistant MCP add-on, cloud-hosted MCP servers).
|
||||
|
||||
**Connection parameters:**
|
||||
|
||||
- Connect timeout: 10 seconds
|
||||
- Request timeout: 60 seconds
|
||||
|
||||
### Stdio
|
||||
|
||||
Used when the `command` field is set. OpenJarvis spawns the command as a subprocess and communicates via JSON-RPC lines on stdin/stdout.
|
||||
|
||||
**When to use:** Local MCP servers distributed as CLI tools, development/testing, servers that require filesystem access on the same machine.
|
||||
|
||||
!!! info "SSETransport alias"
|
||||
`SSETransport` is provided as a backward-compatible alias for `StreamableHTTPTransport`. Both refer to the same implementation.
|
||||
|
||||
## Error Handling
|
||||
|
||||
OpenJarvis handles MCP server failures gracefully:
|
||||
|
||||
- **Server unreachable:** A warning is logged and the server is skipped. All other servers and built-in tools continue to load normally.
|
||||
- **Timeout:** HTTP requests time out after 60 seconds. The server is skipped with a warning.
|
||||
- **Invalid config:** If the `servers` JSON is malformed or a server entry has neither `url` nor `command`, a warning is logged and that entry is skipped.
|
||||
- **Tool discovery failure:** If `tools/list` fails on a server, the error is caught and the server is skipped.
|
||||
- **Runtime tool call failure:** If a tool call to an external MCP server fails at runtime, it returns a `ToolResult` with `success=False` and the error message.
|
||||
|
||||
No single server failure causes OpenJarvis to crash or prevents other tools from working.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Server not discovered
|
||||
|
||||
1. Check that `[tools.mcp]` has `enabled = true`.
|
||||
2. Verify the `servers` JSON is valid. A common mistake is using TOML arrays instead of a JSON string.
|
||||
3. Check the OpenJarvis logs for warnings like `Failed to discover external MCP tools`.
|
||||
|
||||
### Connection refused / timeout
|
||||
|
||||
1. Verify the server is running and reachable from the OpenJarvis host: `curl -v http://host:port/`.
|
||||
2. Check firewall rules between the OpenJarvis container and the MCP server.
|
||||
3. For Docker deployments, ensure both containers are on the same network or use host IPs.
|
||||
|
||||
### Tools not appearing
|
||||
|
||||
1. Run with debug logging to see which tools were discovered.
|
||||
2. Check if `include_tools` or `exclude_tools` filters are too restrictive.
|
||||
3. Verify the MCP server actually exposes tools via `tools/list` (some servers only expose resources or prompts).
|
||||
|
||||
### Stdio server crashes immediately
|
||||
|
||||
1. Test the command manually: `python -m my_mcp_server` should start and wait for input on stdin.
|
||||
2. Check stderr output in the OpenJarvis logs for error messages from the subprocess.
|
||||
3. Ensure all dependencies for the MCP server are installed in the same environment.
|
||||
@@ -204,7 +204,7 @@ All built-in tools are registered via `@ToolRegistry.register()` and are availab
|
||||
| **Scheduler** | `pause_scheduled_task` | Pause an active scheduled task |
|
||||
| **Scheduler** | `resume_scheduled_task` | Resume a paused scheduled task |
|
||||
| **Scheduler** | `cancel_scheduled_task` | Cancel a scheduled task permanently |
|
||||
| **Integration** | `mcp_adapter` | Bridge to external MCP tool servers |
|
||||
| **Integration** | `mcp_adapter` | Bridge to external MCP tool servers (see [External MCP Servers](mcp-external-servers.md)) |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -67,10 +67,10 @@ export default function App() {
|
||||
.then((data) => {
|
||||
setSavings(data);
|
||||
if (optInEnabled && optInDisplayName && data) {
|
||||
const dollarSavings = data.per_provider.reduce(
|
||||
(sum, p) => sum + p.total_cost,
|
||||
0,
|
||||
const claudeEntry = data.per_provider.find(
|
||||
(p) => p.provider === 'claude-opus-4.6',
|
||||
);
|
||||
const dollarSavings = claudeEntry ? claudeEntry.total_cost : 0;
|
||||
const energySaved = data.per_provider.reduce(
|
||||
(sum, p) => sum + (p.energy_wh || 0),
|
||||
0,
|
||||
|
||||
@@ -100,6 +100,12 @@ export async function fetchModels(): Promise<ModelInfo[]> {
|
||||
return data.data || [];
|
||||
}
|
||||
|
||||
export async function fetchRecommendedModel(): Promise<{ model: string; reason: string }> {
|
||||
const res = await fetch(`${getBase()}/v1/recommended-model`);
|
||||
if (!res.ok) return { model: '', reason: 'Failed to fetch' };
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function pullModel(modelName: string): Promise<void> {
|
||||
// In Tauri, go through the Rust backend directly (avoids CORS / timeout
|
||||
// issues with long model downloads via fetch).
|
||||
@@ -296,6 +302,8 @@ export interface ManagedAgent {
|
||||
total_runs?: number;
|
||||
total_cost?: number;
|
||||
total_tokens?: number;
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
last_run_at?: number | null;
|
||||
// Schedule
|
||||
schedule_type?: string;
|
||||
@@ -304,6 +312,8 @@ export interface ManagedAgent {
|
||||
budget?: number;
|
||||
// Learning
|
||||
learning_enabled?: boolean;
|
||||
// Live progress
|
||||
current_activity?: string;
|
||||
}
|
||||
|
||||
export interface AgentTask {
|
||||
|
||||
+423
-749
File diff suppressed because it is too large
Load Diff
@@ -1,417 +0,0 @@
|
||||
# Agent Runtime Manual Test Plan
|
||||
|
||||
**Branch:** `main`
|
||||
**PR Reference:** [#32](https://github.com/open-jarvis/OpenJarvis/pull/32)
|
||||
|
||||
---
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
git checkout main && git pull
|
||||
uv sync --extra dev
|
||||
```
|
||||
|
||||
Create `~/.openjarvis/config.toml`:
|
||||
|
||||
```toml
|
||||
[engine]
|
||||
type = "cloud"
|
||||
|
||||
[intelligence]
|
||||
default_model = "Qwen/Qwen3.5-35B-A3B"
|
||||
|
||||
[engine.cloud]
|
||||
provider = "openai"
|
||||
api_key = "sk-..."
|
||||
```
|
||||
|
||||
For every test case, record: **Pass / Fail / Partial / Blocked**, what you actually saw, and screenshots for any UI issues.
|
||||
|
||||
---
|
||||
|
||||
## Part 1: CLI (`jarvis agents`)
|
||||
|
||||
### 1.1 Commands exist
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 1 | `jarvis agents --help` | Shows all subcommands: `launch`, `start`, `stop`, `run`, `status`, `logs`, `daemon`, `watch`, `recover`, `errors`, `ask`, `instruct`, `messages`, `list`, `info`, `create`, `pause`, `resume`, `delete`, `bind`, `channels`, `search`, `templates`, `tasks` |
|
||||
|
||||
### 1.2 Agent lifecycle: create → run → pause → resume → delete
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 2 | `jarvis agents launch` | Wizard: template list → name/schedule/tools/budget/learning prompts → creates agent, prints ID |
|
||||
| 3 | `jarvis agents list` | Agent appears, status=`idle` |
|
||||
| 4 | `jarvis agents status` | Table: name, status dot, schedule, last run, runs=0, cost=$0 |
|
||||
| 5 | `jarvis agents run <id>` | Prints progress then "Tick complete. Status: idle, runs: 1" |
|
||||
| 6 | `jarvis agents status` | runs=1, last run time updated |
|
||||
| 7 | `jarvis agents pause <id>` then `status` | Status shows `paused` |
|
||||
| 8 | `jarvis agents resume <id>` then `status` | Status back to `idle` |
|
||||
| 9 | `jarvis agents delete <id>` then `list` | Agent gone (soft-deleted/archived, not in list) |
|
||||
|
||||
### 1.3 Agent creation variants
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 10 | `jarvis agents create "Test Agent"` | Creates agent by name, prints ID |
|
||||
| 11 | `jarvis agents create --template <template_name>` | Creates from template, inherits template config |
|
||||
| 12 | `jarvis agents launch` → pick a template | Wizard pre-fills config from template |
|
||||
| 13 | `jarvis agents launch` → pick "Custom Agent" | Wizard starts with blank config |
|
||||
| 14 | `jarvis agents templates` | Lists built-in + user templates with descriptions |
|
||||
|
||||
### 1.4 Scheduling
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 15 | Create agent with `schedule_type=interval`, `schedule_value=30` | Created |
|
||||
| 16 | `jarvis agents start <id>` | "Agent registered with scheduler" |
|
||||
| 17 | `jarvis agents stop <id>` | "Agent deregistered from scheduler" |
|
||||
| 18 | `jarvis agents daemon` | Starts, prints agent count, blocks. Ctrl+C → "Daemon stopped." clean exit |
|
||||
| 19 | Create agent with `schedule_type=cron`, `schedule_value="*/5 * * * *"` | Created |
|
||||
| 20 | `jarvis agents start <id>` (cron agent) | Registered, next fire time displayed or logged |
|
||||
| 21 | Create agent with `schedule_type=manual` then `start <id>` | Agent registered but never auto-fires |
|
||||
|
||||
### 1.5 Interaction: ask / instruct / messages
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 22 | `jarvis agents ask <id> "What is 2+2?"` | Runs tick, prints agent response inline |
|
||||
| 23 | `jarvis agents messages <id>` | Shows user→agent ask + agent→user response |
|
||||
| 24 | `jarvis agents instruct <id> "Focus on ML papers"` | "Instruction queued for next tick" |
|
||||
| 25 | `jarvis agents messages <id>` | Queued instruction shows `[queued]`, status=pending |
|
||||
| 26 | `jarvis agents run <id>` then `messages <id>` | Queued message now delivered, status changes from pending |
|
||||
| 27 | `jarvis agents ask <id> ""` (empty message) | Graceful error or rejection, no crash |
|
||||
| 28 | `jarvis agents instruct <id>` with very long message (>1000 chars) | Accepted and stored correctly |
|
||||
|
||||
### 1.6 Error recovery & monitoring
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 29 | `jarvis agents errors` | Lists agents in error/needs_attention/stalled/budget_exceeded (or empty table) |
|
||||
| 30 | `jarvis agents recover <id>` (on errored agent) | Restores checkpoint, status → `idle` |
|
||||
| 31 | `jarvis agents recover <id>` (on idle agent) | Clear message: "Agent is not in error state" or similar |
|
||||
| 32 | `jarvis agents logs <id>` | Recent traces with tick IDs and timestamps |
|
||||
| 33 | `jarvis agents logs <nonexistent_id>` | Clear error: "Agent not found" |
|
||||
| 34 | `jarvis agents watch` (then run a tick in another terminal) | Events stream live: AGENT_TICK_START, AGENT_TICK_END visible. Ctrl+C to stop. |
|
||||
| 35 | `jarvis agents watch <id>` | Same, filtered to one agent only |
|
||||
| 36 | `jarvis agents watch` then Ctrl+C | Clean exit, no traceback, no hanging threads |
|
||||
|
||||
### 1.7 Agent info & inspection
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 37 | `jarvis agents info <id>` | Shows agent type, status, memory snippet, tasks, channels, config details |
|
||||
| 38 | `jarvis agents tasks <id>` | Lists tasks with statuses (or empty state) |
|
||||
| 39 | `jarvis agents channels <id>` | Lists channel bindings (or empty state) |
|
||||
| 40 | `jarvis agents search "keyword"` | Searches across agent traces, returns relevant results |
|
||||
|
||||
### 1.8 Edge cases & invalid input
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 41 | `jarvis agents run <nonexistent_id>` | Clear error: "Agent not found" — no Python traceback |
|
||||
| 42 | `jarvis agents pause <id>` twice | Second pause is no-op or clear message, no crash |
|
||||
| 43 | `jarvis agents resume <id>` (already idle) | No-op or clear message, no crash |
|
||||
| 44 | `jarvis agents run <id>` while another tick is running | Concurrency guard: "Agent is already running" error |
|
||||
| 45 | `jarvis agents delete <id>` then `run <id>` | Clear error about deleted/archived agent |
|
||||
| 46 | Create agent with invalid cron expression | Rejected with clear validation error |
|
||||
| 47 | Create agent with negative budget | Rejected or clamped to 0 |
|
||||
|
||||
### 1.9 CLI aesthetics
|
||||
|
||||
| # | Check | Expected |
|
||||
|---|-------|----------|
|
||||
| 48 | `status` table formatting | Columns aligned, readable at 80-char terminal width |
|
||||
| 49 | Error messages (run with no engine configured) | Clear human-readable message, no Python tracebacks |
|
||||
| 50 | `launch` wizard prompts | Clear labels, sensible defaults, no confusing jargon |
|
||||
| 51 | `watch` event stream | Color-coded, event type + agent name visible, timestamps |
|
||||
| 52 | `list` table with 0 agents | "No agents found" or empty table — not a crash |
|
||||
| 53 | `list` table with 10+ agents | Table remains readable, no column overflow |
|
||||
| 54 | All commands with `--help` | Every subcommand has a help string |
|
||||
|
||||
---
|
||||
|
||||
## Part 2: Web Frontend
|
||||
|
||||
### 2.0 Setup
|
||||
|
||||
```bash
|
||||
# Terminal 1 # Terminal 2
|
||||
uv run jarvis serve cd frontend && npm install && npm run dev
|
||||
```
|
||||
|
||||
Open http://localhost:5173, navigate to **Agents** page via sidebar.
|
||||
|
||||
### 2.1 Navigation & routing
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 55 | Click "Agents" in sidebar | AgentsPage renders, URL is `/agents` |
|
||||
| 56 | Direct navigation to `/agents` | Page loads correctly (no blank screen) |
|
||||
| 57 | Browser back/forward after visiting agent detail | Navigation works, state preserved |
|
||||
|
||||
### 2.2 List view
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 58 | Page loads with backend running | No console errors, agent list renders |
|
||||
| 59 | Page loads with backend **down** | User-visible error message (not blank white screen), no console exceptions |
|
||||
| 60 | Agent cards | Name, color status dot, schedule description, last run time, runs count, cost |
|
||||
| 61 | "Run Now" button | Triggers tick, card updates (runs count increments, last run time updates) |
|
||||
| 62 | Pause/Resume button | Toggles status, dot color changes immediately |
|
||||
| 63 | Agent list auto-refresh | After running a tick via CLI, the web list eventually reflects the updated state |
|
||||
| 64 | 10+ agents in list | Cards render without performance issues, scroll works |
|
||||
|
||||
### 2.3 Launch wizard
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 65 | Click "Launch Agent" | Modal appears: Step 1 template picker with templates + "Custom Agent" option |
|
||||
| 66 | Templates load from API | Template cards display with names and descriptions |
|
||||
| 67 | Select template → Next → Step 2 | Config form: name (pre-filled from template), schedule_type dropdown, schedule_value, tools checkboxes, budget, learning toggle (off) |
|
||||
| 68 | Select "Custom Agent" → Next → Step 2 | Config form with blank name, no pre-filled values |
|
||||
| 69 | Next → Step 3 | Review summary of all config values |
|
||||
| 70 | Click Launch | Agent created, modal closes, new agent appears in list |
|
||||
| 71 | Back button at Step 2 | Returns to Step 1, template selection preserved |
|
||||
| 72 | Back button at Step 3 | Returns to Step 2, all form inputs preserved |
|
||||
| 73 | Launch with empty name | Inline error: "Agent name is required" — modal stays open |
|
||||
| 74 | Launch with all tools selected | All tools included in review and in created agent config |
|
||||
| 75 | Click outside modal / press Escape | Modal closes (or stays open — document behavior) |
|
||||
| 76 | Schedule type = "Manual" | schedule_value input is disabled/hidden |
|
||||
| 77 | Schedule type = "Cron" | schedule_value placeholder shows cron example |
|
||||
| 78 | Schedule type = "Interval" | schedule_value placeholder shows seconds example |
|
||||
|
||||
### 2.4 Detail view (click an agent)
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 79 | Click agent card | Detail view opens with tabbed interface |
|
||||
| 80 | **Overview** tab | Stat cards (Total Runs, Success Rate, Total Cost), config display, channels list, action buttons |
|
||||
| 81 | **Overview** action buttons | Run Now, Pause, Resume visible and functional |
|
||||
| 82 | **Interact** tab | Chat message list, textarea, "Immediate" and "Queue" send buttons |
|
||||
| 83 | Send immediate message | Appears in chat with user styling, agent responds after tick |
|
||||
| 84 | Send queued message | Shows with "queued" badge, status=pending |
|
||||
| 85 | Send empty message | Button disabled or graceful rejection — no empty message sent |
|
||||
| 86 | Rapid-fire send (click Send multiple times quickly) | No duplicate messages, no race condition errors |
|
||||
| 87 | Chat auto-scroll | New messages scroll into view automatically |
|
||||
| 88 | **Tasks** tab | Task list with status badges (completed=green, failed=red, active=blue, pending=gray) |
|
||||
| 89 | **Tasks** tab (no tasks) | Empty state: "No tasks assigned." |
|
||||
| 90 | **Memory** tab | summary_memory text displayed in readable format |
|
||||
| 91 | **Memory** tab (no memory) | Empty state: "Agent has no stored memory yet." |
|
||||
| 92 | **Learning** tab | Toggle switch (read-only, off by default), placeholder text for future events |
|
||||
| 93 | **Logs** tab | Placeholder / empty state message (not a crash or blank) |
|
||||
| 94 | Tab switching — rapid clicks | All 6 tabs render instantly, no layout shift, no flash of wrong content |
|
||||
|
||||
### 2.5 Error states
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 95 | Agent in `error` status | Red status dot/badge, "Recover" button visible |
|
||||
| 96 | Click Recover | Status resets to `idle`, dot turns green |
|
||||
| 97 | Agent in `needs_attention` status | Amber badge visible |
|
||||
| 98 | Agent in `budget_exceeded` status | Orange badge visible |
|
||||
| 99 | Agent in `stalled` status | Yellow badge visible |
|
||||
| 100 | Backend goes down while page is open | Next refresh/action shows error — not silent failure |
|
||||
| 101 | Delete agent → confirm it disappears from list | Agent removed from list immediately (or on next refresh) |
|
||||
| 102 | Delete agent (no confirmation dialog in web) | **Document:** Is instant delete OK or should there be a confirm? |
|
||||
|
||||
### 2.6 Overflow menu
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 103 | Click "..." menu on agent card | Dropdown with Delete + other options |
|
||||
| 104 | Click Delete from menu | Agent deleted, list updates |
|
||||
| 105 | Click outside dropdown | Dropdown closes |
|
||||
|
||||
### 2.7 Web aesthetics & UX
|
||||
|
||||
| # | Check | Expected |
|
||||
|---|-------|----------|
|
||||
| 106 | Status dot colors | idle=#22c55e, running=#3b82f6, paused=#6b7280, error=#ef4444, needs_attention=#f59e0b, budget_exceeded=#f97316, stalled=#eab308 |
|
||||
| 107 | Launch wizard spacing/alignment | Modal centered, steps clearly numbered, form inputs aligned, no overlap |
|
||||
| 108 | Detail view tab switching | Instant, no layout shift or flash |
|
||||
| 109 | Interact tab chat feel | Messages visually distinct (user=right vs agent=left or different colors), auto-scroll, clear input area |
|
||||
| 110 | Responsive at 1024px width | No overflow or cut-off content, agent cards reflow |
|
||||
| 111 | Responsive at 1440px width | Proper use of space, no excessive stretching |
|
||||
| 112 | Responsive at 768px width (tablet) | Still usable, no broken layout |
|
||||
| 113 | Empty states | "No agents yet" + CTA button / "No messages" / "No tasks" — not blank white space |
|
||||
| 114 | Loading states | "Loading agents..." shown during fetch, spinner or skeleton |
|
||||
| 115 | Page title / browser tab | Meaningful title (not just "Vite App") |
|
||||
| 116 | Console errors | Zero console errors during normal usage flow |
|
||||
|
||||
---
|
||||
|
||||
## Part 3: Desktop App
|
||||
|
||||
### 3.0 Setup
|
||||
|
||||
```bash
|
||||
# Terminal 1 # Terminal 2
|
||||
uv run jarvis serve cd desktop && npm install && npm run tauri dev
|
||||
```
|
||||
|
||||
Navigate to the **Agents** tab.
|
||||
|
||||
### 3.1 Functionality
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 117 | Left panel: agent list | Status dots, schedule descriptions, last run times |
|
||||
| 118 | Click agent → right panel | Tabbed detail view (Overview, Interact, Tasks, Memory, Learning, Logs) |
|
||||
| 119 | No agent selected | Right panel shows "Select an agent to view details" |
|
||||
| 120 | "Launch Agent" button | Opens wizard, same 3-step flow as web |
|
||||
| 121 | Launch wizard → Create agent | Agent appears in left panel list |
|
||||
| 122 | **Overview** tab | Key-value stats (Status, Agent Type, Schedule, Last Run, Total Runs, Total Cost, Budget) + action buttons (Run Now, Pause, Resume, Recover) |
|
||||
| 123 | **Interact** tab | Chat UI, mode toggle (immediate/queued), Enter shortcut sends message |
|
||||
| 124 | Send immediate message | Response appears in chat |
|
||||
| 125 | Send queued message | Shows as pending |
|
||||
| 126 | **Tasks** tab | Task list with colored status badges + created-at timestamps |
|
||||
| 127 | **Memory** tab | summary_memory in monospace font |
|
||||
| 128 | **Learning** tab | Shows enabled/disabled status + placeholder text |
|
||||
| 129 | **Logs** tab | Placeholder: "Log streaming not yet connected." |
|
||||
| 130 | Auto-refresh | Agent list refreshes on ~10s interval (verify with CLI-triggered state change) |
|
||||
| 131 | Delete agent via desktop | Confirmation dialog appears, agent removed on confirm |
|
||||
|
||||
### 3.2 Desktop edge cases
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 132 | Backend not running → open desktop app | Error state shown, not a crash |
|
||||
| 133 | Backend dies while desktop is open | Graceful degradation on next action/refresh |
|
||||
| 134 | Selected agent deleted via CLI → desktop refreshes | Selected agent deselects, list updates |
|
||||
|
||||
### 3.3 Desktop aesthetics
|
||||
|
||||
| # | Check | Expected |
|
||||
|---|-------|----------|
|
||||
| 135 | Catppuccin color scheme consistent | idle=#a6e3a1, running=#89b4fa, paused=#6c7086, error=#f38ba8, needs_attention=#fab387, stalled=#f9e2af |
|
||||
| 136 | Left/right panel split | Resizable or fixed at reasonable ratio, no overlap |
|
||||
| 137 | Tab switching | Smooth, no flicker |
|
||||
| 138 | Launch wizard modal | Properly overlays content, dismissible with Escape or outside click |
|
||||
| 139 | Text readability | Font sizes consistent, sufficient contrast against dark background |
|
||||
| 140 | Window resize | Layout adapts, no overflow or clipping |
|
||||
| 141 | Status badge consistency with web | Same statuses map to same semantic colors (green=idle, blue=running, etc.) |
|
||||
|
||||
---
|
||||
|
||||
## Part 4: API Backend (Direct)
|
||||
|
||||
### 4.1 REST endpoint smoke tests
|
||||
|
||||
Run with `uv run jarvis serve` and test via curl or Postman.
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 142 | `GET /v1/managed-agents` | 200, returns `[]` or agent list JSON |
|
||||
| 143 | `POST /v1/managed-agents` with valid body | 200/201, returns created agent JSON with `id` |
|
||||
| 144 | `POST /v1/managed-agents` with empty body | 422 or 400 with validation error |
|
||||
| 145 | `GET /v1/managed-agents/<id>` | 200, returns single agent |
|
||||
| 146 | `GET /v1/managed-agents/<bad_id>` | 404, returns error JSON |
|
||||
| 147 | `POST /v1/managed-agents/<id>/run` | 200, tick executes |
|
||||
| 148 | `POST /v1/managed-agents/<id>/pause` | 200, status changes to paused |
|
||||
| 149 | `POST /v1/managed-agents/<id>/resume` | 200, status changes to idle |
|
||||
| 150 | `POST /v1/managed-agents/<id>/recover` | 200 if errored, appropriate error if not |
|
||||
| 151 | `DELETE /v1/managed-agents/<id>` | 200, agent archived |
|
||||
| 152 | `GET /v1/templates` | 200, returns template list |
|
||||
| 153 | `POST /v1/templates/<id>/instantiate` | 200, creates agent from template |
|
||||
| 154 | `GET /v1/agents/errors` | 200, returns list of problem agents |
|
||||
| 155 | `GET /v1/agents/health` | 200, returns health summary |
|
||||
|
||||
### 4.2 Message endpoints
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 156 | `POST /v1/managed-agents/<id>/messages` with `{"content":"hi","direction":"user_to_agent","mode":"immediate"}` | 200, message stored |
|
||||
| 157 | `GET /v1/managed-agents/<id>/messages` | 200, returns message list |
|
||||
| 158 | `POST /v1/managed-agents/<id>/messages` with `{"content":"","direction":"user_to_agent","mode":"immediate"}` | 422 or graceful handling |
|
||||
| 159 | `POST /v1/managed-agents/<id>/messages` with `{"content":"cmd","direction":"user_to_agent","mode":"queued"}` | 200, message has status=pending |
|
||||
|
||||
### 4.3 Task & channel endpoints
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 160 | `GET /v1/managed-agents/<id>/tasks` | 200, returns task list |
|
||||
| 161 | `POST /v1/managed-agents/<id>/tasks` | 200, creates task |
|
||||
| 162 | `GET /v1/managed-agents/<id>/channels` | 200, returns channel bindings |
|
||||
| 163 | `GET /v1/managed-agents/<id>/state` | 200, returns full agent state |
|
||||
|
||||
### 4.4 WebSocket events
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 164 | Connect to `ws://localhost:8222/v1/agents/events` | Connection established |
|
||||
| 165 | Trigger a tick → observe WS messages | Receive AGENT_TICK_START and AGENT_TICK_END events |
|
||||
| 166 | Connect with `?agent_id=<id>` filter | Only events for that agent |
|
||||
| 167 | Disconnect cleanly | No server error logs |
|
||||
|
||||
---
|
||||
|
||||
## Part 5: Cross-Platform Consistency
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 168 | Create agent via CLI → check web + desktop | Same name, status, config everywhere |
|
||||
| 169 | Run tick via CLI → check web + desktop | Run count and last run time update in both UIs |
|
||||
| 170 | Send message via web Interact → check CLI `messages` | Same content, direction, mode |
|
||||
| 171 | Pause via desktop → check CLI `status` + web | `paused` everywhere |
|
||||
| 172 | Delete via web → check CLI `list` + desktop | Gone everywhere |
|
||||
| 173 | Create via web wizard → check CLI `list` + desktop | Agent visible in all three |
|
||||
| 174 | Recover via CLI → check web + desktop | Status back to idle in all UIs |
|
||||
| 175 | Send queued message via CLI `instruct` → check web Interact | Message shows with pending/queued status |
|
||||
| 176 | Multiple agents created from different clients | All agents appear correctly in all views |
|
||||
|
||||
---
|
||||
|
||||
## Part 6: Stress & Concurrency
|
||||
|
||||
| # | Test | Expected |
|
||||
|---|------|----------|
|
||||
| 177 | Run tick on same agent from two terminals simultaneously | Concurrency guard blocks second tick: "Agent is already running" |
|
||||
| 178 | Create 20+ agents → check list performance | All clients render list without lag |
|
||||
| 179 | Rapidly pause/resume same agent | All state transitions correct, no stuck states |
|
||||
| 180 | Run daemon + manual `run` at same time | No double-ticking, concurrency guard holds |
|
||||
| 181 | Delete agent while tick is in progress | Tick completes or fails gracefully, agent ends up archived |
|
||||
|
||||
---
|
||||
|
||||
## Part 7: Deferred Features (Placeholder Verification)
|
||||
|
||||
Confirm these show placeholders (not crashes):
|
||||
|
||||
| # | Feature | CLI | Web | Desktop |
|
||||
|---|---------|-----|-----|---------|
|
||||
| 182 | Budget enforcement | `run` still works even if cost > budget | No enforcement, budget is display-only | Same |
|
||||
| 183 | Stall detection | No automatic stall detection fires | N/A | N/A |
|
||||
| 184 | Learning event timeline | `Learning` tab shows placeholder text | Same | Same |
|
||||
| 185 | Logs trace replay | `Logs` tab shows placeholder text | Same | Same |
|
||||
| 186 | `POST /v1/skills` | N/A | N/A | Returns `"not_implemented"` |
|
||||
| 187 | `POST /v1/optimize/runs` | N/A | N/A | Returns placeholder `run_id` |
|
||||
| 188 | `GET /v1/feedback/stats` | N/A | N/A | Returns `{total: 0, mean_score: 0.0}` |
|
||||
|
||||
---
|
||||
|
||||
## Deliverables
|
||||
|
||||
**1. Test results** — Spreadsheet with columns: #, Status (Pass/Fail/Partial/Blocked), Actual Behavior, Screenshot (for UI issues).
|
||||
|
||||
**2. Bug list** — Each bug: steps to reproduce, expected vs actual, severity (Critical/Major/Minor), screenshot.
|
||||
|
||||
**3. UX & aesthetics feedback** — Is the launch wizard clear? Are status colors distinguishable? Does the Interact tab feel like chat? Is CLI output readable? Are error messages helpful? Is the delete-without-confirm behavior in web acceptable?
|
||||
|
||||
**4. API error handling audit** — Document all cases where the frontend silently swallows errors (currently: agent list fetch, interact tab sends). Recommend which should show user-visible errors.
|
||||
|
||||
**5. Deferred features check** — Confirm placeholder items in Part 7 show graceful stubs (not crashes or blank screens).
|
||||
|
||||
---
|
||||
|
||||
## Notes
|
||||
|
||||
- Backend (`jarvis serve`) must be running for web and desktop (default port 8222).
|
||||
- Without an engine configured, `run`/`ask` will error — document whether the error message is clear.
|
||||
- `daemon` and `watch` block — Ctrl+C to exit.
|
||||
- Web frontend API client has unused functions (`updateManagedAgent`, `createAgentTask`, `fetchAgentState`, `fetchErrorAgents`) — not a bug, but note for future.
|
||||
- Desktop API client is missing some endpoints that the web client has (`fetchAgentChannels`, `fetchAgentState`, `fetchErrorAgents`) — may affect feature parity.
|
||||
- Frontend has **no automated tests** — all testing is manual per this plan.
|
||||
- Both frontends silently catch API errors (`.catch(() => {})`) — this is a known UX gap to evaluate.
|
||||
@@ -143,6 +143,7 @@ nav:
|
||||
- Home: index.md
|
||||
- Getting Started:
|
||||
- Installation: getting-started/installation.md
|
||||
- macOS Guide: getting-started/macos.md
|
||||
- Quick Start: getting-started/quickstart.md
|
||||
- Code Snippets: getting-started/snippets.md
|
||||
- Configuration: getting-started/configuration.md
|
||||
@@ -163,5 +164,8 @@ nav:
|
||||
- Security: architecture/security.md
|
||||
- Design Principles: architecture/design-principles.md
|
||||
- API Reference: api-reference/
|
||||
- User Guide:
|
||||
- Tools: user-guide/tools.md
|
||||
- External MCP Servers: user-guide/mcp-external-servers.md
|
||||
- Leaderboard: leaderboard.md
|
||||
- Roadmap: development/roadmap.md
|
||||
|
||||
@@ -17,6 +17,7 @@ dependencies = [
|
||||
"python-telegram-bot>=22.6",
|
||||
"rich>=13",
|
||||
"tomli>=2.0; python_version < '3.11'",
|
||||
"tomlkit>=0.12",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -83,6 +84,12 @@ channel-zulip = ["zulip>=0.9"]
|
||||
channel-twitch = ["twitchio>=2.6"]
|
||||
channel-nostr = ["pynostr>=0.6"]
|
||||
channel-twilio = ["twilio>=9.0"]
|
||||
channel-gmail = [
|
||||
"google-api-python-client>=2.0",
|
||||
"google-auth-oauthlib>=1.0",
|
||||
"google-auth-httplib2>=0.2",
|
||||
]
|
||||
channel-twitter = ["tweepy>=4.14"]
|
||||
browser = ["playwright>=1.40"]
|
||||
media = ["openai>=1.30"]
|
||||
pdf = ["pdfplumber>=0.10"]
|
||||
@@ -123,6 +130,7 @@ markers = [
|
||||
"apple: requires Apple Silicon",
|
||||
"macos15: requires macOS 15+ (Sequoia) for Apple FM",
|
||||
"slow: long-running test",
|
||||
"live_channel: requires real channel credentials (env vars)",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
-- Migration: Recompute dollar_savings to use only Claude Opus 4.6 pricing
|
||||
-- -----------------------------------------------------------------------
|
||||
-- Previously the frontend summed hypothetical costs across all 3 cloud
|
||||
-- providers (GPT-5.3 + Claude Opus 4.6 + Gemini 3.1 Pro). This
|
||||
-- migration recalculates dollar_savings using Claude Opus 4.6 only.
|
||||
--
|
||||
-- Derivation
|
||||
-- ----------
|
||||
-- Let P = prompt_tokens, C = completion_tokens, T = total_tokens = P + C.
|
||||
--
|
||||
-- old = (P/1M)*(2+5+2) + (C/1M)*(10+25+12) = (P/1M)*9 + (C/1M)*47
|
||||
-- new = (P/1M)*5 + (C/1M)*25
|
||||
--
|
||||
-- Solving the system {T = P + C, old = 9P/1M + 47C/1M} for P and C and
|
||||
-- substituting into the "new" formula gives:
|
||||
--
|
||||
-- new = T / 3_800_000 + 10 * old / 19
|
||||
--
|
||||
-- Run this in the Supabase SQL Editor (Dashboard > SQL Editor).
|
||||
-- -----------------------------------------------------------------------
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- Preview the changes first (uncomment the SELECT, comment the UPDATE)
|
||||
-- SELECT
|
||||
-- display_name,
|
||||
-- dollar_savings AS old_savings,
|
||||
-- total_tokens / 3800000.0 + 10.0 * dollar_savings / 19.0 AS new_savings
|
||||
-- FROM savings_entries
|
||||
-- WHERE dollar_savings > 0
|
||||
-- ORDER BY dollar_savings DESC;
|
||||
|
||||
UPDATE savings_entries
|
||||
SET dollar_savings = total_tokens / 3800000.0 + 10.0 * dollar_savings / 19.0
|
||||
WHERE dollar_savings > 0;
|
||||
|
||||
COMMIT;
|
||||
@@ -47,6 +47,34 @@ class AgentExecutor:
|
||||
"""Deferred system injection — called after JarvisSystem is constructed."""
|
||||
self._system = system
|
||||
|
||||
def _set_activity(self, agent_id: str, activity: str) -> None:
|
||||
"""Update the agent's current_activity for progress visibility."""
|
||||
try:
|
||||
self._manager.update_agent(agent_id, current_activity=activity)
|
||||
except Exception:
|
||||
pass # Non-critical
|
||||
|
||||
def _inject_tool_deps(self, tool: Any) -> None:
|
||||
"""Inject runtime dependencies into a tool instance.
|
||||
|
||||
Mirrors SystemBuilder._inject_tool_deps (system.py:920-945)
|
||||
but uses the lightweight system's references.
|
||||
"""
|
||||
if self._system is None:
|
||||
return
|
||||
name = getattr(getattr(tool, "spec", None), "name", "")
|
||||
if name == "llm":
|
||||
if hasattr(tool, "_engine"):
|
||||
tool._engine = self._system.engine
|
||||
if hasattr(tool, "_model"):
|
||||
tool._model = self._system.model
|
||||
elif name == "retrieval" or name.startswith("memory_"):
|
||||
if hasattr(tool, "_backend"):
|
||||
tool._backend = getattr(self._system, "memory_backend", None)
|
||||
elif name.startswith("channel_"):
|
||||
if hasattr(tool, "_channel"):
|
||||
tool._channel = getattr(self._system, "channel_backend", None)
|
||||
|
||||
def run_ephemeral(
|
||||
self,
|
||||
agent_type: str,
|
||||
@@ -75,6 +103,7 @@ class AgentExecutor:
|
||||
"""
|
||||
try:
|
||||
self._manager.start_tick(agent_id)
|
||||
self._set_activity(agent_id, "Preparing tick...")
|
||||
except ValueError:
|
||||
logger.warning("Agent %s already running, skipping tick", agent_id)
|
||||
return
|
||||
@@ -203,6 +232,13 @@ class AgentExecutor:
|
||||
if not model:
|
||||
raise FatalError("No model configured for agent")
|
||||
|
||||
logger.info(
|
||||
"Agent %s [%s]: using model=%s, engine=%s",
|
||||
agent["name"], agent["id"],
|
||||
model, type(engine).__name__,
|
||||
)
|
||||
self._set_activity(agent["id"], f"Loading model {model}...")
|
||||
|
||||
# Optionally override model via router policy
|
||||
router_policy_key = config.get("router_policy")
|
||||
if router_policy_key and self._system:
|
||||
@@ -224,12 +260,44 @@ class AgentExecutor:
|
||||
except Exception:
|
||||
pass # Fall back to configured model
|
||||
|
||||
# Construct agent instance (BaseAgent requires engine, model as positional args)
|
||||
# Resolve tools from config via ToolRegistry
|
||||
tool_names = config.get("tools", [])
|
||||
if isinstance(tool_names, str):
|
||||
tool_names = [t.strip() for t in tool_names.split(",") if t.strip()]
|
||||
|
||||
tool_instances: list[Any] = []
|
||||
if tool_names:
|
||||
try:
|
||||
from openjarvis.server.agent_manager_routes import (
|
||||
_ensure_registries_populated,
|
||||
)
|
||||
|
||||
_ensure_registries_populated()
|
||||
except ImportError:
|
||||
pass
|
||||
from openjarvis.core.registry import ToolRegistry
|
||||
|
||||
for tname in tool_names:
|
||||
if ToolRegistry.contains(tname):
|
||||
try:
|
||||
tool_cls = ToolRegistry.get(tname)
|
||||
tool = tool_cls()
|
||||
self._inject_tool_deps(tool)
|
||||
tool_instances.append(tool)
|
||||
except Exception:
|
||||
logger.warning("Failed to instantiate tool %s", tname)
|
||||
if tool_instances:
|
||||
logger.info(
|
||||
"Agent %s: resolved %d/%d tools",
|
||||
agent["name"], len(tool_instances), len(tool_names),
|
||||
)
|
||||
|
||||
# Construct agent instance
|
||||
agent_instance = agent_cls(
|
||||
engine,
|
||||
model,
|
||||
system_prompt=config.get("system_prompt"),
|
||||
tools=[],
|
||||
tools=tool_instances,
|
||||
)
|
||||
|
||||
# Build input from instruction + summary_memory + pending messages
|
||||
@@ -247,6 +315,20 @@ class AgentExecutor:
|
||||
input_text = f"{input_text}\n\nNew instructions:\n{user_msgs}"
|
||||
for m in pending:
|
||||
self._manager.mark_message_delivered(m["id"])
|
||||
logger.info(
|
||||
"Agent %s: delivering %d pending message(s)",
|
||||
agent["name"], len(pending),
|
||||
)
|
||||
self._set_activity(
|
||||
agent["id"],
|
||||
f"Delivering {len(pending)} message(s)...",
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Agent %s: no pending messages, running with "
|
||||
"instruction only",
|
||||
agent["name"],
|
||||
)
|
||||
|
||||
# Build AgentContext with memory results from FTS5 backend
|
||||
from openjarvis.agents._stubs import AgentContext
|
||||
@@ -298,7 +380,36 @@ class AgentExecutor:
|
||||
pass # Don't break agent tick if memory retrieval fails
|
||||
|
||||
agent_ctx.memory_results = memory_results
|
||||
return agent_instance.run(input_text, context=agent_ctx)
|
||||
self._set_activity(agent["id"], "Generating response...")
|
||||
logger.info(
|
||||
"Agent %s: calling agent.run() with %d chars input",
|
||||
agent["name"], len(input_text),
|
||||
)
|
||||
_t0 = time.time()
|
||||
result = agent_instance.run(input_text, context=agent_ctx)
|
||||
|
||||
# Retry once if the model returned empty content (common with
|
||||
# Qwen3.5 thinking mode consuming all tokens).
|
||||
if not (result.content or "").strip():
|
||||
self._set_activity(
|
||||
agent["id"], "Retrying (empty response)...",
|
||||
)
|
||||
logger.warning(
|
||||
"Agent %s: empty content, retrying once",
|
||||
agent["name"],
|
||||
)
|
||||
result = agent_instance.run(input_text, context=agent_ctx)
|
||||
|
||||
_elapsed = time.time() - _t0
|
||||
logger.info(
|
||||
"Agent %s: agent.run() completed in %.1fs, "
|
||||
"content_len=%d, turns=%d, tokens=%s",
|
||||
agent["name"], _elapsed,
|
||||
len(result.content or ""),
|
||||
result.turns,
|
||||
result.metadata.get("total_tokens", "?"),
|
||||
)
|
||||
return result
|
||||
|
||||
def _build_error_detail(self, error: AgentTickError) -> dict[str, Any]:
|
||||
"""Build structured error detail for trace metadata."""
|
||||
@@ -334,18 +445,37 @@ class AgentExecutor:
|
||||
duration: float,
|
||||
) -> None:
|
||||
"""Update agent state after tick completion or failure."""
|
||||
self._set_activity(agent_id, "Finalizing...")
|
||||
if error is None:
|
||||
# Success
|
||||
logger.info(
|
||||
"Tick succeeded for agent %s in %.1fs, "
|
||||
"response_len=%d",
|
||||
agent_id, duration,
|
||||
len(result.content or "") if result else 0,
|
||||
)
|
||||
self._manager.end_tick(agent_id)
|
||||
self._manager.update_agent(agent_id, total_runs_increment=1)
|
||||
|
||||
# Accumulate budget metrics from AgentResult metadata
|
||||
if result:
|
||||
tokens = result.metadata.get("tokens_used", 0)
|
||||
tokens = (
|
||||
result.metadata.get("total_tokens")
|
||||
or result.metadata.get("tokens_used")
|
||||
or 0
|
||||
)
|
||||
in_tokens = result.metadata.get("prompt_tokens", 0)
|
||||
out_tokens = result.metadata.get(
|
||||
"completion_tokens", 0,
|
||||
)
|
||||
cost = result.metadata.get("cost", 0.0)
|
||||
budget_kwargs: dict[str, Any] = {"stall_retries": 0}
|
||||
if tokens > 0:
|
||||
budget_kwargs["total_tokens_increment"] = tokens
|
||||
if in_tokens > 0:
|
||||
budget_kwargs["input_tokens_increment"] = in_tokens
|
||||
if out_tokens > 0:
|
||||
budget_kwargs["output_tokens_increment"] = out_tokens
|
||||
if cost > 0:
|
||||
budget_kwargs["total_cost_increment"] = cost
|
||||
self._manager.update_agent(agent_id, **budget_kwargs)
|
||||
@@ -381,6 +511,10 @@ class AgentExecutor:
|
||||
"status": "ok",
|
||||
})
|
||||
elif isinstance(error, EscalateError):
|
||||
logger.warning(
|
||||
"Tick escalated for agent %s after %.1fs: %s",
|
||||
agent_id, duration, error,
|
||||
)
|
||||
self._manager.end_tick(agent_id)
|
||||
self._manager.update_agent(agent_id, status="needs_attention")
|
||||
self._bus.publish(EventType.AGENT_TICK_ERROR, {
|
||||
@@ -390,6 +524,10 @@ class AgentExecutor:
|
||||
"duration": duration,
|
||||
})
|
||||
else:
|
||||
logger.error(
|
||||
"Tick failed for agent %s after %.1fs: %s",
|
||||
agent_id, duration, error, exc_info=error,
|
||||
)
|
||||
self._manager.end_tick(agent_id)
|
||||
self._manager.update_agent(agent_id, status="error")
|
||||
# Write error detail to summary_memory so frontend can display it
|
||||
|
||||
@@ -111,6 +111,9 @@ class AgentManager:
|
||||
"ALTER TABLE managed_agents ADD COLUMN last_run_at REAL",
|
||||
"ALTER TABLE managed_agents ADD COLUMN last_activity_at REAL",
|
||||
"ALTER TABLE managed_agents ADD COLUMN stall_retries INTEGER DEFAULT 0",
|
||||
"ALTER TABLE managed_agents ADD COLUMN current_activity TEXT DEFAULT ''",
|
||||
"ALTER TABLE managed_agents ADD COLUMN input_tokens INTEGER DEFAULT 0",
|
||||
"ALTER TABLE managed_agents ADD COLUMN output_tokens INTEGER DEFAULT 0",
|
||||
]
|
||||
for migration in _MIGRATIONS:
|
||||
try:
|
||||
@@ -160,7 +163,7 @@ class AgentManager:
|
||||
def update_agent(self, agent_id: str, **kwargs: Any) -> Dict[str, Any]:
|
||||
sets: List[str] = []
|
||||
vals: List[Any] = []
|
||||
for key in ("name", "agent_type", "status"):
|
||||
for key in ("name", "agent_type", "status", "current_activity"):
|
||||
if key in kwargs:
|
||||
sets.append(f"{key} = ?")
|
||||
vals.append(kwargs[key])
|
||||
@@ -181,6 +184,14 @@ class AgentManager:
|
||||
if total_tokens_increment:
|
||||
sets.append("total_tokens = total_tokens + ?")
|
||||
vals.append(total_tokens_increment)
|
||||
input_tokens_increment = kwargs.get("input_tokens_increment", 0)
|
||||
if input_tokens_increment:
|
||||
sets.append("input_tokens = input_tokens + ?")
|
||||
vals.append(input_tokens_increment)
|
||||
output_tokens_increment = kwargs.get("output_tokens_increment", 0)
|
||||
if output_tokens_increment:
|
||||
sets.append("output_tokens = output_tokens + ?")
|
||||
vals.append(output_tokens_increment)
|
||||
if "last_activity_at" in kwargs:
|
||||
sets.append("last_activity_at = ?")
|
||||
vals.append(kwargs["last_activity_at"])
|
||||
@@ -222,7 +233,12 @@ class AgentManager:
|
||||
self._set_status(agent_id, "running")
|
||||
|
||||
def end_tick(self, agent_id: str) -> None:
|
||||
self._set_status(agent_id, "idle")
|
||||
self._conn.execute(
|
||||
"UPDATE managed_agents SET status = 'idle', "
|
||||
"current_activity = '', updated_at = ? WHERE id = ?",
|
||||
(time.time(), agent_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
# ── Checkpoints ───────────────────────────────────────────────
|
||||
|
||||
@@ -472,6 +488,15 @@ class AgentManager:
|
||||
if overrides:
|
||||
config.update(overrides)
|
||||
agent_type = config.pop("agent_type", "monitor_operative")
|
||||
|
||||
# Expand system_prompt_template with instruction
|
||||
prompt_tpl = config.pop("system_prompt_template", "")
|
||||
if prompt_tpl:
|
||||
instruction = config.get("instruction", "")
|
||||
config["system_prompt"] = prompt_tpl.format(
|
||||
instruction=instruction or "(No specific instruction provided)",
|
||||
)
|
||||
|
||||
return self.create_agent(name=name, agent_type=agent_type, config=config)
|
||||
|
||||
# ── Message queue ─────────────────────────────────────────────
|
||||
@@ -638,6 +663,9 @@ class AgentManager:
|
||||
"last_run_at": row["last_run_at"],
|
||||
"last_activity_at": row["last_activity_at"],
|
||||
"stall_retries": row["stall_retries"] or 0,
|
||||
"current_activity": row["current_activity"] or "",
|
||||
"input_tokens": row["input_tokens"] or 0,
|
||||
"output_tokens": row["output_tokens"] or 0,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -283,14 +283,25 @@ class NativeOpenHandsAgent(ToolUsingAgent):
|
||||
last_content = ""
|
||||
total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
||||
|
||||
# Build OpenAI-format tool schemas for native function calling
|
||||
openai_tools = (
|
||||
self._executor.get_openai_tools() if self._tools else []
|
||||
)
|
||||
# Side dict for Gemini thought_signatures (ToolCall uses slots)
|
||||
_thought_sigs: dict[str, bytes] = {}
|
||||
|
||||
for _turn in range(self._max_turns):
|
||||
turns += 1
|
||||
# Truncate before every generate call -- tool results may have
|
||||
# expanded the context beyond what the model supports.
|
||||
messages = self._truncate_if_needed(messages)
|
||||
|
||||
gen_kwargs: dict[str, Any] = {}
|
||||
if openai_tools:
|
||||
gen_kwargs["tools"] = openai_tools
|
||||
|
||||
try:
|
||||
result = self._generate(messages)
|
||||
result = self._generate(messages, **gen_kwargs)
|
||||
except Exception as exc:
|
||||
error_str = str(exc)
|
||||
if "400" in error_str:
|
||||
@@ -318,6 +329,46 @@ class NativeOpenHandsAgent(ToolUsingAgent):
|
||||
content = self._strip_think_tags(content)
|
||||
last_content = content
|
||||
|
||||
# --- Native function-calling path (OpenAI, Anthropic, etc.) ---
|
||||
raw_tool_calls = result.get("tool_calls", [])
|
||||
if raw_tool_calls:
|
||||
native_calls = []
|
||||
for i, tc in enumerate(raw_tool_calls):
|
||||
call = ToolCall(
|
||||
id=tc.get("id", f"call_{turns}_{i}"),
|
||||
name=tc.get("name", ""),
|
||||
arguments=tc.get("arguments", "{}"),
|
||||
)
|
||||
# Preserve thought_signature for Gemini reasoning
|
||||
sig = tc.get("thought_signature")
|
||||
if sig is not None:
|
||||
_thought_sigs[call.id] = sig
|
||||
native_calls.append(call)
|
||||
messages.append(
|
||||
Message(
|
||||
role=Role.ASSISTANT,
|
||||
content=content,
|
||||
tool_calls=native_calls,
|
||||
)
|
||||
)
|
||||
for tc in native_calls:
|
||||
tool_result = self._executor.execute(tc)
|
||||
all_tool_results.append(tool_result)
|
||||
obs_text = tool_result.content
|
||||
if len(obs_text) > 4000:
|
||||
obs_text = obs_text[:4000] + "\n\n[Output truncated]"
|
||||
messages.append(
|
||||
Message(
|
||||
role=Role.TOOL,
|
||||
content=obs_text,
|
||||
tool_call_id=tc.id,
|
||||
name=tc.name,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# --- Text-based fallback (CodeAct / Action-Input format) ---
|
||||
|
||||
# Try to extract code
|
||||
code = self._extract_code(content)
|
||||
if code:
|
||||
|
||||
@@ -1,15 +1,48 @@
|
||||
[template]
|
||||
id = "code_reviewer"
|
||||
name = "Code Reviewer"
|
||||
description = "Monitors a repository for changes, reviews code, and stores feedback."
|
||||
description = "Monitors a repository for changes, reviews code quality, and identifies bugs."
|
||||
icon = "🔍"
|
||||
agent_type = "monitor_operative"
|
||||
schedule_type = "interval"
|
||||
schedule_value = "3600"
|
||||
tools = ["file_read", "shell_exec", "memory_store", "think"]
|
||||
tools = ["file_read", "file_write", "shell_exec", "git_status", "git_diff", "git_commit", "git_log", "apply_patch", "code_interpreter", "memory_store", "memory_retrieve", "think"]
|
||||
max_turns = 15
|
||||
temperature = 0.2
|
||||
temperature = 0.1
|
||||
memory_extraction = "scratchpad"
|
||||
observation_compression = "summarize"
|
||||
retrieval_strategy = "keyword"
|
||||
retrieval_strategy = "sqlite"
|
||||
task_decomposition = "monolithic"
|
||||
system_prompt = "You are a code review agent. Monitor the repository for recent changes, review code quality, identify potential bugs, and store your findings."
|
||||
system_prompt_template = """You are a Code Reviewer agent. Your job is to monitor a repository for recent changes, review code quality, identify potential bugs, suggest improvements, and store your findings.
|
||||
|
||||
## Your Review Focus
|
||||
{instruction}
|
||||
|
||||
## Available Tools
|
||||
You have access to these tools. Use them systematically:
|
||||
|
||||
- **git_status()** — Check the current state of the repository. Start here.
|
||||
- **git_diff()** — View uncommitted changes or diffs between commits.
|
||||
- **git_log()** — View recent commit history to understand what changed.
|
||||
- **file_read(path)** — Read source files for detailed review.
|
||||
- **file_write(path, content)** — Write review notes or suggested patches.
|
||||
- **shell_exec(command)** — Run linters, tests, or other analysis commands.
|
||||
- **code_interpreter(code)** — Execute code snippets to verify behavior.
|
||||
- **apply_patch(patch)** — Apply a suggested fix as a patch.
|
||||
- **memory_store(key, content)** / **memory_retrieve(query)** — Track review history across sessions.
|
||||
- **think(thought)** — Reason through complex code logic before commenting.
|
||||
|
||||
## How to Work
|
||||
1. Run git_status and git_log to understand recent changes.
|
||||
2. For each significant change, read the affected files with file_read.
|
||||
3. Analyze for: bugs, security issues, performance problems, readability.
|
||||
4. Run relevant tests or linters via shell_exec if available.
|
||||
5. Store findings in memory for tracking across sessions.
|
||||
6. Produce a summary: what changed, what's good, what needs attention.
|
||||
|
||||
## Quality Standards
|
||||
- Focus on substance: real bugs and security issues over style nitpicks.
|
||||
- Be specific: reference exact file paths and line numbers.
|
||||
- Be constructive: suggest fixes, not just problems.
|
||||
- Prioritize severity: critical bugs > performance > readability.
|
||||
- Don't comment on formatting if a linter handles it."""
|
||||
|
||||
@@ -1,15 +1,44 @@
|
||||
[template]
|
||||
id = "inbox_triager"
|
||||
name = "Inbox Triager"
|
||||
description = "Monitors email and messaging channels, categorizes and summarizes incoming messages."
|
||||
description = "Monitors email and messaging channels, categorizes and summarizes by priority."
|
||||
icon = "📥"
|
||||
agent_type = "monitor_operative"
|
||||
schedule_type = "interval"
|
||||
schedule_value = "1800"
|
||||
tools = ["web_search", "memory_store", "memory_retrieve", "think"]
|
||||
tools = ["channel_send", "channel_list", "memory_store", "memory_retrieve", "think", "web_search", "file_write"]
|
||||
max_turns = 20
|
||||
temperature = 0.3
|
||||
memory_extraction = "structured_json"
|
||||
observation_compression = "summarize"
|
||||
retrieval_strategy = "semantic"
|
||||
retrieval_strategy = "sqlite"
|
||||
task_decomposition = "phased"
|
||||
system_prompt = "You are an inbox triage agent. Monitor incoming messages, categorize them by priority and topic, summarize key information, and flag items that need attention."
|
||||
system_prompt_template = """You are an Inbox Triager agent. Your job is to monitor incoming messages across email and messaging channels, categorize them by priority and topic, summarize key information, and flag items that need immediate attention.
|
||||
|
||||
## Your Triage Instructions
|
||||
{instruction}
|
||||
|
||||
## Available Tools
|
||||
You have access to these tools. Use them to process incoming messages:
|
||||
|
||||
- **channel_list()** — List available messaging channels and their recent messages.
|
||||
- **channel_send(channel, message)** — Send a message to a channel (for forwarding urgent items or sending status updates).
|
||||
- **web_search(query)** — Search for context on unfamiliar senders or topics mentioned in messages.
|
||||
- **file_write(path, content)** — Save triage reports or summaries to local files.
|
||||
- **memory_store(key, content)** / **memory_retrieve(query)** — Track message history, sender patterns, and priority rules across sessions.
|
||||
- **think(thought)** — Reason through priority decisions before categorizing.
|
||||
|
||||
## How to Work
|
||||
1. Check memory for your existing triage rules and sender patterns.
|
||||
2. List channels to see new incoming messages.
|
||||
3. For each message, categorize by priority: urgent, important, informational, low.
|
||||
4. For urgent items, forward via channel_send with a brief summary.
|
||||
5. Store triage decisions in memory for pattern learning.
|
||||
6. Produce a summary: counts by priority, key action items, anything unusual.
|
||||
|
||||
## Quality Standards
|
||||
- Never miss urgent items: err on the side of flagging too much.
|
||||
- Be concise: triage summaries should be scannable in 30 seconds.
|
||||
- Learn patterns: remember which senders/topics are usually important.
|
||||
- Respect context: a message from your boss is higher priority than a newsletter.
|
||||
- Group related messages: thread continuations should be triaged together."""
|
||||
|
||||
@@ -1,15 +1,45 @@
|
||||
[template]
|
||||
id = "research_monitor"
|
||||
name = "Research Monitor"
|
||||
description = "Periodically searches for papers/news on a topic, stores findings in memory, reports via channel."
|
||||
description = "Searches papers, news, blogs on your topic. Stores findings in memory."
|
||||
icon = "🔬"
|
||||
agent_type = "monitor_operative"
|
||||
schedule_type = "cron"
|
||||
schedule_value = "0 9 * * *"
|
||||
tools = ["web_search", "memory_store", "memory_retrieve", "think"]
|
||||
tools = ["web_search", "http_request", "file_read", "file_write", "memory_store", "memory_retrieve", "think"]
|
||||
max_turns = 25
|
||||
temperature = 0.3
|
||||
memory_extraction = "structured_json"
|
||||
observation_compression = "summarize"
|
||||
retrieval_strategy = "hybrid_with_self_eval"
|
||||
retrieval_strategy = "sqlite"
|
||||
task_decomposition = "phased"
|
||||
system_prompt = "You are a research monitor agent. Your job is to search for new papers, articles, and developments on your assigned topics. Store important findings in memory. Be thorough but concise in your summaries."
|
||||
system_prompt_template = """You are a Research Monitor agent. Your job is to systematically search for new papers, articles, and developments on your assigned topics, store important findings in memory, and produce concise summaries.
|
||||
|
||||
## Your Assigned Topic
|
||||
{instruction}
|
||||
|
||||
## Available Tools
|
||||
You have access to these tools. Use them proactively:
|
||||
|
||||
- **web_search(query)** — Search the web for recent articles, papers, and news. Use specific, targeted queries. Try multiple search angles to get comprehensive coverage.
|
||||
- **http_request(url, method)** — Fetch a specific URL to read full article content. Use after finding promising URLs via web_search.
|
||||
- **file_read(path)** / **file_write(path, content)** — Read and write local files. Use to save detailed reports or read reference material.
|
||||
- **memory_store(key, content)** — Store findings for future reference across sessions. Use structured keys like "finding:YYYY-MM-DD:topic-name".
|
||||
- **memory_retrieve(query)** — Recall previously stored findings. Always check what you already know before searching again.
|
||||
- **think(thought)** — Reason through complex decisions before acting. Use when planning search strategy or evaluating source quality.
|
||||
|
||||
## How to Work
|
||||
1. Start by checking memory (memory_retrieve) for what you already know about this topic.
|
||||
2. Search the web with 2-3 different query angles using web_search.
|
||||
3. For promising results, fetch the full content via http_request.
|
||||
4. Extract key findings: title, authors/source, date, main contribution, relevance to your assigned topic.
|
||||
5. Store each significant finding in memory with a structured key.
|
||||
6. Produce a concise summary of new developments found.
|
||||
|
||||
## Quality Standards
|
||||
- Be thorough: try multiple search queries, not just one.
|
||||
- Be concise: summaries should be 2-4 paragraphs, not essays.
|
||||
- Be structured: use headers, bullet points, and dates.
|
||||
- Be honest: if you cannot find recent information, say so clearly. Never fabricate or hallucinate sources.
|
||||
- Prioritize recency: newer findings are more valuable than old ones.
|
||||
- Cite sources: include URLs or paper titles for every claim."""
|
||||
|
||||
@@ -39,6 +39,8 @@ _CHANNEL_MODULES = [
|
||||
"twitch_channel",
|
||||
"nostr_channel",
|
||||
"twilio_sms",
|
||||
"twitter",
|
||||
"gmail",
|
||||
]
|
||||
|
||||
for _mod in _CHANNEL_MODULES:
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
"""GmailChannel — native Gmail API adapter using OAuth2."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from email.mime.text import MIMEText
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from openjarvis.channels._stubs import (
|
||||
BaseChannel,
|
||||
ChannelHandler,
|
||||
ChannelMessage,
|
||||
ChannelStatus,
|
||||
)
|
||||
from openjarvis.core.events import EventBus, EventType
|
||||
from openjarvis.core.registry import ChannelRegistry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@ChannelRegistry.register("gmail")
|
||||
class GmailChannel(BaseChannel):
|
||||
"""Native Gmail channel adapter using the Gmail API with OAuth2.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
credentials_path:
|
||||
Path to the OAuth2 ``credentials.json`` file.
|
||||
Falls back to ``GMAIL_CREDENTIALS_PATH`` env var.
|
||||
token_path:
|
||||
Path to the stored OAuth2 token file.
|
||||
Falls back to ``GMAIL_TOKEN_PATH`` env var.
|
||||
user_id:
|
||||
Gmail user ID (default ``"me"`` for the authenticated user).
|
||||
poll_interval:
|
||||
Seconds between inbox polls (default 30).
|
||||
bus:
|
||||
Optional event bus for publishing channel events.
|
||||
"""
|
||||
|
||||
channel_id = "gmail"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
credentials_path: str = "",
|
||||
*,
|
||||
token_path: str = "",
|
||||
user_id: str = "me",
|
||||
poll_interval: int = 30,
|
||||
bus: Optional[EventBus] = None,
|
||||
) -> None:
|
||||
self._credentials_path = credentials_path or os.environ.get(
|
||||
"GMAIL_CREDENTIALS_PATH", "",
|
||||
)
|
||||
self._token_path = token_path or os.environ.get(
|
||||
"GMAIL_TOKEN_PATH", "",
|
||||
)
|
||||
self._user_id = user_id
|
||||
self._poll_interval = poll_interval
|
||||
self._bus = bus
|
||||
self._handlers: List[ChannelHandler] = []
|
||||
self._status = ChannelStatus.DISCONNECTED
|
||||
self._service: Any = None
|
||||
self._listener_thread: Optional[threading.Thread] = None
|
||||
self._stop_event = threading.Event()
|
||||
|
||||
# -- connection lifecycle ---------------------------------------------------
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Load OAuth2 credentials and build the Gmail API service."""
|
||||
if not self._credentials_path and not self._token_path:
|
||||
logger.warning("No Gmail credentials configured")
|
||||
self._status = ChannelStatus.ERROR
|
||||
return
|
||||
|
||||
self._stop_event.clear()
|
||||
self._status = ChannelStatus.CONNECTING
|
||||
|
||||
try:
|
||||
from google.oauth2.credentials import Credentials
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
SCOPES = ["https://www.googleapis.com/auth/gmail.modify"]
|
||||
|
||||
creds: Any = None
|
||||
if self._token_path and os.path.exists(self._token_path):
|
||||
creds = Credentials.from_authorized_user_file(
|
||||
self._token_path, SCOPES,
|
||||
)
|
||||
|
||||
if not creds or not creds.valid:
|
||||
if creds and creds.expired and creds.refresh_token:
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
creds.refresh(Request())
|
||||
elif self._credentials_path and os.path.exists(
|
||||
self._credentials_path,
|
||||
):
|
||||
flow = InstalledAppFlow.from_client_secrets_file(
|
||||
self._credentials_path, SCOPES,
|
||||
)
|
||||
creds = flow.run_local_server(port=0)
|
||||
else:
|
||||
logger.warning("Gmail credentials not found or invalid")
|
||||
self._status = ChannelStatus.ERROR
|
||||
return
|
||||
|
||||
# Save token for future use
|
||||
if self._token_path and creds:
|
||||
with open(self._token_path, "w") as token_file:
|
||||
token_file.write(creds.to_json())
|
||||
|
||||
self._service = build("gmail", "v1", credentials=creds)
|
||||
self._status = ChannelStatus.CONNECTED
|
||||
logger.info("Gmail channel connected")
|
||||
|
||||
# Start polling thread
|
||||
self._listener_thread = threading.Thread(
|
||||
target=self._poll_loop, daemon=True,
|
||||
)
|
||||
self._listener_thread.start()
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Google API libraries not installed; "
|
||||
"install with: pip install openjarvis[channel-gmail]",
|
||||
)
|
||||
self._status = ChannelStatus.ERROR
|
||||
except Exception:
|
||||
logger.debug("Gmail connect failed", exc_info=True)
|
||||
self._status = ChannelStatus.ERROR
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Stop the polling thread and clear the service."""
|
||||
self._stop_event.set()
|
||||
if self._listener_thread is not None:
|
||||
self._listener_thread.join(timeout=5.0)
|
||||
self._listener_thread = None
|
||||
self._service = None
|
||||
self._status = ChannelStatus.DISCONNECTED
|
||||
|
||||
# -- send / receive --------------------------------------------------------
|
||||
|
||||
def send(
|
||||
self,
|
||||
channel: str,
|
||||
content: str,
|
||||
*,
|
||||
conversation_id: str = "",
|
||||
metadata: Dict[str, Any] | None = None,
|
||||
) -> bool:
|
||||
"""Send an email via the Gmail API.
|
||||
|
||||
``channel`` is the recipient email address.
|
||||
"""
|
||||
if self._service is None:
|
||||
logger.warning("Cannot send: Gmail service not connected")
|
||||
return False
|
||||
|
||||
try:
|
||||
msg = MIMEText(content)
|
||||
msg["To"] = channel
|
||||
msg["From"] = self._user_id
|
||||
msg["Subject"] = (metadata or {}).get(
|
||||
"subject", "Message from OpenJarvis",
|
||||
)
|
||||
|
||||
raw = base64.urlsafe_b64encode(msg.as_bytes()).decode("utf-8")
|
||||
body: Dict[str, Any] = {"raw": raw}
|
||||
if conversation_id:
|
||||
body["threadId"] = conversation_id
|
||||
|
||||
self._service.users().messages().send(
|
||||
userId=self._user_id, body=body,
|
||||
).execute()
|
||||
|
||||
self._publish_sent(channel, content, conversation_id)
|
||||
return True
|
||||
except Exception:
|
||||
logger.debug("Gmail send failed", exc_info=True)
|
||||
return False
|
||||
|
||||
def status(self) -> ChannelStatus:
|
||||
"""Return the current connection status."""
|
||||
if self._status == ChannelStatus.CONNECTED and self._service is None:
|
||||
return ChannelStatus.ERROR
|
||||
return self._status
|
||||
|
||||
def list_channels(self) -> List[str]:
|
||||
"""Return available channel identifiers."""
|
||||
return ["inbox"]
|
||||
|
||||
def on_message(self, handler: ChannelHandler) -> None:
|
||||
"""Register a callback for incoming Gmail messages."""
|
||||
self._handlers.append(handler)
|
||||
|
||||
# -- internal helpers -------------------------------------------------------
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
"""Poll Gmail inbox for unread messages."""
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
if self._service is None:
|
||||
break
|
||||
|
||||
results = (
|
||||
self._service.users()
|
||||
.messages()
|
||||
.list(userId=self._user_id, q="is:unread")
|
||||
.execute()
|
||||
)
|
||||
messages = results.get("messages", [])
|
||||
|
||||
for msg_ref in messages:
|
||||
msg_id = msg_ref["id"]
|
||||
msg = (
|
||||
self._service.users()
|
||||
.messages()
|
||||
.get(userId=self._user_id, id=msg_id, format="full")
|
||||
.execute()
|
||||
)
|
||||
|
||||
headers = {
|
||||
h["name"]: h["value"]
|
||||
for h in msg.get("payload", {}).get("headers", [])
|
||||
}
|
||||
|
||||
# Extract body text
|
||||
body = ""
|
||||
payload = msg.get("payload", {})
|
||||
if payload.get("body", {}).get("data"):
|
||||
body = base64.urlsafe_b64decode(
|
||||
payload["body"]["data"],
|
||||
).decode("utf-8", errors="replace")
|
||||
elif payload.get("parts"):
|
||||
for part in payload["parts"]:
|
||||
if part.get("mimeType") == "text/plain":
|
||||
data = part.get("body", {}).get("data", "")
|
||||
if data:
|
||||
body = base64.urlsafe_b64decode(
|
||||
data,
|
||||
).decode("utf-8", errors="replace")
|
||||
break
|
||||
|
||||
cm = ChannelMessage(
|
||||
channel="gmail",
|
||||
sender=headers.get("From", ""),
|
||||
content=body,
|
||||
message_id=msg_id,
|
||||
conversation_id=msg.get("threadId", ""),
|
||||
)
|
||||
|
||||
for handler in self._handlers:
|
||||
try:
|
||||
handler(cm)
|
||||
except Exception:
|
||||
logger.exception("Gmail handler error")
|
||||
|
||||
if self._bus is not None:
|
||||
self._bus.publish(
|
||||
EventType.CHANNEL_MESSAGE_RECEIVED,
|
||||
{
|
||||
"channel": cm.channel,
|
||||
"sender": cm.sender,
|
||||
"content": cm.content,
|
||||
"message_id": cm.message_id,
|
||||
},
|
||||
)
|
||||
|
||||
# Mark message as read
|
||||
self._service.users().messages().modify(
|
||||
userId=self._user_id,
|
||||
id=msg_id,
|
||||
body={"removeLabelIds": ["UNREAD"]},
|
||||
).execute()
|
||||
|
||||
except Exception:
|
||||
logger.debug("Gmail poll error", exc_info=True)
|
||||
|
||||
self._stop_event.wait(self._poll_interval)
|
||||
|
||||
def _publish_sent(self, channel: str, content: str, conversation_id: str) -> None:
|
||||
"""Publish a CHANNEL_MESSAGE_SENT event on the bus."""
|
||||
if self._bus is not None:
|
||||
self._bus.publish(
|
||||
EventType.CHANNEL_MESSAGE_SENT,
|
||||
{
|
||||
"channel": channel,
|
||||
"content": content,
|
||||
"conversation_id": conversation_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["GmailChannel"]
|
||||
@@ -0,0 +1,242 @@
|
||||
"""TwitterChannel — native Twitter/X API adapter using tweepy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from openjarvis.channels._stubs import (
|
||||
BaseChannel,
|
||||
ChannelHandler,
|
||||
ChannelMessage,
|
||||
ChannelStatus,
|
||||
)
|
||||
from openjarvis.core.events import EventBus, EventType
|
||||
from openjarvis.core.registry import ChannelRegistry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@ChannelRegistry.register("twitter")
|
||||
class TwitterChannel(BaseChannel):
|
||||
"""Native Twitter/X channel adapter using tweepy.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
bearer_token:
|
||||
Twitter API v2 Bearer Token. Falls back to ``TWITTER_BEARER_TOKEN``.
|
||||
api_key:
|
||||
Twitter API Key (consumer key). Falls back to ``TWITTER_API_KEY``.
|
||||
api_secret:
|
||||
Twitter API Secret (consumer secret). Falls back to ``TWITTER_API_SECRET``.
|
||||
access_token:
|
||||
Twitter Access Token. Falls back to ``TWITTER_ACCESS_TOKEN``.
|
||||
access_secret:
|
||||
Twitter Access Token Secret. Falls back to ``TWITTER_ACCESS_SECRET``.
|
||||
poll_interval:
|
||||
Seconds between mention polls (default 60).
|
||||
bus:
|
||||
Optional event bus for publishing channel events.
|
||||
"""
|
||||
|
||||
channel_id = "twitter"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bearer_token: str = "",
|
||||
*,
|
||||
api_key: str = "",
|
||||
api_secret: str = "",
|
||||
access_token: str = "",
|
||||
access_secret: str = "",
|
||||
poll_interval: int = 60,
|
||||
bus: Optional[EventBus] = None,
|
||||
) -> None:
|
||||
self._bearer_token = bearer_token or os.environ.get(
|
||||
"TWITTER_BEARER_TOKEN", "",
|
||||
)
|
||||
self._api_key = api_key or os.environ.get("TWITTER_API_KEY", "")
|
||||
self._api_secret = api_secret or os.environ.get("TWITTER_API_SECRET", "")
|
||||
self._access_token = access_token or os.environ.get(
|
||||
"TWITTER_ACCESS_TOKEN", "",
|
||||
)
|
||||
self._access_secret = access_secret or os.environ.get(
|
||||
"TWITTER_ACCESS_SECRET", "",
|
||||
)
|
||||
self._poll_interval = poll_interval
|
||||
self._bus = bus
|
||||
self._handlers: List[ChannelHandler] = []
|
||||
self._status = ChannelStatus.DISCONNECTED
|
||||
self._client: Any = None
|
||||
self._poll_thread: Optional[threading.Thread] = None
|
||||
self._stop_event = threading.Event()
|
||||
|
||||
# -- connection lifecycle ---------------------------------------------------
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Build a tweepy Client and optionally start polling for mentions."""
|
||||
if not self._bearer_token:
|
||||
logger.warning("No Twitter bearer token configured")
|
||||
self._status = ChannelStatus.ERROR
|
||||
return
|
||||
|
||||
self._stop_event.clear()
|
||||
|
||||
try:
|
||||
import tweepy # noqa: F401
|
||||
|
||||
self._client = tweepy.Client(
|
||||
bearer_token=self._bearer_token,
|
||||
consumer_key=self._api_key or None,
|
||||
consumer_secret=self._api_secret or None,
|
||||
access_token=self._access_token or None,
|
||||
access_token_secret=self._access_secret or None,
|
||||
)
|
||||
self._status = ChannelStatus.CONNECTED
|
||||
logger.info("Twitter channel connected")
|
||||
|
||||
if self._access_token:
|
||||
self._poll_thread = threading.Thread(
|
||||
target=self._poll_loop, daemon=True,
|
||||
)
|
||||
self._poll_thread.start()
|
||||
except ImportError:
|
||||
logger.info("tweepy not installed; Twitter channel unavailable")
|
||||
self._status = ChannelStatus.ERROR
|
||||
except Exception:
|
||||
logger.debug("Twitter connect failed", exc_info=True)
|
||||
self._status = ChannelStatus.ERROR
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Stop the polling thread and disconnect."""
|
||||
self._stop_event.set()
|
||||
if self._poll_thread is not None:
|
||||
self._poll_thread.join(timeout=5.0)
|
||||
self._poll_thread = None
|
||||
self._client = None
|
||||
self._status = ChannelStatus.DISCONNECTED
|
||||
|
||||
# -- send / receive --------------------------------------------------------
|
||||
|
||||
def send(
|
||||
self,
|
||||
channel: str,
|
||||
content: str,
|
||||
*,
|
||||
conversation_id: str = "",
|
||||
metadata: Dict[str, Any] | None = None,
|
||||
) -> bool:
|
||||
"""Send a tweet or direct message.
|
||||
|
||||
If *channel* is numeric, sends a DM via ``create_direct_message()``.
|
||||
Otherwise sends a tweet via ``create_tweet()``. If *conversation_id*
|
||||
is provided, it is used as ``in_reply_to_tweet_id``.
|
||||
"""
|
||||
if self._client is None:
|
||||
logger.warning("Cannot send: Twitter client not connected")
|
||||
return False
|
||||
|
||||
try:
|
||||
if channel.isdigit():
|
||||
# Direct message to a user ID
|
||||
self._client.create_direct_message(
|
||||
participant_id=int(channel),
|
||||
text=content,
|
||||
)
|
||||
else:
|
||||
kwargs: Dict[str, Any] = {"text": content}
|
||||
if conversation_id:
|
||||
kwargs["in_reply_to_tweet_id"] = int(conversation_id)
|
||||
self._client.create_tweet(**kwargs)
|
||||
|
||||
self._publish_sent(channel, content, conversation_id)
|
||||
return True
|
||||
except Exception:
|
||||
logger.debug("Twitter send failed", exc_info=True)
|
||||
return False
|
||||
|
||||
def status(self) -> ChannelStatus:
|
||||
"""Return the current connection status."""
|
||||
return self._status
|
||||
|
||||
def list_channels(self) -> List[str]:
|
||||
"""Return available channel identifiers."""
|
||||
return ["timeline", "dm"]
|
||||
|
||||
def on_message(self, handler: ChannelHandler) -> None:
|
||||
"""Register a callback for incoming messages."""
|
||||
self._handlers.append(handler)
|
||||
|
||||
# -- internal helpers -------------------------------------------------------
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
"""Poll for new mentions in a background thread."""
|
||||
since_id: Optional[str] = None
|
||||
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
kwargs: Dict[str, Any] = {}
|
||||
if since_id:
|
||||
kwargs["since_id"] = since_id
|
||||
|
||||
me = self._client.get_me()
|
||||
if me and me.data:
|
||||
user_id = me.data.id
|
||||
else:
|
||||
self._stop_event.wait(self._poll_interval)
|
||||
continue
|
||||
|
||||
response = self._client.get_users_mentions(
|
||||
id=user_id, **kwargs,
|
||||
)
|
||||
|
||||
if response and response.data:
|
||||
for tweet in response.data:
|
||||
since_id = str(tweet.id)
|
||||
cm = ChannelMessage(
|
||||
channel="twitter",
|
||||
sender=str(getattr(tweet, "author_id", "")),
|
||||
content=tweet.text,
|
||||
message_id=str(tweet.id),
|
||||
conversation_id=str(
|
||||
getattr(tweet, "conversation_id", ""),
|
||||
),
|
||||
)
|
||||
for handler in self._handlers:
|
||||
try:
|
||||
handler(cm)
|
||||
except Exception:
|
||||
logger.exception("Twitter handler error")
|
||||
if self._bus is not None:
|
||||
self._bus.publish(
|
||||
EventType.CHANNEL_MESSAGE_RECEIVED,
|
||||
{
|
||||
"channel": cm.channel,
|
||||
"sender": cm.sender,
|
||||
"content": cm.content,
|
||||
"message_id": cm.message_id,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Twitter poll error", exc_info=True)
|
||||
|
||||
self._stop_event.wait(self._poll_interval)
|
||||
|
||||
def _publish_sent(
|
||||
self, channel: str, content: str, conversation_id: str,
|
||||
) -> None:
|
||||
"""Publish a CHANNEL_MESSAGE_SENT event on the bus."""
|
||||
if self._bus is not None:
|
||||
self._bus.publish(
|
||||
EventType.CHANNEL_MESSAGE_SENT,
|
||||
{
|
||||
"channel": channel,
|
||||
"content": content,
|
||||
"conversation_id": conversation_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["TwitterChannel"]
|
||||
+83
-41
@@ -44,7 +44,8 @@ def _get_memory_backend(config):
|
||||
|
||||
if key == "sqlite":
|
||||
backend = MemoryRegistry.create(
|
||||
key, db_path=config.memory.db_path,
|
||||
key,
|
||||
db_path=config.memory.db_path,
|
||||
)
|
||||
else:
|
||||
backend = MemoryRegistry.create(key)
|
||||
@@ -106,8 +107,7 @@ def _run_agent(
|
||||
|
||||
if not AgentRegistry.contains(agent_name):
|
||||
raise click.ClickException(
|
||||
f"Unknown agent: {agent_name}. "
|
||||
f"Available: {', '.join(AgentRegistry.keys())}"
|
||||
f"Unknown agent: {agent_name}. Available: {', '.join(AgentRegistry.keys())}"
|
||||
)
|
||||
|
||||
agent_cls = AgentRegistry.get(agent_name)
|
||||
@@ -117,6 +117,7 @@ def _run_agent(
|
||||
if tool_names:
|
||||
# Trigger tool registration
|
||||
import openjarvis.tools # noqa: F401
|
||||
|
||||
tools = _build_tools(tool_names, config, engine, model_name)
|
||||
|
||||
# Build agent with appropriate kwargs
|
||||
@@ -149,7 +150,10 @@ def _run_agent(
|
||||
max_context_tokens=config.memory.context_max_tokens,
|
||||
)
|
||||
context_messages = inject_context(
|
||||
query_text, [], backend, config=ctx_cfg,
|
||||
query_text,
|
||||
[],
|
||||
backend,
|
||||
config=ctx_cfg,
|
||||
)
|
||||
for msg in context_messages:
|
||||
ctx.conversation.add(msg)
|
||||
@@ -169,9 +173,7 @@ def _print_profile(
|
||||
) -> None:
|
||||
"""Print an inference telemetry profile table from EventBus history."""
|
||||
# Collect all INFERENCE_END events (agents may fire multiple)
|
||||
inf_events = [
|
||||
e for e in bus.history if e.event_type == EventType.INFERENCE_END
|
||||
]
|
||||
inf_events = [e for e in bus.history if e.event_type == EventType.INFERENCE_END]
|
||||
if not inf_events:
|
||||
console.print("[dim]No inference telemetry recorded.[/dim]")
|
||||
return
|
||||
@@ -186,13 +188,15 @@ def _print_profile(
|
||||
for e in inf_events
|
||||
)
|
||||
total_prompt = sum(
|
||||
e.data.get("usage", {}).get("prompt_tokens", 0)
|
||||
for e in inf_events
|
||||
e.data.get("usage", {}).get("prompt_tokens", 0) for e in inf_events
|
||||
)
|
||||
total_energy = sum(e.data.get("energy_joules", 0.0) for e in inf_events)
|
||||
avg_power = 0.0
|
||||
power_vals = [e.data.get("power_watts", 0.0) for e in inf_events
|
||||
if e.data.get("power_watts", 0.0) > 0]
|
||||
power_vals = [
|
||||
e.data.get("power_watts", 0.0)
|
||||
for e in inf_events
|
||||
if e.data.get("power_watts", 0.0) > 0
|
||||
]
|
||||
if power_vals:
|
||||
avg_power = sum(power_vals) / len(power_vals)
|
||||
|
||||
@@ -284,29 +288,42 @@ def _print_profile(
|
||||
@click.option("-m", "--model", "model_name", default=None, help="Model to use.")
|
||||
@click.option("-e", "--engine", "engine_key", default=None, help="Engine backend.")
|
||||
@click.option(
|
||||
"-t", "--temperature", default=None, type=float,
|
||||
"-t",
|
||||
"--temperature",
|
||||
default=None,
|
||||
type=float,
|
||||
help="Sampling temperature (default: from config).",
|
||||
)
|
||||
@click.option(
|
||||
"--max-tokens", default=None, type=int,
|
||||
"--max-tokens",
|
||||
default=None,
|
||||
type=int,
|
||||
help="Max tokens to generate (default: from config).",
|
||||
)
|
||||
@click.option("--json", "output_json", is_flag=True, help="Output raw JSON result.")
|
||||
@click.option("--no-stream", is_flag=True, help="Disable streaming (sync mode).")
|
||||
@click.option(
|
||||
"--no-context", is_flag=True,
|
||||
"--no-context",
|
||||
is_flag=True,
|
||||
help="Disable memory context injection.",
|
||||
)
|
||||
@click.option(
|
||||
"-a", "--agent", "agent_name", default=None,
|
||||
"-a",
|
||||
"--agent",
|
||||
"agent_name",
|
||||
default=None,
|
||||
help="Agent to use (simple, orchestrator).",
|
||||
)
|
||||
@click.option(
|
||||
"--tools", "tool_names", default=None,
|
||||
"--tools",
|
||||
"tool_names",
|
||||
default=None,
|
||||
help="Comma-separated tool names to enable (e.g. calculator,think).",
|
||||
)
|
||||
@click.option(
|
||||
"--profile", "enable_profile", is_flag=True,
|
||||
"--profile",
|
||||
"enable_profile",
|
||||
is_flag=True,
|
||||
help="Print inference telemetry profile (latency, tokens, energy, IPW).",
|
||||
)
|
||||
def ask(
|
||||
@@ -377,7 +394,10 @@ def ask(
|
||||
" [cyan]ollama serve[/cyan] — start Ollama\n"
|
||||
" [cyan]vllm serve <model>[/cyan] — start vLLM\n"
|
||||
" [cyan]llama-server -m <gguf>[/cyan] — start llama.cpp\n\n"
|
||||
"Or set OPENAI_API_KEY / ANTHROPIC_API_KEY for cloud inference."
|
||||
"Or set OPENAI_API_KEY / ANTHROPIC_API_KEY for cloud inference.\n\n"
|
||||
"[dim]To use a remote engine:[/dim]\n"
|
||||
" [cyan]jarvis config set engine.ollama.host http://<remote-ip>:11434[/cyan]\n"
|
||||
" [dim]or[/dim] [cyan]export OLLAMA_HOST=http://<remote-ip>:11434[/cyan]"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
@@ -385,6 +405,7 @@ def ask(
|
||||
|
||||
# Apply security guardrails
|
||||
from openjarvis.security import setup_security
|
||||
|
||||
sec = setup_security(config, engine, bus)
|
||||
engine = sec.engine
|
||||
|
||||
@@ -427,12 +448,14 @@ def ask(
|
||||
# the user would have gotten without the analyzer.
|
||||
if not user_set_max_tokens:
|
||||
suggested = adjust_tokens_for_model(
|
||||
complexity_result.suggested_max_tokens, model_name,
|
||||
complexity_result.suggested_max_tokens,
|
||||
model_name,
|
||||
)
|
||||
max_tokens = max(suggested, config.intelligence.max_tokens)
|
||||
logger.debug(
|
||||
"Using complexity-suggested max_tokens=%d (model=%s)",
|
||||
max_tokens, model_name,
|
||||
max_tokens,
|
||||
model_name,
|
||||
)
|
||||
|
||||
# Agent mode
|
||||
@@ -444,8 +467,15 @@ def ask(
|
||||
)
|
||||
try:
|
||||
result = _run_agent(
|
||||
agent_name, query_text, engine, model_name,
|
||||
parsed_tools, config, bus, temperature, max_tokens,
|
||||
agent_name,
|
||||
query_text,
|
||||
engine,
|
||||
model_name,
|
||||
parsed_tools,
|
||||
config,
|
||||
bus,
|
||||
temperature,
|
||||
max_tokens,
|
||||
capability_policy=sec.capability_policy,
|
||||
)
|
||||
except EngineConnectionError as exc:
|
||||
@@ -454,25 +484,33 @@ def ask(
|
||||
sys.exit(1)
|
||||
|
||||
if output_json:
|
||||
click.echo(json_mod.dumps({
|
||||
"content": result.content,
|
||||
"turns": result.turns,
|
||||
"tool_results": [
|
||||
click.echo(
|
||||
json_mod.dumps(
|
||||
{
|
||||
"tool_name": tr.tool_name,
|
||||
"content": tr.content,
|
||||
"success": tr.success,
|
||||
}
|
||||
for tr in result.tool_results
|
||||
],
|
||||
}, indent=2))
|
||||
"content": result.content,
|
||||
"turns": result.turns,
|
||||
"tool_results": [
|
||||
{
|
||||
"tool_name": tr.tool_name,
|
||||
"content": tr.content,
|
||||
"success": tr.success,
|
||||
}
|
||||
for tr in result.tool_results
|
||||
],
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
else:
|
||||
click.echo(result.content)
|
||||
|
||||
if enable_profile:
|
||||
_print_profile(
|
||||
bus, time.monotonic() - wall_start,
|
||||
engine_name, model_name, console,
|
||||
bus,
|
||||
time.monotonic() - wall_start,
|
||||
engine_name,
|
||||
model_name,
|
||||
console,
|
||||
complexity_result=complexity_result,
|
||||
)
|
||||
|
||||
@@ -493,17 +531,18 @@ def ask(
|
||||
ContextConfig,
|
||||
inject_context,
|
||||
)
|
||||
|
||||
backend = _get_memory_backend(config)
|
||||
if backend is not None:
|
||||
ctx_cfg = ContextConfig(
|
||||
top_k=config.memory.context_top_k,
|
||||
min_score=config.memory.context_min_score,
|
||||
max_context_tokens=(
|
||||
config.memory.context_max_tokens
|
||||
),
|
||||
max_context_tokens=(config.memory.context_max_tokens),
|
||||
)
|
||||
messages = inject_context(
|
||||
query_text, messages, backend,
|
||||
query_text,
|
||||
messages,
|
||||
backend,
|
||||
config=ctx_cfg,
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -531,8 +570,11 @@ def ask(
|
||||
|
||||
if enable_profile:
|
||||
_print_profile(
|
||||
bus, time.monotonic() - wall_start,
|
||||
engine_name, model_name, console,
|
||||
bus,
|
||||
time.monotonic() - wall_start,
|
||||
engine_name,
|
||||
model_name,
|
||||
console,
|
||||
complexity_result=complexity_result,
|
||||
)
|
||||
|
||||
|
||||
@@ -4,9 +4,11 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import httpx
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.syntax import Syntax
|
||||
@@ -274,4 +276,100 @@ def hardware() -> None:
|
||||
config.add_command(show_group, "show")
|
||||
|
||||
|
||||
def _probe_engine_host(url: str, console: Console) -> None:
|
||||
"""Probe an engine host URL and print reachability status."""
|
||||
try:
|
||||
resp = httpx.get(url.rstrip("/") + "/", timeout=2.0)
|
||||
if resp.status_code < 500:
|
||||
console.print(f" [green]Reachable[/green] ({url})")
|
||||
else:
|
||||
console.print(
|
||||
f" [yellow]Warning:[/yellow] Host returned status "
|
||||
f"{resp.status_code} — config saved anyway."
|
||||
)
|
||||
except Exception:
|
||||
console.print(
|
||||
f" [yellow]Warning:[/yellow] Host unreachable ({url}) "
|
||||
f"— config saved anyway."
|
||||
)
|
||||
|
||||
|
||||
def _coerce_value(value: str, target_type: type) -> object:
|
||||
"""Coerce a CLI string value to the target Python type."""
|
||||
if target_type is bool:
|
||||
low = value.lower()
|
||||
if low in ("true", "1", "yes"):
|
||||
return True
|
||||
if low in ("false", "0", "no"):
|
||||
return False
|
||||
raise ValueError(
|
||||
f"Invalid boolean value: {value!r} (expected: true/false, yes/no, 1/0)"
|
||||
)
|
||||
if target_type is int:
|
||||
return int(value)
|
||||
if target_type is float:
|
||||
return float(value)
|
||||
return value
|
||||
|
||||
|
||||
@click.command("set")
|
||||
@click.argument("key")
|
||||
@click.argument("value")
|
||||
def set_config(key: str, value: str) -> None:
|
||||
"""Set a configuration value (e.g. jarvis config set engine.ollama.host URL)."""
|
||||
import tomlkit
|
||||
|
||||
from openjarvis.core.config import DEFAULT_CONFIG_DIR, validate_config_key
|
||||
|
||||
console = Console(stderr=True)
|
||||
|
||||
# Validate key
|
||||
try:
|
||||
target_type = validate_config_key(key)
|
||||
except ValueError as exc:
|
||||
console.print(f"[red]Error:[/red] {exc}")
|
||||
raise SystemExit(1)
|
||||
|
||||
# Coerce value
|
||||
try:
|
||||
typed_value = _coerce_value(value, target_type)
|
||||
except (ValueError, TypeError) as exc:
|
||||
console.print(
|
||||
f"[red]Error:[/red] Cannot convert {value!r} to "
|
||||
f"{target_type.__name__}: {exc}"
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
# Load or create TOML document
|
||||
config_path = Path(
|
||||
os.environ.get("OPENJARVIS_CONFIG", DEFAULT_CONFIG_DIR / "config.toml")
|
||||
)
|
||||
if config_path.exists():
|
||||
doc = tomlkit.parse(config_path.read_text())
|
||||
else:
|
||||
doc = tomlkit.document()
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Set nested key
|
||||
parts = key.split(".")
|
||||
current = doc
|
||||
for part in parts[:-1]:
|
||||
if part not in current:
|
||||
current.add(part, tomlkit.table())
|
||||
current = current[part]
|
||||
current[parts[-1]] = typed_value
|
||||
|
||||
# Write back
|
||||
config_path.write_text(tomlkit.dumps(doc))
|
||||
|
||||
console.print(f"[green]Set[/green] {key} = {value!r}")
|
||||
|
||||
# Probe engine host if applicable
|
||||
if re.match(r"^engine\.\w+\.host$", key):
|
||||
_probe_engine_host(value, console)
|
||||
|
||||
|
||||
config.add_command(set_config, "set")
|
||||
|
||||
|
||||
__all__ = ["config"]
|
||||
|
||||
@@ -35,17 +35,13 @@ def _check_python_version() -> CheckResult:
|
||||
version_str = f"{ver.major}.{ver.minor}.{ver.micro}"
|
||||
if (ver.major, ver.minor) >= (3, 10):
|
||||
return CheckResult("Python version", "ok", version_str)
|
||||
return CheckResult(
|
||||
"Python version", "fail", f"{version_str} (requires >= 3.10)"
|
||||
)
|
||||
return CheckResult("Python version", "fail", f"{version_str} (requires >= 3.10)")
|
||||
|
||||
|
||||
def _check_config_exists() -> CheckResult:
|
||||
"""Check that the config file exists."""
|
||||
if DEFAULT_CONFIG_PATH.exists():
|
||||
return CheckResult(
|
||||
"Config file", "ok", str(DEFAULT_CONFIG_PATH)
|
||||
)
|
||||
return CheckResult("Config file", "ok", str(DEFAULT_CONFIG_PATH))
|
||||
return CheckResult(
|
||||
"Config file",
|
||||
"warn",
|
||||
@@ -57,16 +53,12 @@ def _check_config_exists() -> CheckResult:
|
||||
def _check_config_parses() -> CheckResult:
|
||||
"""Check that the config file parses successfully."""
|
||||
if not DEFAULT_CONFIG_PATH.exists():
|
||||
return CheckResult(
|
||||
"Config parsing", "warn", "Skipped (no config file)"
|
||||
)
|
||||
return CheckResult("Config parsing", "warn", "Skipped (no config file)")
|
||||
try:
|
||||
load_config()
|
||||
return CheckResult("Config parsing", "ok", "Config loaded successfully")
|
||||
except Exception as exc:
|
||||
return CheckResult(
|
||||
"Config parsing", "fail", f"Parse error: {exc}"
|
||||
)
|
||||
return CheckResult("Config parsing", "fail", f"Parse error: {exc}")
|
||||
|
||||
|
||||
def _ensure_engines_imported() -> None:
|
||||
@@ -102,22 +94,16 @@ def _check_engines() -> List[CheckResult]:
|
||||
try:
|
||||
engine = _discovery._make_engine(key, config)
|
||||
if engine.health():
|
||||
results.append(
|
||||
CheckResult(f"Engine: {key}", "ok", "Reachable")
|
||||
)
|
||||
results.append(CheckResult(f"Engine: {key}", "ok", "Reachable"))
|
||||
else:
|
||||
results.append(
|
||||
CheckResult(f"Engine: {key}", "warn", "Unreachable")
|
||||
)
|
||||
results.append(CheckResult(f"Engine: {key}", "warn", "Unreachable"))
|
||||
except Exception as exc:
|
||||
results.append(
|
||||
CheckResult(f"Engine: {key}", "warn", f"Unreachable ({exc})")
|
||||
)
|
||||
|
||||
if not results:
|
||||
results.append(
|
||||
CheckResult("Engines", "warn", "No engines registered")
|
||||
)
|
||||
results.append(CheckResult("Engines", "warn", "No engines registered"))
|
||||
|
||||
return results
|
||||
|
||||
@@ -154,7 +140,7 @@ def _check_models() -> List[CheckResult]:
|
||||
f"Models: {key}",
|
||||
"warn",
|
||||
"No models available",
|
||||
details="Pull a model (e.g. `ollama pull qwen3.5:3b`).",
|
||||
details="Pull a model (e.g. `ollama pull qwen3.5:2b`).",
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
@@ -168,9 +154,7 @@ def _check_default_model() -> CheckResult:
|
||||
try:
|
||||
config = load_config()
|
||||
except Exception:
|
||||
return CheckResult(
|
||||
"Default model", "warn", "Skipped (config unavailable)"
|
||||
)
|
||||
return CheckResult("Default model", "warn", "Skipped (config unavailable)")
|
||||
|
||||
default_model = config.intelligence.default_model
|
||||
if not default_model:
|
||||
@@ -221,9 +205,7 @@ def _check_optional_deps() -> List[CheckResult]:
|
||||
for pkg, install_hint, description in optional_packages:
|
||||
try:
|
||||
__import__(pkg)
|
||||
results.append(
|
||||
CheckResult(f"Optional: {description}", "ok", "Installed")
|
||||
)
|
||||
results.append(CheckResult(f"Optional: {description}", "ok", "Installed"))
|
||||
except Exception:
|
||||
results.append(
|
||||
CheckResult(
|
||||
@@ -266,8 +248,7 @@ def _check_nodejs() -> CheckResult:
|
||||
"warn",
|
||||
f"{version_str} (requires >= v22)",
|
||||
details=(
|
||||
"Upgrade Node.js for ClaudeCodeAgent and WhatsApp "
|
||||
"Baileys support."
|
||||
"Upgrade Node.js for ClaudeCodeAgent and WhatsApp Baileys support."
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -335,7 +316,5 @@ def doctor(as_json: bool) -> None:
|
||||
warn_count = sum(1 for c in checks if c.status == "warn")
|
||||
fail_count = sum(1 for c in checks if c.status == "fail")
|
||||
console.print()
|
||||
console.print(
|
||||
f" {ok_count} passed, {warn_count} warnings, {fail_count} failures"
|
||||
)
|
||||
console.print(f" {ok_count} passed, {warn_count} warnings, {fail_count} failures")
|
||||
console.print()
|
||||
|
||||
@@ -22,7 +22,11 @@ def hint_no_engine(engine_name: Optional[str] = None) -> str:
|
||||
f"[yellow]Hint:[/yellow] Engine '{name}' is not reachable.\n"
|
||||
f" Make sure the {name} server is running.\n"
|
||||
" Run [bold]jarvis doctor[/bold] to check all engines.\n"
|
||||
" Run [bold]jarvis quickstart[/bold] for guided setup."
|
||||
" Run [bold]jarvis quickstart[/bold] for guided setup.\n"
|
||||
"\n"
|
||||
" [dim]To use a remote engine:[/dim]\n"
|
||||
f" [cyan]jarvis config set engine.{name}.host http://<remote-ip>:<port>[/cyan]\n"
|
||||
f" [dim]or[/dim] [cyan]export OLLAMA_HOST=http://<remote-ip>:11434[/cyan]"
|
||||
)
|
||||
|
||||
|
||||
@@ -36,6 +40,6 @@ def hint_no_model(model_name: Optional[str] = None) -> str:
|
||||
)
|
||||
return (
|
||||
"[yellow]Hint:[/yellow] No models available.\n"
|
||||
" Pull a model first: [bold]ollama pull qwen3.5:3b[/bold]\n"
|
||||
" Pull a model first: [bold]ollama pull qwen3.5:2b[/bold]\n"
|
||||
" Run [bold]jarvis model list[/bold] to see available models."
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import click
|
||||
import httpx
|
||||
from rich.console import Console
|
||||
from rich.markup import escape
|
||||
from rich.panel import Panel
|
||||
@@ -25,8 +26,14 @@ from openjarvis.core.config import (
|
||||
|
||||
# Engines supported by ``jarvis init --engine``.
|
||||
_SUPPORTED_ENGINES = [
|
||||
"ollama", "vllm", "sglang", "llamacpp", "mlx", "lmstudio",
|
||||
"exo", "nexa",
|
||||
"ollama",
|
||||
"vllm",
|
||||
"sglang",
|
||||
"llamacpp",
|
||||
"mlx",
|
||||
"lmstudio",
|
||||
"exo",
|
||||
"nexa",
|
||||
]
|
||||
|
||||
|
||||
@@ -57,7 +64,7 @@ def _detect_running_engines() -> list[str]:
|
||||
|
||||
def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
"""Return engine-specific next-steps guidance after init."""
|
||||
pull_model = model or "qwen3.5:3b"
|
||||
pull_model = model or "qwen3.5:2b"
|
||||
steps: dict[str, str] = {
|
||||
"ollama": (
|
||||
"Next steps:\n"
|
||||
@@ -70,7 +77,7 @@ def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
f" ollama pull {pull_model}\n"
|
||||
"\n"
|
||||
" 3. Try it out:\n"
|
||||
" jarvis ask \"Hello\"\n"
|
||||
' jarvis ask "Hello"\n'
|
||||
"\n"
|
||||
" Run `jarvis doctor` to verify your setup."
|
||||
),
|
||||
@@ -82,7 +89,7 @@ def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
" vllm serve Qwen/Qwen3-4B\n"
|
||||
"\n"
|
||||
" 2. Try it out:\n"
|
||||
" jarvis ask \"Hello\"\n"
|
||||
' jarvis ask "Hello"\n'
|
||||
"\n"
|
||||
" Run `jarvis doctor` to verify your setup."
|
||||
),
|
||||
@@ -94,7 +101,7 @@ def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
" llama-server -m path/to/model.gguf\n"
|
||||
"\n"
|
||||
" 2. Try it out:\n"
|
||||
" jarvis ask \"Hello\"\n"
|
||||
' jarvis ask "Hello"\n'
|
||||
"\n"
|
||||
" Run `jarvis doctor` to verify your setup."
|
||||
),
|
||||
@@ -106,7 +113,7 @@ def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
" python -m sglang.launch_server --model-path Qwen/Qwen3-8B\n"
|
||||
"\n"
|
||||
" 2. Try it out:\n"
|
||||
" jarvis ask \"Hello\"\n"
|
||||
' jarvis ask "Hello"\n'
|
||||
"\n"
|
||||
" Run `jarvis doctor` to verify your setup."
|
||||
),
|
||||
@@ -118,7 +125,7 @@ def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
" mlx_lm.server --model mlx-community/Qwen2.5-7B-4bit\n"
|
||||
"\n"
|
||||
" 2. Try it out:\n"
|
||||
" jarvis ask \"Hello\"\n"
|
||||
' jarvis ask "Hello"\n'
|
||||
"\n"
|
||||
" Run `jarvis doctor` to verify your setup."
|
||||
),
|
||||
@@ -131,7 +138,7 @@ def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
" 2. Load a model and start the local server (port 1234)\n"
|
||||
"\n"
|
||||
" 3. Try it out:\n"
|
||||
" jarvis ask \"Hello\"\n"
|
||||
' jarvis ask "Hello"\n'
|
||||
"\n"
|
||||
" Run `jarvis doctor` to verify your setup."
|
||||
),
|
||||
@@ -141,7 +148,7 @@ def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
" pip install exo\n"
|
||||
" exo\n\n"
|
||||
" 2. Try it out:\n"
|
||||
" jarvis ask \"Hello\"\n\n"
|
||||
' jarvis ask "Hello"\n\n'
|
||||
" Run `jarvis doctor` to verify your setup."
|
||||
),
|
||||
"nexa": (
|
||||
@@ -150,7 +157,7 @@ def _next_steps_text(engine: str, model: str = "") -> str:
|
||||
" pip install nexaai\n"
|
||||
" nexa server\n\n"
|
||||
" 2. Try it out:\n"
|
||||
" jarvis ask \"Hello\"\n\n"
|
||||
' jarvis ask "Hello"\n\n'
|
||||
" Run `jarvis doctor` to verify your setup."
|
||||
),
|
||||
}
|
||||
@@ -177,6 +184,7 @@ def _quick_privacy_check(console: Console) -> None:
|
||||
def _do_download(engine: str, model: str, spec, console: Console) -> None:
|
||||
"""Dispatch model download based on engine type."""
|
||||
import os
|
||||
|
||||
if engine == "ollama":
|
||||
host = os.environ.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/")
|
||||
ollama_pull(host, model, console)
|
||||
@@ -228,12 +236,26 @@ def _do_download(engine: str, model: str, spec, console: Console) -> None:
|
||||
@click.option(
|
||||
"--no-download", is_flag=True, default=False, help="Skip the model download prompt."
|
||||
)
|
||||
@click.option(
|
||||
"--no-scan",
|
||||
"skip_scan",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Skip the post-init security environment audit.",
|
||||
)
|
||||
@click.option(
|
||||
"--host",
|
||||
default=None,
|
||||
help="Remote engine host URL (e.g. http://192.168.1.50:11434).",
|
||||
)
|
||||
def init(
|
||||
force: bool,
|
||||
config: Optional[Path],
|
||||
full_config: bool = False,
|
||||
engine: Optional[str] = None,
|
||||
no_download: bool = False,
|
||||
skip_scan: bool = False,
|
||||
host: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Detect hardware and generate ~/.openjarvis/config.toml."""
|
||||
console = Console()
|
||||
@@ -267,9 +289,7 @@ def init(
|
||||
console.print("[bold]Detecting running inference engines...[/bold]")
|
||||
running = _detect_running_engines()
|
||||
if running:
|
||||
console.print(
|
||||
f" Found running: [green]{', '.join(running)}[/green]"
|
||||
)
|
||||
console.print(f" Found running: [green]{', '.join(running)}[/green]")
|
||||
else:
|
||||
console.print(" No running engines detected.")
|
||||
|
||||
@@ -313,13 +333,31 @@ def init(
|
||||
default=default,
|
||||
)
|
||||
|
||||
# Probe remote host if specified
|
||||
if host:
|
||||
console.print("\n[bold]Checking remote host...[/bold]")
|
||||
try:
|
||||
resp = httpx.get(host.rstrip("/") + "/", timeout=2.0)
|
||||
if resp.status_code < 500:
|
||||
console.print(f" [green]Reachable[/green] ({host})")
|
||||
else:
|
||||
console.print(
|
||||
f" [yellow]Warning:[/yellow] Host returned status "
|
||||
f"{resp.status_code} — writing config anyway."
|
||||
)
|
||||
except Exception:
|
||||
console.print(
|
||||
f" [yellow]Warning:[/yellow] Host unreachable ({host}) "
|
||||
f"— writing config anyway."
|
||||
)
|
||||
|
||||
if config:
|
||||
toml_content = config.read_text()
|
||||
else:
|
||||
if full_config:
|
||||
toml_content = generate_default_toml(hw, engine=engine)
|
||||
toml_content = generate_default_toml(hw, engine=engine, host=host)
|
||||
else:
|
||||
toml_content = generate_minimal_toml(hw, engine=engine)
|
||||
toml_content = generate_minimal_toml(hw, engine=engine, host=host)
|
||||
|
||||
DEFAULT_CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if config:
|
||||
@@ -341,8 +379,7 @@ def init(
|
||||
soul_path = DEFAULT_CONFIG_DIR / "SOUL.md"
|
||||
if not soul_path.exists():
|
||||
soul_path.write_text(
|
||||
"# Agent Persona\n\n"
|
||||
"You are Jarvis, a helpful personal AI assistant.\n"
|
||||
"# Agent Persona\n\nYou are Jarvis, a helpful personal AI assistant.\n"
|
||||
)
|
||||
|
||||
memory_path = DEFAULT_CONFIG_DIR / "MEMORY.md"
|
||||
@@ -376,7 +413,8 @@ def init(
|
||||
if click.confirm(prompt, default=True):
|
||||
_do_download(selected_engine, model, spec, console)
|
||||
|
||||
_quick_privacy_check(console)
|
||||
if not skip_scan:
|
||||
_quick_privacy_check(console)
|
||||
console.print()
|
||||
console.print(
|
||||
Panel(
|
||||
|
||||
@@ -106,8 +106,7 @@ def quickstart(force: bool) -> None:
|
||||
console.print("[bold cyan][2/5][/bold cyan] Writing config...")
|
||||
if DEFAULT_CONFIG_PATH.exists() and not force:
|
||||
console.print(
|
||||
f" [dim]Config already exists at"
|
||||
f" {DEFAULT_CONFIG_PATH} (skip)[/dim]"
|
||||
f" [dim]Config already exists at {DEFAULT_CONFIG_PATH} (skip)[/dim]"
|
||||
)
|
||||
else:
|
||||
toml_content = generate_default_toml(hw)
|
||||
@@ -144,7 +143,7 @@ def quickstart(force: bool) -> None:
|
||||
if not _check_model_available(active_engine):
|
||||
console.print(" [yellow]No models found.[/yellow]")
|
||||
console.print(
|
||||
" Pull a model first (e.g. [bold]ollama pull qwen3.5:3b[/bold])."
|
||||
" Pull a model first (e.g. [bold]ollama pull qwen3.5:2b[/bold])."
|
||||
)
|
||||
raise SystemExit(1)
|
||||
console.print(" [green]Models available.[/green]")
|
||||
@@ -157,6 +156,5 @@ def quickstart(force: bool) -> None:
|
||||
|
||||
console.print()
|
||||
console.print(
|
||||
'[bold green]Setup complete![/bold green]'
|
||||
' Try: [bold]jarvis ask "Hello"[/bold]'
|
||||
'[bold green]Setup complete![/bold green] Try: [bold]jarvis ask "Hello"[/bold]'
|
||||
)
|
||||
|
||||
+163
-27
@@ -19,11 +19,24 @@ _CLOUD_SYNC_PROCS = ["Dropbox", "OneDrive", "Google Drive", "iCloudDrive"]
|
||||
|
||||
# Screen-recording / remote-access processes (macOS).
|
||||
_SCREEN_RECORDING_PROCS = [
|
||||
"TeamViewer", "AnyDesk", "ScreenConnect", "vncviewer", "Vine"
|
||||
"TeamViewer",
|
||||
"AnyDesk",
|
||||
"ScreenConnect",
|
||||
"vncviewer",
|
||||
"Vine",
|
||||
]
|
||||
|
||||
# Remote-access processes (Linux).
|
||||
_REMOTE_ACCESS_PROCS = ["xrdp", "x11vnc", "vncserver", "AnyDesk"]
|
||||
# Remote-access processes (cross-platform).
|
||||
_REMOTE_ACCESS_PROCS = [
|
||||
"xrdp",
|
||||
"x11vnc",
|
||||
"vncserver",
|
||||
"AnyDesk",
|
||||
"ngrok",
|
||||
"tailscaled",
|
||||
"cloudflared",
|
||||
"ZeroTier",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -295,12 +308,83 @@ class PrivacyScanner:
|
||||
)
|
||||
|
||||
def check_remote_access(self) -> ScanResult:
|
||||
"""Check for running remote-access processes (Linux)."""
|
||||
"""Check for running remote-access / tunneling processes."""
|
||||
return self._check_processes(
|
||||
names=_REMOTE_ACCESS_PROCS,
|
||||
check_name="Remote Access",
|
||||
warn_msg="{name} is running — system may be accessible remotely.",
|
||||
platform="linux",
|
||||
platform="all",
|
||||
)
|
||||
|
||||
def check_dns(self) -> ScanResult:
|
||||
"""Check DNS configuration for encrypted resolvers (macOS)."""
|
||||
if sys.platform != "darwin":
|
||||
return ScanResult(
|
||||
name="DNS Configuration",
|
||||
status="skip",
|
||||
message="DNS check not yet implemented for this platform.",
|
||||
platform="darwin",
|
||||
)
|
||||
try:
|
||||
proc = self._run(["scutil", "--dns"])
|
||||
output = proc.stdout
|
||||
except Exception:
|
||||
return ScanResult(
|
||||
name="DNS Configuration",
|
||||
status="skip",
|
||||
message="scutil command not available.",
|
||||
platform="darwin",
|
||||
)
|
||||
|
||||
_DOH_INDICATORS = [
|
||||
"dns-over-https",
|
||||
"dns-over-tls",
|
||||
"encrypted",
|
||||
"doh",
|
||||
"dot",
|
||||
]
|
||||
if any(ind in output.lower() for ind in _DOH_INDICATORS):
|
||||
return ScanResult(
|
||||
name="DNS Configuration",
|
||||
status="ok",
|
||||
message="Encrypted DNS (DoH/DoT) appears to be active.",
|
||||
platform="darwin",
|
||||
)
|
||||
|
||||
# Extract nameserver IPs
|
||||
nameservers: list[str] = []
|
||||
for line in output.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("nameserver["):
|
||||
parts = stripped.split(":", 1)
|
||||
if len(parts) == 2:
|
||||
nameservers.append(parts[1].strip())
|
||||
|
||||
if not nameservers:
|
||||
return ScanResult(
|
||||
name="DNS Configuration",
|
||||
status="ok",
|
||||
message="Could not parse DNS nameservers from scutil output.",
|
||||
platform="darwin",
|
||||
)
|
||||
|
||||
_PLAIN_DNS = {"8.8.8.8", "8.8.4.4", "1.1.1.1", "1.0.0.1", "9.9.9.9"}
|
||||
plain = [ns for ns in nameservers if ns in _PLAIN_DNS]
|
||||
if plain:
|
||||
return ScanResult(
|
||||
name="DNS Configuration",
|
||||
status="warn",
|
||||
message=(
|
||||
f"Plain DNS in use ({', '.join(plain)}). "
|
||||
"Queries may be visible to your ISP or resolver."
|
||||
),
|
||||
platform="darwin",
|
||||
)
|
||||
return ScanResult(
|
||||
name="DNS Configuration",
|
||||
status="ok",
|
||||
message=f"DNS resolvers: {', '.join(nameservers[:3])}.",
|
||||
platform="darwin",
|
||||
)
|
||||
|
||||
# -- Orchestration -------------------------------------------------------
|
||||
@@ -315,6 +399,7 @@ class PrivacyScanner:
|
||||
self.check_luks,
|
||||
self.check_screen_recording,
|
||||
self.check_remote_access,
|
||||
self.check_dns,
|
||||
]
|
||||
|
||||
def run_all(self) -> list[ScanResult]:
|
||||
@@ -350,37 +435,88 @@ class PrivacyScanner:
|
||||
# CLI command
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_STATUS_ICONS = {"ok": "✓", "warn": "!", "fail": "✗", "skip": "-"}
|
||||
_RICH_ICONS = {
|
||||
"ok": "[green]\u2713[/green]",
|
||||
"warn": "[yellow]![/yellow]",
|
||||
"fail": "[red]\u2717[/red]",
|
||||
"skip": "[dim]-[/dim]",
|
||||
}
|
||||
|
||||
|
||||
def _render_results(results: List[ScanResult]) -> None:
|
||||
"""Render scan results as a Rich table."""
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
console.print()
|
||||
console.print("[bold]OpenJarvis Security Scan[/bold]")
|
||||
console.print()
|
||||
|
||||
table = Table(show_header=True, header_style="bold", show_lines=True)
|
||||
table.add_column("", width=3, justify="center")
|
||||
table.add_column("Check")
|
||||
table.add_column("Finding")
|
||||
|
||||
for r in results:
|
||||
icon = _RICH_ICONS.get(r.status, "?")
|
||||
style = {"ok": "green", "warn": "yellow", "fail": "red"}.get(r.status, "white")
|
||||
table.add_row(icon, r.name, f"[{style}]{r.message}[/{style}]")
|
||||
|
||||
console.print(table)
|
||||
|
||||
ok_count = sum(1 for r in results if r.status == "ok")
|
||||
warn_count = sum(1 for r in results if r.status == "warn")
|
||||
fail_count = sum(1 for r in results if r.status == "fail")
|
||||
|
||||
console.print()
|
||||
parts = [f"[green]{ok_count} ok[/green]"]
|
||||
if warn_count:
|
||||
parts.append(f"[yellow]{warn_count} warning(s)[/yellow]")
|
||||
if fail_count:
|
||||
parts.append(f"[red]{fail_count} issue(s)[/red]")
|
||||
console.print(" " + ", ".join(parts))
|
||||
console.print()
|
||||
|
||||
if fail_count:
|
||||
console.print(
|
||||
"[red bold]Action required:[/red bold] address critical findings "
|
||||
"before storing sensitive data with OpenJarvis."
|
||||
)
|
||||
console.print()
|
||||
elif warn_count:
|
||||
console.print(
|
||||
"[yellow]Review warnings above[/yellow] — they may not block "
|
||||
"usage but could affect your privacy posture."
|
||||
)
|
||||
console.print()
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--quick", is_flag=True, default=False, help="Run only critical checks.")
|
||||
def scan(quick: bool) -> None:
|
||||
@click.option("--json", "as_json", is_flag=True, default=False, help="Output as JSON.")
|
||||
def scan(quick: bool, as_json: bool) -> None:
|
||||
"""Audit your environment for privacy and security risks."""
|
||||
scanner = PrivacyScanner()
|
||||
results: List[ScanResult] = scanner.run_quick() if quick else scanner.run_all()
|
||||
|
||||
if as_json:
|
||||
import json as json_mod
|
||||
|
||||
output = [
|
||||
{
|
||||
"name": r.name,
|
||||
"status": r.status,
|
||||
"message": r.message,
|
||||
"platform": r.platform,
|
||||
}
|
||||
for r in results
|
||||
]
|
||||
click.echo(json_mod.dumps(output, indent=2))
|
||||
return
|
||||
|
||||
if not results:
|
||||
click.echo("No applicable checks for this platform.")
|
||||
return
|
||||
|
||||
warnings = 0
|
||||
failures = 0
|
||||
for r in results:
|
||||
icon = _STATUS_ICONS.get(r.status, "?")
|
||||
click.echo(f" [{icon}] {r.name}: {r.message}")
|
||||
if r.status == "warn":
|
||||
warnings += 1
|
||||
elif r.status == "fail":
|
||||
failures += 1
|
||||
|
||||
click.echo("")
|
||||
parts = []
|
||||
if warnings:
|
||||
parts.append(f"{warnings} warning(s)")
|
||||
if failures:
|
||||
parts.append(f"{failures} issue(s)")
|
||||
if parts:
|
||||
click.echo("Summary: " + ", ".join(parts) + ".")
|
||||
else:
|
||||
click.echo("Summary: all checks passed.")
|
||||
_render_results(results)
|
||||
|
||||
@@ -181,10 +181,16 @@ def serve(
|
||||
_DEFAULT_TOOLS = {"think", "calculator", "web_search"}
|
||||
configured = config.agent.tools
|
||||
if configured:
|
||||
allowed = {
|
||||
t.strip() for t in configured.split(",")
|
||||
if t.strip()
|
||||
}
|
||||
if isinstance(configured, list):
|
||||
allowed = {
|
||||
t.strip() for t in configured
|
||||
if isinstance(t, str) and t.strip()
|
||||
}
|
||||
else:
|
||||
allowed = {
|
||||
t.strip() for t in configured.split(",")
|
||||
if t.strip()
|
||||
}
|
||||
else:
|
||||
allowed = _DEFAULT_TOOLS
|
||||
|
||||
|
||||
@@ -1157,6 +1157,80 @@ class JarvisConfig:
|
||||
self.tools.storage = value
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config key validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Sections that users may set via ``jarvis config set``.
|
||||
# ``hardware`` is auto-detected and not user-settable.
|
||||
_SETTABLE_SECTIONS = frozenset(JarvisConfig.__dataclass_fields__.keys()) - {"hardware"}
|
||||
|
||||
|
||||
def validate_config_key(dotted_key: str) -> type:
|
||||
"""Validate a dotted config key and return the leaf field's Python type.
|
||||
|
||||
Raises :class:`ValueError` when the key does not map to a known field.
|
||||
The function walks the ``JarvisConfig`` dataclass hierarchy using
|
||||
``dataclasses.fields()``.
|
||||
|
||||
Examples::
|
||||
|
||||
validate_config_key("engine.ollama.host") # -> str
|
||||
validate_config_key("intelligence.temperature") # -> float
|
||||
"""
|
||||
from dataclasses import fields as dc_fields
|
||||
|
||||
parts = dotted_key.split(".")
|
||||
if len(parts) < 2:
|
||||
raise ValueError(
|
||||
f"Config key must have at least two segments (e.g. engine.default), "
|
||||
f"got: {dotted_key!r}"
|
||||
)
|
||||
|
||||
if parts[0] not in _SETTABLE_SECTIONS:
|
||||
raise ValueError(
|
||||
f"Unknown config key: {dotted_key!r} "
|
||||
f"(valid top-level sections: {sorted(_SETTABLE_SECTIONS)})"
|
||||
)
|
||||
|
||||
# Walk the dataclass tree
|
||||
current_cls = JarvisConfig
|
||||
for i, part in enumerate(parts):
|
||||
field_map = {f.name: f for f in dc_fields(current_cls)}
|
||||
if part not in field_map:
|
||||
path_so_far = ".".join(parts[: i + 1])
|
||||
raise ValueError(
|
||||
f"Unknown config key: {dotted_key!r} "
|
||||
f"(no field {part!r} at {path_so_far}; "
|
||||
f"valid fields: {sorted(field_map.keys())})"
|
||||
)
|
||||
fld = field_map[part]
|
||||
# Resolve the type — unwrap Optional, etc.
|
||||
fld_type = fld.type
|
||||
if isinstance(fld_type, str):
|
||||
# Evaluate forward references in the config module namespace
|
||||
import openjarvis.core.config as _cfg_mod
|
||||
|
||||
fld_type = eval(fld_type, vars(_cfg_mod)) # noqa: S307
|
||||
|
||||
if i == len(parts) - 1:
|
||||
# Leaf — return the primitive type
|
||||
return fld_type
|
||||
else:
|
||||
# Must be a nested dataclass
|
||||
if not hasattr(fld_type, "__dataclass_fields__"):
|
||||
path_so_far = ".".join(parts[: i + 1])
|
||||
raise ValueError(
|
||||
f"Unknown config key: {dotted_key!r} "
|
||||
f"({path_so_far} is a leaf of type {fld_type.__name__}, "
|
||||
f"not a section)"
|
||||
)
|
||||
current_cls = fld_type
|
||||
|
||||
# Should not reach here, but satisfy type checker
|
||||
raise ValueError(f"Unknown config key: {dotted_key!r}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TOML loading
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1166,7 +1240,9 @@ def _apply_toml_section(target: Any, section: Dict[str, Any]) -> None:
|
||||
"""Overlay TOML key/value pairs onto a dataclass instance.
|
||||
|
||||
Recursively handles nested dicts when the target attribute is itself
|
||||
a dataclass.
|
||||
a dataclass. Normalises TOML arrays to comma-separated strings — both
|
||||
for dataclass fields annotated as ``str`` and for backward-compat
|
||||
property setters that expect string input.
|
||||
"""
|
||||
for key, value in section.items():
|
||||
if hasattr(target, key):
|
||||
@@ -1177,6 +1253,20 @@ def _apply_toml_section(target: Any, section: Dict[str, Any]) -> None:
|
||||
else:
|
||||
setattr(target, key, value)
|
||||
else:
|
||||
# Normalise TOML arrays → comma-separated string.
|
||||
# Covers both real dataclass fields and backward-compat
|
||||
# property setters (e.g. reward_weights, default_tools).
|
||||
if isinstance(value, list):
|
||||
is_str_field = False
|
||||
if hasattr(target, "__dataclass_fields__"):
|
||||
field_obj = target.__dataclass_fields__.get(key)
|
||||
if field_obj is not None and field_obj.type in ("str", str):
|
||||
is_str_field = True
|
||||
elif field_obj is None:
|
||||
# Property, not a real field — normalise to string
|
||||
is_str_field = True
|
||||
if is_str_field:
|
||||
value = ",".join(str(v) for v in value)
|
||||
setattr(target, key, value)
|
||||
|
||||
|
||||
@@ -1283,7 +1373,9 @@ def load_config(path: Optional[Path] = None) -> JarvisConfig:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_minimal_toml(hw: HardwareInfo, engine: str | None = None) -> str:
|
||||
def generate_minimal_toml(
|
||||
hw: HardwareInfo, engine: str | None = None, *, host: str | None = None
|
||||
) -> str:
|
||||
"""Render a minimal TOML config with only essential settings."""
|
||||
engine = engine or recommend_engine(hw)
|
||||
model = recommend_model(hw, engine)
|
||||
@@ -1291,6 +1383,14 @@ def generate_minimal_toml(hw: HardwareInfo, engine: str | None = None) -> str:
|
||||
if hw.gpu:
|
||||
mem_label = "unified memory" if hw.gpu.vendor == "apple" else "VRAM"
|
||||
gpu_comment = f"\n# GPU: {hw.gpu.name} ({hw.gpu.vram_gb} GB {mem_label})"
|
||||
if host:
|
||||
engine_host_section = f'\n[engine.{engine}]\nhost = "{host}"\n'
|
||||
else:
|
||||
engine_host_section = (
|
||||
f"\n[engine.{engine}]\n"
|
||||
f'# host = "http://localhost:11434" '
|
||||
f"# set to remote URL if engine runs elsewhere\n"
|
||||
)
|
||||
return f"""\
|
||||
# OpenJarvis configuration
|
||||
# Hardware: {hw.cpu_brand} ({hw.cpu_count} cores, {hw.ram_gb} GB RAM){gpu_comment}
|
||||
@@ -1298,7 +1398,7 @@ def generate_minimal_toml(hw: HardwareInfo, engine: str | None = None) -> str:
|
||||
|
||||
[engine]
|
||||
default = "{engine}"
|
||||
|
||||
{engine_host_section}
|
||||
[intelligence]
|
||||
default_model = "{model}"
|
||||
|
||||
@@ -1310,7 +1410,9 @@ enabled = ["code_interpreter", "web_search", "file_read", "shell_exec"]
|
||||
"""
|
||||
|
||||
|
||||
def generate_default_toml(hw: HardwareInfo, engine: str | None = None) -> str:
|
||||
def generate_default_toml(
|
||||
hw: HardwareInfo, engine: str | None = None, *, host: str | None = None
|
||||
) -> str:
|
||||
"""Render a commented TOML string suitable for ``~/.openjarvis/config.toml``."""
|
||||
engine = engine or recommend_engine(hw)
|
||||
model = recommend_model(hw, engine)
|
||||
@@ -1322,7 +1424,7 @@ def generate_default_toml(hw: HardwareInfo, engine: str | None = None) -> str:
|
||||
if model:
|
||||
model_comment = " # recommended for your hardware"
|
||||
|
||||
return f"""\
|
||||
result = f"""\
|
||||
# OpenJarvis configuration
|
||||
# Generated by `jarvis init`
|
||||
#
|
||||
@@ -1517,6 +1619,13 @@ ssrf_protection = true
|
||||
# assistant_name = "Jarvis"
|
||||
# assistant_has_own_number = false
|
||||
"""
|
||||
if host:
|
||||
import re as _re
|
||||
|
||||
pattern = _re.escape(f"[engine.{engine}]") + r"\nhost = \"[^\"]*\""
|
||||
replacement = f'[engine.{engine}]\\nhost = "{host}"'
|
||||
result = _re.sub(pattern, replacement, result)
|
||||
return result
|
||||
|
||||
|
||||
__all__ = [
|
||||
@@ -1581,4 +1690,5 @@ __all__ = [
|
||||
"load_config",
|
||||
"recommend_engine",
|
||||
"recommend_model",
|
||||
"validate_config_key",
|
||||
]
|
||||
|
||||
@@ -16,6 +16,7 @@ from openjarvis.engine._base import (
|
||||
estimate_prompt_tokens,
|
||||
messages_to_dicts,
|
||||
)
|
||||
from openjarvis.engine._stubs import StreamChunk
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -50,6 +51,9 @@ class _OpenAICompatibleEngine(InferenceEngine):
|
||||
"stream": False,
|
||||
**kwargs,
|
||||
}
|
||||
# Default to tool_choice=auto when tools are provided
|
||||
if "tools" in payload and "tool_choice" not in payload:
|
||||
payload["tool_choice"] = "auto"
|
||||
try:
|
||||
url = f"{self._api_prefix}/chat/completions"
|
||||
resp = self._client.post(url, json=payload)
|
||||
@@ -122,6 +126,9 @@ class _OpenAICompatibleEngine(InferenceEngine):
|
||||
"stream": True,
|
||||
**kwargs,
|
||||
}
|
||||
# Default to tool_choice=auto when tools are provided
|
||||
if "tools" in payload and "tool_choice" not in payload:
|
||||
payload["tool_choice"] = "auto"
|
||||
try:
|
||||
url = f"{self._api_prefix}/chat/completions"
|
||||
with self._client.stream("POST", url, json=payload) as resp:
|
||||
@@ -129,7 +136,7 @@ class _OpenAICompatibleEngine(InferenceEngine):
|
||||
for line in resp.iter_lines():
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data_str = line[len("data:"):].strip()
|
||||
data_str = line[len("data:") :].strip()
|
||||
if data_str == "[DONE]":
|
||||
break
|
||||
try:
|
||||
@@ -145,16 +152,74 @@ class _OpenAICompatibleEngine(InferenceEngine):
|
||||
f"{self.engine_id} engine not reachable at {self._host}"
|
||||
) from exc
|
||||
|
||||
async def stream_full(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: int = 1024,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator["StreamChunk"]:
|
||||
"""Yield StreamChunks with content, tool_calls, and finish_reason."""
|
||||
msg_dicts = messages_to_dicts(messages)
|
||||
payload: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": msg_dicts,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
"stream": True,
|
||||
**kwargs,
|
||||
}
|
||||
if "tools" in payload and "tool_choice" not in payload:
|
||||
payload["tool_choice"] = "auto"
|
||||
try:
|
||||
url = f"{self._api_prefix}/chat/completions"
|
||||
with self._client.stream("POST", url, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
for line in resp.iter_lines():
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data_str = line[len("data:") :].strip()
|
||||
if data_str == "[DONE]":
|
||||
break
|
||||
try:
|
||||
chunk = json.loads(data_str)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
choice = chunk.get("choices", [{}])[0]
|
||||
delta = choice.get("delta", {})
|
||||
finish = choice.get("finish_reason")
|
||||
content = delta.get("content")
|
||||
tool_calls = delta.get("tool_calls")
|
||||
usage = chunk.get("usage")
|
||||
|
||||
if content or tool_calls or finish or usage:
|
||||
yield StreamChunk(
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
finish_reason=finish,
|
||||
usage=usage,
|
||||
)
|
||||
except (httpx.ConnectError, httpx.TimeoutException) as exc:
|
||||
raise EngineConnectionError(
|
||||
f"{self.engine_id} engine not reachable at {self._host}"
|
||||
) from exc
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
try:
|
||||
resp = self._client.get(f"{self._api_prefix}/models")
|
||||
resp.raise_for_status()
|
||||
except (
|
||||
httpx.ConnectError, httpx.TimeoutException, httpx.HTTPStatusError,
|
||||
httpx.ConnectError,
|
||||
httpx.TimeoutException,
|
||||
httpx.HTTPStatusError,
|
||||
) as exc:
|
||||
logger.warning(
|
||||
"Failed to list models from %s at %s: %s",
|
||||
self.engine_id, self._host, exc,
|
||||
self.engine_id,
|
||||
self._host,
|
||||
exc,
|
||||
)
|
||||
return []
|
||||
data = resp.json()
|
||||
@@ -167,7 +232,9 @@ class _OpenAICompatibleEngine(InferenceEngine):
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"%s health check failed at %s: %s",
|
||||
self.engine_id, self._host, exc,
|
||||
self.engine_id,
|
||||
self._host,
|
||||
exc,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -14,6 +14,21 @@ from typing import Any, Dict, List, Optional, Sequence
|
||||
from openjarvis.core.types import Message
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class StreamChunk:
|
||||
"""A single chunk from a streaming LLM response.
|
||||
|
||||
Used by ``stream_full()`` to yield rich streaming data including
|
||||
tool_calls fragments and finish_reason, unlike ``stream()`` which
|
||||
only yields plain content strings.
|
||||
"""
|
||||
|
||||
content: Optional[str] = None
|
||||
tool_calls: Optional[List[Dict[str, Any]]] = None
|
||||
finish_reason: Optional[str] = None
|
||||
usage: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ResponseFormat:
|
||||
"""Structured output configuration for inference engines.
|
||||
@@ -65,6 +80,30 @@ class InferenceEngine(ABC):
|
||||
# NOTE: must contain a yield to satisfy the type checker
|
||||
yield "" # pragma: no cover
|
||||
|
||||
async def stream_full(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: int = 1024,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator["StreamChunk"]:
|
||||
"""Yield full StreamChunks including tool_calls and finish_reason.
|
||||
|
||||
Default implementation wraps ``stream()`` for backward compatibility.
|
||||
Engines with native tool-call streaming should override this.
|
||||
"""
|
||||
async for token in self.stream(
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
yield StreamChunk(content=token)
|
||||
yield StreamChunk(finish_reason="stop")
|
||||
|
||||
@abstractmethod
|
||||
def list_models(self) -> List[str]:
|
||||
"""Return identifiers of models available on this engine."""
|
||||
@@ -80,4 +119,4 @@ class InferenceEngine(ABC):
|
||||
"""Optional warm-up hook called before the first request."""
|
||||
|
||||
|
||||
__all__ = ["InferenceEngine", "ResponseFormat"]
|
||||
__all__ = ["InferenceEngine", "ResponseFormat", "StreamChunk"]
|
||||
|
||||
+497
-101
@@ -6,7 +6,9 @@ import json
|
||||
import os
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
from openjarvis.core.registry import EngineRegistry
|
||||
from openjarvis.core.types import Message
|
||||
@@ -15,6 +17,7 @@ from openjarvis.engine._base import (
|
||||
InferenceEngine,
|
||||
messages_to_dicts,
|
||||
)
|
||||
from openjarvis.engine._stubs import StreamChunk
|
||||
|
||||
# Pricing per million tokens (input, output)
|
||||
PRICING: Dict[str, tuple[float, float]] = {
|
||||
@@ -46,7 +49,12 @@ PRICING: Dict[str, tuple[float, float]] = {
|
||||
|
||||
# Well-known model IDs per provider
|
||||
_OPENAI_MODELS = [
|
||||
"gpt-4o", "gpt-4o-mini", "gpt-5", "gpt-5.4", "gpt-5-mini", "o3-mini",
|
||||
"gpt-4o",
|
||||
"gpt-4o-mini",
|
||||
"gpt-5",
|
||||
"gpt-5.4",
|
||||
"gpt-5-mini",
|
||||
"o3-mini",
|
||||
]
|
||||
_ANTHROPIC_MODELS = [
|
||||
"claude-sonnet-4-20250514",
|
||||
@@ -85,6 +93,16 @@ _OPENROUTER_POPULAR = [
|
||||
"openrouter/qwen/qwen3-235b-a22b",
|
||||
]
|
||||
|
||||
# Codex models — prefixed with "codex/" for ChatGPT Plus/Pro subscribers.
|
||||
# Uses the Responses API at chatgpt.com, not the standard OpenAI API.
|
||||
_CODEX_MODELS = [
|
||||
"codex/gpt-4o",
|
||||
"codex/gpt-4o-mini",
|
||||
"codex/o3-mini",
|
||||
"codex/gpt-5-mini",
|
||||
"codex/gpt-5-mini-2025-08-07",
|
||||
]
|
||||
|
||||
|
||||
def _is_minimax_model(model: str) -> bool:
|
||||
return model.lower().startswith("minimax")
|
||||
@@ -94,6 +112,10 @@ def _is_openrouter_model(model: str) -> bool:
|
||||
return model.startswith("openrouter/")
|
||||
|
||||
|
||||
def _is_codex_model(model: str) -> bool:
|
||||
return model.startswith("codex/")
|
||||
|
||||
|
||||
def _is_anthropic_model(model: str) -> bool:
|
||||
return "claude" in model.lower() and not _is_openrouter_model(model)
|
||||
|
||||
@@ -159,11 +181,13 @@ def _convert_tools_to_anthropic(
|
||||
result = []
|
||||
for tool in openai_tools:
|
||||
func = tool.get("function", {})
|
||||
result.append({
|
||||
"name": func.get("name", ""),
|
||||
"description": func.get("description", ""),
|
||||
"input_schema": func.get("parameters", {}),
|
||||
})
|
||||
result.append(
|
||||
{
|
||||
"name": func.get("name", ""),
|
||||
"description": func.get("description", ""),
|
||||
"input_schema": func.get("parameters", {}),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@@ -174,11 +198,13 @@ def _convert_tools_to_google(
|
||||
declarations = []
|
||||
for tool in openai_tools:
|
||||
func = tool.get("function", {})
|
||||
declarations.append({
|
||||
"name": func.get("name", ""),
|
||||
"description": func.get("description", ""),
|
||||
"parameters": func.get("parameters", {}),
|
||||
})
|
||||
declarations.append(
|
||||
{
|
||||
"name": func.get("name", ""),
|
||||
"description": func.get("description", ""),
|
||||
"parameters": func.get("parameters", {}),
|
||||
}
|
||||
)
|
||||
return declarations
|
||||
|
||||
|
||||
@@ -194,28 +220,33 @@ class CloudEngine(InferenceEngine):
|
||||
self._google_client: Any = None
|
||||
self._openrouter_client: Any = None
|
||||
self._minimax_client: Any = None
|
||||
self._codex_client: Any = None
|
||||
# Gemini thought_signatures: tool_call_id -> signature bytes
|
||||
self._thought_sigs: Dict[str, bytes] = {}
|
||||
self._init_clients()
|
||||
|
||||
def _init_clients(self) -> None:
|
||||
if os.environ.get("OPENAI_API_KEY"):
|
||||
try:
|
||||
import openai
|
||||
|
||||
self._openai_client = openai.OpenAI()
|
||||
except ImportError:
|
||||
pass
|
||||
if os.environ.get("ANTHROPIC_API_KEY"):
|
||||
try:
|
||||
import anthropic
|
||||
|
||||
self._anthropic_client = anthropic.Anthropic()
|
||||
except ImportError:
|
||||
pass
|
||||
gemini_key = (
|
||||
os.environ.get("GEMINI_API_KEY")
|
||||
or os.environ.get("GOOGLE_API_KEY")
|
||||
gemini_key = os.environ.get("GEMINI_API_KEY") or os.environ.get(
|
||||
"GOOGLE_API_KEY"
|
||||
)
|
||||
if gemini_key:
|
||||
try:
|
||||
from google import genai
|
||||
|
||||
self._google_client = genai.Client(api_key=gemini_key)
|
||||
except ImportError:
|
||||
pass
|
||||
@@ -223,6 +254,7 @@ class CloudEngine(InferenceEngine):
|
||||
if openrouter_key:
|
||||
try:
|
||||
import openai
|
||||
|
||||
self._openrouter_client = openai.OpenAI(
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
api_key=openrouter_key,
|
||||
@@ -233,12 +265,180 @@ class CloudEngine(InferenceEngine):
|
||||
if minimax_key:
|
||||
try:
|
||||
import openai
|
||||
|
||||
self._minimax_client = openai.OpenAI(
|
||||
base_url="https://api.minimax.io/v1",
|
||||
api_key=minimax_key,
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
# Codex — uses the OpenAI Responses API.
|
||||
# Supports both standard API keys (api.openai.com) and ChatGPT
|
||||
# OAuth tokens (chatgpt.com) via OPENAI_CODEX_BASE_URL override.
|
||||
codex_token = os.environ.get("OPENAI_CODEX_API_KEY")
|
||||
if codex_token:
|
||||
codex_url = os.environ.get(
|
||||
"OPENAI_CODEX_BASE_URL",
|
||||
"https://api.openai.com/v1",
|
||||
).rstrip("/")
|
||||
if not codex_url.endswith("/responses"):
|
||||
codex_url += "/responses"
|
||||
self._codex_client = {
|
||||
"token": codex_token,
|
||||
"url": codex_url,
|
||||
}
|
||||
|
||||
def _prepare_anthropic_messages(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
) -> Tuple[str, List[Dict[str, Any]]]:
|
||||
"""Extract system text and convert messages to Anthropic format."""
|
||||
system_text = ""
|
||||
chat_msgs: List[Dict[str, Any]] = []
|
||||
for m in messages:
|
||||
if m.role.value == "system":
|
||||
system_text = m.content
|
||||
elif m.role.value == "tool":
|
||||
tool_result_block = {
|
||||
"type": "tool_result",
|
||||
"tool_use_id": m.tool_call_id or "",
|
||||
"content": m.content,
|
||||
}
|
||||
if (
|
||||
chat_msgs
|
||||
and chat_msgs[-1]["role"] == "user"
|
||||
and isinstance(chat_msgs[-1]["content"], list)
|
||||
and chat_msgs[-1]["content"]
|
||||
and chat_msgs[-1]["content"][-1].get("type") == "tool_result"
|
||||
):
|
||||
chat_msgs[-1]["content"].append(tool_result_block)
|
||||
else:
|
||||
chat_msgs.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [tool_result_block],
|
||||
}
|
||||
)
|
||||
elif m.role.value == "assistant" and m.tool_calls:
|
||||
content_blocks: List[Dict[str, Any]] = []
|
||||
if m.content:
|
||||
content_blocks.append({"type": "text", "text": m.content})
|
||||
for tc in m.tool_calls:
|
||||
args = tc.arguments
|
||||
if isinstance(args, str):
|
||||
try:
|
||||
args = json.loads(args)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
args = {"input": args}
|
||||
content_blocks.append(
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": tc.id,
|
||||
"name": tc.name,
|
||||
"input": args if isinstance(args, dict) else {},
|
||||
}
|
||||
)
|
||||
chat_msgs.append({"role": "assistant", "content": content_blocks})
|
||||
else:
|
||||
chat_msgs.append({"role": m.role.value, "content": m.content})
|
||||
return system_text, chat_msgs
|
||||
|
||||
@staticmethod
|
||||
def _codex_build_input(
|
||||
messages: Sequence[Message],
|
||||
) -> tuple[str, List[Dict[str, Any]]]:
|
||||
"""Convert Message list to Codex Responses API format.
|
||||
|
||||
Returns (system_instructions, input_messages).
|
||||
"""
|
||||
instructions = ""
|
||||
input_msgs: List[Dict[str, Any]] = []
|
||||
for m in messages:
|
||||
if m.role.value == "system":
|
||||
instructions = m.content
|
||||
elif m.role.value in ("user", "assistant"):
|
||||
input_msgs.append(
|
||||
{
|
||||
"role": m.role.value,
|
||||
"content": [{"type": "input_text", "text": m.content}],
|
||||
}
|
||||
)
|
||||
return instructions, input_msgs
|
||||
|
||||
def _generate_codex(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Generate via Codex Responses API (ChatGPT Plus/Pro)."""
|
||||
if self._codex_client is None:
|
||||
raise EngineConnectionError(
|
||||
"Codex client not available — set OPENAI_CODEX_API_KEY"
|
||||
)
|
||||
actual_model = model.removeprefix("codex/")
|
||||
instructions, input_msgs = self._codex_build_input(messages)
|
||||
|
||||
body: Dict[str, Any] = {
|
||||
"model": actual_model,
|
||||
"input": input_msgs,
|
||||
"store": False,
|
||||
"stream": False,
|
||||
}
|
||||
if instructions:
|
||||
body["instructions"] = instructions
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._codex_client['token']}",
|
||||
"Content-Type": "application/json",
|
||||
"OpenAI-Beta": "responses=experimental",
|
||||
}
|
||||
|
||||
t0 = time.monotonic()
|
||||
resp = httpx.post(
|
||||
self._codex_client["url"],
|
||||
json=body,
|
||||
headers=headers,
|
||||
timeout=120.0,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# Extract text from Responses API output.
|
||||
# The output array contains items of type "reasoning" and "message";
|
||||
# we want the "message" item's content blocks.
|
||||
content = data.get("output_text", "")
|
||||
if not content:
|
||||
for item in data.get("output", []):
|
||||
if item.get("type") not in ("message", None):
|
||||
continue
|
||||
for block in item.get("content", []):
|
||||
if block.get("type") == "output_text":
|
||||
content = block.get("text", "")
|
||||
break
|
||||
if content:
|
||||
break
|
||||
|
||||
usage_data = data.get("usage", {})
|
||||
prompt_tokens = usage_data.get("input_tokens", 0)
|
||||
completion_tokens = usage_data.get("output_tokens", 0)
|
||||
|
||||
return {
|
||||
"content": content,
|
||||
"usage": {
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens,
|
||||
},
|
||||
"model": actual_model,
|
||||
"finish_reason": "stop",
|
||||
"cost_usd": 0.0,
|
||||
"ttft": elapsed,
|
||||
}
|
||||
|
||||
def _generate_openai(
|
||||
self,
|
||||
@@ -333,56 +533,7 @@ class CloudEngine(InferenceEngine):
|
||||
"ANTHROPIC_API_KEY and install "
|
||||
"openjarvis[inference-cloud]"
|
||||
)
|
||||
# Separate system message and convert to Anthropic message format
|
||||
system_text = ""
|
||||
chat_msgs: List[Dict[str, Any]] = []
|
||||
for m in messages:
|
||||
if m.role.value == "system":
|
||||
system_text = m.content
|
||||
elif m.role.value == "tool":
|
||||
# Anthropic expects tool results as role="user" with
|
||||
# tool_result content blocks
|
||||
tool_result_block = {
|
||||
"type": "tool_result",
|
||||
"tool_use_id": m.tool_call_id or "",
|
||||
"content": m.content,
|
||||
}
|
||||
# Merge consecutive tool results into a single user message
|
||||
if (
|
||||
chat_msgs
|
||||
and chat_msgs[-1]["role"] == "user"
|
||||
and isinstance(chat_msgs[-1]["content"], list)
|
||||
and chat_msgs[-1]["content"]
|
||||
and chat_msgs[-1]["content"][-1].get("type") == "tool_result"
|
||||
):
|
||||
chat_msgs[-1]["content"].append(tool_result_block)
|
||||
else:
|
||||
chat_msgs.append({
|
||||
"role": "user",
|
||||
"content": [tool_result_block],
|
||||
})
|
||||
elif m.role.value == "assistant" and m.tool_calls:
|
||||
# Convert assistant messages with tool_calls to Anthropic
|
||||
# content blocks (text + tool_use)
|
||||
content_blocks: List[Dict[str, Any]] = []
|
||||
if m.content:
|
||||
content_blocks.append({"type": "text", "text": m.content})
|
||||
for tc in m.tool_calls:
|
||||
args = tc.arguments
|
||||
if isinstance(args, str):
|
||||
try:
|
||||
args = json.loads(args)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
args = {"input": args}
|
||||
content_blocks.append({
|
||||
"type": "tool_use",
|
||||
"id": tc.id,
|
||||
"name": tc.name,
|
||||
"input": args if isinstance(args, dict) else {},
|
||||
})
|
||||
chat_msgs.append({"role": "assistant", "content": content_blocks})
|
||||
else:
|
||||
chat_msgs.append({"role": m.role.value, "content": m.content})
|
||||
system_text, chat_msgs = self._prepare_anthropic_messages(messages)
|
||||
create_kwargs: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": chat_msgs,
|
||||
@@ -426,13 +577,15 @@ class CloudEngine(InferenceEngine):
|
||||
tool_calls: list[Dict[str, Any]] = []
|
||||
for block in resp.content:
|
||||
if getattr(block, "type", None) == "tool_use":
|
||||
tool_calls.append({
|
||||
"id": block.id,
|
||||
"name": block.name,
|
||||
"arguments": json.dumps(block.input)
|
||||
if isinstance(block.input, dict)
|
||||
else str(block.input),
|
||||
})
|
||||
tool_calls.append(
|
||||
{
|
||||
"id": block.id,
|
||||
"name": block.name,
|
||||
"arguments": json.dumps(block.input)
|
||||
if isinstance(block.input, dict)
|
||||
else str(block.input),
|
||||
}
|
||||
)
|
||||
elif hasattr(block, "text"):
|
||||
content_parts.append(block.text)
|
||||
|
||||
@@ -510,12 +663,17 @@ class CloudEngine(InferenceEngine):
|
||||
args = json.loads(args)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
args = {"input": args}
|
||||
parts.append({
|
||||
fc_part: Dict[str, Any] = {
|
||||
"function_call": {
|
||||
"name": tc.name,
|
||||
"args": args if isinstance(args, dict) else {},
|
||||
}
|
||||
})
|
||||
}
|
||||
# Replay thought_signature for Gemini reasoning models
|
||||
sig = self._thought_sigs.get(tc.id)
|
||||
if sig is not None:
|
||||
fc_part["thought_signature"] = sig
|
||||
parts.append(fc_part)
|
||||
contents.append({"role": "model", "parts": parts})
|
||||
elif m.role.value == "assistant":
|
||||
contents.append({"role": "model", "parts": [{"text": m.content}]})
|
||||
@@ -564,14 +722,18 @@ class CloudEngine(InferenceEngine):
|
||||
for part in parts:
|
||||
if hasattr(part, "function_call") and part.function_call:
|
||||
fc = part.function_call
|
||||
fc_args = (
|
||||
dict(fc.args) if hasattr(fc.args, "items") else {}
|
||||
)
|
||||
tool_calls.append({
|
||||
fc_args = dict(fc.args) if hasattr(fc.args, "items") else {}
|
||||
tc_dict: Dict[str, Any] = {
|
||||
"id": f"google_{fc.name}",
|
||||
"name": fc.name,
|
||||
"arguments": json.dumps(fc_args),
|
||||
})
|
||||
}
|
||||
# Preserve thought_signature for Gemini reasoning models
|
||||
sig = getattr(part, "thought_signature", None)
|
||||
if sig is not None:
|
||||
tc_dict["thought_signature"] = sig
|
||||
self._thought_sigs[tc_dict["id"]] = sig
|
||||
tool_calls.append(tc_dict)
|
||||
elif hasattr(part, "text") and part.text:
|
||||
text_parts.append(part.text)
|
||||
|
||||
@@ -585,12 +747,8 @@ class CloudEngine(InferenceEngine):
|
||||
content = ""
|
||||
|
||||
um = resp.usage_metadata
|
||||
prompt_tokens = (
|
||||
getattr(um, "prompt_token_count", 0) if um else 0
|
||||
)
|
||||
completion_tokens = (
|
||||
getattr(um, "candidates_token_count", 0) if um else 0
|
||||
)
|
||||
prompt_tokens = getattr(um, "prompt_token_count", 0) if um else 0
|
||||
completion_tokens = getattr(um, "candidates_token_count", 0) if um else 0
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"content": content,
|
||||
@@ -719,6 +877,8 @@ class CloudEngine(InferenceEngine):
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
if _is_codex_model(model):
|
||||
return self._generate_codex(messages, **kw)
|
||||
if _is_openrouter_model(model):
|
||||
return self._generate_openrouter(messages, **kw)
|
||||
if _is_minimax_model(model):
|
||||
@@ -744,32 +904,81 @@ class CloudEngine(InferenceEngine):
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
if _is_openrouter_model(model):
|
||||
async for token in self._stream_openrouter(
|
||||
messages, **kw
|
||||
):
|
||||
if _is_codex_model(model):
|
||||
async for token in self._stream_codex(messages, **kw):
|
||||
yield token
|
||||
elif _is_openrouter_model(model):
|
||||
async for token in self._stream_openrouter(messages, **kw):
|
||||
yield token
|
||||
elif _is_minimax_model(model):
|
||||
async for token in self._stream_minimax(
|
||||
messages, **kw
|
||||
):
|
||||
async for token in self._stream_minimax(messages, **kw):
|
||||
yield token
|
||||
elif _is_anthropic_model(model):
|
||||
async for token in self._stream_anthropic(
|
||||
messages, **kw
|
||||
):
|
||||
async for token in self._stream_anthropic(messages, **kw):
|
||||
yield token
|
||||
elif _is_google_model(model):
|
||||
async for token in self._stream_google(
|
||||
messages, **kw
|
||||
):
|
||||
async for token in self._stream_google(messages, **kw):
|
||||
yield token
|
||||
else:
|
||||
async for token in self._stream_openai(
|
||||
messages, **kw
|
||||
):
|
||||
async for token in self._stream_openai(messages, **kw):
|
||||
yield token
|
||||
|
||||
async def _stream_codex(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream via Codex Responses API (SSE)."""
|
||||
if self._codex_client is None:
|
||||
raise EngineConnectionError("Codex client not available")
|
||||
actual_model = model.removeprefix("codex/")
|
||||
instructions, input_msgs = self._codex_build_input(messages)
|
||||
|
||||
body: Dict[str, Any] = {
|
||||
"model": actual_model,
|
||||
"input": input_msgs,
|
||||
"store": False,
|
||||
"stream": True,
|
||||
}
|
||||
if instructions:
|
||||
body["instructions"] = instructions
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._codex_client['token']}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream",
|
||||
"OpenAI-Beta": "responses=experimental",
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
self._codex_client["url"],
|
||||
json=body,
|
||||
headers=headers,
|
||||
timeout=120.0,
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload = line[6:]
|
||||
if payload == "[DONE]":
|
||||
break
|
||||
try:
|
||||
event = json.loads(payload)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
etype = event.get("type", "")
|
||||
if etype == "response.output_text.delta":
|
||||
delta = event.get("delta", "")
|
||||
if delta:
|
||||
yield delta
|
||||
|
||||
async def _stream_openai(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
@@ -915,6 +1124,188 @@ class CloudEngine(InferenceEngine):
|
||||
if delta and delta.content:
|
||||
yield delta.content
|
||||
|
||||
# -- stream_full: rich streaming with tool_calls support ----------------
|
||||
|
||||
async def _stream_full_openai(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator[StreamChunk]:
|
||||
"""Yield StreamChunks from an OpenAI-compatible streaming response.
|
||||
|
||||
Works for OpenAI, OpenRouter, MiniMax, and Codex.
|
||||
"""
|
||||
if _is_codex_model(model):
|
||||
# Codex uses Responses API — fall back to base stream_full wrapper
|
||||
async for chunk in super().stream_full(
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
yield chunk
|
||||
return
|
||||
if _is_openrouter_model(model):
|
||||
client = self._openrouter_client
|
||||
if client is None:
|
||||
raise EngineConnectionError("OpenRouter client not available")
|
||||
actual_model = model.removeprefix("openrouter/")
|
||||
create_kwargs: Dict[str, Any] = {
|
||||
"model": actual_model,
|
||||
"messages": messages_to_dicts(messages),
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
"stream": True,
|
||||
**kwargs,
|
||||
}
|
||||
elif _is_minimax_model(model):
|
||||
client = self._minimax_client
|
||||
if client is None:
|
||||
raise EngineConnectionError("MiniMax client not available")
|
||||
temperature = max(temperature, 0.01)
|
||||
temperature = min(temperature, 1.0)
|
||||
create_kwargs = {
|
||||
"model": model,
|
||||
"messages": messages_to_dicts(messages),
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
"stream": True,
|
||||
**kwargs,
|
||||
}
|
||||
else:
|
||||
client = self._openai_client
|
||||
if client is None:
|
||||
raise EngineConnectionError("OpenAI client not available")
|
||||
create_kwargs = {
|
||||
"model": model,
|
||||
"messages": messages_to_dicts(messages),
|
||||
"max_completion_tokens": max_tokens,
|
||||
"stream": True,
|
||||
**kwargs,
|
||||
}
|
||||
if not _is_openai_reasoning_model(model):
|
||||
create_kwargs["temperature"] = temperature
|
||||
resp = client.chat.completions.create(**create_kwargs)
|
||||
for chunk in resp:
|
||||
choice = chunk.choices[0] if chunk.choices else None
|
||||
if not choice:
|
||||
continue
|
||||
delta = choice.delta
|
||||
content = delta.content if delta else None
|
||||
tool_calls = None
|
||||
if delta and delta.tool_calls:
|
||||
tool_calls = [
|
||||
{
|
||||
"index": tc.index,
|
||||
"id": tc.id or "",
|
||||
"function": {
|
||||
"name": (tc.function.name or "") if tc.function else "",
|
||||
"arguments": (
|
||||
(tc.function.arguments or "") if tc.function else ""
|
||||
),
|
||||
},
|
||||
}
|
||||
for tc in delta.tool_calls
|
||||
]
|
||||
finish = choice.finish_reason
|
||||
if content or tool_calls or finish:
|
||||
yield StreamChunk(
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
finish_reason=finish,
|
||||
)
|
||||
|
||||
async def _stream_full_anthropic(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator[StreamChunk]:
|
||||
"""Yield StreamChunks from an Anthropic streaming response."""
|
||||
if self._anthropic_client is None:
|
||||
raise EngineConnectionError("Anthropic client not available")
|
||||
system_text, chat_msgs = self._prepare_anthropic_messages(messages)
|
||||
create_kwargs: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": chat_msgs,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if system_text:
|
||||
create_kwargs["system"] = system_text
|
||||
raw_tools = kwargs.pop("tools", None)
|
||||
if raw_tools:
|
||||
create_kwargs["tools"] = _convert_tools_to_anthropic(raw_tools)
|
||||
kwargs.pop("tool_choice", None)
|
||||
|
||||
with self._anthropic_client.messages.stream(**create_kwargs) as stream:
|
||||
tool_index = -1
|
||||
for event in stream:
|
||||
if event.type == "content_block_start":
|
||||
block = event.content_block
|
||||
if block.type == "tool_use":
|
||||
tool_index += 1
|
||||
yield StreamChunk(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": tool_index,
|
||||
"id": block.id,
|
||||
"function": {"name": block.name, "arguments": ""},
|
||||
}
|
||||
]
|
||||
)
|
||||
elif event.type == "content_block_delta":
|
||||
delta = event.delta
|
||||
if delta.type == "text_delta":
|
||||
yield StreamChunk(content=delta.text)
|
||||
elif delta.type == "input_json_delta":
|
||||
yield StreamChunk(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": tool_index,
|
||||
"function": {"arguments": delta.partial_json},
|
||||
}
|
||||
]
|
||||
)
|
||||
elif event.type == "message_delta":
|
||||
stop_reason = event.delta.stop_reason
|
||||
finish = "tool_calls" if stop_reason == "tool_use" else "stop"
|
||||
yield StreamChunk(finish_reason=finish)
|
||||
|
||||
async def stream_full(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: int = 1024,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator[StreamChunk]:
|
||||
"""Yield StreamChunks with content, tool_calls, and finish_reason."""
|
||||
kw = dict(
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
if _is_anthropic_model(model):
|
||||
async for chunk in self._stream_full_anthropic(messages, **kw):
|
||||
yield chunk
|
||||
elif _is_google_model(model):
|
||||
async for chunk in super().stream_full(messages, **kw):
|
||||
yield chunk
|
||||
else:
|
||||
async for chunk in self._stream_full_openai(messages, **kw):
|
||||
yield chunk
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
models: List[str] = []
|
||||
if self._openai_client is not None:
|
||||
@@ -927,6 +1318,8 @@ class CloudEngine(InferenceEngine):
|
||||
models.extend(_OPENROUTER_POPULAR)
|
||||
if self._minimax_client is not None:
|
||||
models.extend(_MINIMAX_MODELS)
|
||||
if self._codex_client is not None:
|
||||
models.extend(_CODEX_MODELS)
|
||||
return models
|
||||
|
||||
def health(self) -> bool:
|
||||
@@ -936,6 +1329,7 @@ class CloudEngine(InferenceEngine):
|
||||
or self._google_client is not None
|
||||
or self._openrouter_client is not None
|
||||
or self._minimax_client is not None
|
||||
or self._codex_client is not None
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
@@ -957,6 +1351,8 @@ class CloudEngine(InferenceEngine):
|
||||
if hasattr(self._minimax_client, "close"):
|
||||
self._minimax_client.close()
|
||||
self._minimax_client = None
|
||||
if self._codex_client is not None:
|
||||
self._codex_client = None
|
||||
|
||||
|
||||
__all__ = ["CloudEngine", "PRICING", "_annotate_anthropic_cache", "estimate_cost"]
|
||||
|
||||
@@ -6,7 +6,9 @@ import logging
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from openjarvis.core.types import Message
|
||||
from openjarvis.engine._base import InferenceEngine
|
||||
from openjarvis.engine._stubs import StreamChunk
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -65,7 +67,7 @@ class MultiEngine(InferenceEngine):
|
||||
|
||||
def generate(
|
||||
self,
|
||||
messages: Sequence[Any],
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float = 0.7,
|
||||
@@ -73,13 +75,16 @@ class MultiEngine(InferenceEngine):
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
return self._engine_for(model).generate(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def stream(
|
||||
self,
|
||||
messages: Sequence[Any],
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float = 0.7,
|
||||
@@ -87,11 +92,26 @@ class MultiEngine(InferenceEngine):
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator[str]:
|
||||
async for token in self._engine_for(model).stream(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
yield token
|
||||
|
||||
async def stream_full(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator["StreamChunk"]:
|
||||
"""Delegate stream_full() to the engine that owns the model."""
|
||||
engine = self._engine_for(model)
|
||||
async for chunk in engine.stream_full(messages, model=model, **kwargs):
|
||||
yield chunk
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
self._refresh_map()
|
||||
return list(self._model_map.keys())
|
||||
|
||||
@@ -75,6 +75,13 @@ class OllamaEngine(InferenceEngine):
|
||||
"num_ctx": kwargs.get("num_ctx", 8192),
|
||||
},
|
||||
}
|
||||
# Disable extended thinking by default (Qwen3.5 etc.).
|
||||
# When enabled, thinking tokens consume the entire budget and
|
||||
# the visible content comes back empty.
|
||||
if "think" not in kwargs:
|
||||
payload["think"] = False
|
||||
elif kwargs["think"] is not None:
|
||||
payload["think"] = kwargs["think"]
|
||||
# Pass tools if provided
|
||||
tools = kwargs.get("tools")
|
||||
if tools:
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# PinchBench eval: Claude Opus 4.6 (cloud)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-claude-opus"
|
||||
description = "PinchBench on claude-opus-4-6"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/pinchbench-claude-opus/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "claude-opus-4-6"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
# PinchBench eval: Gemini 3.1 Pro Preview (cloud)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-gemini-31-pro"
|
||||
description = "PinchBench on gemini-3.1-pro-preview"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/pinchbench-gemini-31-pro/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "gemini-3.1-pro-preview"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
# PinchBench eval: gpt-5.4 (cloud)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-gpt54"
|
||||
description = "PinchBench on gpt-5.4-2026-03-05"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/pinchbench-gpt54/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "gpt-5.4-2026-03-05"
|
||||
engine = "cloud"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: NVIDIA-Nemotron-3-Super-120B-A12B-FP8 (vLLM, TP=4)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-nemotron3-super"
|
||||
description = "PinchBench on NVIDIA-Nemotron-3-Super-120B-A12B-FP8 (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/pinchbench-nemotron3-super/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "nemotron"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-qwen122b"
|
||||
description = "PinchBench on Qwen/Qwen3.5-122B-A10B-FP8 (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/pinchbench-qwen122b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-122B-A10B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-qwen35b"
|
||||
description = "PinchBench on Qwen/Qwen3.5-35B-A3B-FP8 (vLLM, TP=4)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/pinchbench-qwen35b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 4
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
# PinchBench eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)
|
||||
# Agent: native_openhands — all PinchBench-required tools enabled
|
||||
|
||||
[meta]
|
||||
name = "pinchbench-qwen397b"
|
||||
description = "PinchBench on Qwen/Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)"
|
||||
|
||||
[defaults]
|
||||
temperature = 0.6
|
||||
max_tokens = 8192
|
||||
|
||||
[judge]
|
||||
model = "claude-opus-4-5"
|
||||
temperature = 0.0
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "results/pinchbench-qwen397b/"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "Qwen/Qwen3.5-397B-A17B-FP8"
|
||||
engine = "vllm"
|
||||
num_gpus = 8
|
||||
|
||||
[[benchmarks]]
|
||||
name = "pinchbench"
|
||||
backend = "jarvis-agent"
|
||||
agent = "native_openhands"
|
||||
tools = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
]
|
||||
@@ -32,9 +32,9 @@ name = "mlx-community/Qwen2.5-7B-4bit"
|
||||
engine = "mlx"
|
||||
param_count_b = 7.0
|
||||
|
||||
# Ollama GGUF — pull with: ollama pull qwen3.5:3b
|
||||
# Ollama GGUF — pull with: ollama pull qwen3.5:2b
|
||||
[[models]]
|
||||
name = "qwen3.5:3b"
|
||||
name = "qwen3.5:2b"
|
||||
engine = "ollama"
|
||||
|
||||
# ── Benchmarks ────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -43,6 +43,7 @@ KNOWN_BENCHMARKS = {
|
||||
"knowledge_base", "coding_task",
|
||||
"coding_assistant", "security_scanner", "daily_digest",
|
||||
"doc_qa", "browser_assistant",
|
||||
"pinchbench",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -96,11 +96,19 @@ class EvalRunner:
|
||||
seed=cfg.seed,
|
||||
)
|
||||
|
||||
# Auto-enable episode_mode when the dataset has iter_episodes()
|
||||
# (i.e. it is a lifelong/sequential benchmark like LifelongAgentBench).
|
||||
# This is enforced at the runner level so it applies regardless of
|
||||
# how the runner is invoked (CLI, SDK, tests, etc.).
|
||||
if not cfg.episode_mode and hasattr(self._dataset, "iter_episodes"):
|
||||
# Auto-enable episode_mode when the dataset *overrides*
|
||||
# iter_episodes() (i.e. it is a lifelong/sequential benchmark like
|
||||
# LifelongAgentBench). The base DatasetProvider always defines a
|
||||
# default iter_episodes() that wraps each record in its own episode,
|
||||
# so hasattr() is always True — we must check for a real override.
|
||||
from openjarvis.evals.core.dataset import DatasetProvider as _DP
|
||||
try:
|
||||
_overrides_episodes = (
|
||||
type(self._dataset).iter_episodes is not _DP.iter_episodes
|
||||
)
|
||||
except AttributeError:
|
||||
_overrides_episodes = False
|
||||
if not cfg.episode_mode and _overrides_episodes:
|
||||
LOGGER.info(
|
||||
"%s requires sequential episode processing — "
|
||||
"auto-enabling episode_mode.",
|
||||
@@ -109,6 +117,15 @@ class EvalRunner:
|
||||
cfg = dataclasses.replace(cfg, episode_mode=True)
|
||||
self._config = cfg
|
||||
|
||||
# Detect if dataset provides task environments (e.g. PinchBench)
|
||||
try:
|
||||
self._has_task_env = (
|
||||
type(self._dataset).create_task_env
|
||||
is not _DP.create_task_env
|
||||
)
|
||||
except AttributeError:
|
||||
self._has_task_env = False
|
||||
|
||||
records = list(self._dataset.iter_records())
|
||||
LOGGER.info(
|
||||
"Running %s: %d samples, backend=%s, model=%s, workers=%d, "
|
||||
@@ -145,6 +162,15 @@ class EvalRunner:
|
||||
try:
|
||||
if cfg.episode_mode:
|
||||
self._run_episode_mode(records, progress_callback, total)
|
||||
elif self._has_task_env:
|
||||
# Task environments (PinchBench etc.) change CWD —
|
||||
# must process sequentially for thread safety.
|
||||
for record in records:
|
||||
result = self._process_one(record)
|
||||
self._results.append(result)
|
||||
self._flush_result(result)
|
||||
if progress_callback is not None:
|
||||
progress_callback(len(self._results), total)
|
||||
else:
|
||||
with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool:
|
||||
futures = {
|
||||
@@ -224,17 +250,60 @@ class EvalRunner:
|
||||
)
|
||||
if cfg.system_prompt:
|
||||
gen_kwargs["system"] = cfg.system_prompt
|
||||
full = self._backend.generate_full(
|
||||
record.problem,
|
||||
**gen_kwargs,
|
||||
)
|
||||
content = full.get("content", "")
|
||||
|
||||
if getattr(self, "_has_task_env", False):
|
||||
from contextlib import nullcontext
|
||||
task_env = self._dataset.create_task_env(record)
|
||||
ctx = task_env if task_env is not None else nullcontext()
|
||||
with ctx:
|
||||
full = self._backend.generate_full(
|
||||
record.problem,
|
||||
**gen_kwargs,
|
||||
)
|
||||
full = full or {}
|
||||
all_tool_results = list(full.get(
|
||||
"tool_results", [],
|
||||
))
|
||||
|
||||
# Multi-session: execute remaining sessions
|
||||
sessions = record.metadata.get("sessions", [])
|
||||
if (
|
||||
record.metadata.get("multi_session")
|
||||
and len(sessions) > 1
|
||||
):
|
||||
for session in sessions[1:]:
|
||||
prompt = session.get("prompt", "")
|
||||
if not prompt:
|
||||
continue
|
||||
sfull = self._backend.generate_full(
|
||||
prompt, **gen_kwargs,
|
||||
)
|
||||
sfull = sfull or {}
|
||||
all_tool_results.extend(
|
||||
sfull.get("tool_results", []),
|
||||
)
|
||||
|
||||
record.metadata["tool_results"] = all_tool_results
|
||||
# Score INSIDE context so workspace files still exist
|
||||
content = full.get("content", "")
|
||||
is_correct, scoring_meta = self._scorer.score(
|
||||
record, content,
|
||||
)
|
||||
else:
|
||||
full = self._backend.generate_full(
|
||||
record.problem,
|
||||
**gen_kwargs,
|
||||
)
|
||||
full = full or {}
|
||||
content = full.get("content", "")
|
||||
is_correct, scoring_meta = self._scorer.score(
|
||||
record, content,
|
||||
)
|
||||
|
||||
usage = full.get("usage", {})
|
||||
latency = full.get("latency_seconds", 0.0)
|
||||
cost = full.get("cost_usd", 0.0)
|
||||
|
||||
is_correct, scoring_meta = self._scorer.score(record, content)
|
||||
|
||||
energy_j = full.get("energy_joules", 0.0)
|
||||
power_w = full.get("power_watts", 0.0)
|
||||
throughput = full.get("throughput_tok_per_sec", 0.0)
|
||||
|
||||
@@ -74,6 +74,8 @@ def _parse_task_markdown(content: str, filename: str = "") -> Dict[str, Any]:
|
||||
"timeout_seconds": frontmatter.get("timeout_seconds", 180),
|
||||
"workspace_files": frontmatter.get("workspace_files", []),
|
||||
"grading_weights": frontmatter.get("grading_weights"),
|
||||
"multi_session": frontmatter.get("multi_session", False),
|
||||
"sessions": frontmatter.get("sessions", []),
|
||||
"prompt": sections.get("Prompt", ""),
|
||||
"expected_behavior": sections.get("Expected Behavior", ""),
|
||||
"grading_criteria": sections.get("Grading Criteria", ""),
|
||||
@@ -160,7 +162,11 @@ class PinchBenchDataset(DatasetProvider):
|
||||
self._records = [
|
||||
EvalRecord(
|
||||
record_id=t["id"],
|
||||
problem=t["prompt"],
|
||||
problem=(
|
||||
t["sessions"][0]["prompt"]
|
||||
if t.get("multi_session") and t.get("sessions")
|
||||
else t["prompt"]
|
||||
),
|
||||
reference=t["expected_behavior"],
|
||||
category=t["category"],
|
||||
subject=t["name"],
|
||||
@@ -172,6 +178,8 @@ class PinchBenchDataset(DatasetProvider):
|
||||
"timeout_seconds": t["timeout_seconds"],
|
||||
"workspace_files": t["workspace_files"],
|
||||
"pinchbench_repo_dir": str(repo_dir),
|
||||
"multi_session": t.get("multi_session", False),
|
||||
"sessions": t.get("sessions", []),
|
||||
},
|
||||
)
|
||||
for t in tasks
|
||||
|
||||
@@ -52,12 +52,12 @@ def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]:
|
||||
if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value:
|
||||
tool_name = event.metadata.get("tool", "unknown")
|
||||
mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)
|
||||
arguments = event.metadata.get("arguments", {})
|
||||
arguments = event.metadata.get("arguments") or {}
|
||||
transcript.append({
|
||||
"type": "message",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [{"type": "toolCall", "name": mapped, "arguments": arguments}],
|
||||
"content": [{"type": "toolCall", "name": mapped, "params": arguments}],
|
||||
},
|
||||
})
|
||||
elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value:
|
||||
@@ -78,12 +78,14 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:
|
||||
transcript: List[Dict[str, Any]] = []
|
||||
for turn in trace.turns:
|
||||
for tc in getattr(turn, "tool_calls", []):
|
||||
if tc is None:
|
||||
continue
|
||||
mapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"])
|
||||
transcript.append({
|
||||
"type": "message",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [{"type": "toolCall", "name": mapped, "arguments": tc.get("arguments", {})}],
|
||||
"content": [{"type": "toolCall", "name": mapped, "params": tc.get("arguments") or {}}],
|
||||
},
|
||||
})
|
||||
transcript.append({
|
||||
@@ -93,6 +95,42 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:
|
||||
"content": [{"text": tc.get("result", "")}],
|
||||
},
|
||||
})
|
||||
|
||||
# Capture final assistant text response (for tasks graded on text output)
|
||||
response_text = getattr(trace, "response_text", "")
|
||||
if response_text:
|
||||
transcript.append({
|
||||
"type": "message",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [{"type": "text", "text": response_text}],
|
||||
},
|
||||
})
|
||||
return transcript
|
||||
|
||||
|
||||
def _tool_results_to_transcript(
|
||||
tool_results: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Build transcript from JarvisAgentBackend tool_results list."""
|
||||
transcript: List[Dict[str, Any]] = []
|
||||
for tr in tool_results:
|
||||
tool_name = tr.get("tool_name", "unknown")
|
||||
mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)
|
||||
transcript.append({
|
||||
"type": "message",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [{"type": "toolCall", "name": mapped, "params": tr.get("arguments", {})}],
|
||||
},
|
||||
})
|
||||
transcript.append({
|
||||
"type": "message",
|
||||
"message": {
|
||||
"role": "toolResult",
|
||||
"content": [{"text": tr.get("content", "")}],
|
||||
},
|
||||
})
|
||||
return transcript
|
||||
|
||||
|
||||
@@ -118,8 +156,10 @@ def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str:
|
||||
for item in msg.get("content", []):
|
||||
if item.get("type") == "toolCall":
|
||||
parts.append(
|
||||
f"Tool: {item.get('name')}({json.dumps(item.get('arguments', {}))})"
|
||||
f"Tool: {item.get('name')}({json.dumps(item.get('params', {}))})"
|
||||
)
|
||||
elif item.get("type") == "text":
|
||||
parts.append(f"Assistant: {item.get('text', '')}")
|
||||
elif role == "toolResult":
|
||||
content = msg.get("content", [])
|
||||
if content:
|
||||
@@ -372,7 +412,7 @@ def _grade_hybrid(
|
||||
judge_model: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run both automated and LLM judge grading, combine with weights."""
|
||||
weights = record.metadata.get("grading_weights", {"automated": 0.5, "llm_judge": 0.5})
|
||||
weights = record.metadata.get("grading_weights") or {"automated": 0.5, "llm_judge": 0.5}
|
||||
auto = _grade_automated(record, transcript, workspace_path)
|
||||
llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model)
|
||||
|
||||
@@ -426,7 +466,24 @@ class PinchBenchScorer(LLMJudgeScorer):
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
trace = record.metadata.get("query_trace")
|
||||
transcript = _trace_to_transcript(trace) if trace else []
|
||||
if trace:
|
||||
transcript = _trace_to_transcript(trace)
|
||||
else:
|
||||
# No trace — build transcript from tool_results if available
|
||||
tool_results = record.metadata.get("tool_results", [])
|
||||
transcript = _tool_results_to_transcript(tool_results)
|
||||
|
||||
# Always append final model answer as assistant text message
|
||||
# so grading functions that check for text responses can find it
|
||||
if model_answer:
|
||||
transcript.append({
|
||||
"type": "message",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [{"type": "text", "text": model_answer}],
|
||||
},
|
||||
})
|
||||
|
||||
result = grade_pinchbench_task(
|
||||
record=record,
|
||||
transcript=transcript,
|
||||
@@ -442,4 +499,5 @@ __all__ = [
|
||||
"PinchBenchScorer",
|
||||
"events_to_transcript",
|
||||
"grade_pinchbench_task",
|
||||
"_tool_results_to_transcript",
|
||||
]
|
||||
|
||||
@@ -40,48 +40,62 @@ BUILTIN_MODELS: List[ModelSpec] = [
|
||||
# Local models — Qwen3.5 (MoE)
|
||||
# -----------------------------------------------------------------------
|
||||
ModelSpec(
|
||||
model_id="qwen3.5:3b",
|
||||
name="Qwen3.5 3B",
|
||||
parameter_count_b=3.0,
|
||||
active_parameter_count_b=0.6,
|
||||
model_id="qwen3.5:0.8b",
|
||||
name="Qwen3.5 0.8B",
|
||||
parameter_count_b=0.8,
|
||||
active_parameter_count_b=0.15,
|
||||
context_length=131072,
|
||||
supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx"),
|
||||
provider="alibaba",
|
||||
metadata={
|
||||
"architecture": "moe",
|
||||
"hf_repo": "Qwen/Qwen3.5-3B",
|
||||
"gguf_file": "qwen3.5-3b-q4_k_m.gguf",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-3B-4bit",
|
||||
"hf_repo": "Qwen/Qwen3.5-0.8B",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-0.8B-OptiQ-4bit",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
model_id="qwen3.5:8b",
|
||||
name="Qwen3.5 8B",
|
||||
parameter_count_b=8.0,
|
||||
active_parameter_count_b=1.0,
|
||||
model_id="qwen3.5:2b",
|
||||
name="Qwen3.5 2B",
|
||||
parameter_count_b=2.0,
|
||||
active_parameter_count_b=0.4,
|
||||
context_length=131072,
|
||||
supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx"),
|
||||
provider="alibaba",
|
||||
metadata={
|
||||
"architecture": "moe",
|
||||
"hf_repo": "Qwen/Qwen3.5-8B",
|
||||
"gguf_file": "qwen3.5-8b-q4_k_m.gguf",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-8B-4bit",
|
||||
"hf_repo": "Qwen/Qwen3.5-2B",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-2B-OptiQ-4bit",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
model_id="qwen3.5:14b",
|
||||
name="Qwen3.5 14B",
|
||||
parameter_count_b=14.0,
|
||||
active_parameter_count_b=2.0,
|
||||
model_id="qwen3.5:9b",
|
||||
name="Qwen3.5 9B",
|
||||
parameter_count_b=9.0,
|
||||
active_parameter_count_b=1.5,
|
||||
context_length=131072,
|
||||
supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx"),
|
||||
provider="alibaba",
|
||||
metadata={
|
||||
"architecture": "moe",
|
||||
"hf_repo": "Qwen/Qwen3.5-14B",
|
||||
"gguf_file": "qwen3.5-14b-q4_k_m.gguf",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-14B-4bit",
|
||||
"hf_repo": "Qwen/Qwen3.5-9B",
|
||||
"gguf_file": "qwen3.5-9b-q4_k_m.gguf",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-9B-MLX-4bit",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
model_id="qwen3.5:27b",
|
||||
name="Qwen3.5 27B",
|
||||
parameter_count_b=27.0,
|
||||
active_parameter_count_b=3.0,
|
||||
context_length=131072,
|
||||
min_vram_gb=16.0,
|
||||
supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx"),
|
||||
provider="alibaba",
|
||||
metadata={
|
||||
"architecture": "moe",
|
||||
"hf_repo": "Qwen/Qwen3.5-27B",
|
||||
"gguf_file": "qwen3.5-27b-q4_k_m.gguf",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-27B-4bit-DWQ",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
@@ -235,7 +249,7 @@ BUILTIN_MODELS: List[ModelSpec] = [
|
||||
"architecture": "moe",
|
||||
"hf_repo": "Qwen/Qwen3.5-4B",
|
||||
"gguf_file": "qwen3.5-4b-q4_k_m.gguf",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-4B-4bit",
|
||||
"mlx_repo": "mlx-community/Qwen3.5-4B-OptiQ-4bit",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
@@ -490,8 +504,7 @@ BUILTIN_MODELS: List[ModelSpec] = [
|
||||
metadata={
|
||||
"architecture": "moe",
|
||||
"hf_repo": (
|
||||
"TeichAI/GLM-4.7-Flash-Claude-"
|
||||
"Opus-4.5-High-Reasoning-Distill-GGUF"
|
||||
"TeichAI/GLM-4.7-Flash-Claude-Opus-4.5-High-Reasoning-Distill-GGUF"
|
||||
),
|
||||
"teacher": "Claude Opus 4.5",
|
||||
"quantization": "GGUF Q4_K_M / Q8_0",
|
||||
@@ -637,6 +650,89 @@ BUILTIN_MODELS: List[ModelSpec] = [
|
||||
},
|
||||
),
|
||||
# -----------------------------------------------------------------------
|
||||
# Cloud models — OpenAI Codex (ChatGPT Plus/Pro subscription)
|
||||
# -----------------------------------------------------------------------
|
||||
ModelSpec(
|
||||
model_id="codex/gpt-4o",
|
||||
name="GPT-4o (Codex)",
|
||||
parameter_count_b=0.0,
|
||||
context_length=128000,
|
||||
supported_engines=("cloud",),
|
||||
provider="openai-codex",
|
||||
requires_api_key=True,
|
||||
metadata={
|
||||
"architecture": "proprietary",
|
||||
"auth": "OPENAI_CODEX_API_KEY",
|
||||
"pricing_input": 0.0,
|
||||
"pricing_output": 0.0,
|
||||
"url": "https://platform.openai.com/docs/models/gpt-4o",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
model_id="codex/gpt-4o-mini",
|
||||
name="GPT-4o Mini (Codex)",
|
||||
parameter_count_b=0.0,
|
||||
context_length=128000,
|
||||
supported_engines=("cloud",),
|
||||
provider="openai-codex",
|
||||
requires_api_key=True,
|
||||
metadata={
|
||||
"architecture": "proprietary",
|
||||
"auth": "OPENAI_CODEX_API_KEY",
|
||||
"pricing_input": 0.0,
|
||||
"pricing_output": 0.0,
|
||||
"url": "https://platform.openai.com/docs/models/gpt-4o-mini",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
model_id="codex/o3-mini",
|
||||
name="o3-mini (Codex)",
|
||||
parameter_count_b=0.0,
|
||||
context_length=200000,
|
||||
supported_engines=("cloud",),
|
||||
provider="openai-codex",
|
||||
requires_api_key=True,
|
||||
metadata={
|
||||
"architecture": "proprietary",
|
||||
"auth": "OPENAI_CODEX_API_KEY",
|
||||
"pricing_input": 0.0,
|
||||
"pricing_output": 0.0,
|
||||
"url": "https://platform.openai.com/docs/models",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
model_id="codex/gpt-5-mini",
|
||||
name="GPT-5 Mini (Codex)",
|
||||
parameter_count_b=0.0,
|
||||
context_length=400000,
|
||||
supported_engines=("cloud",),
|
||||
provider="openai-codex",
|
||||
requires_api_key=True,
|
||||
metadata={
|
||||
"architecture": "proprietary",
|
||||
"auth": "OPENAI_CODEX_API_KEY",
|
||||
"pricing_input": 0.0,
|
||||
"pricing_output": 0.0,
|
||||
"url": "https://platform.openai.com/docs/models",
|
||||
},
|
||||
),
|
||||
ModelSpec(
|
||||
model_id="codex/gpt-5-mini-2025-08-07",
|
||||
name="GPT-5 Mini 2025-08-07 (Codex)",
|
||||
parameter_count_b=0.0,
|
||||
context_length=400000,
|
||||
supported_engines=("cloud",),
|
||||
provider="openai-codex",
|
||||
requires_api_key=True,
|
||||
metadata={
|
||||
"architecture": "proprietary",
|
||||
"auth": "OPENAI_CODEX_API_KEY",
|
||||
"pricing_input": 0.0,
|
||||
"pricing_output": 0.0,
|
||||
"url": "https://platform.openai.com/docs/models",
|
||||
},
|
||||
),
|
||||
# -----------------------------------------------------------------------
|
||||
# Cloud models — Anthropic
|
||||
# -----------------------------------------------------------------------
|
||||
ModelSpec(
|
||||
|
||||
@@ -8,6 +8,7 @@ from openjarvis.mcp.transport import (
|
||||
MCPTransport,
|
||||
SSETransport,
|
||||
StdioTransport,
|
||||
StreamableHTTPTransport,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -21,4 +22,5 @@ __all__ = [
|
||||
"InProcessTransport",
|
||||
"SSETransport",
|
||||
"StdioTransport",
|
||||
"StreamableHTTPTransport",
|
||||
]
|
||||
|
||||
@@ -47,13 +47,36 @@ class MCPClient:
|
||||
def initialize(self) -> Dict[str, Any]:
|
||||
"""Perform the MCP initialize handshake.
|
||||
|
||||
Sends the required client info and protocol version, then
|
||||
confirms with a ``notifications/initialized`` notification
|
||||
as required by the MCP specification.
|
||||
|
||||
Returns the server capabilities.
|
||||
"""
|
||||
response = self._send("initialize")
|
||||
params = {
|
||||
"protocolVersion": "2025-03-26",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "openjarvis", "version": "0.1.0"},
|
||||
}
|
||||
response = self._send("initialize", params)
|
||||
self._initialized = True
|
||||
self._capabilities = response.result.get("capabilities", {})
|
||||
# Send the required initialized notification per MCP spec
|
||||
self.notify("notifications/initialized")
|
||||
return response.result
|
||||
|
||||
def notify(self, method: str, params: Dict[str, Any] | None = None) -> None:
|
||||
"""Send a JSON-RPC notification (no response expected).
|
||||
|
||||
Per JSON-RPC 2.0 spec, notifications omit the ``id`` field entirely.
|
||||
"""
|
||||
request = MCPRequest(
|
||||
method=method,
|
||||
params=params or {},
|
||||
id=None, # None → no id field in JSON (notification)
|
||||
)
|
||||
self._transport.send_notification(request)
|
||||
|
||||
def list_tools(self) -> List[ToolSpec]:
|
||||
"""Discover available tools from the server.
|
||||
|
||||
@@ -71,7 +94,9 @@ class MCPClient:
|
||||
]
|
||||
|
||||
def call_tool(
|
||||
self, name: str, arguments: Dict[str, Any] | None = None,
|
||||
self,
|
||||
name: str,
|
||||
arguments: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Call a tool on the server.
|
||||
|
||||
@@ -87,5 +112,11 @@ class MCPClient:
|
||||
"""Close the transport connection."""
|
||||
self._transport.close()
|
||||
|
||||
def __enter__(self) -> MCPClient:
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc: Any) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
__all__ = ["MCPClient"]
|
||||
|
||||
@@ -16,23 +16,34 @@ INTERNAL_ERROR = -32603
|
||||
|
||||
@dataclass
|
||||
class MCPRequest:
|
||||
"""JSON-RPC 2.0 request message."""
|
||||
"""JSON-RPC 2.0 request message.
|
||||
|
||||
Set *id* to ``None`` to create a JSON-RPC **notification** (no ``id``
|
||||
field will appear in the serialized output, and no response is expected).
|
||||
"""
|
||||
|
||||
method: str
|
||||
params: Dict[str, Any] = field(default_factory=dict)
|
||||
id: int | str = 0
|
||||
id: Optional[int | str] = 0
|
||||
jsonrpc: str = "2.0"
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Return a dict suitable for JSON serialization.
|
||||
|
||||
Omits the ``id`` key when it is ``None`` (notification).
|
||||
"""
|
||||
obj: Dict[str, Any] = {
|
||||
"jsonrpc": self.jsonrpc,
|
||||
"method": self.method,
|
||||
"params": self.params,
|
||||
}
|
||||
if self.id is not None:
|
||||
obj["id"] = self.id
|
||||
return obj
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Serialize to JSON string."""
|
||||
return json.dumps(
|
||||
{
|
||||
"jsonrpc": self.jsonrpc,
|
||||
"id": self.id,
|
||||
"method": self.method,
|
||||
"params": self.params,
|
||||
}
|
||||
)
|
||||
return json.dumps(self.to_dict())
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, data: str) -> MCPRequest:
|
||||
|
||||
@@ -70,31 +70,37 @@ class MCPServer:
|
||||
# Built-in API tools
|
||||
try:
|
||||
from openjarvis.tools.calculator import CalculatorTool
|
||||
|
||||
_tool_classes.append(CalculatorTool)
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
from openjarvis.tools.think import ThinkTool
|
||||
|
||||
_tool_classes.append(ThinkTool)
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
from openjarvis.tools.file_read import FileReadTool
|
||||
|
||||
_tool_classes.append(FileReadTool)
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
from openjarvis.tools.web_search import WebSearchTool
|
||||
|
||||
_tool_classes.append(WebSearchTool)
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
from openjarvis.tools.code_interpreter import CodeInterpreterTool
|
||||
|
||||
_tool_classes.append(CodeInterpreterTool)
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
from openjarvis.tools.repl import ReplTool
|
||||
|
||||
_tool_classes.append(ReplTool)
|
||||
except ImportError:
|
||||
pass
|
||||
@@ -107,10 +113,15 @@ class MCPServer:
|
||||
MemorySearchTool,
|
||||
MemoryStoreTool,
|
||||
)
|
||||
_tool_classes.extend([
|
||||
MemoryStoreTool, MemoryRetrieveTool,
|
||||
MemorySearchTool, MemoryIndexTool,
|
||||
])
|
||||
|
||||
_tool_classes.extend(
|
||||
[
|
||||
MemoryStoreTool,
|
||||
MemoryRetrieveTool,
|
||||
MemorySearchTool,
|
||||
MemoryIndexTool,
|
||||
]
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
@@ -121,15 +132,21 @@ class MCPServer:
|
||||
ChannelSendTool,
|
||||
ChannelStatusTool,
|
||||
)
|
||||
_tool_classes.extend([
|
||||
ChannelSendTool, ChannelListTool, ChannelStatusTool,
|
||||
])
|
||||
|
||||
_tool_classes.extend(
|
||||
[
|
||||
ChannelSendTool,
|
||||
ChannelListTool,
|
||||
ChannelStatusTool,
|
||||
]
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# LM tool (needs engine/model — instantiate with None)
|
||||
try:
|
||||
from openjarvis.tools.llm_tool import LLMTool
|
||||
|
||||
_tool_classes.append(LLMTool)
|
||||
except ImportError:
|
||||
pass
|
||||
@@ -137,6 +154,7 @@ class MCPServer:
|
||||
# Retrieval tool (needs backend — instantiate with None)
|
||||
try:
|
||||
from openjarvis.tools.retrieval import RetrievalTool
|
||||
|
||||
_tool_classes.append(RetrievalTool)
|
||||
except ImportError:
|
||||
pass
|
||||
@@ -150,6 +168,7 @@ class MCPServer:
|
||||
# Also check ToolRegistry for any user-registered tools
|
||||
try:
|
||||
from openjarvis.core.registry import ToolRegistry
|
||||
|
||||
known_names = {t.spec.name for t in tools}
|
||||
for key in ToolRegistry.keys():
|
||||
if key not in known_names:
|
||||
|
||||
+130
-18
@@ -2,10 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, List, Optional
|
||||
|
||||
from openjarvis.mcp.protocol import MCPRequest, MCPResponse
|
||||
|
||||
@@ -20,6 +19,15 @@ class MCPTransport(ABC):
|
||||
def send(self, request: MCPRequest) -> MCPResponse:
|
||||
"""Send a request and return the response."""
|
||||
|
||||
def send_notification(self, request: MCPRequest) -> None:
|
||||
"""Send a JSON-RPC notification (no response expected).
|
||||
|
||||
The default implementation delegates to :meth:`send` and discards the
|
||||
response. Transports may override this when the server returns no
|
||||
body for notifications (e.g. HTTP 202 Accepted).
|
||||
"""
|
||||
self.send(request)
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Release transport resources."""
|
||||
@@ -88,30 +96,133 @@ class StdioTransport(MCPTransport):
|
||||
self._process = None
|
||||
|
||||
|
||||
class SSETransport(MCPTransport):
|
||||
"""JSON-RPC over HTTP with Server-Sent Events.
|
||||
class StreamableHTTPTransport(MCPTransport):
|
||||
"""MCP Streamable HTTP transport (JSON-RPC over HTTP).
|
||||
|
||||
Sends requests via HTTP POST and reads SSE responses.
|
||||
Uses a persistent ``httpx.Client`` session, tracks the
|
||||
``Mcp-Session-Id`` header, and sends the ``Accept`` header
|
||||
required by the MCP Streamable HTTP specification.
|
||||
"""
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
self._url = url
|
||||
|
||||
def send(self, request: MCPRequest) -> MCPResponse:
|
||||
"""Send request via HTTP POST."""
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
connect_timeout: float = 10.0,
|
||||
request_timeout: float = 60.0,
|
||||
) -> None:
|
||||
import httpx
|
||||
|
||||
response = httpx.post(
|
||||
self._url,
|
||||
json=json.loads(request.to_json()),
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=30.0,
|
||||
self._url = url
|
||||
self._session_id: Optional[str] = None
|
||||
self._client = httpx.Client(
|
||||
timeout=httpx.Timeout(
|
||||
connect=connect_timeout,
|
||||
read=request_timeout,
|
||||
write=request_timeout,
|
||||
pool=connect_timeout,
|
||||
),
|
||||
)
|
||||
response.raise_for_status()
|
||||
return MCPResponse.from_json(response.text)
|
||||
|
||||
def _safe_url(self) -> str:
|
||||
"""Return scheme://host:port without path or query (avoids leaking tokens)."""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(self._url)
|
||||
return f"{parsed.scheme}://{parsed.netloc}"
|
||||
|
||||
def _build_headers(self) -> dict:
|
||||
"""Build common request headers."""
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json, text/event-stream",
|
||||
}
|
||||
if self._session_id is not None:
|
||||
headers["Mcp-Session-Id"] = self._session_id
|
||||
return headers
|
||||
|
||||
def _post(self, request: MCPRequest) -> Any:
|
||||
"""Post a request and return the raw httpx response."""
|
||||
import httpx
|
||||
|
||||
headers = self._build_headers()
|
||||
try:
|
||||
response = self._client.post(
|
||||
self._url,
|
||||
json=request.to_dict(),
|
||||
headers=headers,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except httpx.ConnectError as exc:
|
||||
raise RuntimeError(
|
||||
f"Failed to connect to MCP server at {self._safe_url()}: {exc}"
|
||||
) from exc
|
||||
except httpx.TimeoutException as exc:
|
||||
raise RuntimeError(
|
||||
f"Timeout communicating with MCP server at {self._safe_url()}: {exc}"
|
||||
) from exc
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise RuntimeError(
|
||||
f"MCP server at {self._safe_url()} returned HTTP "
|
||||
f"{exc.response.status_code}"
|
||||
) from exc
|
||||
|
||||
# Track session id from the first response
|
||||
new_session_id = response.headers.get("mcp-session-id")
|
||||
if new_session_id is not None:
|
||||
self._session_id = new_session_id
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _extract_json_from_sse(text: str) -> str:
|
||||
"""Extract JSON payload from an SSE response body.
|
||||
|
||||
MCP Streamable HTTP servers may respond with ``text/event-stream``
|
||||
instead of ``application/json``. In that case the body looks like::
|
||||
|
||||
event: message
|
||||
data: {"jsonrpc":"2.0", ...}
|
||||
|
||||
This helper finds the last ``data:`` line and returns its content,
|
||||
which is the actual JSON-RPC response.
|
||||
"""
|
||||
last_data = ""
|
||||
for line in text.splitlines():
|
||||
if line.startswith("data:"):
|
||||
last_data = line[len("data:") :].strip()
|
||||
if not last_data:
|
||||
raise RuntimeError(
|
||||
"SSE response contained no 'data:' lines"
|
||||
" — cannot extract JSON-RPC payload"
|
||||
)
|
||||
return last_data
|
||||
|
||||
def send(self, request: MCPRequest) -> MCPResponse:
|
||||
"""Send request via HTTP POST following the MCP Streamable HTTP spec.
|
||||
|
||||
Handles both ``application/json`` and ``text/event-stream`` responses
|
||||
as allowed by the MCP Streamable HTTP specification.
|
||||
"""
|
||||
response = self._post(request)
|
||||
content_type = response.headers.get("content-type", "")
|
||||
body = response.text
|
||||
if "text/event-stream" in content_type or body.lstrip().startswith("event:"):
|
||||
body = self._extract_json_from_sse(body)
|
||||
return MCPResponse.from_json(body)
|
||||
|
||||
def send_notification(self, request: MCPRequest) -> None:
|
||||
"""Send a notification — accept any 2xx, don't parse the body."""
|
||||
# Track session id but don't try to parse a JSON-RPC response.
|
||||
# Servers may return 202 Accepted with an empty body.
|
||||
self._post(request)
|
||||
|
||||
def close(self) -> None:
|
||||
"""No persistent connection to close."""
|
||||
"""Close the underlying httpx client."""
|
||||
self._client.close()
|
||||
|
||||
|
||||
# Backward-compatible alias
|
||||
SSETransport = StreamableHTTPTransport
|
||||
|
||||
|
||||
__all__ = [
|
||||
@@ -119,4 +230,5 @@ __all__ = [
|
||||
"MCPTransport",
|
||||
"SSETransport",
|
||||
"StdioTransport",
|
||||
"StreamableHTTPTransport",
|
||||
]
|
||||
|
||||
@@ -218,8 +218,13 @@ class TaskScheduler:
|
||||
|
||||
try:
|
||||
if self._system is not None:
|
||||
raw_tools = (
|
||||
task.tools
|
||||
if isinstance(task.tools, list)
|
||||
else task.tools.split(",")
|
||||
)
|
||||
tools_list = (
|
||||
[t.strip() for t in task.tools.split(",") if t.strip()]
|
||||
[t.strip() for t in raw_tools if t.strip()]
|
||||
if task.tools
|
||||
else []
|
||||
)
|
||||
|
||||
@@ -30,11 +30,27 @@ def is_sensitive_file(path: Union[str, Path]) -> bool:
|
||||
|
||||
Checks both the filename and the full name against
|
||||
``DEFAULT_SENSITIVE_PATTERNS`` using :func:`fnmatch.fnmatch`.
|
||||
Uses the Rust implementation when available, falls back to Python.
|
||||
"""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
try:
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
|
||||
_rust = get_rust_module()
|
||||
return _rust.is_sensitive_file(str(path))
|
||||
_rust = get_rust_module()
|
||||
return _rust.is_sensitive_file(str(path))
|
||||
except ImportError:
|
||||
return _is_sensitive_file_py(str(path))
|
||||
|
||||
|
||||
def _is_sensitive_file_py(path_str: str) -> bool:
|
||||
"""Pure-Python fallback for sensitive file detection."""
|
||||
import fnmatch
|
||||
|
||||
p = Path(path_str)
|
||||
name = p.name
|
||||
for pattern in DEFAULT_SENSITIVE_PATTERNS:
|
||||
if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(str(p), pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def filter_sensitive_paths(paths: Iterable[Union[str, Path]]) -> List[Path]:
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from openjarvis.core.events import EventBus, EventType
|
||||
from openjarvis.core.types import Message
|
||||
from openjarvis.engine._stubs import InferenceEngine
|
||||
from openjarvis.engine._stubs import InferenceEngine, StreamChunk
|
||||
from openjarvis.security._stubs import BaseScanner
|
||||
from openjarvis.security.scanner import PIIScanner, SecretScanner
|
||||
from openjarvis.security.types import RedactionMode, ScanResult
|
||||
@@ -50,10 +50,14 @@ class GuardrailsEngine(InferenceEngine):
|
||||
bus: Optional[EventBus] = None,
|
||||
) -> None:
|
||||
self._engine = engine
|
||||
self._scanners: List[BaseScanner] = scanners if scanners is not None else [
|
||||
SecretScanner(),
|
||||
PIIScanner(),
|
||||
]
|
||||
self._scanners: List[BaseScanner] = (
|
||||
scanners
|
||||
if scanners is not None
|
||||
else [
|
||||
SecretScanner(),
|
||||
PIIScanner(),
|
||||
]
|
||||
)
|
||||
self._mode = mode
|
||||
self._scan_input = scan_input
|
||||
self._scan_output = scan_output
|
||||
@@ -180,7 +184,9 @@ class GuardrailsEngine(InferenceEngine):
|
||||
processed[i] = Message(
|
||||
role=msg.role,
|
||||
content=self._handle_findings(
|
||||
msg.content, result, "input",
|
||||
msg.content,
|
||||
result,
|
||||
"input",
|
||||
),
|
||||
name=msg.name,
|
||||
tool_calls=msg.tool_calls,
|
||||
@@ -191,8 +197,11 @@ class GuardrailsEngine(InferenceEngine):
|
||||
|
||||
# Call wrapped engine
|
||||
response = self._engine.generate(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Scan output
|
||||
@@ -219,8 +228,11 @@ class GuardrailsEngine(InferenceEngine):
|
||||
"""Yield tokens in real-time, scan accumulated output post-hoc."""
|
||||
accumulated = []
|
||||
async for token in self._engine.stream(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
accumulated.append(token)
|
||||
yield token
|
||||
@@ -248,6 +260,51 @@ class GuardrailsEngine(InferenceEngine):
|
||||
},
|
||||
)
|
||||
|
||||
async def stream_full(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: int = 1024,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator["StreamChunk"]:
|
||||
"""Delegate to wrapped engine, scan accumulated output post-hoc."""
|
||||
accumulated: list[str] = []
|
||||
async for chunk in self._engine.stream_full(
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
if chunk.content:
|
||||
accumulated.append(chunk.content)
|
||||
yield chunk
|
||||
|
||||
# Post-hoc scan of accumulated output
|
||||
if self._scan_output:
|
||||
full_output = "".join(accumulated)
|
||||
if full_output:
|
||||
result = self._scan_text(full_output)
|
||||
if not result.clean and self._bus:
|
||||
finding_dicts = [
|
||||
{
|
||||
"pattern": f.pattern_name,
|
||||
"threat": f.threat_level.value,
|
||||
"description": f.description,
|
||||
}
|
||||
for f in result.findings
|
||||
]
|
||||
self._bus.publish(
|
||||
EventType.SECURITY_ALERT,
|
||||
{
|
||||
"direction": "output",
|
||||
"findings": finding_dicts,
|
||||
"mode": "stream_full_post_hoc",
|
||||
},
|
||||
)
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
"""Delegate to wrapped engine."""
|
||||
return self._engine.list_models()
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re as _re
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from openjarvis.agents.manager import AgentManager
|
||||
@@ -59,11 +60,91 @@ class FeedbackRequest(BaseModel):
|
||||
|
||||
|
||||
_BROWSER_SUB_TOOLS = {
|
||||
"browser_navigate", "browser_click", "browser_type",
|
||||
"browser_screenshot", "browser_extract", "browser_axtree",
|
||||
"browser_navigate",
|
||||
"browser_click",
|
||||
"browser_type",
|
||||
"browser_screenshot",
|
||||
"browser_extract",
|
||||
"browser_axtree",
|
||||
}
|
||||
|
||||
|
||||
class _LightweightSystem:
|
||||
"""Minimal system facade for the executor — avoids rebuilding the
|
||||
full JarvisSystem (which picks a random model from Ollama)."""
|
||||
|
||||
def __init__(self, engine: Any, model: str, config: Any = None):
|
||||
self.engine = engine
|
||||
self.model = model
|
||||
self.config = config
|
||||
self.memory_backend = None
|
||||
|
||||
|
||||
def _make_lightweight_system(
|
||||
engine: Any,
|
||||
model: str,
|
||||
config: Any = None,
|
||||
) -> _LightweightSystem:
|
||||
"""Build a minimal system with a plain OllamaEngine.
|
||||
|
||||
The server's ``app.state.engine`` is heavily wrapped
|
||||
(MultiEngine -> InstrumentedEngine -> GuardrailsEngine) and can
|
||||
return empty content from background threads. Create a fresh
|
||||
OllamaEngine directly (no health checks or model discovery that
|
||||
could interfere with in-flight Ollama requests).
|
||||
"""
|
||||
try:
|
||||
from openjarvis.engine.ollama import OllamaEngine
|
||||
|
||||
cfg = config
|
||||
if cfg is None:
|
||||
from openjarvis.core.config import load_config
|
||||
|
||||
cfg = load_config()
|
||||
host = cfg.engine.ollama.host if cfg else ""
|
||||
plain_engine = OllamaEngine(host=host) if host else OllamaEngine()
|
||||
return _LightweightSystem(plain_engine, model, cfg)
|
||||
except Exception:
|
||||
pass
|
||||
return _LightweightSystem(engine, model, config)
|
||||
|
||||
|
||||
def _parse_param_count(model_name: str) -> float:
|
||||
"""Extract parameter count in billions from model name.
|
||||
|
||||
Examples: 'qwen3.5:9b' -> 9.0, 'qwen3.5:0.8b' -> 0.8
|
||||
"""
|
||||
m = _re.search(r":(\d+(?:\.\d+)?)b", model_name.lower())
|
||||
return float(m.group(1)) if m else 0.0
|
||||
|
||||
|
||||
_CLOUD_PREFIXES = ("gpt-", "claude-", "gemini-", "o1-", "o3-", "o4-")
|
||||
|
||||
|
||||
def _pick_recommended_model(
|
||||
model_ids: list[str],
|
||||
) -> dict[str, str]:
|
||||
"""Pick the second-largest local model from a list."""
|
||||
local = [
|
||||
m for m in model_ids
|
||||
if not any(m.startswith(p) for p in _CLOUD_PREFIXES)
|
||||
]
|
||||
if not local:
|
||||
return {
|
||||
"model": model_ids[0] if model_ids else "",
|
||||
"reason": "Only model available",
|
||||
}
|
||||
sized = sorted(local, key=_parse_param_count, reverse=True)
|
||||
if len(sized) == 1:
|
||||
return {"model": sized[0], "reason": "Only local model available"}
|
||||
pick = sized[1] # second-largest
|
||||
params = _parse_param_count(pick)
|
||||
return {
|
||||
"model": pick,
|
||||
"reason": f"Second-largest local model ({params}B parameters)",
|
||||
}
|
||||
|
||||
|
||||
def _ensure_registries_populated() -> None:
|
||||
"""Ensure ToolRegistry and ChannelRegistry are populated.
|
||||
|
||||
@@ -97,9 +178,8 @@ def _ensure_registries_populated() -> None:
|
||||
# If registries are still empty, reload individual submodules from sys.modules
|
||||
if not ChannelRegistry.keys():
|
||||
for mod_name in list(sys.modules):
|
||||
if (
|
||||
mod_name.startswith("openjarvis.channels.")
|
||||
and not mod_name.endswith("_stubs")
|
||||
if mod_name.startswith("openjarvis.channels.") and not mod_name.endswith(
|
||||
"_stubs"
|
||||
):
|
||||
try:
|
||||
importlib.reload(sys.modules[mod_name])
|
||||
@@ -154,53 +234,63 @@ def build_tools_list() -> List[Dict[str, Any]]:
|
||||
except Exception:
|
||||
spec = None
|
||||
cred_keys = TOOL_CREDENTIALS.get(name, [])
|
||||
items.append({
|
||||
"name": name,
|
||||
"description": spec.description if spec else "",
|
||||
"category": spec.category if spec else "",
|
||||
"source": "tool",
|
||||
"requires_credentials": len(cred_keys) > 0,
|
||||
"credential_keys": cred_keys,
|
||||
"configured": (
|
||||
all(bool(os.environ.get(k)) for k in cred_keys)
|
||||
if cred_keys else True
|
||||
),
|
||||
})
|
||||
items.append(
|
||||
{
|
||||
"name": name,
|
||||
"description": spec.description if spec else "",
|
||||
"category": spec.category if spec else "",
|
||||
"source": "tool",
|
||||
"requires_credentials": len(cred_keys) > 0,
|
||||
"credential_keys": cred_keys,
|
||||
"configured": (
|
||||
all(bool(os.environ.get(k)) for k in cred_keys)
|
||||
if cred_keys
|
||||
else True
|
||||
),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
if any(ToolRegistry.contains(n) for n in _BROWSER_SUB_TOOLS):
|
||||
items.append({
|
||||
"name": "browser",
|
||||
"description": (
|
||||
"Web browser automation"
|
||||
" (navigate, click, type, screenshot, extract)"
|
||||
),
|
||||
"category": "browser",
|
||||
"source": "tool",
|
||||
"requires_credentials": False,
|
||||
"credential_keys": [],
|
||||
"configured": True,
|
||||
})
|
||||
items.append(
|
||||
{
|
||||
"name": "browser",
|
||||
"description": (
|
||||
"Web browser automation"
|
||||
" (navigate, click, type, screenshot, extract)"
|
||||
),
|
||||
"category": "browser",
|
||||
"source": "tool",
|
||||
"requires_credentials": False,
|
||||
"credential_keys": [],
|
||||
"configured": True,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
for name, _cls in ChannelRegistry.items():
|
||||
cred_keys = TOOL_CREDENTIALS.get(name, [])
|
||||
items.append({
|
||||
"name": name,
|
||||
"description": f"{name.replace('_', ' ').title()} messaging channel",
|
||||
"category": "communication",
|
||||
"source": "channel",
|
||||
"requires_credentials": len(cred_keys) > 0,
|
||||
"credential_keys": cred_keys,
|
||||
"configured": (
|
||||
all(bool(os.environ.get(k)) for k in cred_keys)
|
||||
if cred_keys else True
|
||||
),
|
||||
})
|
||||
items.append(
|
||||
{
|
||||
"name": name,
|
||||
"description": (
|
||||
f"{name.replace('_', ' ').title()} messaging channel"
|
||||
),
|
||||
"category": "communication",
|
||||
"source": "channel",
|
||||
"requires_credentials": len(cred_keys) > 0,
|
||||
"credential_keys": cred_keys,
|
||||
"configured": (
|
||||
all(bool(os.environ.get(k)) for k in cred_keys)
|
||||
if cred_keys
|
||||
else True
|
||||
),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -242,6 +332,142 @@ def _build_deep_research_tools(
|
||||
]
|
||||
|
||||
|
||||
def _merge_tool_call_fragments(
|
||||
accumulated: Dict[int, Dict[str, Any]],
|
||||
fragments: List[Dict[str, Any]],
|
||||
) -> None:
|
||||
"""Merge incremental tool_call delta fragments into accumulated state.
|
||||
|
||||
OpenAI-compatible APIs send tool_calls as incremental fragments keyed
|
||||
by ``index``. Each fragment may contain partial ``function.name`` and/or
|
||||
``function.arguments`` strings that must be concatenated.
|
||||
"""
|
||||
for frag in fragments:
|
||||
idx = frag.get("index", 0)
|
||||
if idx not in accumulated:
|
||||
accumulated[idx] = {
|
||||
"id": frag.get("id", ""),
|
||||
"type": "function",
|
||||
"function": {"name": "", "arguments": ""},
|
||||
}
|
||||
entry = accumulated[idx]
|
||||
if frag.get("id"):
|
||||
entry["id"] = frag["id"]
|
||||
fn = frag.get("function", {})
|
||||
if fn.get("name"):
|
||||
entry["function"]["name"] += fn["name"]
|
||||
if fn.get("arguments"):
|
||||
entry["function"]["arguments"] += fn["arguments"]
|
||||
|
||||
|
||||
def _get_mcp_tools(app_state: Any) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
|
||||
"""Return (openai_tools_list, mcp_adapters_by_name).
|
||||
|
||||
Lazily discovers MCP tools from config and caches them on ``app_state``
|
||||
so that subsequent requests reuse the same connections.
|
||||
"""
|
||||
cached = getattr(app_state, "_mcp_tools_cache", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
import json as _json
|
||||
|
||||
from openjarvis.core.config import load_config
|
||||
|
||||
openai_tools: List[Dict[str, Any]] = []
|
||||
adapters_by_name: Dict[str, Any] = {}
|
||||
|
||||
try:
|
||||
app_config = load_config()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to load config for MCP discovery: %s", exc)
|
||||
return openai_tools, adapters_by_name
|
||||
|
||||
if not app_config.tools.mcp.enabled or not app_config.tools.mcp.servers:
|
||||
return openai_tools, adapters_by_name
|
||||
|
||||
from openjarvis.mcp.client import MCPClient
|
||||
from openjarvis.mcp.transport import StdioTransport, StreamableHTTPTransport
|
||||
from openjarvis.tools.mcp_adapter import MCPToolProvider
|
||||
|
||||
# Keep clients alive so transports persist for tool calls at runtime
|
||||
mcp_clients: list = getattr(app_state, "_mcp_clients", [])
|
||||
|
||||
try:
|
||||
server_list = _json.loads(app_config.tools.mcp.servers)
|
||||
except (_json.JSONDecodeError, TypeError) as exc:
|
||||
logger.warning("Failed to parse MCP server config: %s", exc)
|
||||
return openai_tools, adapters_by_name
|
||||
|
||||
if not isinstance(server_list, list):
|
||||
return openai_tools, adapters_by_name
|
||||
|
||||
for server_cfg in server_list:
|
||||
cfg = _json.loads(server_cfg) if isinstance(server_cfg, str) else server_cfg
|
||||
name = cfg.get("name", "<unnamed>")
|
||||
url = cfg.get("url")
|
||||
command = cfg.get("command", "")
|
||||
args = cfg.get("args", [])
|
||||
|
||||
try:
|
||||
if url:
|
||||
transport = StreamableHTTPTransport(url=url)
|
||||
elif command:
|
||||
transport = StdioTransport(command=[command] + args)
|
||||
else:
|
||||
logger.warning(
|
||||
"MCP server '%s' has neither 'url' nor 'command' — skipping",
|
||||
name,
|
||||
)
|
||||
continue
|
||||
|
||||
client = MCPClient(transport)
|
||||
client.initialize()
|
||||
mcp_clients.append(client)
|
||||
|
||||
provider = MCPToolProvider(client)
|
||||
discovered = provider.discover()
|
||||
|
||||
# Per-server tool filtering
|
||||
include_tools = set(cfg.get("include_tools", []))
|
||||
exclude_tools = set(cfg.get("exclude_tools", []))
|
||||
if include_tools:
|
||||
discovered = [t for t in discovered if t.spec.name in include_tools]
|
||||
if exclude_tools:
|
||||
discovered = [t for t in discovered if t.spec.name not in exclude_tools]
|
||||
|
||||
for adapter in discovered:
|
||||
spec = adapter.spec
|
||||
openai_tools.append(
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": spec.name,
|
||||
"description": spec.description,
|
||||
"parameters": spec.parameters,
|
||||
},
|
||||
}
|
||||
)
|
||||
adapters_by_name[spec.name] = adapter
|
||||
|
||||
logger.info(
|
||||
"Discovered %d MCP tools from server '%s'",
|
||||
len(discovered),
|
||||
name,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to discover MCP tools from '%s': %s",
|
||||
name,
|
||||
exc,
|
||||
)
|
||||
|
||||
app_state._mcp_clients = mcp_clients
|
||||
if openai_tools:
|
||||
app_state._mcp_tools_cache = (openai_tools, adapters_by_name)
|
||||
return openai_tools, adapters_by_name
|
||||
|
||||
|
||||
async def _stream_managed_agent(
|
||||
*,
|
||||
manager: AgentManager,
|
||||
@@ -250,36 +476,31 @@ async def _stream_managed_agent(
|
||||
message_id: str,
|
||||
engine: Any,
|
||||
bus: Any,
|
||||
app_state: Any = None,
|
||||
) -> StreamingResponse:
|
||||
"""Run a managed agent and stream the response as SSE.
|
||||
"""Run a managed agent with real LLM token streaming via SSE.
|
||||
|
||||
Instantiates the agent from its stored config, builds conversation
|
||||
context from message history, executes the agent in a background
|
||||
thread, and yields SSE-formatted chunks. After completion the
|
||||
full response is persisted via ``manager.store_agent_response()``.
|
||||
Uses ``engine.stream_full()`` to yield tokens as they arrive from the
|
||||
LLM. Supports multi-turn tool-calling: when the model emits tool_calls,
|
||||
they are executed and the results fed back for the next turn.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from openjarvis.agents._stubs import AgentContext
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
from openjarvis.core.types import Message, Role
|
||||
|
||||
agent_id = agent_record["id"]
|
||||
config = agent_record.get("config", {})
|
||||
agent_type = agent_record.get("agent_type", "orchestrator")
|
||||
model = config.get("model", getattr(engine, "_model", ""))
|
||||
system_prompt = config.get("system_prompt")
|
||||
temperature = config.get("temperature", 0.7)
|
||||
max_tokens = config.get("max_tokens", 1024)
|
||||
max_turns = config.get("max_turns", 10)
|
||||
|
||||
# Resolve the agent class from registry
|
||||
agent_cls = AgentRegistry.get(agent_type)
|
||||
if agent_cls is None:
|
||||
# Fallback to orchestrator if the type is not registered
|
||||
agent_cls = AgentRegistry.get("orchestrator")
|
||||
if agent_cls is None:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Agent type '{agent_type}' not found in registry",
|
||||
)
|
||||
# Build conversation messages from history + current input
|
||||
llm_messages: List[Message] = []
|
||||
if system_prompt:
|
||||
llm_messages.append(Message(role=Role.SYSTEM, content=system_prompt))
|
||||
|
||||
# Build agent constructor kwargs from config
|
||||
agent_kwargs: Dict[str, Any] = {
|
||||
@@ -317,85 +538,234 @@ async def _stream_managed_agent(
|
||||
messages = manager.list_messages(agent_id, limit=50)
|
||||
# Messages come in DESC order, reverse for chronological
|
||||
for m in reversed(messages):
|
||||
# Skip the message we just stored (it will be the input)
|
||||
if m["id"] == message_id:
|
||||
continue
|
||||
if m["direction"] == "user_to_agent":
|
||||
ctx.conversation.add(Message(role=Role.USER, content=m["content"]))
|
||||
llm_messages.append(Message(role=Role.USER, content=m["content"]))
|
||||
elif m["direction"] == "agent_to_user":
|
||||
ctx.conversation.add(Message(role=Role.ASSISTANT, content=m["content"]))
|
||||
llm_messages.append(Message(role=Role.ASSISTANT, content=m["content"]))
|
||||
|
||||
# Append the current user message
|
||||
llm_messages.append(Message(role=Role.USER, content=user_content))
|
||||
|
||||
# Mark the user message as delivered
|
||||
manager.mark_message_delivered(message_id)
|
||||
|
||||
chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
|
||||
async def generate():
|
||||
"""Async generator yielding SSE-formatted chunks."""
|
||||
collected_content = ""
|
||||
# Build extra kwargs for stream_full (e.g. tools from config)
|
||||
stream_kwargs: Dict[str, Any] = {}
|
||||
if config.get("tools"):
|
||||
stream_kwargs["tools"] = config["tools"]
|
||||
|
||||
# Run agent.run() in a background thread
|
||||
# Discover MCP tools and merge into stream_kwargs
|
||||
mcp_adapters: Dict[str, Any] = {}
|
||||
if app_state is not None:
|
||||
try:
|
||||
result = await asyncio.to_thread(agent.run, user_content, context=ctx)
|
||||
mcp_openai_tools, mcp_adapters = _get_mcp_tools(app_state)
|
||||
if mcp_openai_tools:
|
||||
existing_tools = stream_kwargs.get("tools", [])
|
||||
stream_kwargs["tools"] = existing_tools + mcp_openai_tools
|
||||
logger.info(
|
||||
"Added %d MCP tools to streaming request",
|
||||
len(mcp_openai_tools),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Managed agent stream error: %s", exc, exc_info=True)
|
||||
error_data = {
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": {"content": f"Error: {exc}"},
|
||||
"finish_reason": "stop",
|
||||
}],
|
||||
}
|
||||
yield f"data: {json.dumps(error_data)}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
return
|
||||
logger.warning(
|
||||
"Failed to get MCP tools for streaming: %s", exc, exc_info=True
|
||||
)
|
||||
|
||||
content = result.content or ""
|
||||
collected_content = content
|
||||
async def generate():
|
||||
"""Async generator yielding SSE-formatted chunks with real token streaming."""
|
||||
|
||||
# Emit tool results metadata if any
|
||||
if result.tool_results:
|
||||
tool_data = []
|
||||
for tr in result.tool_results:
|
||||
tool_data.append({
|
||||
"tool_name": tr.tool_name,
|
||||
"success": tr.success,
|
||||
"output": tr.content,
|
||||
"latency_ms": tr.latency_seconds * 1000,
|
||||
})
|
||||
yield f"event: tool_results\ndata: {json.dumps({'results': tool_data})}\n\n"
|
||||
collected_content = ""
|
||||
messages_for_llm = list(llm_messages)
|
||||
turns = 0
|
||||
|
||||
# Stream content word-by-word for real-time feel
|
||||
if content:
|
||||
words = content.split(" ")
|
||||
for i, word in enumerate(words):
|
||||
token = word if i == 0 else " " + word
|
||||
chunk_data = {
|
||||
while turns < max_turns:
|
||||
turns += 1
|
||||
turn_content = ""
|
||||
tool_call_fragments: Dict[int, Dict[str, Any]] = {}
|
||||
current_finish_reason = None
|
||||
|
||||
try:
|
||||
async for chunk in engine.stream_full(
|
||||
messages_for_llm,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**stream_kwargs,
|
||||
):
|
||||
# Stream content tokens immediately to the client
|
||||
if chunk.content:
|
||||
turn_content += chunk.content
|
||||
chunk_data = {
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": chunk.content},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
yield f"data: {json.dumps(chunk_data)}\n\n"
|
||||
|
||||
# Accumulate tool_call fragments
|
||||
if chunk.tool_calls:
|
||||
_merge_tool_call_fragments(
|
||||
tool_call_fragments,
|
||||
chunk.tool_calls,
|
||||
)
|
||||
|
||||
if chunk.finish_reason:
|
||||
current_finish_reason = chunk.finish_reason
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Managed agent stream error: %s", exc, exc_info=True)
|
||||
error_data = {
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": {"content": token},
|
||||
"finish_reason": None,
|
||||
}],
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": f"Error: {exc}"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
}
|
||||
yield f"data: {json.dumps(chunk_data)}\n\n"
|
||||
await asyncio.sleep(0.012)
|
||||
yield f"data: {json.dumps(error_data)}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
return
|
||||
|
||||
# Handle tool calls: execute tools and loop for next turn
|
||||
if tool_call_fragments and current_finish_reason == "tool_calls":
|
||||
# Build the assistant message with tool_calls
|
||||
sorted_tcs = [
|
||||
tool_call_fragments[i] for i in sorted(tool_call_fragments.keys())
|
||||
]
|
||||
|
||||
# Emit tool_calls metadata as SSE event
|
||||
tool_meta = []
|
||||
for tc in sorted_tcs:
|
||||
tool_meta.append(
|
||||
{
|
||||
"tool_name": tc["function"]["name"],
|
||||
"arguments": tc["function"]["arguments"],
|
||||
}
|
||||
)
|
||||
yield (
|
||||
f"event: tool_calls\ndata: {json.dumps({'calls': tool_meta})}\n\n"
|
||||
)
|
||||
|
||||
# Add assistant message with tool_calls to conversation
|
||||
from openjarvis.core.types import ToolCall as MsgToolCall
|
||||
|
||||
assistant_msg = Message(
|
||||
role=Role.ASSISTANT,
|
||||
content=turn_content or None,
|
||||
tool_calls=[
|
||||
MsgToolCall(
|
||||
id=tc["id"],
|
||||
name=tc["function"]["name"],
|
||||
arguments=tc["function"]["arguments"],
|
||||
)
|
||||
for tc in sorted_tcs
|
||||
],
|
||||
)
|
||||
messages_for_llm.append(assistant_msg)
|
||||
|
||||
# Execute each tool call and append results
|
||||
for tc in sorted_tcs:
|
||||
tool_name = tc["function"]["name"]
|
||||
tool_args = tc["function"]["arguments"]
|
||||
tool_result_content = f"Tool '{tool_name}' not available"
|
||||
|
||||
try:
|
||||
# Try MCP adapter first (external tools)
|
||||
mcp_adapter = mcp_adapters.get(tool_name)
|
||||
if mcp_adapter is not None:
|
||||
try:
|
||||
parsed_args = json.loads(tool_args) if tool_args else {}
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
parsed_args = {}
|
||||
result = mcp_adapter.execute(**parsed_args)
|
||||
tool_result_content = result.content
|
||||
else:
|
||||
# Try to use ToolExecutor if tools are configured
|
||||
from openjarvis.core.registry import ToolRegistry
|
||||
from openjarvis.tools._stubs import (
|
||||
ToolCall as StubToolCall,
|
||||
)
|
||||
from openjarvis.tools._stubs import (
|
||||
ToolExecutor,
|
||||
)
|
||||
|
||||
tool_cls = ToolRegistry.get(tool_name)
|
||||
if tool_cls is not None:
|
||||
tool_instance = tool_cls()
|
||||
executor = ToolExecutor(tools=[tool_instance], bus=bus)
|
||||
result = executor.execute(
|
||||
StubToolCall(
|
||||
id=tc["id"],
|
||||
name=tool_name,
|
||||
arguments=tool_args,
|
||||
),
|
||||
)
|
||||
tool_result_content = result.content
|
||||
else:
|
||||
logger.warning(
|
||||
"Tool '%s' not found in registry or MCP adapters",
|
||||
tool_name,
|
||||
)
|
||||
except Exception as tool_exc:
|
||||
logger.error(
|
||||
"Tool execution error for %s: %s",
|
||||
tool_name,
|
||||
tool_exc,
|
||||
exc_info=True,
|
||||
)
|
||||
tool_result_content = f"Error executing {tool_name}: {tool_exc}"
|
||||
|
||||
# Emit tool result as SSE event
|
||||
tool_event_data = json.dumps(
|
||||
{"tool_name": tool_name, "output": tool_result_content}
|
||||
)
|
||||
yield (f"event: tool_result\ndata: {tool_event_data}\n\n")
|
||||
|
||||
# Add tool result message to conversation
|
||||
messages_for_llm.append(
|
||||
Message(
|
||||
role=Role.TOOL,
|
||||
content=tool_result_content,
|
||||
tool_call_id=tc["id"],
|
||||
name=tool_name,
|
||||
)
|
||||
)
|
||||
|
||||
# Continue to next turn (loop back to stream_full)
|
||||
collected_content += turn_content
|
||||
continue
|
||||
|
||||
# No tool calls — this is the final response
|
||||
collected_content += turn_content
|
||||
break
|
||||
|
||||
# Final chunk with finish_reason
|
||||
final_data = {
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": {},
|
||||
"finish_reason": "stop",
|
||||
}],
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
}
|
||||
yield f"data: {json.dumps(final_data)}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
@@ -406,7 +776,9 @@ async def _stream_managed_agent(
|
||||
manager.store_agent_response(agent_id, collected_content)
|
||||
except Exception as store_exc:
|
||||
logger.error(
|
||||
"Failed to store agent response: %s", store_exc, exc_info=True,
|
||||
"Failed to store agent response: %s",
|
||||
store_exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
@@ -493,7 +865,7 @@ def create_agent_manager_router(
|
||||
return {"status": "idle"}
|
||||
|
||||
@agents_router.post("/{agent_id}/run")
|
||||
async def run_agent(agent_id: str):
|
||||
async def run_agent(agent_id: str, request: Request):
|
||||
import threading
|
||||
|
||||
agent = manager.get_agent(agent_id)
|
||||
@@ -501,30 +873,46 @@ def create_agent_manager_router(
|
||||
raise HTTPException(status_code=404, detail="Agent not found")
|
||||
if agent["status"] == "archived":
|
||||
raise HTTPException(status_code=400, detail="Agent is archived")
|
||||
if agent["status"] == "running":
|
||||
|
||||
# Auto-recover from error/needs_attention state
|
||||
if agent["status"] in ("error", "needs_attention"):
|
||||
manager.update_agent(agent_id, status="idle")
|
||||
|
||||
# Acquire tick BEFORE spawning thread — prevents race
|
||||
try:
|
||||
manager.start_tick(agent_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=409, detail="Agent is already running")
|
||||
|
||||
# Re-use the server's engine + model so we don't pick a
|
||||
# random model from Ollama's list.
|
||||
server_engine = getattr(request.app.state, "engine", None)
|
||||
server_model = getattr(request.app.state, "model", "")
|
||||
server_config = getattr(request.app.state, "config", None)
|
||||
|
||||
def _run_tick():
|
||||
try:
|
||||
from openjarvis.agents.executor import AgentExecutor
|
||||
from openjarvis.core.events import get_event_bus
|
||||
from openjarvis.system import SystemBuilder
|
||||
|
||||
executor = AgentExecutor(
|
||||
manager=manager, event_bus=get_event_bus(),
|
||||
manager=manager,
|
||||
event_bus=get_event_bus(),
|
||||
)
|
||||
try:
|
||||
system = SystemBuilder().build()
|
||||
executor.set_system(system)
|
||||
except Exception as build_err:
|
||||
manager.update_agent(agent_id, status="error")
|
||||
manager.update_summary_memory(
|
||||
agent_id,
|
||||
f"ERROR: Failed to build system: {build_err}",
|
||||
)
|
||||
return
|
||||
system = _make_lightweight_system(
|
||||
server_engine,
|
||||
server_model,
|
||||
server_config,
|
||||
)
|
||||
executor.set_system(system)
|
||||
executor.execute_tick(agent_id)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Run-tick failed for agent %s: %s",
|
||||
agent_id,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
manager.end_tick(agent_id)
|
||||
except Exception:
|
||||
@@ -618,10 +1006,84 @@ def create_agent_manager_router(
|
||||
if not agent_record:
|
||||
raise HTTPException(status_code=404, detail="Agent not found")
|
||||
|
||||
# Auto-recover error-state agents on immediate messages
|
||||
if req.mode == "immediate" and agent_record["status"] in (
|
||||
"error",
|
||||
"needs_attention",
|
||||
):
|
||||
manager.update_agent(agent_id, status="idle")
|
||||
|
||||
# Store user message in DB (always, regardless of stream mode)
|
||||
msg = manager.send_message(agent_id, req.content, mode=req.mode)
|
||||
|
||||
if not req.stream:
|
||||
if not req.stream and req.mode != "immediate":
|
||||
return msg
|
||||
|
||||
if not req.stream and req.mode == "immediate":
|
||||
# Non-streaming immediate: trigger a background tick so the
|
||||
# agent processes the message, then return the stored msg.
|
||||
# Re-use the server's existing system (correct model/engine).
|
||||
import threading
|
||||
import time as _time
|
||||
|
||||
from openjarvis.agents.executor import AgentExecutor
|
||||
from openjarvis.core.events import get_event_bus
|
||||
|
||||
_srv_engine = getattr(request.app.state, "engine", None)
|
||||
_srv_model = getattr(request.app.state, "model", "")
|
||||
_srv_config = getattr(request.app.state, "config", None)
|
||||
|
||||
def _immediate_tick():
|
||||
_start = _time.time()
|
||||
logger.info(
|
||||
"Immediate tick starting for agent %s (model=%s)",
|
||||
agent_id,
|
||||
_srv_model,
|
||||
)
|
||||
try:
|
||||
executor = AgentExecutor(
|
||||
manager=manager,
|
||||
event_bus=get_event_bus(),
|
||||
)
|
||||
system = _make_lightweight_system(
|
||||
_srv_engine,
|
||||
_srv_model,
|
||||
_srv_config,
|
||||
)
|
||||
executor.set_system(system)
|
||||
logger.info(
|
||||
"Immediate tick: system ready in %.1fs, "
|
||||
"executing tick for agent %s",
|
||||
_time.time() - _start,
|
||||
agent_id,
|
||||
)
|
||||
executor.execute_tick(agent_id)
|
||||
logger.info(
|
||||
"Immediate tick completed for agent %s in %.1fs",
|
||||
agent_id,
|
||||
_time.time() - _start,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Immediate tick failed for agent %s: %s",
|
||||
agent_id,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
manager.end_tick(agent_id)
|
||||
except Exception:
|
||||
pass
|
||||
manager.update_agent(agent_id, status="error")
|
||||
manager.update_summary_memory(
|
||||
agent_id,
|
||||
f"ERROR: {exc}",
|
||||
)
|
||||
|
||||
threading.Thread(
|
||||
target=_immediate_tick,
|
||||
daemon=True,
|
||||
).start()
|
||||
return msg
|
||||
|
||||
# --- Streaming mode: run agent and return SSE response ---
|
||||
@@ -640,6 +1102,7 @@ def create_agent_manager_router(
|
||||
message_id=msg["id"],
|
||||
engine=engine,
|
||||
bus=bus,
|
||||
app_state=request.app.state,
|
||||
)
|
||||
|
||||
# ── State inspection ─────────────────────────────────────
|
||||
@@ -745,9 +1208,7 @@ def create_agent_manager_router(
|
||||
|
||||
@templates_router.post("/{template_id}/instantiate")
|
||||
async def instantiate_template(template_id: str, req: CreateAgentRequest):
|
||||
return manager.create_from_template(
|
||||
template_id, req.name, overrides=req.config
|
||||
)
|
||||
return manager.create_from_template(template_id, req.name, overrides=req.config)
|
||||
|
||||
# ── Global agent endpoints ───────────────────────────────
|
||||
|
||||
@@ -774,13 +1235,42 @@ def create_agent_manager_router(
|
||||
"by_status": dict(counts),
|
||||
}
|
||||
|
||||
@global_router.get("/v1/recommended-model")
|
||||
def recommended_model(request: Request):
|
||||
engine = getattr(request.app.state, "engine", None)
|
||||
if engine is None:
|
||||
return {"model": "", "reason": "No engine available"}
|
||||
try:
|
||||
models = engine.list_models()
|
||||
except Exception:
|
||||
models = []
|
||||
return _pick_recommended_model(models)
|
||||
|
||||
# ── Tools & credentials ──────────────────────────────────
|
||||
|
||||
tools_router = APIRouter(prefix="/v1/tools", tags=["tools"])
|
||||
|
||||
@tools_router.get("")
|
||||
def list_tools():
|
||||
return {"tools": build_tools_list()}
|
||||
def list_tools(request: Request):
|
||||
items = build_tools_list()
|
||||
try:
|
||||
mcp_tools, _ = _get_mcp_tools(request.app.state)
|
||||
for tool in mcp_tools:
|
||||
fn = tool.get("function", {})
|
||||
items.append(
|
||||
{
|
||||
"name": fn.get("name", ""),
|
||||
"description": fn.get("description", ""),
|
||||
"category": "mcp",
|
||||
"source": "mcp",
|
||||
"requires_credentials": False,
|
||||
"credential_keys": [],
|
||||
"configured": True,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return {"tools": items}
|
||||
|
||||
@tools_router.post("/{tool_name}/credentials")
|
||||
async def save_tool_credentials(tool_name: str, request: Request):
|
||||
@@ -796,6 +1286,7 @@ def create_agent_manager_router(
|
||||
@tools_router.get("/{tool_name}/credentials/status")
|
||||
def credential_status(tool_name: str):
|
||||
from openjarvis.core.credentials import get_credential_status
|
||||
|
||||
return get_credential_status(tool_name)
|
||||
|
||||
return agents_router, templates_router, global_router, tools_router
|
||||
|
||||
+127
-55
@@ -32,12 +32,14 @@ def _to_messages(chat_messages) -> list[Message]:
|
||||
messages = []
|
||||
for m in chat_messages:
|
||||
role = Role(m.role) if m.role in {r.value for r in Role} else Role.USER
|
||||
messages.append(Message(
|
||||
role=role,
|
||||
content=m.content or "",
|
||||
name=m.name,
|
||||
tool_call_id=m.tool_call_id,
|
||||
))
|
||||
messages.append(
|
||||
Message(
|
||||
role=role,
|
||||
content=m.content or "",
|
||||
name=m.name,
|
||||
tool_call_id=m.tool_call_id,
|
||||
)
|
||||
)
|
||||
return messages
|
||||
|
||||
|
||||
@@ -75,7 +77,10 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request
|
||||
max_context_tokens=config.memory.context_max_tokens,
|
||||
)
|
||||
enriched = inject_context(
|
||||
query_text, messages, memory_backend, config=ctx_cfg,
|
||||
query_text,
|
||||
messages,
|
||||
memory_backend,
|
||||
config=ctx_cfg,
|
||||
)
|
||||
# Rebuild request messages from enriched Message objects
|
||||
if len(enriched) > len(messages):
|
||||
@@ -83,16 +88,19 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request
|
||||
|
||||
new_msgs = []
|
||||
for msg in enriched:
|
||||
new_msgs.append(ChatMessage(
|
||||
role=msg.role.value,
|
||||
content=msg.content,
|
||||
name=msg.name,
|
||||
tool_call_id=getattr(msg, "tool_call_id", None),
|
||||
))
|
||||
new_msgs.append(
|
||||
ChatMessage(
|
||||
role=msg.role.value,
|
||||
content=msg.content,
|
||||
name=msg.name,
|
||||
tool_call_id=getattr(msg, "tool_call_id", None),
|
||||
)
|
||||
)
|
||||
request_body.messages = new_msgs
|
||||
except Exception:
|
||||
logging.getLogger("openjarvis.server").debug(
|
||||
"Memory context injection failed", exc_info=True,
|
||||
"Memory context injection failed",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# Run complexity analysis on the last user message
|
||||
@@ -111,7 +119,8 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request
|
||||
|
||||
cr = score_complexity(query_text_for_complexity)
|
||||
suggested = adjust_tokens_for_model(
|
||||
cr.suggested_max_tokens, model,
|
||||
cr.suggested_max_tokens,
|
||||
model,
|
||||
)
|
||||
complexity_info = ComplexityInfo(
|
||||
score=cr.score,
|
||||
@@ -124,7 +133,8 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request
|
||||
request_body.max_tokens = suggested
|
||||
except Exception:
|
||||
logging.getLogger("openjarvis.server").debug(
|
||||
"Complexity analysis failed", exc_info=True,
|
||||
"Complexity analysis failed",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if request_body.stream:
|
||||
@@ -143,13 +153,19 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request
|
||||
|
||||
bus = getattr(request.app.state, "bus", None)
|
||||
return _handle_direct(
|
||||
engine, model, request_body,
|
||||
bus=bus, complexity_info=complexity_info,
|
||||
engine,
|
||||
model,
|
||||
request_body,
|
||||
bus=bus,
|
||||
complexity_info=complexity_info,
|
||||
)
|
||||
|
||||
|
||||
def _handle_direct(
|
||||
engine, model: str, req: ChatCompletionRequest, bus=None,
|
||||
engine,
|
||||
model: str,
|
||||
req: ChatCompletionRequest,
|
||||
bus=None,
|
||||
complexity_info=None,
|
||||
) -> ChatCompletionResponse:
|
||||
"""Direct engine call without agent."""
|
||||
@@ -161,8 +177,12 @@ def _handle_direct(
|
||||
from openjarvis.telemetry.wrapper import instrumented_generate
|
||||
|
||||
result = instrumented_generate(
|
||||
engine, messages, model=model, bus=bus,
|
||||
temperature=req.temperature, max_tokens=req.max_tokens,
|
||||
engine,
|
||||
messages,
|
||||
model=model,
|
||||
bus=bus,
|
||||
temperature=req.temperature,
|
||||
max_tokens=req.max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
@@ -194,10 +214,12 @@ def _handle_direct(
|
||||
|
||||
return ChatCompletionResponse(
|
||||
model=model,
|
||||
choices=[Choice(
|
||||
message=choice_msg,
|
||||
finish_reason=result.get("finish_reason", "stop"),
|
||||
)],
|
||||
choices=[
|
||||
Choice(
|
||||
message=choice_msg,
|
||||
finish_reason=result.get("finish_reason", "stop"),
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(
|
||||
prompt_tokens=usage.get("prompt_tokens", 0),
|
||||
completion_tokens=usage.get("completion_tokens", 0),
|
||||
@@ -208,7 +230,9 @@ def _handle_direct(
|
||||
|
||||
|
||||
def _handle_agent(
|
||||
agent, model: str, req: ChatCompletionRequest,
|
||||
agent,
|
||||
model: str,
|
||||
req: ChatCompletionRequest,
|
||||
complexity_info=None,
|
||||
) -> ChatCompletionResponse:
|
||||
"""Run through agent."""
|
||||
@@ -241,10 +265,12 @@ def _handle_agent(
|
||||
|
||||
return ChatCompletionResponse(
|
||||
model=model,
|
||||
choices=[Choice(
|
||||
message=ChoiceMessage(role="assistant", content=result.content),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
choices=[
|
||||
Choice(
|
||||
message=ChoiceMessage(role="assistant", content=result.content),
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
usage=usage,
|
||||
complexity=complexity_info,
|
||||
)
|
||||
@@ -258,7 +284,9 @@ async def _handle_agent_stream(agent, bus, model, req):
|
||||
|
||||
|
||||
async def _handle_stream(
|
||||
engine, model: str, req: ChatCompletionRequest,
|
||||
engine,
|
||||
model: str,
|
||||
req: ChatCompletionRequest,
|
||||
complexity_info=None,
|
||||
):
|
||||
"""Stream response using SSE format."""
|
||||
@@ -270,9 +298,11 @@ async def _handle_stream(
|
||||
first_chunk = ChatCompletionChunk(
|
||||
id=chunk_id,
|
||||
model=model,
|
||||
choices=[StreamChoice(
|
||||
delta=DeltaMessage(role="assistant"),
|
||||
)],
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(role="assistant"),
|
||||
)
|
||||
],
|
||||
)
|
||||
yield f"data: {first_chunk.model_dump_json()}\n\n"
|
||||
|
||||
@@ -287,27 +317,34 @@ async def _handle_stream(
|
||||
chunk = ChatCompletionChunk(
|
||||
id=chunk_id,
|
||||
model=model,
|
||||
choices=[StreamChoice(
|
||||
delta=DeltaMessage(content=token),
|
||||
)],
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(content=token),
|
||||
)
|
||||
],
|
||||
)
|
||||
yield f"data: {chunk.model_dump_json()}\n\n"
|
||||
except Exception as exc:
|
||||
# Surface errors as a content chunk so the frontend can
|
||||
# display them instead of silently failing.
|
||||
import logging
|
||||
|
||||
logging.getLogger("openjarvis.server").error(
|
||||
"Stream error: %s", exc, exc_info=True,
|
||||
"Stream error: %s",
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
error_chunk = ChatCompletionChunk(
|
||||
id=chunk_id,
|
||||
model=model,
|
||||
choices=[StreamChoice(
|
||||
delta=DeltaMessage(
|
||||
content=f"\n\nError during generation: {exc}",
|
||||
),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(
|
||||
content=f"\n\nError during generation: {exc}",
|
||||
),
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
)
|
||||
yield f"data: {error_chunk.model_dump_json()}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
@@ -315,13 +352,16 @@ async def _handle_stream(
|
||||
|
||||
# Send finish chunk with usage data if available
|
||||
import json as _json
|
||||
|
||||
finish_data = ChatCompletionChunk(
|
||||
id=chunk_id,
|
||||
model=model,
|
||||
choices=[StreamChoice(
|
||||
delta=DeltaMessage(),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(),
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
)
|
||||
finish_dict = _json.loads(finish_data.model_dump_json())
|
||||
|
||||
@@ -456,18 +496,22 @@ async def savings(request: Request):
|
||||
# Exclude cloud model tokens from savings — only local
|
||||
# inference counts toward cost savings.
|
||||
_cloud_prefixes = (
|
||||
"gpt-", "o1-", "o3-", "o4-",
|
||||
"claude-", "gemini-", "openrouter/",
|
||||
"gpt-",
|
||||
"o1-",
|
||||
"o3-",
|
||||
"o4-",
|
||||
"claude-",
|
||||
"gemini-",
|
||||
"openrouter/",
|
||||
)
|
||||
local_models = [
|
||||
m for m in summary.per_model
|
||||
m
|
||||
for m in summary.per_model
|
||||
if not any(m.model_id.startswith(p) for p in _cloud_prefixes)
|
||||
]
|
||||
result = compute_savings(
|
||||
prompt_tokens=sum(m.prompt_tokens for m in local_models),
|
||||
completion_tokens=sum(
|
||||
m.completion_tokens for m in local_models
|
||||
),
|
||||
completion_tokens=sum(m.completion_tokens for m in local_models),
|
||||
total_calls=sum(m.call_count for m in local_models),
|
||||
session_start=session_start if session_start else 0.0,
|
||||
prompt_tokens_evaluated=sum(
|
||||
@@ -557,7 +601,8 @@ async def channel_send(request: Request):
|
||||
|
||||
if not channel_name or not content:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="'channel' and 'content' are required",
|
||||
status_code=400,
|
||||
detail="'channel' and 'content' are required",
|
||||
)
|
||||
|
||||
ok = bridge.send(channel_name, content, conversation_id=conversation_id)
|
||||
@@ -575,4 +620,31 @@ async def channel_status(request: Request):
|
||||
return {"status": bridge.status().value}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Security scan endpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/v1/security/scan")
|
||||
async def security_scan():
|
||||
"""Run a read-only security environment audit and return findings."""
|
||||
from openjarvis.cli.scan_cmd import PrivacyScanner
|
||||
|
||||
scanner = PrivacyScanner()
|
||||
results = scanner.run_all()
|
||||
return {
|
||||
"has_warnings": any(r.status == "warn" for r in results),
|
||||
"has_failures": any(r.status == "fail" for r in results),
|
||||
"findings": [
|
||||
{
|
||||
"name": r.name,
|
||||
"status": r.status,
|
||||
"message": r.message,
|
||||
"platform": r.platform,
|
||||
}
|
||||
for r in results
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["router"]
|
||||
|
||||
@@ -119,17 +119,15 @@ class AgentStreamBridge:
|
||||
from openjarvis.core.types import Message, Role
|
||||
|
||||
for m in self._request.messages[:-1]:
|
||||
role = (
|
||||
Role(m.role)
|
||||
if m.role in {r.value for r in Role}
|
||||
else Role.USER
|
||||
role = Role(m.role) if m.role in {r.value for r in Role} else Role.USER
|
||||
ctx.conversation.add(
|
||||
Message(
|
||||
role=role,
|
||||
content=m.content or "",
|
||||
name=m.name,
|
||||
tool_call_id=m.tool_call_id,
|
||||
)
|
||||
)
|
||||
ctx.conversation.add(Message(
|
||||
role=role,
|
||||
content=m.content or "",
|
||||
name=m.name,
|
||||
tool_call_id=m.tool_call_id,
|
||||
))
|
||||
|
||||
input_text = (
|
||||
self._request.messages[-1].content if self._request.messages else ""
|
||||
@@ -166,9 +164,11 @@ class AgentStreamBridge:
|
||||
first_chunk = ChatCompletionChunk(
|
||||
id=self._chunk_id,
|
||||
model=self._model,
|
||||
choices=[StreamChoice(
|
||||
delta=DeltaMessage(role="assistant"),
|
||||
)],
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(role="assistant"),
|
||||
)
|
||||
],
|
||||
)
|
||||
yield f"data: {first_chunk.model_dump_json()}\n\n"
|
||||
|
||||
@@ -202,18 +202,18 @@ class AgentStreamBridge:
|
||||
"Please try a shorter message."
|
||||
)
|
||||
elif "400" in error_str:
|
||||
error_content = (
|
||||
f"The model returned an error: {error_str}"
|
||||
)
|
||||
error_content = f"The model returned an error: {error_str}"
|
||||
else:
|
||||
error_content = f"Sorry, an error occurred: {error_str}"
|
||||
error_chunk = ChatCompletionChunk(
|
||||
id=self._chunk_id,
|
||||
model=self._model,
|
||||
choices=[StreamChoice(
|
||||
delta=DeltaMessage(content=error_content),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(content=error_content),
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
)
|
||||
yield f"data: {error_chunk.model_dump_json()}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
@@ -222,31 +222,88 @@ class AgentStreamBridge:
|
||||
# Emit tool results metadata if any
|
||||
tool_results_data = []
|
||||
for tr in agent_result.tool_results:
|
||||
tool_results_data.append({
|
||||
"tool_name": tr.tool_name,
|
||||
"success": tr.success,
|
||||
"output": tr.content,
|
||||
"latency_ms": tr.latency_seconds * 1000,
|
||||
})
|
||||
tool_results_data.append(
|
||||
{
|
||||
"tool_name": tr.tool_name,
|
||||
"success": tr.success,
|
||||
"output": tr.content,
|
||||
"latency_ms": tr.latency_seconds * 1000,
|
||||
}
|
||||
)
|
||||
|
||||
if tool_results_data:
|
||||
yield self._format_named_event(
|
||||
"tool_results", {"results": tool_results_data},
|
||||
"tool_results",
|
||||
{"results": tool_results_data},
|
||||
)
|
||||
|
||||
# Stream content progressively (word-by-word) for a
|
||||
# real-time feel, then send a final chunk with usage.
|
||||
# Stream content using real LLM token streaming via
|
||||
# engine.stream_full() when the engine is available.
|
||||
content = agent_result.content or ""
|
||||
if content:
|
||||
engine = getattr(self._agent, "_engine", None)
|
||||
used_real_streaming = False
|
||||
|
||||
if engine is not None and hasattr(engine, "stream_full") and content:
|
||||
# Re-stream using the engine for real token delivery.
|
||||
# Build the same messages the agent used for its final turn.
|
||||
try:
|
||||
from openjarvis.core.types import Message as MsgType
|
||||
from openjarvis.core.types import Role as RoleType
|
||||
|
||||
replay_messages = []
|
||||
for m in self._request.messages:
|
||||
role = (
|
||||
RoleType(m.role)
|
||||
if m.role in {r.value for r in RoleType}
|
||||
else RoleType.USER
|
||||
)
|
||||
replay_messages.append(
|
||||
MsgType(
|
||||
role=role,
|
||||
content=m.content or "",
|
||||
name=m.name,
|
||||
tool_call_id=m.tool_call_id,
|
||||
)
|
||||
)
|
||||
|
||||
async for sc in engine.stream_full(
|
||||
replay_messages,
|
||||
model=self._model,
|
||||
):
|
||||
if sc.content:
|
||||
chunk = ChatCompletionChunk(
|
||||
id=self._chunk_id,
|
||||
model=self._model,
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(content=sc.content),
|
||||
)
|
||||
],
|
||||
)
|
||||
yield f"data: {chunk.model_dump_json()}\n\n"
|
||||
used_real_streaming = True
|
||||
except Exception as stream_exc:
|
||||
import logging as _logging
|
||||
|
||||
_logger = _logging.getLogger("openjarvis.server")
|
||||
_logger.warning(
|
||||
"Real streaming failed, falling back to word replay: %s",
|
||||
stream_exc,
|
||||
)
|
||||
|
||||
# Fallback: word-by-word replay if real streaming was not used
|
||||
if not used_real_streaming and content:
|
||||
words = content.split(" ")
|
||||
for i, word in enumerate(words):
|
||||
token = word if i == 0 else " " + word
|
||||
chunk = ChatCompletionChunk(
|
||||
id=self._chunk_id,
|
||||
model=self._model,
|
||||
choices=[StreamChoice(
|
||||
delta=DeltaMessage(content=token),
|
||||
)],
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(content=token),
|
||||
)
|
||||
],
|
||||
)
|
||||
yield f"data: {chunk.model_dump_json()}\n\n"
|
||||
await asyncio.sleep(0.012)
|
||||
@@ -254,7 +311,8 @@ class AgentStreamBridge:
|
||||
# Final chunk: finish_reason + usage
|
||||
prompt_tokens = agent_result.metadata.get("prompt_tokens", 0)
|
||||
completion_tokens = agent_result.metadata.get(
|
||||
"completion_tokens", 0,
|
||||
"completion_tokens",
|
||||
0,
|
||||
)
|
||||
total_tokens = agent_result.metadata.get("total_tokens", 0)
|
||||
if total_tokens == 0:
|
||||
@@ -266,10 +324,12 @@ class AgentStreamBridge:
|
||||
final_chunk = ChatCompletionChunk(
|
||||
id=self._chunk_id,
|
||||
model=self._model,
|
||||
choices=[StreamChoice(
|
||||
delta=DeltaMessage(),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
choices=[
|
||||
StreamChoice(
|
||||
delta=DeltaMessage(),
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
)
|
||||
final_data = json.loads(final_chunk.model_dump_json())
|
||||
final_data["usage"] = UsageInfo(
|
||||
|
||||
@@ -49,6 +49,7 @@ class JarvisSystem:
|
||||
agent_executor: Optional[Any] = None # AgentExecutor
|
||||
speech_backend: Optional[Any] = None # SpeechBackend
|
||||
_learning_orchestrator: Optional[Any] = None # LearningOrchestrator
|
||||
_mcp_clients: List = field(default_factory=list)
|
||||
|
||||
def ask(
|
||||
self,
|
||||
@@ -251,6 +252,7 @@ class JarvisSystem:
|
||||
"tool_name": tr.tool_name,
|
||||
"content": tr.content,
|
||||
"success": tr.success,
|
||||
"arguments": tr.metadata.get("arguments", {}),
|
||||
}
|
||||
for tr in getattr(result, "tool_results", [])
|
||||
],
|
||||
@@ -370,6 +372,14 @@ class JarvisSystem:
|
||||
|
||||
channel_bridge.on_message(_on_channel_message)
|
||||
|
||||
def _close_mcp_clients(self) -> None:
|
||||
"""Close all persistent MCP client connections."""
|
||||
for client in self._mcp_clients:
|
||||
try:
|
||||
client.close()
|
||||
except Exception:
|
||||
logger.debug("Error closing MCP client", exc_info=True)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release resources."""
|
||||
if self.scheduler and hasattr(self.scheduler, "stop"):
|
||||
@@ -392,6 +402,7 @@ class JarvisSystem:
|
||||
self.agent_manager.close()
|
||||
if self.agent_scheduler is not None:
|
||||
self.agent_scheduler.stop()
|
||||
self._close_mcp_clients()
|
||||
|
||||
def __enter__(self) -> JarvisSystem:
|
||||
return self
|
||||
@@ -430,6 +441,7 @@ class SystemBuilder:
|
||||
self._workflow: Optional[bool] = None
|
||||
self._sessions: Optional[bool] = None
|
||||
self._speech: Optional[bool] = None
|
||||
self._mcp_clients: List = []
|
||||
|
||||
def engine(self, key: str) -> SystemBuilder:
|
||||
self._engine_key = key
|
||||
@@ -670,6 +682,8 @@ class SystemBuilder:
|
||||
speech_backend=speech_backend,
|
||||
)
|
||||
system._learning_orchestrator = learning_orchestrator
|
||||
# Transfer MCP clients so JarvisSystem.close() can shut them down
|
||||
system._mcp_clients = list(getattr(self, "_mcp_clients", []))
|
||||
# Wire system reference — must happen before scheduler.start()
|
||||
if system.agent_executor is not None:
|
||||
system.agent_executor.set_system(system)
|
||||
@@ -1068,24 +1082,60 @@ class SystemBuilder:
|
||||
logger.warning("Failed to set up learning orchestrator: %s", exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _discover_external_mcp(server_cfg) -> List[BaseTool]:
|
||||
"""Discover tools from an external MCP server configuration."""
|
||||
def _discover_external_mcp(self, server_cfg) -> List[BaseTool]:
|
||||
"""Discover tools from an external MCP server configuration.
|
||||
|
||||
Supports both stdio (command + args) and Streamable HTTP (url)
|
||||
transports. Persists MCP clients on ``self._mcp_clients`` so
|
||||
that transports stay alive for runtime tool calls.
|
||||
"""
|
||||
import json
|
||||
|
||||
from openjarvis.mcp.client import MCPClient
|
||||
from openjarvis.mcp.transport import StdioTransport
|
||||
from openjarvis.mcp.transport import StdioTransport, StreamableHTTPTransport
|
||||
from openjarvis.tools.mcp_adapter import MCPToolProvider
|
||||
|
||||
cfg = json.loads(server_cfg) if isinstance(server_cfg, str) else server_cfg
|
||||
name = cfg.get("name", "<unnamed>")
|
||||
url = cfg.get("url")
|
||||
command = cfg.get("command", "")
|
||||
args = cfg.get("args", [])
|
||||
if not command:
|
||||
|
||||
# Build transport based on config keys
|
||||
if url:
|
||||
transport = StreamableHTTPTransport(url=url)
|
||||
elif command:
|
||||
transport = StdioTransport(command=[command] + args)
|
||||
else:
|
||||
logger.warning(
|
||||
"MCP server '%s' has neither 'url' nor 'command' — skipping",
|
||||
name,
|
||||
)
|
||||
return []
|
||||
transport = StdioTransport(command=command, args=args)
|
||||
|
||||
client = MCPClient(transport)
|
||||
client.initialize()
|
||||
|
||||
# Persist client so the transport stays alive for tool calls
|
||||
self._mcp_clients.append(client)
|
||||
|
||||
provider = MCPToolProvider(client)
|
||||
return provider.discover()
|
||||
discovered = provider.discover()
|
||||
|
||||
# Per-server tool filtering
|
||||
include_tools = set(cfg.get("include_tools", []))
|
||||
exclude_tools = set(cfg.get("exclude_tools", []))
|
||||
if include_tools:
|
||||
discovered = [t for t in discovered if t.spec.name in include_tools]
|
||||
if exclude_tools:
|
||||
discovered = [t for t in discovered if t.spec.name not in exclude_tools]
|
||||
|
||||
logger.info(
|
||||
"Discovered %d tools from MCP server '%s'",
|
||||
len(discovered),
|
||||
name,
|
||||
)
|
||||
return discovered
|
||||
|
||||
|
||||
__all__ = ["JarvisSystem", "SystemBuilder"]
|
||||
|
||||
@@ -4,11 +4,12 @@ from __future__ import annotations
|
||||
|
||||
import statistics
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from openjarvis.core.events import EventBus, EventType
|
||||
from openjarvis.core.types import Message, TelemetryRecord
|
||||
from openjarvis.engine._stubs import InferenceEngine
|
||||
from openjarvis.engine._stubs import InferenceEngine, StreamChunk
|
||||
from openjarvis.telemetry.gpu_monitor import GpuSample
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -30,8 +31,14 @@ def _percentile(data: list[float], p: float) -> float:
|
||||
def _compute_itl_stats(itl_values_ms: list[float]) -> dict:
|
||||
"""Compute ITL summary statistics from a list of inter-token latencies in ms."""
|
||||
if not itl_values_ms:
|
||||
return {"mean": 0.0, "median": 0.0, "p90": 0.0,
|
||||
"p95": 0.0, "p99": 0.0, "std": 0.0}
|
||||
return {
|
||||
"mean": 0.0,
|
||||
"median": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0,
|
||||
"std": 0.0,
|
||||
}
|
||||
return {
|
||||
"mean": statistics.mean(itl_values_ms),
|
||||
"median": statistics.median(itl_values_ms),
|
||||
@@ -78,9 +85,13 @@ class InstrumentedEngine(InferenceEngine):
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Generate with telemetry recording."""
|
||||
self._bus.publish(EventType.INFERENCE_START, {
|
||||
"model": model, "message_count": len(messages),
|
||||
})
|
||||
self._bus.publish(
|
||||
EventType.INFERENCE_START,
|
||||
{
|
||||
"model": model,
|
||||
"message_count": len(messages),
|
||||
},
|
||||
)
|
||||
|
||||
gpu_sample: Optional[GpuSample] = None
|
||||
energy_sample: Optional[Any] = None
|
||||
@@ -90,19 +101,28 @@ class InstrumentedEngine(InferenceEngine):
|
||||
if self._energy_monitor is not None:
|
||||
with self._energy_monitor.sample() as energy_sample:
|
||||
result = self._inner.generate(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
elif self._gpu_monitor is not None:
|
||||
with self._gpu_monitor.sample() as gpu_sample:
|
||||
result = self._inner.generate(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
result = self._inner.generate(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
latency = time.time() - t0
|
||||
@@ -155,9 +175,7 @@ class InstrumentedEngine(InferenceEngine):
|
||||
energy_per_output_token = (
|
||||
energy_joules / completion_tokens if completion_tokens > 0 else 0.0
|
||||
)
|
||||
throughput_per_watt = (
|
||||
throughput / power_watts if power_watts > 0 else 0.0
|
||||
)
|
||||
throughput_per_watt = throughput / power_watts if power_watts > 0 else 0.0
|
||||
|
||||
# --- Tier 2.1: Phase energy split ---
|
||||
decode_latency = latency - prefill_latency if prefill_latency > 0 else 0.0
|
||||
@@ -171,13 +189,15 @@ class InstrumentedEngine(InferenceEngine):
|
||||
# --- Tier 3: Non-streaming mean ITL approximation ---
|
||||
mean_itl_ms = (
|
||||
(decode_latency / completion_tokens) * 1000
|
||||
if completion_tokens > 0 and decode_latency > 0 else 0.0
|
||||
if completion_tokens > 0 and decode_latency > 0
|
||||
else 0.0
|
||||
)
|
||||
|
||||
# --- Tier 4: Per-inference efficiency ---
|
||||
tokens_per_joule = (
|
||||
completion_tokens / energy_joules
|
||||
if energy_joules > 0 and completion_tokens > 0 else 0.0
|
||||
if energy_joules > 0 and completion_tokens > 0
|
||||
else 0.0
|
||||
)
|
||||
|
||||
engine_id = getattr(self._inner, "engine_id", "unknown")
|
||||
@@ -275,9 +295,13 @@ class InstrumentedEngine(InferenceEngine):
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Stream with per-token timing and full telemetry recording."""
|
||||
self._bus.publish(EventType.INFERENCE_START, {
|
||||
"model": model, "message_count": len(messages),
|
||||
})
|
||||
self._bus.publish(
|
||||
EventType.INFERENCE_START,
|
||||
{
|
||||
"model": model,
|
||||
"message_count": len(messages),
|
||||
},
|
||||
)
|
||||
|
||||
t0 = time.time()
|
||||
token_timestamps: list[float] = []
|
||||
@@ -289,8 +313,11 @@ class InstrumentedEngine(InferenceEngine):
|
||||
if self._energy_monitor is not None:
|
||||
with self._energy_monitor.sample() as energy_sample:
|
||||
async for token in self._inner.stream(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
token_timestamps.append(time.time())
|
||||
token_count += 1
|
||||
@@ -298,16 +325,22 @@ class InstrumentedEngine(InferenceEngine):
|
||||
elif self._gpu_monitor is not None:
|
||||
with self._gpu_monitor.sample() as gpu_sample:
|
||||
async for token in self._inner.stream(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
token_timestamps.append(time.time())
|
||||
token_count += 1
|
||||
yield token
|
||||
else:
|
||||
async for token in self._inner.stream(
|
||||
messages, model=model, temperature=temperature,
|
||||
max_tokens=max_tokens, **kwargs,
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
token_timestamps.append(time.time())
|
||||
token_count += 1
|
||||
@@ -362,9 +395,7 @@ class InstrumentedEngine(InferenceEngine):
|
||||
energy_per_output_token = (
|
||||
energy_joules / token_count if token_count > 0 else 0.0
|
||||
)
|
||||
throughput_per_watt = (
|
||||
throughput / power_watts if power_watts > 0 else 0.0
|
||||
)
|
||||
throughput_per_watt = throughput / power_watts if power_watts > 0 else 0.0
|
||||
|
||||
# Phase energy split
|
||||
decode_latency = latency - prefill_latency if prefill_latency > 0 else 0.0
|
||||
@@ -378,7 +409,8 @@ class InstrumentedEngine(InferenceEngine):
|
||||
# Per-inference efficiency
|
||||
tokens_per_joule = (
|
||||
token_count / energy_joules
|
||||
if energy_joules > 0 and token_count > 0 else 0.0
|
||||
if energy_joules > 0 and token_count > 0
|
||||
else 0.0
|
||||
)
|
||||
|
||||
engine_id = getattr(self._inner, "engine_id", "unknown")
|
||||
@@ -436,6 +468,25 @@ class InstrumentedEngine(InferenceEngine):
|
||||
self._bus.publish(EventType.INFERENCE_END, event_data)
|
||||
self._bus.publish(EventType.TELEMETRY_RECORD, {"record": record})
|
||||
|
||||
async def stream_full(
|
||||
self,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
model: str,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: int = 1024,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator["StreamChunk"]:
|
||||
"""Delegate to inner engine's stream_full for tool-call support."""
|
||||
async for chunk in self._inner.stream_full(
|
||||
messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
):
|
||||
yield chunk
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
return self._inner.list_models()
|
||||
|
||||
|
||||
@@ -241,6 +241,7 @@ class ToolExecutor:
|
||||
)
|
||||
latency = time.time() - t0
|
||||
result.latency_seconds = latency
|
||||
result.metadata["arguments"] = params
|
||||
|
||||
# Auto-detect taints in results
|
||||
if result.success:
|
||||
|
||||
@@ -89,10 +89,15 @@ def _safe_eval_node(node: ast.AST) -> Any:
|
||||
|
||||
|
||||
def safe_eval(expression: str) -> float:
|
||||
"""Evaluate a math expression safely — always via Rust backend."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
return float(_rust.CalculatorTool().execute(expression))
|
||||
"""Evaluate a math expression safely — Rust backend with Python fallback."""
|
||||
try:
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
return float(_rust.CalculatorTool().execute(expression))
|
||||
except ImportError:
|
||||
import ast as _ast
|
||||
tree = _ast.parse(expression, mode="eval")
|
||||
return float(_safe_eval_node(tree.body))
|
||||
|
||||
|
||||
@ToolRegistry.register("calculator")
|
||||
|
||||
@@ -114,10 +114,15 @@ class FileReadTool(BaseTool):
|
||||
content=f"File too large: {size} bytes (max {_MAX_SIZE_BYTES}).",
|
||||
success=False,
|
||||
)
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
try:
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
text = _rust.FileReadTool().execute(str(path))
|
||||
except ImportError:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_read",
|
||||
|
||||
@@ -155,26 +155,26 @@ class FileWriteTool(BaseTool):
|
||||
success=False,
|
||||
)
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if mode == "write":
|
||||
try:
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
_rust.FileWriteTool().execute(str(path), content)
|
||||
except ImportError:
|
||||
try:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
except OSError as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Write error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Write error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
elif False: # dead code — all write modes go through Rust
|
||||
try:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
except OSError as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Write error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
else:
|
||||
# append mode — always Python
|
||||
try:
|
||||
|
||||
@@ -103,9 +103,13 @@ class HttpRequestTool(BaseTool):
|
||||
body = params.get("body")
|
||||
timeout = params.get("timeout", 30)
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if not headers:
|
||||
_rust = None
|
||||
try:
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
except ImportError:
|
||||
pass
|
||||
if _rust is not None and not headers:
|
||||
try:
|
||||
content = _rust.HttpRequestTool().execute(url, method, body)
|
||||
return ToolResult(
|
||||
|
||||
@@ -125,32 +125,33 @@ class ShellExecTool(BaseTool):
|
||||
if val is not None:
|
||||
env[key] = val
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if True:
|
||||
try:
|
||||
output = _rust.ShellExecTool().execute(command, working_dir)
|
||||
return ToolResult(
|
||||
tool_name="shell_exec",
|
||||
content=output or "(no output)",
|
||||
success=True,
|
||||
metadata={
|
||||
"returncode": 0,
|
||||
"timeout_used": timeout,
|
||||
"working_dir": working_dir,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
return ToolResult(
|
||||
tool_name="shell_exec",
|
||||
content=str(exc),
|
||||
success=False,
|
||||
metadata={
|
||||
"returncode": -1,
|
||||
"timeout_used": timeout,
|
||||
"working_dir": working_dir,
|
||||
},
|
||||
)
|
||||
try:
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
output = _rust.ShellExecTool().execute(command, working_dir)
|
||||
return ToolResult(
|
||||
tool_name="shell_exec",
|
||||
content=output or "(no output)",
|
||||
success=True,
|
||||
metadata={
|
||||
"returncode": 0,
|
||||
"timeout_used": timeout,
|
||||
"working_dir": working_dir,
|
||||
},
|
||||
)
|
||||
except ImportError:
|
||||
pass # Fall through to subprocess below
|
||||
except Exception as exc:
|
||||
return ToolResult(
|
||||
tool_name="shell_exec",
|
||||
content=str(exc),
|
||||
success=False,
|
||||
metadata={
|
||||
"returncode": -1,
|
||||
"timeout_used": timeout,
|
||||
"working_dir": working_dir,
|
||||
},
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Shared fixtures for agent tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.agents.executor import AgentExecutor
|
||||
from openjarvis.agents.manager import AgentManager
|
||||
from openjarvis.agents.scheduler import AgentScheduler
|
||||
from openjarvis.core.events import EventBus
|
||||
from tests.agents.fake_engine import FakeEngine
|
||||
from tests.agents.scenario_harness import FakeSystem, ScenarioHarness
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scenario_harness(tmp_path):
|
||||
"""Wire up real components for agent lifecycle testing."""
|
||||
from openjarvis.agents.monitor_operative import MonitorOperativeAgent
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
|
||||
# Re-register agent types (conftest auto-clears registries)
|
||||
if not AgentRegistry.contains("monitor_operative"):
|
||||
AgentRegistry.register("monitor_operative")(MonitorOperativeAgent)
|
||||
|
||||
db_path = str(tmp_path / "agents.db")
|
||||
bus = EventBus(record_history=True)
|
||||
manager = AgentManager(db_path=db_path)
|
||||
|
||||
engine = FakeEngine([{"content": "Default response."}])
|
||||
system = FakeSystem(engine=engine)
|
||||
|
||||
executor = AgentExecutor(manager=manager, event_bus=bus)
|
||||
executor.set_system(system)
|
||||
|
||||
scheduler = AgentScheduler(
|
||||
manager=manager, executor=executor, event_bus=bus,
|
||||
)
|
||||
|
||||
return ScenarioHarness(
|
||||
manager=manager,
|
||||
executor=executor,
|
||||
scheduler=scheduler,
|
||||
bus=bus,
|
||||
engine=engine,
|
||||
system=system,
|
||||
db_path=db_path,
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Scripted inference engine for deterministic agent testing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, AsyncIterator, Dict, List
|
||||
|
||||
from openjarvis.engine._stubs import InferenceEngine
|
||||
|
||||
|
||||
class FakeEngine(InferenceEngine):
|
||||
"""Returns pre-defined responses in order. Captures prompts for assertions."""
|
||||
|
||||
engine_id = "fake"
|
||||
|
||||
def __init__(self, responses: list[dict]) -> None:
|
||||
self._responses = list(responses)
|
||||
self._call_count = 0
|
||||
self._last_messages: list | None = None
|
||||
|
||||
@property
|
||||
def call_count(self) -> int:
|
||||
return self._call_count
|
||||
|
||||
@property
|
||||
def last_messages(self) -> list | None:
|
||||
return self._last_messages
|
||||
|
||||
def generate(
|
||||
self,
|
||||
messages: list,
|
||||
*,
|
||||
model: str,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: int = 1024,
|
||||
**kw: Any,
|
||||
) -> Dict[str, Any]:
|
||||
self._last_messages = messages
|
||||
idx = min(self._call_count, len(self._responses) - 1)
|
||||
resp = self._responses[idx]
|
||||
self._call_count += 1
|
||||
|
||||
# Support raising exceptions for error testing
|
||||
if "raise" in resp:
|
||||
raise resp["raise"]
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"content": resp.get("content", ""),
|
||||
"usage": resp.get("usage", {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
}),
|
||||
"model": model,
|
||||
"finish_reason": "tool_calls" if resp.get("tool_calls") else "stop",
|
||||
}
|
||||
if resp.get("tool_calls"):
|
||||
result["tool_calls"] = resp["tool_calls"]
|
||||
return result
|
||||
|
||||
async def stream(
|
||||
self, messages: list, *, model: str, **kw: Any,
|
||||
) -> AsyncIterator[str]:
|
||||
result = self.generate(messages, model=model, **kw)
|
||||
yield result["content"]
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
return ["fake-model"]
|
||||
|
||||
def health(self) -> bool:
|
||||
return True
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Shared harness dataclasses for agent lifecycle scenario tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.agents.executor import AgentExecutor
|
||||
from openjarvis.agents.manager import AgentManager
|
||||
from openjarvis.agents.scheduler import AgentScheduler
|
||||
from openjarvis.core.events import EventBus
|
||||
from tests.agents.fake_engine import FakeEngine
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FakeSystem:
|
||||
"""Lightweight stand-in for JarvisSystem — just engine + model."""
|
||||
|
||||
engine: FakeEngine
|
||||
model: str = "fake-model"
|
||||
memory_backend: Any = None
|
||||
channel_backend: Any = None
|
||||
tools: list = field(default_factory=list)
|
||||
config: Any = None
|
||||
session_store: Any = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ScenarioHarness:
|
||||
"""All components needed for an agent lifecycle scenario test."""
|
||||
|
||||
manager: AgentManager
|
||||
executor: AgentExecutor
|
||||
scheduler: AgentScheduler
|
||||
bus: EventBus
|
||||
engine: FakeEngine
|
||||
system: FakeSystem
|
||||
db_path: str
|
||||
@@ -0,0 +1,538 @@
|
||||
"""Agent lifecycle scenario tests.
|
||||
|
||||
Twelve+ scenarios exercising the full managed-agent stack (manager, executor,
|
||||
scheduler) with real SQLite state and a scripted FakeEngine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.agents.errors import RetryableError
|
||||
from openjarvis.core.events import EventType
|
||||
from tests.agents.scenario_harness import ScenarioHarness
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 1: Manual agent full lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_manual_agent_full_lifecycle(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Create a manual agent, run a tick, verify status/runs/memory."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [{"content": "Tick 1 completed."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="manual-agent",
|
||||
config={"schedule_type": "manual", "instruction": "Do something."},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
h.executor.execute_tick(aid)
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["status"] == "idle"
|
||||
assert agent["total_runs"] == 1
|
||||
assert "Tick 1 completed." in agent["summary_memory"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 2: Interval-scheduled agent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_interval_scheduled_agent(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Interval agent does not fire until time advances past interval."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [{"content": "Interval response."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
base_time = 1_000_000.0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="interval-agent",
|
||||
config={
|
||||
"schedule_type": "interval",
|
||||
"schedule_value": 60,
|
||||
"instruction": "Check status.",
|
||||
},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
# Register at base_time — next_fire = base_time + 60
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time
|
||||
h.scheduler.register_agent(aid)
|
||||
|
||||
info = h.scheduler._agents[aid]
|
||||
assert info["next_fire"] == pytest.approx(base_time + 60, abs=1)
|
||||
|
||||
# At base_time + 30 the agent should NOT fire
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time + 30
|
||||
h.scheduler._check_due_agents()
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["total_runs"] == 0
|
||||
|
||||
# At base_time + 61 the agent SHOULD fire
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time + 61
|
||||
h.scheduler._check_due_agents()
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["total_runs"] == 1
|
||||
assert "Interval response." in agent["summary_memory"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 3: Cron-scheduled agent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cron_scheduled_agent(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Cron agent gets next_fire set correctly."""
|
||||
h = scenario_harness
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="cron-agent",
|
||||
config={
|
||||
"schedule_type": "cron",
|
||||
"schedule_value": "0 9 * * *",
|
||||
"instruction": "Daily check.",
|
||||
},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
h.scheduler.register_agent(aid)
|
||||
|
||||
info = h.scheduler._agents[aid]
|
||||
assert info["schedule_type"] == "cron"
|
||||
# next_fire should be in the future
|
||||
import time
|
||||
|
||||
assert info["next_fire"] > time.time()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 4: Queued message delivery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_queued_message_delivery(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Queue 3 messages, run tick, verify all delivered and in prompt."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [{"content": "Processed all messages."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="msg-agent",
|
||||
config={"schedule_type": "manual", "instruction": "Handle messages."},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
h.manager.send_message(aid, "Message one", mode="queued")
|
||||
h.manager.send_message(aid, "Message two", mode="queued")
|
||||
h.manager.send_message(aid, "Message three", mode="queued")
|
||||
|
||||
# Verify 3 pending before tick
|
||||
pending = h.manager.get_pending_messages(aid)
|
||||
assert len(pending) == 3
|
||||
|
||||
h.executor.execute_tick(aid)
|
||||
|
||||
# All should be delivered
|
||||
pending = h.manager.get_pending_messages(aid)
|
||||
assert len(pending) == 0
|
||||
|
||||
# Engine should have been called with messages in the prompt
|
||||
assert h.engine.last_messages is not None
|
||||
prompt_text = " ".join(
|
||||
str(getattr(m, "content", m)) for m in h.engine.last_messages
|
||||
)
|
||||
assert "Message one" in prompt_text
|
||||
assert "Message two" in prompt_text
|
||||
assert "Message three" in prompt_text
|
||||
|
||||
# Response stored
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert "Processed all messages." in agent["summary_memory"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 5: Immediate message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_immediate_message(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Send an immediate message, run tick, assert response stored."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [{"content": "Immediate reply."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="imm-agent",
|
||||
config={"schedule_type": "manual", "instruction": "Reply to user."},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
h.manager.send_message(aid, "Urgent question", mode="immediate")
|
||||
|
||||
h.executor.execute_tick(aid)
|
||||
|
||||
# The response should be stored
|
||||
messages = h.manager.list_messages(aid)
|
||||
responses = [m for m in messages if m["direction"] == "agent_to_user"]
|
||||
assert len(responses) >= 1
|
||||
assert any("Immediate reply." in r["content"] for r in responses)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 6: Budget enforcement
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_budget_enforcement(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Agent with max_cost runs tick, total_runs incremented, stays idle
|
||||
because cost is below threshold."""
|
||||
h = scenario_harness
|
||||
# The default usage in FakeEngine reports 0 cost (no cost key in metadata),
|
||||
# so total_cost stays at 0 which is below max_cost=0.01.
|
||||
h.engine._responses = [{"content": "Budget check."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="budget-agent",
|
||||
config={
|
||||
"schedule_type": "manual",
|
||||
"max_cost": 0.01,
|
||||
"instruction": "Cheap task.",
|
||||
},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
h.executor.execute_tick(aid)
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["total_runs"] == 1
|
||||
# Cost is 0 (FakeEngine doesn't report cost), so stays idle
|
||||
assert agent["status"] == "idle"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 7: Error + retry (success on 3rd attempt)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_error_retry_success(scenario_harness: ScenarioHarness) -> None:
|
||||
"""FakeEngine raises RetryableError first 2 times, succeeds on 3rd."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [
|
||||
{"raise": RetryableError("transient-1")},
|
||||
{"raise": RetryableError("transient-2")},
|
||||
{"content": "Success after retries."},
|
||||
]
|
||||
h.engine._call_count = 0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="retry-agent",
|
||||
config={"schedule_type": "manual", "instruction": "Retry test."},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
# Patch retry_delay to avoid real sleeps
|
||||
with patch("openjarvis.agents.executor.time.sleep"):
|
||||
h.executor.execute_tick(aid)
|
||||
|
||||
assert h.engine.call_count == 3
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["status"] == "idle"
|
||||
assert "Success after retries." in agent["summary_memory"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 7b: Error exhaustion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_error_exhaustion(scenario_harness: ScenarioHarness) -> None:
|
||||
"""FakeEngine always raises RetryableError. Agent ends in error state."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [
|
||||
{"raise": RetryableError("fail-1")},
|
||||
{"raise": RetryableError("fail-2")},
|
||||
{"raise": RetryableError("fail-3")},
|
||||
]
|
||||
h.engine._call_count = 0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="exhaust-agent",
|
||||
config={"schedule_type": "manual", "instruction": "Always fail."},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
with patch("openjarvis.agents.executor.time.sleep"):
|
||||
h.executor.execute_tick(aid)
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["status"] == "error"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 8: Stall detection + recovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stall_detection_and_recovery(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Set agent to running with stale last_activity, reconcile detects stall,
|
||||
then recover resets to idle."""
|
||||
h = scenario_harness
|
||||
import time as real_time
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="stall-agent",
|
||||
config={
|
||||
"schedule_type": "manual",
|
||||
"timeout_seconds": 300,
|
||||
"max_stall_retries": 5,
|
||||
"instruction": "Long task.",
|
||||
},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
# Simulate: agent is running with stale activity
|
||||
h.manager.start_tick(aid)
|
||||
ten_minutes_ago = real_time.time() - 600
|
||||
h.manager.update_agent(aid, last_activity_at=ten_minutes_ago)
|
||||
|
||||
# Reconcile should detect the stall
|
||||
h.scheduler._reconcile()
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
# Stall detection should have released the concurrency guard (end_tick)
|
||||
# and incremented stall_retries
|
||||
assert agent["status"] == "idle"
|
||||
assert agent["stall_retries"] == 1
|
||||
|
||||
# Verify stall event was published
|
||||
stall_events = [
|
||||
e for e in h.bus.history if e.event_type == EventType.AGENT_STALL_DETECTED
|
||||
]
|
||||
assert len(stall_events) >= 1
|
||||
assert stall_events[0].data["agent_id"] == aid
|
||||
|
||||
# Now recover the agent explicitly
|
||||
h.manager.recover_agent(aid)
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["status"] == "idle"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 9: Pause / resume
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pause_resume(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Paused agent does not fire; resumed agent fires normally."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [{"content": "Resumed response."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
base_time = 2_000_000.0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="pause-agent",
|
||||
config={
|
||||
"schedule_type": "interval",
|
||||
"schedule_value": 60,
|
||||
"instruction": "Pause test.",
|
||||
},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
# Register at base_time
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time
|
||||
h.scheduler.register_agent(aid)
|
||||
|
||||
# Pause the agent
|
||||
h.manager.pause_agent(aid)
|
||||
|
||||
# Advance past interval — should NOT fire because paused
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time + 61
|
||||
h.scheduler._check_due_agents()
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["total_runs"] == 0
|
||||
|
||||
# Resume
|
||||
h.manager.resume_agent(aid)
|
||||
|
||||
# Now advance again — should fire
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time + 61
|
||||
h.scheduler._check_due_agents()
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert agent["total_runs"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 10: Multi-agent concurrent scheduling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_multi_agent_scheduling(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Three agents with different intervals fire at the right times."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [{"content": "Agent response."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
base_time = 3_000_000.0
|
||||
|
||||
agents = []
|
||||
for i, interval in enumerate([30, 60, 120]):
|
||||
a = h.manager.create_agent(
|
||||
name=f"multi-{i}",
|
||||
config={
|
||||
"schedule_type": "interval",
|
||||
"schedule_value": interval,
|
||||
"instruction": f"Task {i}.",
|
||||
},
|
||||
)
|
||||
agents.append(a)
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time
|
||||
h.scheduler.register_agent(a["id"])
|
||||
|
||||
# At base_time + 35, only agent 0 (30s interval) should fire
|
||||
h.engine._call_count = 0
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time + 35
|
||||
h.scheduler._check_due_agents()
|
||||
|
||||
a0 = h.manager.get_agent(agents[0]["id"])
|
||||
a1 = h.manager.get_agent(agents[1]["id"])
|
||||
a2 = h.manager.get_agent(agents[2]["id"])
|
||||
assert a0 is not None and a0["total_runs"] == 1
|
||||
assert a1 is not None and a1["total_runs"] == 0
|
||||
assert a2 is not None and a2["total_runs"] == 0
|
||||
|
||||
# At base_time + 65, agents 0 and 1 should have fired (but agent 0
|
||||
# next_fire was reset to base_time + 35 + 30 = base_time + 65, so it
|
||||
# fires again at exactly 65)
|
||||
h.engine._responses = [{"content": "Agent response again."}]
|
||||
h.engine._call_count = 0
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time + 65
|
||||
h.scheduler._check_due_agents()
|
||||
|
||||
a0 = h.manager.get_agent(agents[0]["id"])
|
||||
a1 = h.manager.get_agent(agents[1]["id"])
|
||||
a2 = h.manager.get_agent(agents[2]["id"])
|
||||
assert a0 is not None and a0["total_runs"] == 2
|
||||
assert a1 is not None and a1["total_runs"] == 1
|
||||
assert a2 is not None and a2["total_runs"] == 0
|
||||
|
||||
# At base_time + 125, all three should have fired
|
||||
h.engine._responses = [{"content": "All fired."}]
|
||||
h.engine._call_count = 0
|
||||
with patch("openjarvis.agents.scheduler.time") as mock_time:
|
||||
mock_time.time.return_value = base_time + 125
|
||||
h.scheduler._check_due_agents()
|
||||
|
||||
a0 = h.manager.get_agent(agents[0]["id"])
|
||||
a1 = h.manager.get_agent(agents[1]["id"])
|
||||
a2 = h.manager.get_agent(agents[2]["id"])
|
||||
assert a0 is not None and a0["total_runs"] >= 3
|
||||
assert a1 is not None and a1["total_runs"] >= 2
|
||||
assert a2 is not None and a2["total_runs"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 11: Template instantiation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_template_instantiation(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Each built-in template creates an agent with correct config fields."""
|
||||
h = scenario_harness
|
||||
|
||||
templates = h.manager.list_templates()
|
||||
|
||||
for tpl in templates:
|
||||
tpl_id = tpl.get("id")
|
||||
if not tpl_id:
|
||||
continue
|
||||
|
||||
agent = h.manager.create_from_template(tpl_id, name=f"from-{tpl_id}")
|
||||
config = agent["config"]
|
||||
|
||||
# Every template has a schedule_type and schedule_value
|
||||
assert "schedule_type" in config
|
||||
assert "schedule_value" in config
|
||||
# Every template has a system_prompt
|
||||
assert "system_prompt" in config
|
||||
# Agent type should be set
|
||||
assert agent["agent_type"] == tpl.get("agent_type", "monitor_operative")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 12: Memory persistence across ticks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_memory_persistence_across_ticks(scenario_harness: ScenarioHarness) -> None:
|
||||
"""Tick 1 summary becomes part of tick 2 engine prompt (Previous context)."""
|
||||
h = scenario_harness
|
||||
h.engine._responses = [{"content": "Findings from tick one."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
agent = h.manager.create_agent(
|
||||
name="memory-agent",
|
||||
config={
|
||||
"schedule_type": "manual",
|
||||
"instruction": "Investigate topic X.",
|
||||
},
|
||||
)
|
||||
aid = agent["id"]
|
||||
|
||||
# --- Tick 1 ---
|
||||
h.executor.execute_tick(aid)
|
||||
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert "Findings from tick one." in agent["summary_memory"]
|
||||
|
||||
# --- Tick 2 ---
|
||||
h.engine._responses = [{"content": "Tick two builds on prior context."}]
|
||||
h.engine._call_count = 0
|
||||
|
||||
h.executor.execute_tick(aid)
|
||||
|
||||
# The engine should have received the tick-1 summary in its prompt
|
||||
assert h.engine.last_messages is not None
|
||||
prompt_text = " ".join(
|
||||
str(getattr(m, "content", m)) for m in h.engine.last_messages
|
||||
)
|
||||
assert "Findings from tick one." in prompt_text
|
||||
|
||||
# Tick 2 summary should now be stored
|
||||
agent = h.manager.get_agent(aid)
|
||||
assert agent is not None
|
||||
assert "Tick two builds on prior context." in agent["summary_memory"]
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Tests for tool wiring in AgentExecutor."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openjarvis.agents.executor import AgentExecutor
|
||||
from openjarvis.agents.manager import AgentManager
|
||||
from openjarvis.core.events import EventBus
|
||||
from tests.agents.fake_engine import FakeEngine
|
||||
from tests.agents.scenario_harness import FakeSystem
|
||||
|
||||
|
||||
def _register_agent():
|
||||
"""Re-register MonitorOperativeAgent (cleared by autouse fixture)."""
|
||||
from openjarvis.agents.monitor_operative import MonitorOperativeAgent
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
|
||||
if not AgentRegistry.contains("monitor_operative"):
|
||||
AgentRegistry.register("monitor_operative")(MonitorOperativeAgent)
|
||||
|
||||
|
||||
def test_executor_runs_with_tools_from_config(tmp_path):
|
||||
"""Executor should resolve tool names from config and complete tick."""
|
||||
_register_agent()
|
||||
|
||||
engine = FakeEngine([{"content": "test response"}])
|
||||
system = FakeSystem(engine=engine)
|
||||
|
||||
mgr = AgentManager(db_path=str(tmp_path / "test.db"))
|
||||
agent = mgr.create_agent(
|
||||
"test",
|
||||
agent_type="monitor_operative",
|
||||
config={
|
||||
"system_prompt": "You are a test agent.",
|
||||
"tools": ["think"],
|
||||
"instruction": "test",
|
||||
},
|
||||
)
|
||||
mgr.send_message(agent["id"], "hello", mode="immediate")
|
||||
|
||||
executor = AgentExecutor(manager=mgr, event_bus=EventBus())
|
||||
executor.set_system(system)
|
||||
|
||||
executor.execute_tick(agent["id"])
|
||||
result_agent = mgr.get_agent(agent["id"])
|
||||
assert result_agent["status"] == "idle"
|
||||
assert result_agent["total_runs"] == 1
|
||||
mgr.close()
|
||||
|
||||
|
||||
def test_executor_handles_missing_tools(tmp_path):
|
||||
"""Executor should not crash if tool names don't exist in registry."""
|
||||
_register_agent()
|
||||
|
||||
engine = FakeEngine([{"content": "test response"}])
|
||||
system = FakeSystem(engine=engine)
|
||||
|
||||
mgr = AgentManager(db_path=str(tmp_path / "test.db"))
|
||||
agent = mgr.create_agent(
|
||||
"test",
|
||||
agent_type="monitor_operative",
|
||||
config={
|
||||
"system_prompt": "You are a test agent.",
|
||||
"tools": ["nonexistent_tool_xyz"],
|
||||
"instruction": "test",
|
||||
},
|
||||
)
|
||||
mgr.send_message(agent["id"], "hello", mode="immediate")
|
||||
|
||||
executor = AgentExecutor(manager=mgr, event_bus=EventBus())
|
||||
executor.set_system(system)
|
||||
|
||||
executor.execute_tick(agent["id"])
|
||||
result_agent = mgr.get_agent(agent["id"])
|
||||
assert result_agent["status"] == "idle"
|
||||
assert result_agent["total_runs"] == 1
|
||||
mgr.close()
|
||||
|
||||
|
||||
def test_executor_handles_string_tools(tmp_path):
|
||||
"""Executor should handle comma-separated tool string as well as list."""
|
||||
_register_agent()
|
||||
|
||||
engine = FakeEngine([{"content": "test response"}])
|
||||
system = FakeSystem(engine=engine)
|
||||
|
||||
mgr = AgentManager(db_path=str(tmp_path / "test.db"))
|
||||
agent = mgr.create_agent(
|
||||
"test",
|
||||
agent_type="monitor_operative",
|
||||
config={
|
||||
"system_prompt": "You are a test agent.",
|
||||
"tools": "think,calculator",
|
||||
"instruction": "test",
|
||||
},
|
||||
)
|
||||
mgr.send_message(agent["id"], "hello", mode="immediate")
|
||||
|
||||
executor = AgentExecutor(manager=mgr, event_bus=EventBus())
|
||||
executor.set_system(system)
|
||||
|
||||
executor.execute_tick(agent["id"])
|
||||
result_agent = mgr.get_agent(agent["id"])
|
||||
assert result_agent["status"] == "idle"
|
||||
mgr.close()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Tests for FakeEngine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agents.fake_engine import FakeEngine
|
||||
|
||||
|
||||
def test_fake_engine_returns_responses_in_order():
|
||||
engine = FakeEngine([
|
||||
{"content": "first"},
|
||||
{"content": "second"},
|
||||
])
|
||||
r1 = engine.generate([], model="m")
|
||||
r2 = engine.generate([], model="m")
|
||||
assert r1["content"] == "first"
|
||||
assert r2["content"] == "second"
|
||||
assert engine.call_count == 2
|
||||
|
||||
|
||||
def test_fake_engine_repeats_last_response():
|
||||
engine = FakeEngine([{"content": "only"}])
|
||||
engine.generate([], model="m")
|
||||
r = engine.generate([], model="m")
|
||||
assert r["content"] == "only"
|
||||
|
||||
|
||||
def test_fake_engine_raises_on_request():
|
||||
engine = FakeEngine([{"raise": ValueError("boom")}])
|
||||
with pytest.raises(ValueError, match="boom"):
|
||||
engine.generate([], model="m")
|
||||
|
||||
|
||||
def test_fake_engine_tool_calls():
|
||||
engine = FakeEngine([{
|
||||
"content": "",
|
||||
"tool_calls": [{"id": "1", "function": {"name": "think", "arguments": "{}"}}],
|
||||
}])
|
||||
r = engine.generate([], model="m")
|
||||
assert r["finish_reason"] == "tool_calls"
|
||||
assert len(r["tool_calls"]) == 1
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Smoke test for the scenario harness fixture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.agents.scenario_harness import ScenarioHarness
|
||||
|
||||
|
||||
def test_harness_creates_and_runs_agent(scenario_harness: ScenarioHarness):
|
||||
"""Verify the harness wires up correctly — create agent, run tick."""
|
||||
h = scenario_harness
|
||||
agent = h.manager.create_agent("Smoke Test", config={
|
||||
"schedule_type": "manual",
|
||||
"instruction": "Say hello.",
|
||||
})
|
||||
h.executor.execute_tick(agent["id"])
|
||||
updated = h.manager.get_agent(agent["id"])
|
||||
assert updated["status"] == "idle"
|
||||
assert updated["total_runs"] == 1
|
||||
assert updated["summary_memory"] != ""
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Tests for system_prompt_template expansion in agent creation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openjarvis.agents.manager import AgentManager
|
||||
|
||||
|
||||
def test_create_from_template_expands_system_prompt(tmp_path):
|
||||
"""system_prompt_template should be expanded with the instruction."""
|
||||
mgr = AgentManager(db_path=str(tmp_path / "test.db"))
|
||||
agent = mgr.create_from_template(
|
||||
"research_monitor",
|
||||
"Test Agent",
|
||||
overrides={"instruction": "Monitor AI safety papers"},
|
||||
)
|
||||
config = agent["config"]
|
||||
# system_prompt should contain the expanded instruction
|
||||
assert "Monitor AI safety papers" in config.get("system_prompt", "")
|
||||
# system_prompt_template should NOT be in the stored config
|
||||
assert "system_prompt_template" not in config
|
||||
mgr.close()
|
||||
|
||||
|
||||
def test_create_from_template_without_instruction(tmp_path):
|
||||
"""Template with no instruction should still have a system_prompt."""
|
||||
mgr = AgentManager(db_path=str(tmp_path / "test.db"))
|
||||
agent = mgr.create_from_template("research_monitor", "Test Agent")
|
||||
config = agent["config"]
|
||||
assert "system_prompt" in config
|
||||
assert len(config["system_prompt"]) > 100 # non-trivial prompt
|
||||
mgr.close()
|
||||
|
||||
|
||||
def test_create_from_template_preserves_icon(tmp_path):
|
||||
"""Template icon field should be preserved in config."""
|
||||
mgr = AgentManager(db_path=str(tmp_path / "test.db"))
|
||||
agent = mgr.create_from_template("research_monitor", "Test Agent")
|
||||
config = agent["config"]
|
||||
assert config.get("icon") == "🔬"
|
||||
mgr.close()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Shared fixtures for channel integration tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.core.events import EventBus
|
||||
from tests.agents.fake_engine import FakeEngine
|
||||
from tests.agents.scenario_harness import FakeSystem, ScenarioHarness
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scenario_harness(tmp_path):
|
||||
"""Wire up real components for agent lifecycle testing (channels copy)."""
|
||||
from openjarvis.agents.executor import AgentExecutor
|
||||
from openjarvis.agents.manager import AgentManager
|
||||
from openjarvis.agents.monitor_operative import MonitorOperativeAgent
|
||||
from openjarvis.agents.scheduler import AgentScheduler
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
|
||||
if not AgentRegistry.contains("monitor_operative"):
|
||||
AgentRegistry.register("monitor_operative")(MonitorOperativeAgent)
|
||||
|
||||
db_path = str(tmp_path / "agents.db")
|
||||
bus = EventBus(record_history=True)
|
||||
manager = AgentManager(db_path=db_path)
|
||||
engine = FakeEngine([{"content": "Default response."}])
|
||||
system = FakeSystem(engine=engine)
|
||||
executor = AgentExecutor(manager=manager, event_bus=bus)
|
||||
executor.set_system(system)
|
||||
scheduler = AgentScheduler(manager=manager, executor=executor, event_bus=bus)
|
||||
|
||||
return ScenarioHarness(
|
||||
manager=manager, executor=executor, scheduler=scheduler,
|
||||
bus=bus, engine=engine, system=system, db_path=db_path,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_event_bus() -> EventBus:
|
||||
"""EventBus with history recording for asserting event emissions."""
|
||||
return EventBus(record_history=True)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def credential_env(channel_name: str, **creds: str):
|
||||
"""Context manager that sets env vars for a channel, cleans up after."""
|
||||
prefix = channel_name.upper()
|
||||
old_values = {}
|
||||
for key, value in creds.items():
|
||||
env_key = f"{prefix}_{key.upper()}"
|
||||
old_values[env_key] = os.environ.get(env_key)
|
||||
os.environ[env_key] = value
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for env_key, old in old_values.items():
|
||||
if old is None:
|
||||
os.environ.pop(env_key, None)
|
||||
else:
|
||||
os.environ[env_key] = old
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Agent-channel E2E tests — using WebChatChannel (in-memory, no external deps)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.agents.scenario_harness import ScenarioHarness
|
||||
|
||||
|
||||
def test_agent_sends_to_webchat(scenario_harness: ScenarioHarness):
|
||||
"""Agent runs and completes tick with WebChatChannel available."""
|
||||
from openjarvis.channels.webchat import WebChatChannel
|
||||
|
||||
h = scenario_harness
|
||||
webchat = WebChatChannel()
|
||||
webchat.connect()
|
||||
|
||||
agent = h.manager.create_agent("Channel Agent", config={
|
||||
"schedule_type": "manual",
|
||||
"instruction": "Send a report to the general channel.",
|
||||
})
|
||||
|
||||
h.executor.execute_tick(agent["id"])
|
||||
data = h.manager.get_agent(agent["id"])
|
||||
assert data["status"] == "idle"
|
||||
assert data["total_runs"] == 1
|
||||
webchat.disconnect()
|
||||
|
||||
|
||||
def test_channel_failure_does_not_crash_agent(scenario_harness: ScenarioHarness):
|
||||
"""Agent continues if channel send fails."""
|
||||
h = scenario_harness
|
||||
agent = h.manager.create_agent("Resilient Agent", config={
|
||||
"schedule_type": "manual",
|
||||
"instruction": "Try to send a message.",
|
||||
})
|
||||
|
||||
h.executor.execute_tick(agent["id"])
|
||||
data = h.manager.get_agent(agent["id"])
|
||||
assert data["status"] == "idle"
|
||||
assert data["total_runs"] == 1
|
||||
|
||||
|
||||
def test_agent_with_channel_binding(scenario_harness: ScenarioHarness):
|
||||
"""Agent with a channel binding completes tick without error."""
|
||||
from openjarvis.channels.webchat import WebChatChannel
|
||||
|
||||
h = scenario_harness
|
||||
webchat = WebChatChannel()
|
||||
webchat.connect()
|
||||
|
||||
agent = h.manager.create_agent("Bound Agent", config={
|
||||
"schedule_type": "manual",
|
||||
"instruction": "Monitor and report.",
|
||||
})
|
||||
aid = agent["id"]
|
||||
|
||||
# Bind a channel
|
||||
h.manager.bind_channel(aid, "webchat", config={"channel": "general"})
|
||||
bindings = h.manager.list_channel_bindings(aid)
|
||||
assert len(bindings) == 1
|
||||
|
||||
h.executor.execute_tick(aid)
|
||||
data = h.manager.get_agent(aid)
|
||||
assert data["status"] == "idle"
|
||||
webchat.disconnect()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Parametrized contract tests — verify every channel implements BaseChannel correctly.
|
||||
|
||||
These tests run without any credentials and verify that all channel adapters
|
||||
handle graceful degradation (no crashes on missing auth, idempotent disconnect,
|
||||
valid enum returns).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
import openjarvis.channels # noqa: F401 — trigger registration
|
||||
from openjarvis.channels._stubs import ChannelStatus
|
||||
from openjarvis.core.registry import ChannelRegistry
|
||||
|
||||
# Collect channel classes at import time (before registry gets cleared).
|
||||
# We store the actual class objects, not registry keys, so they survive
|
||||
# the autouse _clean_registries fixture.
|
||||
importlib.reload(openjarvis.channels)
|
||||
_ALL_CHANNELS = [
|
||||
(key, ChannelRegistry.get(key)) for key in ChannelRegistry.keys()
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(params=_ALL_CHANNELS, ids=lambda x: x[0])
|
||||
def channel_entry(request):
|
||||
"""Parametrized fixture yielding (channel_key, channel_cls) tuples.
|
||||
|
||||
Uses the class reference captured at import time — does not depend
|
||||
on the registry being populated at test time.
|
||||
"""
|
||||
return request.param
|
||||
|
||||
|
||||
def test_has_channel_id(channel_entry):
|
||||
key, channel_cls = channel_entry
|
||||
assert hasattr(channel_cls, "channel_id")
|
||||
assert isinstance(channel_cls.channel_id, str)
|
||||
assert len(channel_cls.channel_id) > 0
|
||||
|
||||
|
||||
def test_connect_no_credentials_no_crash(channel_entry):
|
||||
key, channel_cls = channel_entry
|
||||
ch = channel_cls()
|
||||
ch.connect()
|
||||
# Should not raise — just set status to ERROR or DISCONNECTED
|
||||
|
||||
|
||||
def test_disconnect_idempotent(channel_entry):
|
||||
key, channel_cls = channel_entry
|
||||
ch = channel_cls()
|
||||
ch.disconnect()
|
||||
ch.disconnect() # Second call should not raise
|
||||
|
||||
|
||||
def test_status_returns_valid_enum(channel_entry):
|
||||
key, channel_cls = channel_entry
|
||||
ch = channel_cls()
|
||||
s = ch.status()
|
||||
assert isinstance(s, ChannelStatus)
|
||||
|
||||
|
||||
def test_list_channels_returns_list(channel_entry):
|
||||
key, channel_cls = channel_entry
|
||||
ch = channel_cls()
|
||||
result = ch.list_channels()
|
||||
assert isinstance(result, list)
|
||||
|
||||
|
||||
def test_send_no_credentials_returns_false(channel_entry):
|
||||
key, channel_cls = channel_entry
|
||||
if key == "webchat":
|
||||
pytest.skip("WebChatChannel is in-memory and always succeeds")
|
||||
ch = channel_cls()
|
||||
result = ch.send("test", "hello")
|
||||
assert result is False
|
||||
|
||||
|
||||
def test_on_message_accepts_handler(channel_entry):
|
||||
key, channel_cls = channel_entry
|
||||
ch = channel_cls()
|
||||
ch.on_message(lambda msg: None)
|
||||
# Should not raise
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Tests for the GmailChannel adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.channels._stubs import ChannelStatus
|
||||
from openjarvis.channels.gmail import GmailChannel
|
||||
from openjarvis.core.events import EventBus, EventType
|
||||
from openjarvis.core.registry import ChannelRegistry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _register_gmail():
|
||||
"""Re-register after any registry clear."""
|
||||
if not ChannelRegistry.contains("gmail"):
|
||||
ChannelRegistry.register_value("gmail", GmailChannel)
|
||||
|
||||
|
||||
class TestRegistration:
|
||||
def test_gmail_channel_registered(self):
|
||||
assert ChannelRegistry.contains("gmail")
|
||||
|
||||
def test_channel_id(self):
|
||||
ch = GmailChannel()
|
||||
assert ch.channel_id == "gmail"
|
||||
|
||||
|
||||
class TestNoCredentials:
|
||||
def test_gmail_no_credentials_status(self):
|
||||
ch = GmailChannel()
|
||||
ch.connect()
|
||||
assert ch.status() in (ChannelStatus.ERROR, ChannelStatus.DISCONNECTED)
|
||||
|
||||
def test_gmail_send_no_credentials_returns_false(self):
|
||||
ch = GmailChannel()
|
||||
result = ch.send("recipient@example.com", "Hello!")
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestSend:
|
||||
def test_gmail_send_constructs_mime(self):
|
||||
ch = GmailChannel()
|
||||
mock_service = MagicMock()
|
||||
ch._service = mock_service
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
result = ch.send("recipient@example.com", "Hello Gmail!")
|
||||
assert result is True
|
||||
|
||||
# Verify send was called
|
||||
mock_service.users().messages().send.assert_called_once()
|
||||
call_kwargs = mock_service.users().messages().send.call_args[1]
|
||||
assert call_kwargs["userId"] == "me"
|
||||
|
||||
# Verify the body contains base64-encoded MIME message
|
||||
body = call_kwargs["body"]
|
||||
assert "raw" in body
|
||||
decoded = base64.urlsafe_b64decode(body["raw"]).decode("utf-8")
|
||||
assert "recipient@example.com" in decoded
|
||||
assert "Hello Gmail!" in decoded
|
||||
|
||||
def test_gmail_send_with_thread_id(self):
|
||||
ch = GmailChannel()
|
||||
mock_service = MagicMock()
|
||||
ch._service = mock_service
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
result = ch.send(
|
||||
"recipient@example.com",
|
||||
"Thread reply",
|
||||
conversation_id="thread-abc123",
|
||||
)
|
||||
assert result is True
|
||||
|
||||
call_kwargs = mock_service.users().messages().send.call_args[1]
|
||||
body = call_kwargs["body"]
|
||||
assert body["threadId"] == "thread-abc123"
|
||||
|
||||
def test_gmail_send_with_subject_metadata(self):
|
||||
ch = GmailChannel()
|
||||
mock_service = MagicMock()
|
||||
ch._service = mock_service
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
result = ch.send(
|
||||
"recipient@example.com",
|
||||
"Body text",
|
||||
metadata={"subject": "Custom Subject"},
|
||||
)
|
||||
assert result is True
|
||||
|
||||
call_kwargs = mock_service.users().messages().send.call_args[1]
|
||||
decoded = base64.urlsafe_b64decode(
|
||||
call_kwargs["body"]["raw"],
|
||||
).decode("utf-8")
|
||||
assert "Custom Subject" in decoded
|
||||
|
||||
def test_gmail_send_exception_returns_false(self):
|
||||
ch = GmailChannel()
|
||||
mock_service = MagicMock()
|
||||
mock_service.users().messages().send().execute.side_effect = (
|
||||
RuntimeError("API error")
|
||||
)
|
||||
ch._service = mock_service
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
result = ch.send("recipient@example.com", "Hello!")
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestListChannels:
|
||||
def test_gmail_list_channels(self):
|
||||
ch = GmailChannel()
|
||||
assert ch.list_channels() == ["inbox"]
|
||||
|
||||
|
||||
class TestOnMessage:
|
||||
def test_gmail_on_message_registers_handler(self):
|
||||
ch = GmailChannel()
|
||||
handler = MagicMock()
|
||||
ch.on_message(handler)
|
||||
assert handler in ch._handlers
|
||||
|
||||
|
||||
class TestEventBus:
|
||||
def test_gmail_event_bus_integration(self):
|
||||
bus = EventBus(record_history=True)
|
||||
ch = GmailChannel(bus=bus)
|
||||
mock_service = MagicMock()
|
||||
ch._service = mock_service
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
ch.send("recipient@example.com", "Hello!")
|
||||
|
||||
event_types = [e.event_type for e in bus.history]
|
||||
assert EventType.CHANNEL_MESSAGE_SENT in event_types
|
||||
|
||||
|
||||
class TestStatus:
|
||||
def test_disconnected_initially(self):
|
||||
ch = GmailChannel()
|
||||
assert ch.status() == ChannelStatus.DISCONNECTED
|
||||
|
||||
def test_status_error_when_connected_but_no_service(self):
|
||||
ch = GmailChannel()
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
ch._service = None
|
||||
assert ch.status() == ChannelStatus.ERROR
|
||||
|
||||
|
||||
class TestDisconnect:
|
||||
def test_disconnect(self):
|
||||
ch = GmailChannel()
|
||||
ch._service = MagicMock()
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
ch.disconnect()
|
||||
assert ch.status() == ChannelStatus.DISCONNECTED
|
||||
assert ch._service is None
|
||||
|
||||
|
||||
class TestEnvVarFallback:
|
||||
def test_credentials_path_env_var(self):
|
||||
with patch.dict(os.environ, {"GMAIL_CREDENTIALS_PATH": "/tmp/creds.json"}):
|
||||
ch = GmailChannel()
|
||||
assert ch._credentials_path == "/tmp/creds.json"
|
||||
|
||||
def test_token_path_env_var(self):
|
||||
with patch.dict(os.environ, {"GMAIL_TOKEN_PATH": "/tmp/token.json"}):
|
||||
ch = GmailChannel()
|
||||
assert ch._token_path == "/tmp/token.json"
|
||||
|
||||
def test_constructor_overrides_env(self):
|
||||
with patch.dict(os.environ, {"GMAIL_CREDENTIALS_PATH": "/env/creds.json"}):
|
||||
ch = GmailChannel(credentials_path="/explicit/creds.json")
|
||||
assert ch._credentials_path == "/explicit/creds.json"
|
||||
|
||||
|
||||
@pytest.mark.live_channel
|
||||
class TestLive:
|
||||
def test_gmail_send_live(self):
|
||||
creds_path = os.environ.get("GMAIL_CREDENTIALS_PATH", "")
|
||||
token_path = os.environ.get("GMAIL_TOKEN_PATH", "")
|
||||
recipient = os.environ.get("GMAIL_TEST_RECIPIENT", "")
|
||||
|
||||
if not creds_path and not token_path:
|
||||
pytest.skip("No Gmail credentials configured")
|
||||
if not recipient:
|
||||
pytest.skip("No GMAIL_TEST_RECIPIENT configured")
|
||||
|
||||
ch = GmailChannel(
|
||||
credentials_path=creds_path,
|
||||
token_path=token_path,
|
||||
)
|
||||
ch.connect()
|
||||
if ch.status() != ChannelStatus.CONNECTED:
|
||||
pytest.skip("Gmail channel failed to connect")
|
||||
|
||||
result = ch.send(
|
||||
recipient,
|
||||
"OpenJarvis Gmail channel test message",
|
||||
metadata={"subject": "OpenJarvis Test"},
|
||||
)
|
||||
assert result is True
|
||||
ch.disconnect()
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Tests for the TwitterChannel adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.channels._stubs import ChannelStatus
|
||||
from openjarvis.channels.twitter import TwitterChannel
|
||||
from openjarvis.core.events import EventBus, EventType
|
||||
from openjarvis.core.registry import ChannelRegistry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _register_twitter():
|
||||
"""Re-register after any registry clear."""
|
||||
if not ChannelRegistry.contains("twitter"):
|
||||
ChannelRegistry.register_value("twitter", TwitterChannel)
|
||||
|
||||
|
||||
def test_twitter_channel_registered():
|
||||
"""Twitter channel should be discoverable via the registry."""
|
||||
assert ChannelRegistry.contains("twitter")
|
||||
cls = ChannelRegistry.get("twitter")
|
||||
assert cls is TwitterChannel
|
||||
|
||||
|
||||
def test_twitter_no_credentials_status():
|
||||
"""Without credentials, connect() sets status to ERROR."""
|
||||
ch = TwitterChannel()
|
||||
ch.connect()
|
||||
assert ch.status() == ChannelStatus.ERROR
|
||||
|
||||
|
||||
def test_twitter_send_no_credentials_returns_false():
|
||||
"""send() returns False when client is not connected."""
|
||||
ch = TwitterChannel()
|
||||
result = ch.send("timeline", "Hello Twitter!")
|
||||
assert result is False
|
||||
|
||||
|
||||
def test_twitter_send_tweet():
|
||||
"""send() with a non-numeric channel calls create_tweet."""
|
||||
ch = TwitterChannel(bearer_token="test-bearer")
|
||||
mock_client = MagicMock()
|
||||
ch._client = mock_client
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
result = ch.send("timeline", "Hello from Jarvis!")
|
||||
assert result is True
|
||||
mock_client.create_tweet.assert_called_once_with(text="Hello from Jarvis!")
|
||||
|
||||
|
||||
def test_twitter_send_dm_when_channel_is_numeric():
|
||||
"""send() with a numeric channel calls create_direct_message."""
|
||||
ch = TwitterChannel(bearer_token="test-bearer")
|
||||
mock_client = MagicMock()
|
||||
ch._client = mock_client
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
result = ch.send("12345678", "Hello via DM!")
|
||||
assert result is True
|
||||
mock_client.create_direct_message.assert_called_once_with(
|
||||
participant_id=12345678,
|
||||
text="Hello via DM!",
|
||||
)
|
||||
|
||||
|
||||
def test_twitter_send_reply():
|
||||
"""send() with conversation_id sets in_reply_to_tweet_id."""
|
||||
ch = TwitterChannel(bearer_token="test-bearer")
|
||||
mock_client = MagicMock()
|
||||
ch._client = mock_client
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
result = ch.send(
|
||||
"timeline", "This is a reply", conversation_id="999888777",
|
||||
)
|
||||
assert result is True
|
||||
mock_client.create_tweet.assert_called_once_with(
|
||||
text="This is a reply",
|
||||
in_reply_to_tweet_id=999888777,
|
||||
)
|
||||
|
||||
|
||||
def test_twitter_list_channels():
|
||||
"""list_channels() returns the expected identifiers."""
|
||||
ch = TwitterChannel()
|
||||
assert ch.list_channels() == ["timeline", "dm"]
|
||||
|
||||
|
||||
def test_twitter_event_bus_integration():
|
||||
"""Successful send publishes CHANNEL_MESSAGE_SENT on the event bus."""
|
||||
bus = EventBus(record_history=True)
|
||||
ch = TwitterChannel(bearer_token="test-bearer", bus=bus)
|
||||
mock_client = MagicMock()
|
||||
ch._client = mock_client
|
||||
ch._status = ChannelStatus.CONNECTED
|
||||
|
||||
ch.send("timeline", "Event test!")
|
||||
|
||||
event_types = [e.event_type for e in bus.history]
|
||||
assert EventType.CHANNEL_MESSAGE_SENT in event_types
|
||||
|
||||
sent_event = next(
|
||||
e for e in bus.history
|
||||
if e.event_type == EventType.CHANNEL_MESSAGE_SENT
|
||||
)
|
||||
assert sent_event.data["content"] == "Event test!"
|
||||
assert sent_event.data["channel"] == "timeline"
|
||||
|
||||
|
||||
@pytest.mark.live_channel
|
||||
def test_twitter_connect_live():
|
||||
"""Live test: connect with real credentials from env vars.
|
||||
|
||||
Skipped unless all required env vars are set.
|
||||
"""
|
||||
required_vars = [
|
||||
"TWITTER_BEARER_TOKEN",
|
||||
"TWITTER_API_KEY",
|
||||
"TWITTER_API_SECRET",
|
||||
"TWITTER_ACCESS_TOKEN",
|
||||
"TWITTER_ACCESS_SECRET",
|
||||
]
|
||||
for var in required_vars:
|
||||
if not os.environ.get(var):
|
||||
pytest.skip(f"Missing env var {var}")
|
||||
|
||||
ch = TwitterChannel()
|
||||
ch.connect()
|
||||
assert ch.status() == ChannelStatus.CONNECTED
|
||||
ch.disconnect()
|
||||
assert ch.status() == ChannelStatus.DISCONNECTED
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Tests for ``jarvis config set`` command."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from openjarvis.cli import cli
|
||||
|
||||
|
||||
class TestConfigSet:
|
||||
def test_set_creates_config_file(self, tmp_path: Path) -> None:
|
||||
"""config set creates config.toml if it doesn't exist."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["config", "set", "engine.default", "vllm"]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert config_file.exists()
|
||||
content = config_file.read_text()
|
||||
assert "vllm" in content
|
||||
|
||||
def test_set_engine_ollama_host(self, tmp_path: Path) -> None:
|
||||
"""config set writes engine.ollama.host correctly."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
config_file.write_text('[engine]\ndefault = "ollama"\n')
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with (
|
||||
mock.patch.dict(os.environ, env),
|
||||
mock.patch("openjarvis.cli.config_cmd.httpx"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["config", "set", "engine.ollama.host", "http://192.168.1.50:11434"],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
content = config_file.read_text()
|
||||
assert "http://192.168.1.50:11434" in content
|
||||
|
||||
def test_set_preserves_existing_keys(self, tmp_path: Path) -> None:
|
||||
"""config set preserves other keys in the file."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
config_file.write_text(
|
||||
'[engine]\ndefault = "ollama"\n\n'
|
||||
"[intelligence]\n"
|
||||
'default_model = "qwen2.5:3b"\n'
|
||||
)
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["config", "set", "engine.default", "vllm"]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
content = config_file.read_text()
|
||||
assert "vllm" in content
|
||||
assert "qwen2.5:3b" in content
|
||||
|
||||
def test_set_invalid_key_rejected(self, tmp_path: Path) -> None:
|
||||
"""config set rejects unknown keys."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
config_file.write_text("")
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["config", "set", "engine.olllama.host", "http://x:1234"]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "Unknown config key" in result.output
|
||||
|
||||
def test_set_engine_host_probes_reachable(self, tmp_path: Path) -> None:
|
||||
"""config set probes engine host and reports reachability."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
config_file.write_text("")
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with (
|
||||
mock.patch.dict(os.environ, env),
|
||||
mock.patch("openjarvis.cli.config_cmd.httpx") as mock_httpx,
|
||||
):
|
||||
mock_httpx.get.return_value = mock.Mock(status_code=200)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["config", "set", "engine.ollama.host", "http://myserver:11434"],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Reachable" in result.output
|
||||
|
||||
def test_set_engine_host_probes_unreachable(self, tmp_path: Path) -> None:
|
||||
"""config set warns when engine host is unreachable, but still writes."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
config_file.write_text("")
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with (
|
||||
mock.patch.dict(os.environ, env),
|
||||
mock.patch("openjarvis.cli.config_cmd.httpx") as mock_httpx,
|
||||
):
|
||||
mock_httpx.get.side_effect = Exception("Connection refused")
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["config", "set", "engine.ollama.host", "http://myserver:11434"],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
output_lower = result.output.lower()
|
||||
assert "unreachable" in output_lower or "warning" in output_lower
|
||||
content = config_file.read_text()
|
||||
assert "http://myserver:11434" in content
|
||||
|
||||
def test_set_integer_value(self, tmp_path: Path) -> None:
|
||||
"""config set coerces integer values."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
config_file.write_text("")
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["config", "set", "intelligence.max_tokens", "2048"]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
content = config_file.read_text()
|
||||
assert "2048" in content
|
||||
|
||||
def test_set_float_value(self, tmp_path: Path) -> None:
|
||||
"""config set coerces float values."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
config_file.write_text("")
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["config", "set", "intelligence.temperature", "0.9"]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
content = config_file.read_text()
|
||||
assert "0.9" in content
|
||||
|
||||
def test_set_missing_args(self) -> None:
|
||||
"""config set with missing args shows usage error."""
|
||||
result = CliRunner().invoke(cli, ["config", "set"])
|
||||
assert result.exit_code != 0
|
||||
|
||||
def test_set_shows_confirmation(self, tmp_path: Path) -> None:
|
||||
"""config set prints a confirmation message."""
|
||||
config_file = tmp_path / "config.toml"
|
||||
config_file.write_text("")
|
||||
env = {"OPENJARVIS_CONFIG": str(config_file)}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["config", "set", "engine.default", "vllm"]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Set" in result.output
|
||||
assert "engine.default" in result.output
|
||||
@@ -35,3 +35,13 @@ class TestHintFunctions:
|
||||
def test_hint_no_model_with_name(self):
|
||||
msg = hint_no_model("qwen3:8b")
|
||||
assert "qwen3:8b" in msg
|
||||
|
||||
def test_hint_no_engine_includes_remote_tip(self):
|
||||
msg = hint_no_engine()
|
||||
assert "config set" in msg
|
||||
assert "OLLAMA_HOST" in msg
|
||||
|
||||
def test_hint_no_engine_with_name_includes_remote_tip(self):
|
||||
msg = hint_no_engine("vllm")
|
||||
assert "config set" in msg
|
||||
assert "engine.vllm.host" in msg
|
||||
|
||||
@@ -23,9 +23,7 @@ class TestInitShowsNextSteps:
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["init", "--engine", "llamacpp", _NO_DL]
|
||||
)
|
||||
result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp", _NO_DL])
|
||||
assert result.exit_code == 0
|
||||
assert "Getting Started" in result.output
|
||||
assert "jarvis ask" in result.output
|
||||
@@ -40,9 +38,7 @@ class TestInitShowsNextSteps:
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["init", "--engine", "llamacpp", _NO_DL]
|
||||
)
|
||||
result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp", _NO_DL])
|
||||
assert result.exit_code == 0
|
||||
assert "[engine]" in result.output
|
||||
assert "[intelligence]" in result.output
|
||||
@@ -57,12 +53,12 @@ class TestNextStepsOllama:
|
||||
assert "jarvis doctor" in text
|
||||
|
||||
def test_next_steps_ollama_with_model(self) -> None:
|
||||
text = _next_steps_text("ollama", "qwen3.5:14b")
|
||||
assert "ollama pull qwen3.5:14b" in text
|
||||
text = _next_steps_text("ollama", "qwen3.5:27b")
|
||||
assert "ollama pull qwen3.5:27b" in text
|
||||
|
||||
def test_next_steps_ollama_default_model(self) -> None:
|
||||
text = _next_steps_text("ollama")
|
||||
assert "ollama pull qwen3.5:3b" in text
|
||||
assert "ollama pull qwen3.5:2b" in text
|
||||
|
||||
|
||||
class TestNextStepsVllm:
|
||||
@@ -102,9 +98,7 @@ class TestMinimalConfig:
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["init", "--engine", "ollama", _NO_DL]
|
||||
)
|
||||
result = CliRunner().invoke(cli, ["init", "--engine", "ollama", _NO_DL])
|
||||
assert result.exit_code == 0
|
||||
content = config_path.read_text()
|
||||
# Minimal config should be short
|
||||
@@ -158,9 +152,7 @@ class TestInitDownloadPrompt:
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["init", "--engine", "ollama", _NO_DL]
|
||||
)
|
||||
result = CliRunner().invoke(cli, ["init", "--engine", "ollama", _NO_DL])
|
||||
assert result.exit_code == 0
|
||||
assert "Download" not in result.output
|
||||
|
||||
@@ -175,13 +167,10 @@ class TestInitEmptyModelFallback:
|
||||
mock.patch("openjarvis.cli.init_cmd.recommend_model", return_value=""),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["init", "--engine", "llamacpp"]
|
||||
)
|
||||
result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp"])
|
||||
assert result.exit_code == 0
|
||||
assert (
|
||||
"Not enough memory" in result.output
|
||||
or "not enough memory" in result.output
|
||||
"Not enough memory" in result.output or "not enough memory" in result.output
|
||||
)
|
||||
|
||||
|
||||
@@ -200,9 +189,7 @@ class TestNextStepsExoNexa:
|
||||
|
||||
|
||||
class TestInitDownloadDispatch:
|
||||
def test_init_ollama_download_calls_ollama_pull(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
def test_init_ollama_download_calls_ollama_pull(self, tmp_path: Path) -> None:
|
||||
config_dir = tmp_path / ".openjarvis"
|
||||
config_path = config_dir / "config.toml"
|
||||
with (
|
||||
@@ -228,9 +215,7 @@ class TestInitDownloadDispatch:
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli, ["init", "--engine", "vllm"], input="y\n"
|
||||
)
|
||||
result = CliRunner().invoke(cli, ["init", "--engine", "vllm"], input="y\n")
|
||||
assert result.exit_code == 0
|
||||
assert "automatically" in result.output
|
||||
|
||||
@@ -245,12 +230,11 @@ class TestInitPrivacyHook:
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner") as MockScanner,
|
||||
):
|
||||
from openjarvis.cli.scan_cmd import ScanResult
|
||||
|
||||
instance = MockScanner.return_value
|
||||
instance.run_quick.return_value = [
|
||||
ScanResult("FileVault", "ok", "FileVault enabled", "darwin"),
|
||||
]
|
||||
result = CliRunner().invoke(
|
||||
cli, ["init", "--engine", "llamacpp", _NO_DL]
|
||||
)
|
||||
result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp", _NO_DL])
|
||||
assert result.exit_code == 0
|
||||
assert "jarvis scan" in result.output
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Tests for ``jarvis init --host`` option."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from openjarvis.cli import cli
|
||||
from openjarvis.core.config import generate_default_toml, generate_minimal_toml
|
||||
|
||||
_NO_DL = "--no-download"
|
||||
|
||||
|
||||
class TestInitHost:
|
||||
def test_init_host_writes_to_config(self, tmp_path: Path) -> None:
|
||||
"""jarvis init --host writes the host into config.toml."""
|
||||
config_dir = tmp_path / ".openjarvis"
|
||||
config_path = config_dir / "config.toml"
|
||||
with (
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"init",
|
||||
"--engine",
|
||||
"ollama",
|
||||
"--host",
|
||||
"http://192.168.1.50:11434",
|
||||
_NO_DL,
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
content = config_path.read_text()
|
||||
assert "http://192.168.1.50:11434" in content
|
||||
|
||||
def test_init_host_with_vllm(self, tmp_path: Path) -> None:
|
||||
"""jarvis init --host applies to the selected engine."""
|
||||
config_dir = tmp_path / ".openjarvis"
|
||||
config_path = config_dir / "config.toml"
|
||||
with (
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["init", "--engine", "vllm", "--host", "http://10.0.0.5:8000", _NO_DL],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
content = config_path.read_text()
|
||||
assert "http://10.0.0.5:8000" in content
|
||||
|
||||
def test_init_host_probes_and_reports(self, tmp_path: Path) -> None:
|
||||
"""jarvis init --host shows reachability status."""
|
||||
config_dir = tmp_path / ".openjarvis"
|
||||
config_path = config_dir / "config.toml"
|
||||
with (
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
mock.patch("openjarvis.cli.init_cmd.httpx") as mock_httpx,
|
||||
):
|
||||
mock_httpx.get.side_effect = Exception("Connection refused")
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["init", "--engine", "ollama", "--host", "http://bad:11434", _NO_DL],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
output_lower = result.output.lower()
|
||||
assert "unreachable" in output_lower or "warning" in output_lower
|
||||
|
||||
def test_init_without_host_still_works(self, tmp_path: Path) -> None:
|
||||
"""jarvis init without --host still produces valid config."""
|
||||
config_dir = tmp_path / ".openjarvis"
|
||||
config_path = config_dir / "config.toml"
|
||||
with (
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
|
||||
mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
|
||||
mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"),
|
||||
):
|
||||
result = CliRunner().invoke(cli, ["init", "--engine", "ollama", _NO_DL])
|
||||
assert result.exit_code == 0
|
||||
content = config_path.read_text()
|
||||
assert "[engine]" in content
|
||||
|
||||
|
||||
class TestGenerateTomlHost:
|
||||
def test_minimal_toml_with_host(self) -> None:
|
||||
from openjarvis.core.config import HardwareInfo
|
||||
|
||||
hw = HardwareInfo()
|
||||
toml_str = generate_minimal_toml(
|
||||
hw, engine="ollama", host="http://remote:11434"
|
||||
)
|
||||
assert "http://remote:11434" in toml_str
|
||||
assert "[engine.ollama]" in toml_str
|
||||
|
||||
def test_minimal_toml_without_host_has_comment(self) -> None:
|
||||
from openjarvis.core.config import HardwareInfo
|
||||
|
||||
hw = HardwareInfo()
|
||||
toml_str = generate_minimal_toml(hw, engine="ollama")
|
||||
assert "# host" in toml_str
|
||||
|
||||
def test_default_toml_with_host(self) -> None:
|
||||
from openjarvis.core.config import HardwareInfo
|
||||
|
||||
hw = HardwareInfo()
|
||||
toml_str = generate_default_toml(
|
||||
hw, engine="ollama", host="http://remote:11434"
|
||||
)
|
||||
assert "http://remote:11434" in toml_str
|
||||
@@ -15,6 +15,7 @@ class TestOllamaPull:
|
||||
|
||||
def test_ollama_pull_success(self) -> None:
|
||||
import io
|
||||
|
||||
console = Console(file=io.StringIO())
|
||||
mock_lines = [
|
||||
'{"status": "pulling manifest"}',
|
||||
@@ -28,7 +29,7 @@ class TestOllamaPull:
|
||||
mock_resp.__exit__ = mock.MagicMock(return_value=False)
|
||||
|
||||
with mock.patch("httpx.stream", return_value=mock_resp):
|
||||
result = ollama_pull("http://localhost:11434", "qwen3.5:3b", console)
|
||||
result = ollama_pull("http://localhost:11434", "qwen3.5:2b", console)
|
||||
assert result is True
|
||||
|
||||
def test_ollama_pull_connect_error(self) -> None:
|
||||
@@ -38,7 +39,7 @@ class TestOllamaPull:
|
||||
|
||||
console = Console(file=io.StringIO())
|
||||
with mock.patch("httpx.stream", side_effect=httpx.ConnectError("refused")):
|
||||
result = ollama_pull("http://localhost:11434", "qwen3.5:3b", console)
|
||||
result = ollama_pull("http://localhost:11434", "qwen3.5:2b", console)
|
||||
assert result is False
|
||||
|
||||
|
||||
@@ -58,14 +59,14 @@ class TestPullCliMultiEngine:
|
||||
mock_run.return_value = mock.MagicMock(returncode=0)
|
||||
|
||||
result = runner.invoke(
|
||||
cli, ["model", "pull", "qwen3.5:8b", "--engine", "llamacpp"]
|
||||
cli, ["model", "pull", "qwen3.5:9b", "--engine", "llamacpp"]
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
mock_run.assert_called_once()
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "huggingface-cli" in call_args
|
||||
assert "qwen3.5-8b-q4_k_m.gguf" in call_args
|
||||
assert "qwen3.5-9b-q4_k_m.gguf" in call_args
|
||||
|
||||
def test_pull_mlx_uses_huggingface_cli(self) -> None:
|
||||
from openjarvis.cli import cli
|
||||
@@ -80,14 +81,14 @@ class TestPullCliMultiEngine:
|
||||
mock_run.return_value = mock.MagicMock(returncode=0)
|
||||
|
||||
result = runner.invoke(
|
||||
cli, ["model", "pull", "qwen3.5:8b", "--engine", "mlx"]
|
||||
cli, ["model", "pull", "qwen3.5:9b", "--engine", "mlx"]
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
mock_run.assert_called_once()
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "huggingface-cli" in call_args
|
||||
assert "mlx-community/Qwen3.5-8B-4bit" in call_args
|
||||
assert "mlx-community/Qwen3.5-9B-MLX-4bit" in call_args
|
||||
|
||||
def test_pull_llamacpp_huggingface_cli_not_found(self) -> None:
|
||||
from openjarvis.cli import cli
|
||||
@@ -101,7 +102,7 @@ class TestPullCliMultiEngine:
|
||||
mock_cfg.return_value.engine.ollama_host = None
|
||||
|
||||
result = runner.invoke(
|
||||
cli, ["model", "pull", "qwen3.5:8b", "--engine", "llamacpp"]
|
||||
cli, ["model", "pull", "qwen3.5:9b", "--engine", "llamacpp"]
|
||||
)
|
||||
|
||||
assert result.exit_code != 0
|
||||
|
||||
+134
-21
@@ -31,9 +31,7 @@ def _make_proc(
|
||||
|
||||
class TestScanResultDataclass:
|
||||
def test_fields_exist(self) -> None:
|
||||
r = ScanResult(
|
||||
name="test", status="ok", message="all good", platform="all"
|
||||
)
|
||||
r = ScanResult(name="test", status="ok", message="all good", platform="all")
|
||||
assert r.name == "test"
|
||||
assert r.status == "ok"
|
||||
assert r.message == "all good"
|
||||
@@ -246,15 +244,9 @@ class TestPlatformFiltering:
|
||||
other_plat = "linux" if current_plat == "darwin" else "darwin"
|
||||
|
||||
with patch.object(scanner, "_get_all_checks") as mock_get:
|
||||
darwin_ck = MagicMock(
|
||||
return_value=ScanResult("d", "ok", "msg", "darwin")
|
||||
)
|
||||
linux_ck = MagicMock(
|
||||
return_value=ScanResult("l", "ok", "msg", "linux")
|
||||
)
|
||||
all_ck = MagicMock(
|
||||
return_value=ScanResult("a", "ok", "msg", "all")
|
||||
)
|
||||
darwin_ck = MagicMock(return_value=ScanResult("d", "ok", "msg", "darwin"))
|
||||
linux_ck = MagicMock(return_value=ScanResult("l", "ok", "msg", "linux"))
|
||||
all_ck = MagicMock(return_value=ScanResult("a", "ok", "msg", "all"))
|
||||
|
||||
mock_get.return_value = [darwin_ck, linux_ck, all_ck]
|
||||
results = scanner.run_all()
|
||||
@@ -275,23 +267,19 @@ class TestRemoteAccess:
|
||||
def test_no_remote_access(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
with patch.object(scanner, "_run") as mock_run:
|
||||
mock_run.return_value = CompletedProcess(
|
||||
[], 1, stdout="", stderr=""
|
||||
)
|
||||
mock_run.return_value = CompletedProcess([], 1, stdout="", stderr="")
|
||||
result = scanner.check_remote_access()
|
||||
assert result.status == "ok"
|
||||
|
||||
def test_xrdp_running(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
with patch.object(scanner, "_run") as mock_run:
|
||||
|
||||
def side_effect(cmd, **kw):
|
||||
if any("xrdp" in str(c) for c in cmd):
|
||||
return CompletedProcess(
|
||||
cmd, 0, stdout="12345", stderr=""
|
||||
)
|
||||
return CompletedProcess(
|
||||
cmd, 1, stdout="", stderr=""
|
||||
)
|
||||
return CompletedProcess(cmd, 0, stdout="12345", stderr="")
|
||||
return CompletedProcess(cmd, 1, stdout="", stderr="")
|
||||
|
||||
mock_run.side_effect = side_effect
|
||||
result = scanner.check_remote_access()
|
||||
assert result.status == "warn"
|
||||
@@ -348,3 +336,128 @@ class TestRunQuick:
|
||||
names = {r.name for r in results}
|
||||
assert "Network Exposure" not in names
|
||||
assert "Screen Recording" not in names
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestDNS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDNS:
|
||||
"""Tests for check_dns (macOS)."""
|
||||
|
||||
def test_encrypted_dns_detected(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
scutil_out = (
|
||||
"resolver #1\n nameserver[0] : 127.0.0.1\n flags : dns-over-https\n"
|
||||
)
|
||||
with (
|
||||
patch("sys.platform", "darwin"),
|
||||
patch("subprocess.run", return_value=_make_proc(stdout=scutil_out)),
|
||||
):
|
||||
result = scanner.check_dns()
|
||||
assert result.status == "ok"
|
||||
assert "Encrypted" in result.message or "DoH" in result.message
|
||||
|
||||
def test_plain_dns_detected(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
scutil_out = (
|
||||
"resolver #1\n nameserver[0] : 8.8.8.8\n nameserver[1] : 8.8.4.4\n"
|
||||
)
|
||||
with (
|
||||
patch("sys.platform", "darwin"),
|
||||
patch("subprocess.run", return_value=_make_proc(stdout=scutil_out)),
|
||||
):
|
||||
result = scanner.check_dns()
|
||||
assert result.status == "warn"
|
||||
assert "8.8.8.8" in result.message
|
||||
|
||||
def test_private_dns(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
scutil_out = "resolver #1\n nameserver[0] : 192.168.1.1\n"
|
||||
with (
|
||||
patch("sys.platform", "darwin"),
|
||||
patch("subprocess.run", return_value=_make_proc(stdout=scutil_out)),
|
||||
):
|
||||
result = scanner.check_dns()
|
||||
assert result.status == "ok"
|
||||
assert "192.168.1.1" in result.message
|
||||
|
||||
def test_skip_on_linux(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
with patch("sys.platform", "linux"):
|
||||
result = scanner.check_dns()
|
||||
assert result.status == "skip"
|
||||
|
||||
def test_scutil_not_available(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
with (
|
||||
patch("sys.platform", "darwin"),
|
||||
patch("subprocess.run", side_effect=FileNotFoundError),
|
||||
):
|
||||
result = scanner.check_dns()
|
||||
assert result.status == "skip"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestExpandedRemoteAccess
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExpandedRemoteAccess:
|
||||
"""Verify expanded remote-access process list."""
|
||||
|
||||
def test_ngrok_detected(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
with patch.object(scanner, "_run") as mock_run:
|
||||
|
||||
def side_effect(cmd, **kw):
|
||||
if any("ngrok" in str(c) for c in cmd):
|
||||
return CompletedProcess(cmd, 0, stdout="99999", stderr="")
|
||||
return CompletedProcess(cmd, 1, stdout="", stderr="")
|
||||
|
||||
mock_run.side_effect = side_effect
|
||||
result = scanner.check_remote_access()
|
||||
assert result.status == "warn"
|
||||
assert "ngrok" in result.message
|
||||
|
||||
def test_tailscaled_detected(self) -> None:
|
||||
scanner = PrivacyScanner()
|
||||
with patch.object(scanner, "_run") as mock_run:
|
||||
|
||||
def side_effect(cmd, **kw):
|
||||
if any("tailscaled" in str(c) for c in cmd):
|
||||
return CompletedProcess(cmd, 0, stdout="88888", stderr="")
|
||||
return CompletedProcess(cmd, 1, stdout="", stderr="")
|
||||
|
||||
mock_run.side_effect = side_effect
|
||||
result = scanner.check_remote_access()
|
||||
assert result.status == "warn"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestJsonOutput
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestJsonOutput:
|
||||
"""Test --json output flag."""
|
||||
|
||||
def test_json_output_structure(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
from openjarvis.cli.scan_cmd import scan
|
||||
|
||||
runner = CliRunner()
|
||||
with patch(
|
||||
"openjarvis.cli.scan_cmd.PrivacyScanner.run_all",
|
||||
return_value=[
|
||||
ScanResult("Test", "ok", "all good", "all"),
|
||||
],
|
||||
):
|
||||
result = runner.invoke(scan, ["--json"])
|
||||
assert result.exit_code == 0
|
||||
data = json.loads(result.output)
|
||||
assert isinstance(data, list)
|
||||
assert data[0]["name"] == "Test"
|
||||
assert data[0]["status"] == "ok"
|
||||
|
||||
@@ -447,6 +447,41 @@ class TestSchedulerConfig:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApplyTomlSectionListNormalization:
|
||||
def test_apply_toml_section_list_to_str_field(self) -> None:
|
||||
"""TOML arrays assigned to str-typed fields should be joined with ','."""
|
||||
from openjarvis.core.config import ToolsConfig, _apply_toml_section
|
||||
|
||||
target = ToolsConfig()
|
||||
tools = ["code_interpreter", "web_search", "file_read"]
|
||||
_apply_toml_section(target, {"enabled": tools})
|
||||
assert isinstance(target.enabled, str)
|
||||
assert target.enabled == "code_interpreter,web_search,file_read"
|
||||
|
||||
def test_apply_toml_section_list_to_property_setter(self) -> None:
|
||||
"""TOML arrays passed to backward-compat property setters should be
|
||||
normalized to comma-separated strings, not passed as raw lists."""
|
||||
from openjarvis.core.config import _apply_toml_section
|
||||
|
||||
target = LearningConfig()
|
||||
_apply_toml_section(target, {
|
||||
"reward_weights": ["accuracy=0.8", "latency=0.2"],
|
||||
})
|
||||
assert target.metrics.accuracy_weight == 0.8
|
||||
assert target.metrics.latency_weight == 0.2
|
||||
|
||||
def test_apply_toml_section_agent_tools_list(self) -> None:
|
||||
"""Agent tools should work as a TOML array."""
|
||||
from openjarvis.core.config import _apply_toml_section
|
||||
|
||||
target = AgentConfig()
|
||||
_apply_toml_section(target, {
|
||||
"tools": ["web_search", "http_request", "file_read"],
|
||||
})
|
||||
assert isinstance(target.tools, str)
|
||||
assert target.tools == "web_search,http_request,file_read"
|
||||
|
||||
|
||||
class TestWhatsAppBaileysChannelConfig:
|
||||
def test_defaults(self) -> None:
|
||||
wc = WhatsAppBaileysChannelConfig()
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Tests for config key validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.core.config import validate_config_key
|
||||
|
||||
|
||||
class TestValidateConfigKey:
|
||||
def test_valid_engine_default(self):
|
||||
field_type = validate_config_key("engine.default")
|
||||
assert field_type is str
|
||||
|
||||
def test_valid_engine_ollama_host(self):
|
||||
field_type = validate_config_key("engine.ollama.host")
|
||||
assert field_type is str
|
||||
|
||||
def test_valid_intelligence_temperature(self):
|
||||
field_type = validate_config_key("intelligence.temperature")
|
||||
assert field_type is float
|
||||
|
||||
def test_valid_intelligence_max_tokens(self):
|
||||
field_type = validate_config_key("intelligence.max_tokens")
|
||||
assert field_type is int
|
||||
|
||||
def test_valid_agent_default_agent(self):
|
||||
field_type = validate_config_key("agent.default_agent")
|
||||
assert field_type is str
|
||||
|
||||
def test_invalid_top_level_key(self):
|
||||
with pytest.raises(ValueError, match="Unknown config key"):
|
||||
validate_config_key("nonexistent.foo")
|
||||
|
||||
def test_invalid_nested_key(self):
|
||||
with pytest.raises(ValueError, match="Unknown config key"):
|
||||
validate_config_key("engine.olllama.host")
|
||||
|
||||
def test_invalid_leaf_key(self):
|
||||
with pytest.raises(ValueError, match="Unknown config key"):
|
||||
validate_config_key("engine.ollama.nonexistent")
|
||||
|
||||
def test_empty_key(self):
|
||||
with pytest.raises(ValueError):
|
||||
validate_config_key("")
|
||||
|
||||
def test_single_segment(self):
|
||||
with pytest.raises(ValueError):
|
||||
validate_config_key("engine")
|
||||
|
||||
def test_hardware_key_rejected(self):
|
||||
"""Hardware is auto-detected, not user-settable."""
|
||||
with pytest.raises(ValueError, match="Unknown config key"):
|
||||
validate_config_key("hardware.cpu_count")
|
||||
@@ -27,7 +27,7 @@ class TestRecommendModelGpu:
|
||||
result = recommend_model(hw, "ollama")
|
||||
# 14B * 0.5 * 1.1 = 7.7 GB; available = 8 * 0.9 = 7.2 → too big
|
||||
# 8B * 0.5 * 1.1 = 4.4 GB; available = 7.2 → fits
|
||||
assert result == "qwen3.5:8b"
|
||||
assert result == "qwen3.5:9b"
|
||||
|
||||
def test_4gb_gpu_picks_qwen35_4b(self) -> None:
|
||||
hw = HardwareInfo(
|
||||
@@ -47,7 +47,7 @@ class TestRecommendModelGpu:
|
||||
)
|
||||
result = recommend_model(hw, "ollama")
|
||||
# 3B * 0.5 * 1.1 = 1.65 GB; available = 2 * 0.9 = 1.8 → fits
|
||||
assert result == "qwen3.5:3b"
|
||||
assert result == "qwen3.5:2b"
|
||||
|
||||
def test_multi_gpu_picks_larger_model(self) -> None:
|
||||
hw = HardwareInfo(
|
||||
@@ -80,8 +80,9 @@ class TestRecommendModelCpuOnly:
|
||||
hw = HardwareInfo(platform="linux", ram_gb=16.0, gpu=None)
|
||||
result = recommend_model(hw, "llamacpp")
|
||||
# available = (16 - 4) * 0.8 = 9.6 GB
|
||||
# 14B * 0.5 * 1.1 = 7.7 → fits
|
||||
assert result == "qwen3.5:14b"
|
||||
# 27B * 0.5 * 1.1 = 14.85 → too big
|
||||
# 9B * 0.5 * 1.1 = 4.95 → fits
|
||||
assert result == "qwen3.5:9b"
|
||||
|
||||
def test_cpu_only_8gb_ram(self) -> None:
|
||||
hw = HardwareInfo(platform="linux", ram_gb=8.0, gpu=None)
|
||||
@@ -127,7 +128,7 @@ class TestRecommendModelMlx:
|
||||
gpu=GpuInfo(vendor="apple", name="Apple M1", vram_gb=8.0, count=1),
|
||||
)
|
||||
result = recommend_model(hw, "mlx")
|
||||
assert result == "qwen3.5:8b"
|
||||
assert result == "qwen3.5:9b"
|
||||
|
||||
def test_apple_silicon_16gb_mlx(self) -> None:
|
||||
hw = HardwareInfo(
|
||||
@@ -136,7 +137,10 @@ class TestRecommendModelMlx:
|
||||
gpu=GpuInfo(vendor="apple", name="Apple M2", vram_gb=16.0, count=1),
|
||||
)
|
||||
result = recommend_model(hw, "mlx")
|
||||
assert result == "qwen3.5:14b"
|
||||
# available = 16 * 0.9 = 14.4 GB
|
||||
# 27B * 0.5 * 1.1 = 14.85 → too big
|
||||
# 9B * 0.5 * 1.1 = 4.95 → fits
|
||||
assert result == "qwen3.5:9b"
|
||||
|
||||
def test_apple_silicon_32gb_mlx(self) -> None:
|
||||
hw = HardwareInfo(
|
||||
@@ -145,7 +149,7 @@ class TestRecommendModelMlx:
|
||||
gpu=GpuInfo(vendor="apple", name="Apple M2 Pro", vram_gb=32.0, count=1),
|
||||
)
|
||||
result = recommend_model(hw, "mlx")
|
||||
assert result == "qwen3.5:14b"
|
||||
assert result == "qwen3.5:27b"
|
||||
|
||||
def test_apple_silicon_64gb_mlx(self) -> None:
|
||||
hw = HardwareInfo(
|
||||
@@ -154,4 +158,4 @@ class TestRecommendModelMlx:
|
||||
gpu=GpuInfo(vendor="apple", name="Apple M2 Max", vram_gb=64.0, count=1),
|
||||
)
|
||||
result = recommend_model(hw, "mlx")
|
||||
assert result == "qwen3.5:14b"
|
||||
assert result == "qwen3.5:27b"
|
||||
|
||||
+183
-1
@@ -9,7 +9,11 @@ import pytest
|
||||
|
||||
from openjarvis.core.registry import EngineRegistry
|
||||
from openjarvis.core.types import Message, Role
|
||||
from openjarvis.engine.cloud import CloudEngine, estimate_cost
|
||||
from openjarvis.engine.cloud import (
|
||||
CloudEngine,
|
||||
_is_codex_model,
|
||||
estimate_cost,
|
||||
)
|
||||
|
||||
|
||||
class TestEstimateCost:
|
||||
@@ -108,3 +112,181 @@ class TestCloudEngineGenerate:
|
||||
assert result["content"] == "Greetings!"
|
||||
assert result["usage"]["prompt_tokens"] == 12
|
||||
assert result["usage"]["completion_tokens"] == 8
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Codex provider support (OpenAI Responses API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCodexModelDetection:
|
||||
def test_is_codex_model(self) -> None:
|
||||
assert _is_codex_model("codex/gpt-4o") is True
|
||||
assert _is_codex_model("codex/gpt-5-mini") is True
|
||||
assert _is_codex_model("codex/gpt-5-mini-2025-08-07") is True
|
||||
|
||||
def test_not_codex_model(self) -> None:
|
||||
assert _is_codex_model("gpt-4o") is False
|
||||
assert _is_codex_model("openrouter/openai/gpt-4o") is False
|
||||
|
||||
|
||||
class TestCodexClientInit:
|
||||
def test_health_with_codex_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
monkeypatch.setenv("OPENAI_CODEX_API_KEY", "test-token")
|
||||
engine = CloudEngine()
|
||||
assert engine.health() is True
|
||||
assert engine._codex_client is not None
|
||||
assert engine._codex_client["token"] == "test-token"
|
||||
assert "responses" in engine._codex_client["url"]
|
||||
|
||||
def test_custom_codex_base_url(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
monkeypatch.setenv("OPENAI_CODEX_API_KEY", "test-token")
|
||||
monkeypatch.setenv("OPENAI_CODEX_BASE_URL", "http://localhost:9999")
|
||||
engine = CloudEngine()
|
||||
assert engine._codex_client["url"] == "http://localhost:9999/responses"
|
||||
|
||||
def test_list_models_includes_codex(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
monkeypatch.setenv("OPENAI_CODEX_API_KEY", "test-token")
|
||||
engine = CloudEngine()
|
||||
models = engine.list_models()
|
||||
assert "codex/gpt-4o" in models
|
||||
assert "codex/gpt-5-mini" in models
|
||||
assert "codex/gpt-5-mini-2025-08-07" in models
|
||||
|
||||
def test_no_codex_key_means_no_codex(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
monkeypatch.delenv("OPENAI_CODEX_API_KEY", raising=False)
|
||||
engine = CloudEngine()
|
||||
assert engine._codex_client is None
|
||||
assert "codex/gpt-4o" not in engine.list_models()
|
||||
|
||||
|
||||
class TestCodexGenerate:
|
||||
def test_generate_codex_uses_responses_api(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
|
||||
fake_response = mock.MagicMock()
|
||||
fake_response.status_code = 200
|
||||
fake_response.json.return_value = {
|
||||
"output_text": "Codex response!",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
fake_response.raise_for_status = mock.MagicMock()
|
||||
|
||||
engine = CloudEngine()
|
||||
engine._codex_client = {
|
||||
"token": "test-token",
|
||||
"url": "https://api.openai.com/v1/responses",
|
||||
}
|
||||
|
||||
with mock.patch(
|
||||
"openjarvis.engine.cloud.httpx.post",
|
||||
return_value=fake_response,
|
||||
) as mock_post:
|
||||
result = engine.generate(
|
||||
[Message(role=Role.USER, content="Hi")],
|
||||
model="codex/gpt-5-mini-2025-08-07",
|
||||
)
|
||||
|
||||
assert result["content"] == "Codex response!"
|
||||
assert result["model"] == "gpt-5-mini-2025-08-07"
|
||||
assert result["usage"]["prompt_tokens"] == 10
|
||||
assert result["usage"]["completion_tokens"] == 5
|
||||
|
||||
# Verify correct Responses API request format
|
||||
call_kwargs = mock_post.call_args
|
||||
sent_body = call_kwargs.kwargs["json"]
|
||||
assert sent_body["model"] == "gpt-5-mini-2025-08-07"
|
||||
assert sent_body["stream"] is False
|
||||
assert "input" in sent_body # Responses API format
|
||||
assert "messages" not in sent_body # NOT chat completions
|
||||
|
||||
# Verify correct headers
|
||||
sent_headers = call_kwargs.kwargs["headers"]
|
||||
assert sent_headers["Authorization"] == "Bearer test-token"
|
||||
assert sent_headers["OpenAI-Beta"] == "responses=experimental"
|
||||
|
||||
def test_generate_codex_extracts_from_output_blocks(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Fallback extraction from output[].content[] blocks."""
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
|
||||
fake_response = mock.MagicMock()
|
||||
fake_response.json.return_value = {
|
||||
"output": [{"content": [{"type": "output_text", "text": "From blocks!"}]}],
|
||||
"usage": {"input_tokens": 5, "output_tokens": 3},
|
||||
}
|
||||
fake_response.raise_for_status = mock.MagicMock()
|
||||
|
||||
engine = CloudEngine()
|
||||
engine._codex_client = {
|
||||
"token": "t",
|
||||
"url": "https://api.openai.com/v1/responses",
|
||||
}
|
||||
|
||||
with mock.patch(
|
||||
"openjarvis.engine.cloud.httpx.post",
|
||||
return_value=fake_response,
|
||||
):
|
||||
result = engine.generate(
|
||||
[Message(role=Role.USER, content="Hi")],
|
||||
model="codex/gpt-4o",
|
||||
)
|
||||
assert result["content"] == "From blocks!"
|
||||
|
||||
def test_generate_codex_passes_system_as_instructions(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
|
||||
fake_response = mock.MagicMock()
|
||||
fake_response.json.return_value = {
|
||||
"output_text": "ok",
|
||||
"usage": {},
|
||||
}
|
||||
fake_response.raise_for_status = mock.MagicMock()
|
||||
|
||||
engine = CloudEngine()
|
||||
engine._codex_client = {
|
||||
"token": "t",
|
||||
"url": "https://api.openai.com/v1/responses",
|
||||
}
|
||||
|
||||
with mock.patch(
|
||||
"openjarvis.engine.cloud.httpx.post",
|
||||
return_value=fake_response,
|
||||
) as mock_post:
|
||||
engine.generate(
|
||||
[
|
||||
Message(role=Role.SYSTEM, content="Be helpful"),
|
||||
Message(role=Role.USER, content="Hi"),
|
||||
],
|
||||
model="codex/gpt-4o",
|
||||
)
|
||||
|
||||
sent_body = mock_post.call_args.kwargs["json"]
|
||||
assert sent_body["instructions"] == "Be helpful"
|
||||
# System message should NOT appear in input messages
|
||||
roles = [m["role"] for m in sent_body["input"]]
|
||||
assert "system" not in roles
|
||||
|
||||
def test_codex_close(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
engine = CloudEngine()
|
||||
engine._codex_client = {"token": "t", "url": "http://test"}
|
||||
engine.close()
|
||||
assert engine._codex_client is None
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Tests for CloudEngine.stream_full, _stream_full_openai, _stream_full_anthropic,
|
||||
and _prepare_anthropic_messages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, List
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.core.types import Message, Role, ToolCall
|
||||
from openjarvis.engine._stubs import StreamChunk
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_cloud_engine(**overrides: Any) -> Any:
|
||||
"""Create a CloudEngine without calling __init__ (no env vars needed)."""
|
||||
from openjarvis.engine.cloud import CloudEngine
|
||||
|
||||
engine = CloudEngine.__new__(CloudEngine)
|
||||
engine._openai_client = overrides.get("openai_client")
|
||||
engine._anthropic_client = overrides.get("anthropic_client")
|
||||
engine._google_client = overrides.get("google_client")
|
||||
engine._openrouter_client = overrides.get("openrouter_client")
|
||||
engine._minimax_client = overrides.get("minimax_client")
|
||||
return engine
|
||||
|
||||
|
||||
def _openai_chunk(
|
||||
*,
|
||||
content: str | None = None,
|
||||
tool_calls: list | None = None,
|
||||
finish_reason: str | None = None,
|
||||
) -> MagicMock:
|
||||
"""Build a mock OpenAI streaming chunk."""
|
||||
delta = MagicMock()
|
||||
delta.content = content
|
||||
delta.tool_calls = tool_calls
|
||||
choice = MagicMock()
|
||||
choice.delta = delta
|
||||
choice.finish_reason = finish_reason
|
||||
chunk = MagicMock()
|
||||
chunk.choices = [choice]
|
||||
return chunk
|
||||
|
||||
|
||||
def _openai_tool_call_delta(
|
||||
*,
|
||||
index: int = 0,
|
||||
tc_id: str = "",
|
||||
name: str = "",
|
||||
arguments: str = "",
|
||||
) -> MagicMock:
|
||||
tc = MagicMock()
|
||||
tc.index = index
|
||||
tc.id = tc_id
|
||||
tc.function = MagicMock()
|
||||
tc.function.name = name
|
||||
tc.function.arguments = arguments
|
||||
return tc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _stream_full_openai tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_full_openai_content():
|
||||
"""Mock OpenAI streaming response with content chunks."""
|
||||
mock_client = MagicMock()
|
||||
chunks = [
|
||||
_openai_chunk(content="Hello"),
|
||||
_openai_chunk(content=" world"),
|
||||
_openai_chunk(finish_reason="stop"),
|
||||
]
|
||||
mock_client.chat.completions.create.return_value = iter(chunks)
|
||||
|
||||
engine = _make_cloud_engine(openai_client=mock_client)
|
||||
msgs = [Message(role=Role.USER, content="hi")]
|
||||
|
||||
result: List[StreamChunk] = []
|
||||
async for sc in engine._stream_full_openai(
|
||||
msgs,
|
||||
model="gpt-4o",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
):
|
||||
result.append(sc)
|
||||
|
||||
assert len(result) == 3
|
||||
assert result[0].content == "Hello"
|
||||
assert result[1].content == " world"
|
||||
assert result[2].finish_reason == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_full_openai_tool_calls():
|
||||
"""Mock response with tool_call deltas, verify StreamChunk.tool_calls format."""
|
||||
mock_client = MagicMock()
|
||||
tc1 = _openai_tool_call_delta(index=0, tc_id="call_1", name="calc", arguments="")
|
||||
tc2 = _openai_tool_call_delta(index=0, tc_id="", name="", arguments='{"x": 1}')
|
||||
chunks = [
|
||||
_openai_chunk(tool_calls=[tc1]),
|
||||
_openai_chunk(tool_calls=[tc2]),
|
||||
_openai_chunk(finish_reason="tool_calls"),
|
||||
]
|
||||
mock_client.chat.completions.create.return_value = iter(chunks)
|
||||
|
||||
engine = _make_cloud_engine(openai_client=mock_client)
|
||||
msgs = [Message(role=Role.USER, content="calc")]
|
||||
|
||||
result: List[StreamChunk] = []
|
||||
async for sc in engine._stream_full_openai(
|
||||
msgs,
|
||||
model="gpt-4o",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
):
|
||||
result.append(sc)
|
||||
|
||||
assert result[0].tool_calls is not None
|
||||
assert result[0].tool_calls[0]["function"]["name"] == "calc"
|
||||
assert result[0].tool_calls[0]["id"] == "call_1"
|
||||
assert result[1].tool_calls[0]["function"]["arguments"] == '{"x": 1}'
|
||||
assert result[2].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_full_openai_finish_reason():
|
||||
"""Verify finish_reason='tool_calls' and 'stop' propagated correctly."""
|
||||
mock_client = MagicMock()
|
||||
chunks_stop = [
|
||||
_openai_chunk(content="ok"),
|
||||
_openai_chunk(finish_reason="stop"),
|
||||
]
|
||||
mock_client.chat.completions.create.return_value = iter(chunks_stop)
|
||||
|
||||
engine = _make_cloud_engine(openai_client=mock_client)
|
||||
msgs = [Message(role=Role.USER, content="hi")]
|
||||
|
||||
result = []
|
||||
async for sc in engine._stream_full_openai(
|
||||
msgs,
|
||||
model="gpt-4o",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
):
|
||||
result.append(sc)
|
||||
|
||||
assert result[-1].finish_reason == "stop"
|
||||
|
||||
# Now test tool_calls finish
|
||||
tc = _openai_tool_call_delta(index=0, tc_id="c1", name="fn", arguments="{}")
|
||||
chunks_tc = [
|
||||
_openai_chunk(tool_calls=[tc]),
|
||||
_openai_chunk(finish_reason="tool_calls"),
|
||||
]
|
||||
mock_client.chat.completions.create.return_value = iter(chunks_tc)
|
||||
|
||||
result2 = []
|
||||
async for sc in engine._stream_full_openai(
|
||||
msgs,
|
||||
model="gpt-4o",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
):
|
||||
result2.append(sc)
|
||||
|
||||
assert result2[-1].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _stream_full_anthropic tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _anthropic_event(event_type: str, **kwargs: Any) -> MagicMock:
|
||||
"""Build a mock Anthropic stream event."""
|
||||
event = MagicMock()
|
||||
event.type = event_type
|
||||
for k, v in kwargs.items():
|
||||
setattr(event, k, v)
|
||||
return event
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_full_anthropic_content():
|
||||
"""Mock Anthropic stream events with text content."""
|
||||
# Build content_block_start with text type
|
||||
text_block = MagicMock()
|
||||
text_block.type = "text"
|
||||
|
||||
# Build text delta
|
||||
text_delta = MagicMock()
|
||||
text_delta.type = "text_delta"
|
||||
text_delta.text = "Hello world"
|
||||
|
||||
# Build message_delta with stop
|
||||
msg_delta = MagicMock()
|
||||
msg_delta.stop_reason = "end_turn"
|
||||
|
||||
events = [
|
||||
_anthropic_event("content_block_start", content_block=text_block),
|
||||
_anthropic_event("content_block_delta", delta=text_delta),
|
||||
_anthropic_event("message_delta", delta=msg_delta),
|
||||
]
|
||||
|
||||
mock_stream = MagicMock()
|
||||
mock_stream.__enter__ = MagicMock(return_value=iter(events))
|
||||
mock_stream.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
mock_anthropic = MagicMock()
|
||||
mock_anthropic.messages.stream.return_value = mock_stream
|
||||
|
||||
engine = _make_cloud_engine(anthropic_client=mock_anthropic)
|
||||
msgs = [Message(role=Role.USER, content="hi")]
|
||||
|
||||
result: List[StreamChunk] = []
|
||||
async for sc in engine._stream_full_anthropic(
|
||||
msgs,
|
||||
model="claude-sonnet-4-20250514",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
):
|
||||
result.append(sc)
|
||||
|
||||
# Should have text content and a finish reason
|
||||
content_chunks = [r for r in result if r.content is not None]
|
||||
assert len(content_chunks) >= 1
|
||||
assert content_chunks[0].content == "Hello world"
|
||||
|
||||
finish_chunks = [r for r in result if r.finish_reason is not None]
|
||||
assert len(finish_chunks) == 1
|
||||
assert finish_chunks[0].finish_reason == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_full_anthropic_tool_calls():
|
||||
"""Mock Anthropic tool_use events, verify OpenAI-delta-format tool_calls."""
|
||||
# content_block_start with tool_use
|
||||
tool_block = MagicMock()
|
||||
tool_block.type = "tool_use"
|
||||
tool_block.id = "toolu_123"
|
||||
tool_block.name = "get_weather"
|
||||
|
||||
# input_json_delta
|
||||
json_delta = MagicMock()
|
||||
json_delta.type = "input_json_delta"
|
||||
json_delta.partial_json = '{"city": "Berlin"}'
|
||||
|
||||
# message_delta with tool_use stop
|
||||
msg_delta = MagicMock()
|
||||
msg_delta.stop_reason = "tool_use"
|
||||
|
||||
events = [
|
||||
_anthropic_event("content_block_start", content_block=tool_block),
|
||||
_anthropic_event("content_block_delta", delta=json_delta),
|
||||
_anthropic_event("message_delta", delta=msg_delta),
|
||||
]
|
||||
|
||||
mock_stream = MagicMock()
|
||||
mock_stream.__enter__ = MagicMock(return_value=iter(events))
|
||||
mock_stream.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
mock_anthropic = MagicMock()
|
||||
mock_anthropic.messages.stream.return_value = mock_stream
|
||||
|
||||
engine = _make_cloud_engine(anthropic_client=mock_anthropic)
|
||||
msgs = [Message(role=Role.USER, content="weather?")]
|
||||
|
||||
result: List[StreamChunk] = []
|
||||
async for sc in engine._stream_full_anthropic(
|
||||
msgs,
|
||||
model="claude-sonnet-4-20250514",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
):
|
||||
result.append(sc)
|
||||
|
||||
# First chunk: tool_use start with name
|
||||
assert result[0].tool_calls is not None
|
||||
assert result[0].tool_calls[0]["function"]["name"] == "get_weather"
|
||||
assert result[0].tool_calls[0]["id"] == "toolu_123"
|
||||
|
||||
# Second chunk: arguments fragment
|
||||
assert result[1].tool_calls is not None
|
||||
assert result[1].tool_calls[0]["function"]["arguments"] == '{"city": "Berlin"}'
|
||||
|
||||
# Third chunk: finish with tool_calls
|
||||
assert result[2].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_full_anthropic_finish_reason():
|
||||
"""message_delta with stop_reason='tool_use' maps to finish_reason='tool_calls'."""
|
||||
msg_delta_tool = MagicMock()
|
||||
msg_delta_tool.stop_reason = "tool_use"
|
||||
|
||||
msg_delta_stop = MagicMock()
|
||||
msg_delta_stop.stop_reason = "end_turn"
|
||||
|
||||
# Test tool_use -> tool_calls
|
||||
events_tool = [_anthropic_event("message_delta", delta=msg_delta_tool)]
|
||||
mock_stream = MagicMock()
|
||||
mock_stream.__enter__ = MagicMock(return_value=iter(events_tool))
|
||||
mock_stream.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
mock_anthropic = MagicMock()
|
||||
mock_anthropic.messages.stream.return_value = mock_stream
|
||||
|
||||
engine = _make_cloud_engine(anthropic_client=mock_anthropic)
|
||||
msgs = [Message(role=Role.USER, content="test")]
|
||||
|
||||
result = []
|
||||
async for sc in engine._stream_full_anthropic(
|
||||
msgs,
|
||||
model="claude-sonnet-4-20250514",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
):
|
||||
result.append(sc)
|
||||
assert result[0].finish_reason == "tool_calls"
|
||||
|
||||
# Test end_turn -> stop
|
||||
events_stop = [_anthropic_event("message_delta", delta=msg_delta_stop)]
|
||||
mock_stream2 = MagicMock()
|
||||
mock_stream2.__enter__ = MagicMock(return_value=iter(events_stop))
|
||||
mock_stream2.__exit__ = MagicMock(return_value=False)
|
||||
mock_anthropic.messages.stream.return_value = mock_stream2
|
||||
|
||||
result2 = []
|
||||
async for sc in engine._stream_full_anthropic(
|
||||
msgs,
|
||||
model="claude-sonnet-4-20250514",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
):
|
||||
result2.append(sc)
|
||||
assert result2[0].finish_reason == "stop"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _prepare_anthropic_messages tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prepare_anthropic_messages_system():
|
||||
"""System message extracted separately from chat messages."""
|
||||
engine = _make_cloud_engine()
|
||||
msgs = [
|
||||
Message(role=Role.SYSTEM, content="You are helpful"),
|
||||
Message(role=Role.USER, content="Hello"),
|
||||
]
|
||||
|
||||
system_text, chat_msgs = engine._prepare_anthropic_messages(msgs)
|
||||
assert system_text == "You are helpful"
|
||||
assert len(chat_msgs) == 1
|
||||
assert chat_msgs[0]["role"] == "user"
|
||||
assert chat_msgs[0]["content"] == "Hello"
|
||||
|
||||
|
||||
def test_prepare_anthropic_messages_tool_result():
|
||||
"""Tool role converted to user + tool_result content block."""
|
||||
engine = _make_cloud_engine()
|
||||
msgs = [
|
||||
Message(role=Role.USER, content="What's the weather?"),
|
||||
Message(
|
||||
role=Role.TOOL,
|
||||
content='{"temp": 20}',
|
||||
tool_call_id="call_abc",
|
||||
),
|
||||
]
|
||||
|
||||
system_text, chat_msgs = engine._prepare_anthropic_messages(msgs)
|
||||
assert system_text == ""
|
||||
assert len(chat_msgs) == 2
|
||||
# Second message is the tool result wrapped as user
|
||||
tool_msg = chat_msgs[1]
|
||||
assert tool_msg["role"] == "user"
|
||||
assert isinstance(tool_msg["content"], list)
|
||||
assert tool_msg["content"][0]["type"] == "tool_result"
|
||||
assert tool_msg["content"][0]["tool_use_id"] == "call_abc"
|
||||
assert tool_msg["content"][0]["content"] == '{"temp": 20}'
|
||||
|
||||
|
||||
def test_prepare_anthropic_messages_tool_calls():
|
||||
"""Assistant with tool_calls converted to content blocks with tool_use."""
|
||||
engine = _make_cloud_engine()
|
||||
msgs = [
|
||||
Message(
|
||||
role=Role.ASSISTANT,
|
||||
content="Let me check.",
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id="call_1", name="get_weather", arguments='{"city": "Berlin"}'
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
system_text, chat_msgs = engine._prepare_anthropic_messages(msgs)
|
||||
assert len(chat_msgs) == 1
|
||||
blocks = chat_msgs[0]["content"]
|
||||
assert blocks[0]["type"] == "text"
|
||||
assert blocks[0]["text"] == "Let me check."
|
||||
assert blocks[1]["type"] == "tool_use"
|
||||
assert blocks[1]["id"] == "call_1"
|
||||
assert blocks[1]["name"] == "get_weather"
|
||||
assert blocks[1]["input"] == {"city": "Berlin"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# stream_full routing tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_full_routes_to_anthropic():
|
||||
"""model='claude-xxx' routes to _stream_full_anthropic."""
|
||||
msg_delta = MagicMock()
|
||||
msg_delta.stop_reason = "end_turn"
|
||||
events = [_anthropic_event("message_delta", delta=msg_delta)]
|
||||
mock_stream = MagicMock()
|
||||
mock_stream.__enter__ = MagicMock(return_value=iter(events))
|
||||
mock_stream.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
mock_anthropic = MagicMock()
|
||||
mock_anthropic.messages.stream.return_value = mock_stream
|
||||
|
||||
engine = _make_cloud_engine(anthropic_client=mock_anthropic)
|
||||
msgs = [Message(role=Role.USER, content="test")]
|
||||
|
||||
result = []
|
||||
async for sc in engine.stream_full(msgs, model="claude-sonnet-4-20250514"):
|
||||
result.append(sc)
|
||||
|
||||
# Verify Anthropic client was used
|
||||
mock_anthropic.messages.stream.assert_called_once()
|
||||
assert any(r.finish_reason is not None for r in result)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_full_routes_to_openai():
|
||||
"""model='gpt-xxx' routes to _stream_full_openai."""
|
||||
mock_client = MagicMock()
|
||||
chunks = [
|
||||
_openai_chunk(content="hi"),
|
||||
_openai_chunk(finish_reason="stop"),
|
||||
]
|
||||
mock_client.chat.completions.create.return_value = iter(chunks)
|
||||
|
||||
engine = _make_cloud_engine(openai_client=mock_client)
|
||||
msgs = [Message(role=Role.USER, content="test")]
|
||||
|
||||
result = []
|
||||
async for sc in engine.stream_full(msgs, model="gpt-4o"):
|
||||
result.append(sc)
|
||||
|
||||
# Verify OpenAI client was used
|
||||
mock_client.chat.completions.create.assert_called_once()
|
||||
assert result[0].content == "hi"
|
||||
assert result[1].finish_reason == "stop"
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Tests for InstrumentedEngine, GuardrailsEngine, and MultiEngine stream_full
|
||||
delegation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.core.events import EventBus
|
||||
from openjarvis.core.types import Message, Role
|
||||
from openjarvis.engine._stubs import InferenceEngine, StreamChunk
|
||||
from openjarvis.engine.multi import MultiEngine
|
||||
from openjarvis.security.guardrails import GuardrailsEngine
|
||||
from openjarvis.telemetry.instrumented_engine import InstrumentedEngine
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fake engine that yields predetermined StreamChunks via stream_full
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeStreamFullEngine(InferenceEngine):
|
||||
engine_id = "fake-sf"
|
||||
|
||||
def __init__(self, chunks: list[StreamChunk]) -> None:
|
||||
self._chunks = chunks
|
||||
|
||||
def generate(self, messages, *, model, **kwargs) -> Dict[str, Any]:
|
||||
return {"content": "ok", "usage": {}}
|
||||
|
||||
async def stream(self, messages, *, model, **kwargs) -> AsyncIterator[str]:
|
||||
yield "ok"
|
||||
|
||||
async def stream_full(
|
||||
self, messages, *, model, **kwargs
|
||||
) -> AsyncIterator[StreamChunk]:
|
||||
for c in self._chunks:
|
||||
yield c
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
return ["fake-model"]
|
||||
|
||||
def health(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# InstrumentedEngine.stream_full delegation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_instrumented_delegates_stream_full():
|
||||
"""InstrumentedEngine.stream_full delegates to inner engine."""
|
||||
expected = [
|
||||
StreamChunk(content="Hello"),
|
||||
StreamChunk(content=" world"),
|
||||
StreamChunk(finish_reason="stop"),
|
||||
]
|
||||
inner = _FakeStreamFullEngine(expected)
|
||||
bus = EventBus(record_history=True)
|
||||
engine = InstrumentedEngine(inner, bus)
|
||||
|
||||
result = []
|
||||
async for chunk in engine.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="fake-model",
|
||||
):
|
||||
result.append(chunk)
|
||||
|
||||
assert len(result) == 3
|
||||
assert result[0].content == "Hello"
|
||||
assert result[1].content == " world"
|
||||
assert result[2].finish_reason == "stop"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GuardrailsEngine.stream_full delegation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrails_delegates_stream_full():
|
||||
"""GuardrailsEngine.stream_full delegates to wrapped engine."""
|
||||
expected = [
|
||||
StreamChunk(content="safe output"),
|
||||
StreamChunk(finish_reason="stop"),
|
||||
]
|
||||
inner = _FakeStreamFullEngine(expected)
|
||||
engine = GuardrailsEngine(inner, scanners=[])
|
||||
|
||||
result = []
|
||||
async for chunk in engine.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="fake-model",
|
||||
):
|
||||
result.append(chunk)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0].content == "safe output"
|
||||
assert result[1].finish_reason == "stop"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MultiEngine.stream_full routing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_routes_stream_full_by_model():
|
||||
"""MultiEngine routes stream_full to the correct engine by model name."""
|
||||
chunks_a = [StreamChunk(content="from A"), StreamChunk(finish_reason="stop")]
|
||||
chunks_b = [StreamChunk(content="from B"), StreamChunk(finish_reason="stop")]
|
||||
|
||||
engine_a = _FakeStreamFullEngine(chunks_a)
|
||||
engine_a.list_models = lambda: ["model-a"]
|
||||
|
||||
engine_b = _FakeStreamFullEngine(chunks_b)
|
||||
engine_b.list_models = lambda: ["model-b"]
|
||||
|
||||
multi = MultiEngine([("a", engine_a), ("b", engine_b)])
|
||||
|
||||
# Route to engine A
|
||||
result_a = []
|
||||
async for chunk in multi.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="model-a",
|
||||
):
|
||||
result_a.append(chunk)
|
||||
|
||||
assert result_a[0].content == "from A"
|
||||
|
||||
# Route to engine B
|
||||
result_b = []
|
||||
async for chunk in multi.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="model-b",
|
||||
):
|
||||
result_b.append(chunk)
|
||||
|
||||
assert result_b[0].content == "from B"
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Tests for StreamChunk dataclass and stream_full() engine method."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, Dict, List
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.core.types import Message, Role
|
||||
from openjarvis.engine._stubs import InferenceEngine, StreamChunk
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# StreamChunk dataclass tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStreamChunk:
|
||||
def test_defaults(self):
|
||||
chunk = StreamChunk()
|
||||
assert chunk.content is None
|
||||
assert chunk.tool_calls is None
|
||||
assert chunk.finish_reason is None
|
||||
assert chunk.usage is None
|
||||
|
||||
def test_content_only(self):
|
||||
chunk = StreamChunk(content="hello")
|
||||
assert chunk.content == "hello"
|
||||
assert chunk.finish_reason is None
|
||||
|
||||
def test_finish_reason(self):
|
||||
chunk = StreamChunk(finish_reason="stop")
|
||||
assert chunk.content is None
|
||||
assert chunk.finish_reason == "stop"
|
||||
|
||||
def test_tool_calls(self):
|
||||
tc = [{"index": 0, "function": {"name": "calc", "arguments": "{}"}}]
|
||||
chunk = StreamChunk(tool_calls=tc)
|
||||
assert chunk.tool_calls == tc
|
||||
|
||||
def test_usage(self):
|
||||
usage = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
chunk = StreamChunk(usage=usage)
|
||||
assert chunk.usage == usage
|
||||
|
||||
def test_all_fields(self):
|
||||
chunk = StreamChunk(
|
||||
content="hi",
|
||||
tool_calls=[{"index": 0}],
|
||||
finish_reason="tool_calls",
|
||||
usage={"total_tokens": 1},
|
||||
)
|
||||
assert chunk.content == "hi"
|
||||
assert chunk.tool_calls is not None
|
||||
assert chunk.finish_reason == "tool_calls"
|
||||
assert chunk.usage is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Concrete engine stub for testing default stream_full()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeEngine(InferenceEngine):
|
||||
"""Minimal engine that yields predefined tokens via stream()."""
|
||||
|
||||
engine_id = "fake"
|
||||
|
||||
def __init__(self, tokens: list[str]) -> None:
|
||||
self._tokens = tokens
|
||||
|
||||
def generate(self, messages, *, model, **kwargs) -> Dict[str, Any]:
|
||||
return {"content": "".join(self._tokens), "usage": {}}
|
||||
|
||||
async def stream(self, messages, *, model, **kwargs) -> AsyncIterator[str]:
|
||||
for t in self._tokens:
|
||||
yield t
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
return ["fake-model"]
|
||||
|
||||
def health(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class TestDefaultStreamFull:
|
||||
"""Test the default stream_full() implementation that wraps stream()."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wraps_stream_tokens(self):
|
||||
engine = _FakeEngine(["Hello", " world", "!"])
|
||||
chunks = []
|
||||
async for chunk in engine.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="fake-model",
|
||||
):
|
||||
chunks.append(chunk)
|
||||
|
||||
# Should have 3 content chunks + 1 finish chunk
|
||||
assert len(chunks) == 4
|
||||
assert chunks[0].content == "Hello"
|
||||
assert chunks[1].content == " world"
|
||||
assert chunks[2].content == "!"
|
||||
assert chunks[3].finish_reason == "stop"
|
||||
assert chunks[3].content is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_stream(self):
|
||||
engine = _FakeEngine([])
|
||||
chunks = []
|
||||
async for chunk in engine.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="fake-model",
|
||||
):
|
||||
chunks.append(chunk)
|
||||
|
||||
# Should have just the finish chunk
|
||||
assert len(chunks) == 1
|
||||
assert chunks[0].finish_reason == "stop"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_kwargs_passed_through(self):
|
||||
"""Verify that temperature/max_tokens reach stream()."""
|
||||
engine = _FakeEngine(["ok"])
|
||||
chunks = []
|
||||
async for chunk in engine.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="fake-model",
|
||||
temperature=0.1,
|
||||
max_tokens=50,
|
||||
):
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) == 2
|
||||
assert chunks[0].content == "ok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenAI-compatible stream_full() with mock HTTP response
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOpenAICompatStreamFull:
|
||||
"""Test _OpenAICompatibleEngine.stream_full() with mocked HTTP."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parses_sse_with_content_and_finish(self):
|
||||
from openjarvis.engine._openai_compat import _OpenAICompatibleEngine
|
||||
|
||||
# Build mock SSE lines
|
||||
sse_lines = []
|
||||
for token in ["Hello", " world"]:
|
||||
chunk = {
|
||||
"choices": [{"delta": {"content": token}, "finish_reason": None}],
|
||||
}
|
||||
sse_lines.append(f"data: {json.dumps(chunk)}")
|
||||
# Final chunk with finish_reason
|
||||
final = {
|
||||
"choices": [{"delta": {}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
|
||||
}
|
||||
sse_lines.append(f"data: {json.dumps(final)}")
|
||||
sse_lines.append("data: [DONE]")
|
||||
|
||||
# Mock the httpx client stream context manager
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.iter_lines.return_value = iter(sse_lines)
|
||||
|
||||
engine = _OpenAICompatibleEngine.__new__(_OpenAICompatibleEngine)
|
||||
engine.engine_id = "test"
|
||||
engine._host = "http://localhost:8000"
|
||||
engine._api_prefix = "/v1"
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_stream_ctx = MagicMock()
|
||||
mock_stream_ctx.__enter__ = MagicMock(return_value=mock_resp)
|
||||
mock_stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||
mock_client.stream.return_value = mock_stream_ctx
|
||||
engine._client = mock_client
|
||||
|
||||
chunks = []
|
||||
async for chunk in engine.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="test-model",
|
||||
):
|
||||
chunks.append(chunk)
|
||||
|
||||
# Should have: "Hello", " world", finish+usage
|
||||
assert len(chunks) == 3
|
||||
assert chunks[0].content == "Hello"
|
||||
assert chunks[1].content == " world"
|
||||
assert chunks[2].finish_reason == "stop"
|
||||
assert chunks[2].usage is not None
|
||||
assert chunks[2].usage["total_tokens"] == 7
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parses_tool_call_fragments(self):
|
||||
from openjarvis.engine._openai_compat import _OpenAICompatibleEngine
|
||||
|
||||
# Simulate streamed tool_call fragments
|
||||
_tc1 = (
|
||||
'{"choices": [{"delta": {"tool_calls": [{"index": 0, "id": "call_1",'
|
||||
' "function": {"name": "calc", "arguments": ""}}]},'
|
||||
' "finish_reason": null}]}'
|
||||
)
|
||||
_tc2 = (
|
||||
'{"choices": [{"delta": {"tool_calls": [{"index": 0,'
|
||||
' "function": {"name": "", "arguments": "{\\"x\\": 1}"}}]},'
|
||||
' "finish_reason": null}]}'
|
||||
)
|
||||
sse_lines = [
|
||||
f"data: {_tc1}",
|
||||
f"data: {_tc2}",
|
||||
'data: {"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}',
|
||||
"data: [DONE]",
|
||||
]
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.iter_lines.return_value = iter(sse_lines)
|
||||
|
||||
engine = _OpenAICompatibleEngine.__new__(_OpenAICompatibleEngine)
|
||||
engine.engine_id = "test"
|
||||
engine._host = "http://localhost:8000"
|
||||
engine._api_prefix = "/v1"
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_stream_ctx = MagicMock()
|
||||
mock_stream_ctx.__enter__ = MagicMock(return_value=mock_resp)
|
||||
mock_stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||
mock_client.stream.return_value = mock_stream_ctx
|
||||
engine._client = mock_client
|
||||
|
||||
chunks = []
|
||||
async for chunk in engine.stream_full(
|
||||
[Message(role=Role.USER, content="test")],
|
||||
model="test-model",
|
||||
):
|
||||
chunks.append(chunk)
|
||||
|
||||
# First chunk has tool_calls with name
|
||||
assert chunks[0].tool_calls is not None
|
||||
assert chunks[0].tool_calls[0]["function"]["name"] == "calc"
|
||||
# Second chunk has arguments fragment
|
||||
assert chunks[1].tool_calls is not None
|
||||
# Third chunk has finish_reason="tool_calls"
|
||||
assert chunks[2].finish_reason == "tool_calls"
|
||||
@@ -313,7 +313,7 @@ class TestConfigBenchmarks:
|
||||
def test_benchmarks_count(self) -> None:
|
||||
from openjarvis.evals.core.config import KNOWN_BENCHMARKS
|
||||
|
||||
assert len(KNOWN_BENCHMARKS) == 25
|
||||
assert len(KNOWN_BENCHMARKS) == 26
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -74,7 +74,7 @@ def grade(transcript, workspace_path):
|
||||
{
|
||||
"type": "toolCall",
|
||||
"name": "read_file",
|
||||
"arguments": {"path": "a.txt"},
|
||||
"params": {"path": "a.txt"},
|
||||
}
|
||||
],
|
||||
},
|
||||
@@ -124,7 +124,7 @@ class TestSummarizeTranscript:
|
||||
{
|
||||
"type": "toolCall",
|
||||
"name": "read_file",
|
||||
"arguments": {"path": "a.txt"},
|
||||
"params": {"path": "a.txt"},
|
||||
}
|
||||
],
|
||||
},
|
||||
|
||||
@@ -186,26 +186,26 @@ class TestCloudModelSpecs:
|
||||
class TestQwen35ModelSpecs:
|
||||
"""Verify Qwen3.5 MoE model entries."""
|
||||
|
||||
def test_qwen35_3b(self) -> None:
|
||||
spec = _get_spec("qwen3.5:3b")
|
||||
assert spec.parameter_count_b == 3.0
|
||||
assert spec.active_parameter_count_b == 0.6
|
||||
def test_qwen35_2b(self) -> None:
|
||||
spec = _get_spec("qwen3.5:2b")
|
||||
assert spec.parameter_count_b == 2.0
|
||||
assert spec.active_parameter_count_b == 0.4
|
||||
assert spec.context_length == 131072
|
||||
assert spec.provider == "alibaba"
|
||||
assert spec.metadata["architecture"] == "moe"
|
||||
for e in ("ollama", "vllm", "llamacpp", "sglang"):
|
||||
assert e in spec.supported_engines
|
||||
|
||||
def test_qwen35_8b(self) -> None:
|
||||
spec = _get_spec("qwen3.5:8b")
|
||||
assert spec.parameter_count_b == 8.0
|
||||
assert spec.active_parameter_count_b == 1.0
|
||||
def test_qwen35_9b(self) -> None:
|
||||
spec = _get_spec("qwen3.5:9b")
|
||||
assert spec.parameter_count_b == 9.0
|
||||
assert spec.active_parameter_count_b == 1.5
|
||||
assert spec.context_length == 131072
|
||||
|
||||
def test_qwen35_14b(self) -> None:
|
||||
spec = _get_spec("qwen3.5:14b")
|
||||
assert spec.parameter_count_b == 14.0
|
||||
assert spec.active_parameter_count_b == 2.0
|
||||
def test_qwen35_27b(self) -> None:
|
||||
spec = _get_spec("qwen3.5:27b")
|
||||
assert spec.parameter_count_b == 27.0
|
||||
assert spec.active_parameter_count_b == 3.0
|
||||
|
||||
def test_qwen35_35b(self) -> None:
|
||||
spec = _get_spec("qwen3.5:35b")
|
||||
@@ -300,9 +300,9 @@ class TestModelDiscovery:
|
||||
"gpt-oss:120b",
|
||||
"glm-4.7-flash",
|
||||
"trinity-mini",
|
||||
"qwen3.5:3b",
|
||||
"qwen3.5:8b",
|
||||
"qwen3.5:14b",
|
||||
"qwen3.5:2b",
|
||||
"qwen3.5:9b",
|
||||
"qwen3.5:27b",
|
||||
"qwen3.5:35b",
|
||||
"qwen3.5:122b",
|
||||
"qwen3.5:397b",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user