mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 10:52:15 +00:00
merge: resolve conflicts with main (keep both benchmark sets)
Merge origin/main into feat/savings-meter-benchmarks-comparison. Both sides added new benchmarks to cli.py — keep all: - Our branch: use-case benchmarks (email_triage, morning_brief, etc.) - Main: agentic benchmarks (loghub, ama-bench, workarena, etc.) dataset.py: keep both create_task_env/verify_requirements and iter_episodes. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jon Saad-Falcon
@@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
|
||||
## Project Status
|
||||
|
||||
OpenJarvis is a research framework for studying on-device AI systems. Phase 24 complete. Five composable pillars: Intelligence, Engine, Agents, Tools (with storage + MCP), and Learning — with trace-driven learning as a cross-cutting concern. Speech subsystem (STT) with pluggable backends. ~3295 tests pass (~44 skipped for optional deps). Python SDK (`Jarvis` class), composition layer (`SystemBuilder`/`JarvisSystem`), eval framework (15 real benchmarks), composable recipes, agent templates, bundled skills, operator recipes, trace-driven learning pipeline, Docker deployment, Tauri desktop app, 40+ tools, 20+ CLI commands, 40+ API endpoints all ready.
|
||||
OpenJarvis is a research framework for studying on-device AI systems. Phase 25 complete. Five composable pillars: Intelligence, Engine, Agents, Tools (with storage + MCP), and Learning — with trace-driven learning as a cross-cutting concern. Speech subsystem (STT) with pluggable backends. ~3405 tests pass (~44 skipped for optional deps). Python SDK (`Jarvis` class), composition layer (`SystemBuilder`/`JarvisSystem`), eval framework (20 real benchmarks), composable recipes, agent templates, bundled skills, operator recipes, trace-driven learning pipeline, Docker deployment, Tauri desktop app, 41+ tools, 20+ CLI commands, 40+ API endpoints all ready.
|
||||
|
||||
## Build & Development Commands
|
||||
|
||||
@@ -116,7 +116,7 @@ OpenJarvis is a research framework for on-device AI organized around **five comp
|
||||
|
||||
1. **Intelligence** (`src/openjarvis/intelligence/`) — Model definition, catalog, and generation defaults. `ModelRegistry` maps model keys to `ModelSpec`. `IntelligenceConfig` holds model identity (default/fallback model, model_path, checkpoint_path, quantization, preferred_engine, provider) and generation defaults (temperature, max_tokens, top_p, top_k, repetition_penalty, stop_sequences). Model catalog maintains `BUILTIN_MODELS` with auto-discovery via `merge_discovered_models()`.
|
||||
2. **Engine** (`src/openjarvis/engine/`) — The inference runtime. Backends: vLLM, SGLang, Ollama, llama.cpp, MLX, LM Studio. All implement `InferenceEngine` ABC with `generate()`, `stream()`, `list_models()`, `health()`. Engines extract and pass through `tool_calls` in OpenAI format.
|
||||
3. **Agents** (`src/openjarvis/agents/`) — Pluggable logic for queries, tool/API calls, memory. Hierarchy: `BaseAgent` ABC (helpers: `_emit_turn_start/end`, `_build_messages`, `_generate`, `_max_turns_result`, `_strip_think_tags`, `_check_continuation`) → `ToolUsingAgent` (adds `tools`, `ToolExecutor`, `max_turns`). Agents: `SimpleAgent` (single-turn), `OrchestratorAgent` (multi-turn tool loop), `NativeReActAgent` (Thought-Action-Observation, key `"native_react"`, alias `"react"`), `NativeOpenHandsAgent` (CodeAct, key `"native_openhands"`), `RLMAgent` (recursive LM), `OpenHandsAgent` (real `openhands-sdk`, key `"openhands"`, requires Python 3.12+), `OpenClawAgent` (HTTP/subprocess transport), `ClaudeCodeAgent` (Claude Agent SDK via Node.js, key `"claude_code"`), `SandboxedAgent` (Docker wrapper, key `"sandboxed"`). `accepts_tools` class attribute for CLI/SDK auto-detection. Agents call `engine.generate()` directly — telemetry handled by `InstrumentedEngine` wrapper.
|
||||
3. **Agents** (`src/openjarvis/agents/`) — Pluggable logic for queries, tool/API calls, memory. Hierarchy: `BaseAgent` ABC (helpers: `_emit_turn_start/end`, `_build_messages`, `_generate`, `_max_turns_result`, `_strip_think_tags`, `_check_continuation`) → `ToolUsingAgent` (adds `tools`, `ToolExecutor`, `max_turns`). Agents: `SimpleAgent` (single-turn), `OrchestratorAgent` (multi-turn tool loop), `NativeReActAgent` (Thought-Action-Observation, key `"native_react"`, alias `"react"`), `NativeOpenHandsAgent` (CodeAct, key `"native_openhands"`), `RLMAgent` (recursive LM), `OpenHandsAgent` (real `openhands-sdk`, key `"openhands"`, requires Python 3.12+), `OpenClawAgent` (HTTP/subprocess transport), `ClaudeCodeAgent` (Claude Agent SDK via Node.js, key `"claude_code"`), `SandboxedAgent` (Docker wrapper, key `"sandboxed"`), `MonitorOperativeAgent` (long-horizon monitoring with 4 configurable strategies: memory extraction, observation compression, retrieval, task decomposition, key `"monitor_operative"`). `accepts_tools` class attribute for CLI/SDK auto-detection. Agents call `engine.generate()` directly — telemetry handled by `InstrumentedEngine` wrapper.
|
||||
4. **Tools** (`src/openjarvis/tools/`) — All tools managed via MCP (Model Context Protocol).
|
||||
- **API tools**: `CalculatorTool`, `ThinkTool`, `FileReadTool`, `FileWriteTool`, `WebSearchTool`, `CodeInterpreterTool`, `LLMTool`, `ShellExecTool`, `ApplyPatchTool`, `HttpRequestTool`, `DatabaseQueryTool`, `PDFExtractTool`, `ImageGenerateTool`, `AudioTranscribeTool` — all implement `BaseTool` ABC
|
||||
- **Git tools** (`git_tool.py`): `GitStatusTool`, `GitDiffTool`, `GitCommitTool`, `GitLogTool`
|
||||
@@ -148,8 +148,8 @@ OpenJarvis is a research framework for on-device AI organized around **five comp
|
||||
- **Composition Layer** (`system.py`) — `SystemBuilder` fluent builder → `JarvisSystem` with `ask()`, `close()`. Wires engine, model, agent, tools, telemetry, traces, workflow, sessions, capability policy.
|
||||
- **SDK** (`sdk.py`) — `Jarvis` class: high-level sync API with `ask()`/`ask_full()`, `MemoryHandle`, lazy init, telemetry. Also exports `JarvisSystem`/`SystemBuilder`.
|
||||
- **Benchmarks** (`bench/`) — `LatencyBenchmark`, `ThroughputBenchmark`, `EnergyBenchmark`. All registered via `BenchmarkRegistry`. CLI: `jarvis bench run`.
|
||||
- **Eval Framework** (`src/openjarvis/evals/`) — 15 real benchmark datasets from IPW: SuperGPQA, GPQA, MMLU-Pro, MATH-500, Natural Reasoning, HLE, SimpleQA, WildChat, IPW, GAIA, FRAMES, SWE-bench, SWEfficiency, TerminalBench, TerminalBench Native. Scorer types: MCQ letter extraction, LLM-judge, exact match, structural validation. `EvalRunner` with parallel execution. CLI: `jarvis eval list|run|compare|report`.
|
||||
- **Recipes** (`src/openjarvis/recipes/`) — Composable TOML configs that wire all 5 pillars. `Recipe` dataclass with `to_builder_kwargs()`. `load_recipe()`, `discover_recipes()`, `resolve_recipe()`. 3 built-in recipes in `data/`: coding_assistant, research_assistant, general_assistant. Operator recipes in `data/operators/`: researcher (4h cycle), correspondent (5min interval), sentinel (2h cycle).
|
||||
- **Eval Framework** (`src/openjarvis/evals/`) — 20 real benchmark datasets: SuperGPQA, GPQA, MMLU-Pro, MATH-500, Natural Reasoning, HLE, SimpleQA, WildChat, IPW, GAIA, FRAMES, SWE-bench, SWEfficiency, TerminalBench, TerminalBench Native, LogHub, AMA-Bench, LifelongAgentBench, WebChoreArena, WorkArena++. Scorer types: MCQ letter extraction, LLM-judge, exact match, structural validation. `EvalRunner` with parallel execution and episode mode (sequential processing with shared agent state). `EnvironmentProvider` ABC for Docker/ServiceNow environments. CLI: `jarvis eval list|run|compare|report`.
|
||||
- **Recipes** (`src/openjarvis/recipes/`) — Composable TOML configs that wire all 5 pillars. `Recipe` dataclass with `to_builder_kwargs()`. `load_recipe()`, `discover_recipes()`, `resolve_recipe()`. 3 built-in recipes in `data/`: coding_assistant, research_assistant, general_assistant. Operator recipes in `data/operators/`: researcher (4h cycle), correspondent (5min interval), sentinel (2h cycle), monitor (2h cycle, causality graph + hybrid retrieval).
|
||||
- **Agent Templates** (`src/openjarvis/templates/`) — Pre-configured TOML manifests with system prompts, tool sets, behavioral parameters. `AgentTemplate` dataclass, `load_template()`, `discover_templates()`. 15 built-in templates in `data/` (code-reviewer, debugger, architect, deep-researcher, fact-checker, summarizer, etc.).
|
||||
- **Bundled Skills** (`src/openjarvis/skills/data/`) — 20 ready-to-use TOML skill manifests. Categories: file management (organizer, deduplicator, backup), research (web-summarize, topic-research, knowledge-extract), code quality (lint, test-gen, security-scan, dependency-audit), productivity (email-draft, meeting-notes, daily-digest), document processing (compare, translate, data-analyze).
|
||||
- **OpenClaw** (`agents/openclaw*.py`) — `OpenClawAgent` with `HttpTransport`/`SubprocessTransport`, JSON-line protocol, `ProviderPlugin`, `MemorySearchManager`.
|
||||
@@ -244,3 +244,4 @@ OpenAI-compatible server via `jarvis serve`:
|
||||
| v2.7 | 22 | Operators: persistent, scheduled autonomous agents with recipe + schedule + channel output |
|
||||
| v2.8 | 23 | Differentiated functionalities: trace-driven learning pipeline (TrainingDataMiner, LoRATrainer, AgentConfigEvolver, LearningOrchestrator), 15 real IPW benchmarks, composable recipes, 15 agent templates, 20 bundled skills, 3 operator recipes |
|
||||
| v2.9 | 24 | Speech subsystem: SpeechBackend ABC, SpeechRegistry, 3 backends (FasterWhisper local, OpenAI cloud, Deepgram cloud), auto-discovery, API endpoints, frontend MicButton + useSpeech hook, Tauri commands, SystemBuilder wiring |
|
||||
| v3.0 | 25 | Operator benchmarks: LogHub, AMA-Bench, LifelongAgentBench, WebChoreArena, WorkArena++. MonitorOperativeAgent with 4 configurable strategies (memory extraction, observation compression, retrieval, task decomposition). EvalRunner episode mode. EnvironmentProvider ABC. browser_axtree tool |
|
||||
|
||||
@@ -54,6 +54,32 @@ host = "http://localhost:30000"
|
||||
[engine.llamacpp]
|
||||
host = "http://localhost:8080"
|
||||
|
||||
[engine.exo]
|
||||
host = "http://localhost:52415"
|
||||
|
||||
[engine.nexa]
|
||||
host = "http://localhost:18181"
|
||||
# device = ""
|
||||
|
||||
[engine.uzu]
|
||||
host = "http://localhost:8080"
|
||||
|
||||
[engine.apple_fm]
|
||||
host = "http://localhost:8079"
|
||||
|
||||
[engine.exo]
|
||||
host = "http://localhost:52415"
|
||||
|
||||
[engine.nexa]
|
||||
host = "http://localhost:18181"
|
||||
# device = "npu" # optional: cpu, gpu, npu
|
||||
|
||||
[engine.uzu]
|
||||
host = "http://localhost:8080"
|
||||
|
||||
[engine.apple_fm]
|
||||
host = "http://localhost:8079"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# PILLAR 5: Learning — Improvement Methodologies
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
+41
-12
@@ -11,31 +11,60 @@ OpenJarvis runs entirely on your hardware. Choose the interface that fits your w
|
||||
|
||||
## Desktop App
|
||||
|
||||
The native desktop app bundles Ollama (the inference engine) and the OpenJarvis Python backend
|
||||
into a single installer. Download, open, and start chatting — no terminal required.
|
||||
The desktop app is a native window for the OpenJarvis chat UI. All inference and backend
|
||||
processing happens on your local machine — the app connects to the backend you start locally.
|
||||
|
||||
!!! info "Backend required"
|
||||
Start the backend before opening the desktop app. The quickstart script handles everything:
|
||||
```bash
|
||||
git clone https://github.com/HazyResearch/OpenJarvis.git && cd OpenJarvis
|
||||
./scripts/quickstart.sh
|
||||
```
|
||||
|
||||
### Download
|
||||
|
||||
| Platform | Download | Notes |
|
||||
|----------|----------|-------|
|
||||
| macOS (Apple Silicon) | [:material-download: **OpenJarvis.dmg**](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_aarch64.dmg) | M1/M2/M3/M4 Macs |
|
||||
| macOS (Intel) | [:material-download: **OpenJarvis.dmg**](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_x64.dmg) | Intel Macs (2020 and earlier) |
|
||||
| Windows (64-bit) | [:material-download: **OpenJarvis-setup.exe**](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_x64-setup.exe) | Windows 10+ |
|
||||
| Linux (DEB) | [:material-download: **OpenJarvis.deb**](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_amd64.deb) | Ubuntu, Debian |
|
||||
| Linux (RPM) | [:material-download: **OpenJarvis.rpm**](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_amd64.rpm) | Fedora, RHEL |
|
||||
| macOS (Apple Silicon) | [:material-download: **OpenJarvis.dmg**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_aarch64.dmg) | M1/M2/M3/M4 Macs |
|
||||
| Windows (64-bit) | [:material-download: **OpenJarvis-setup.exe**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_x64-setup.exe) | Windows 10+ |
|
||||
| Linux (DEB) | [:material-download: **OpenJarvis.deb**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_amd64.deb) | Ubuntu, Debian |
|
||||
| Linux (RPM) | [:material-download: **OpenJarvis.rpm**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis-1.0.0-1.x86_64.rpm) | Fedora, RHEL |
|
||||
| Linux (AppImage) | [:material-download: **OpenJarvis.AppImage**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_amd64.AppImage) | Any distro |
|
||||
|
||||
!!! tip "All releases"
|
||||
Browse all versions on the [GitHub Releases](https://github.com/HazyResearch/OpenJarvis/releases) page.
|
||||
|
||||
### macOS: "app is damaged" fix
|
||||
|
||||
macOS Gatekeeper quarantines apps downloaded from the internet that aren't notarized
|
||||
by Apple. If you see **"OpenJarvis is damaged and can't be opened"**, run this in
|
||||
Terminal to clear the quarantine flag:
|
||||
|
||||
```bash
|
||||
xattr -cr /Applications/OpenJarvis.app
|
||||
```
|
||||
|
||||
Then open the app normally. If you installed from the DMG but haven't moved it to
|
||||
`/Applications` yet, point the command at wherever the `.app` bundle is:
|
||||
|
||||
```bash
|
||||
xattr -cr ~/Downloads/OpenJarvis.app
|
||||
```
|
||||
|
||||
!!! note
|
||||
This is standard for open-source macOS apps distributed outside the App Store.
|
||||
The command removes the quarantine extended attribute — it does not modify the app.
|
||||
|
||||
### What's included
|
||||
|
||||
The desktop app ships with:
|
||||
The desktop app provides:
|
||||
|
||||
- **Ollama** sidecar — inference engine runs automatically in the background
|
||||
- **OpenJarvis backend** — Python API server managed by the app
|
||||
- **Full chat UI** — same interface as the browser app
|
||||
- **Full chat UI** — same interface as the browser app, in a native window
|
||||
- **Energy monitoring** — real-time power consumption tracking
|
||||
- **Telemetry dashboard** — token throughput, latency, and cost comparison
|
||||
- **Telemetry dashboard** — token throughput, latency, and cost comparison vs. cloud models
|
||||
- **System tray** — quick access without keeping a terminal open
|
||||
|
||||
The backend (Ollama, Python API server, inference) runs separately on your machine.
|
||||
|
||||
### Build from source
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Generate the code reference pages."""
|
||||
from pathlib import Path
|
||||
|
||||
import mkdocs_gen_files
|
||||
|
||||
nav = mkdocs_gen_files.Nav()
|
||||
src = Path("src")
|
||||
|
||||
for path in sorted(src.rglob("*.py")):
|
||||
module_path = path.relative_to(src).with_suffix("")
|
||||
doc_path = path.relative_to(src).with_suffix(".md")
|
||||
full_doc_path = Path("api-reference", doc_path)
|
||||
|
||||
parts = tuple(module_path.parts)
|
||||
if parts[-1] == "__init__":
|
||||
parts = parts[:-1]
|
||||
doc_path = doc_path.with_name("index.md")
|
||||
full_doc_path = full_doc_path.with_name("index.md")
|
||||
elif parts[-1].startswith("_"):
|
||||
continue
|
||||
|
||||
nav[parts] = doc_path.as_posix()
|
||||
|
||||
with mkdocs_gen_files.open(full_doc_path, "w") as fd:
|
||||
identifier = ".".join(parts)
|
||||
fd.write(f"::: {identifier}")
|
||||
|
||||
mkdocs_gen_files.set_edit_path(full_doc_path, path)
|
||||
|
||||
with mkdocs_gen_files.open("api-reference/SUMMARY.md", "w") as nav_file:
|
||||
nav_file.writelines(nav.build_literate_nav())
|
||||
@@ -1,15 +1,20 @@
|
||||
---
|
||||
title: Installation
|
||||
description: Install OpenJarvis and set up an inference backend
|
||||
description: Get OpenJarvis running — browser app, desktop app, CLI, or Python SDK
|
||||
---
|
||||
|
||||
# Installation
|
||||
|
||||
This guide covers installing OpenJarvis, its optional extras, and setting up an inference backend.
|
||||
OpenJarvis runs entirely on your hardware. Choose the interface that fits your workflow.
|
||||
|
||||
## Quickstart (Recommended)
|
||||
---
|
||||
|
||||
The fastest way to get everything running — browser UI, backend, and inference engine — with a single command:
|
||||
## Browser App
|
||||
|
||||
Run the full chat UI in your browser. Everything stays local — the backend runs on
|
||||
your machine and the frontend connects via `localhost`.
|
||||
|
||||
### One-command setup
|
||||
|
||||
```bash
|
||||
git clone https://github.com/HazyResearch/OpenJarvis.git
|
||||
@@ -17,32 +22,108 @@ cd OpenJarvis
|
||||
./scripts/quickstart.sh
|
||||
```
|
||||
|
||||
This script checks for Python 3.10+, Node.js, and Ollama (installing what's missing), pulls a starter model, installs all dependencies, starts the backend and frontend servers, and opens the chat UI in your browser.
|
||||
The script handles everything:
|
||||
|
||||
!!! tip "Desktop app"
|
||||
Prefer a native app? Download the [Desktop App](../downloads.md#desktop-app) instead — it bundles everything into a single installer.
|
||||
1. Checks for Python 3.10+ and Node.js 18+
|
||||
2. Installs Ollama if not present and pulls a starter model
|
||||
3. Installs Python and frontend dependencies
|
||||
4. Starts the backend API server and frontend dev server
|
||||
5. Opens `http://localhost:5173` in your browser
|
||||
|
||||
---
|
||||
### Manual setup
|
||||
|
||||
## Requirements
|
||||
If you prefer to run each step yourself:
|
||||
|
||||
| Requirement | Version | Notes |
|
||||
|-------------|---------|-------|
|
||||
| Python | 3.10+ | Required |
|
||||
| Inference backend | Any | At least one of Ollama, vLLM, llama.cpp, SGLang, or a cloud API |
|
||||
| Node.js | 18+ | Required for the browser UI; 22+ for OpenClaw agent |
|
||||
|
||||
## Installing OpenJarvis
|
||||
|
||||
=== "Quickstart script"
|
||||
=== "Step 1: Clone and install"
|
||||
|
||||
```bash
|
||||
git clone https://github.com/HazyResearch/OpenJarvis.git
|
||||
cd OpenJarvis
|
||||
./scripts/quickstart.sh
|
||||
uv sync --extra server
|
||||
cd frontend && npm install && cd ..
|
||||
```
|
||||
|
||||
Handles everything: deps, Ollama, model pull, backend, frontend, browser open.
|
||||
=== "Step 2: Start Ollama"
|
||||
|
||||
```bash
|
||||
# Install from https://ollama.com if not already installed
|
||||
ollama serve &
|
||||
ollama pull qwen3:0.6b
|
||||
```
|
||||
|
||||
=== "Step 3: Start backend"
|
||||
|
||||
```bash
|
||||
uv run jarvis serve --port 8000
|
||||
```
|
||||
|
||||
=== "Step 4: Start frontend"
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Then open [http://localhost:5173](http://localhost:5173).
|
||||
|
||||
---
|
||||
|
||||
## Desktop App
|
||||
|
||||
The desktop app is a native window for the OpenJarvis chat UI. All inference and backend
|
||||
processing happens on your local machine — the app connects to the backend you start locally.
|
||||
|
||||
### Setup
|
||||
|
||||
**Step 1.** Start the backend (same as Browser App):
|
||||
|
||||
```bash
|
||||
git clone https://github.com/HazyResearch/OpenJarvis.git
|
||||
cd OpenJarvis
|
||||
./scripts/quickstart.sh
|
||||
```
|
||||
|
||||
**Step 2.** Download and open the desktop app:
|
||||
|
||||
| Platform | Download |
|
||||
|----------|----------|
|
||||
| macOS (Apple Silicon) | [:material-download: **OpenJarvis.dmg**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_aarch64.dmg) |
|
||||
| Windows (64-bit) | [:material-download: **OpenJarvis-setup.exe**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_x64-setup.exe) |
|
||||
| Linux (DEB) | [:material-download: **OpenJarvis.deb**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_amd64.deb) |
|
||||
| Linux (RPM) | [:material-download: **OpenJarvis.rpm**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis-1.0.0-1.x86_64.rpm) |
|
||||
| Linux (AppImage) | [:material-download: **OpenJarvis.AppImage**](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_amd64.AppImage) |
|
||||
|
||||
The app connects to `http://localhost:8000` automatically.
|
||||
|
||||
!!! warning "macOS: \"app is damaged\""
|
||||
If macOS says the app is damaged, clear the Gatekeeper quarantine flag:
|
||||
```bash
|
||||
xattr -cr /Applications/OpenJarvis.app
|
||||
```
|
||||
This is normal for open-source apps distributed outside the App Store.
|
||||
|
||||
!!! tip "All releases"
|
||||
Browse all versions on the [GitHub Releases](https://github.com/HazyResearch/OpenJarvis/releases) page.
|
||||
|
||||
### Build from source
|
||||
|
||||
```bash
|
||||
git clone https://github.com/HazyResearch/OpenJarvis.git
|
||||
cd OpenJarvis/desktop
|
||||
npm install
|
||||
npm run tauri build
|
||||
```
|
||||
|
||||
The built installer will be in `desktop/src-tauri/target/release/bundle/`.
|
||||
|
||||
---
|
||||
|
||||
## CLI
|
||||
|
||||
The command-line interface is the fastest way to interact with OpenJarvis
|
||||
programmatically. Every feature is accessible from the terminal.
|
||||
|
||||
### Install
|
||||
|
||||
=== "uv (recommended)"
|
||||
|
||||
@@ -64,64 +145,134 @@ This script checks for Python 3.10+, Node.js, and Ollama (installing what's miss
|
||||
uv sync
|
||||
```
|
||||
|
||||
For development with all dev tools:
|
||||
### Verify
|
||||
|
||||
```bash
|
||||
uv sync --extra dev
|
||||
```
|
||||
```bash
|
||||
jarvis --version
|
||||
# jarvis, version 1.0.0
|
||||
```
|
||||
|
||||
### First commands
|
||||
|
||||
```bash
|
||||
jarvis ask "What is the capital of France?"
|
||||
|
||||
jarvis ask --agent orchestrator --tools calculator "What is 137 * 42?"
|
||||
|
||||
jarvis serve --port 8000
|
||||
|
||||
jarvis doctor
|
||||
|
||||
jarvis model list
|
||||
|
||||
jarvis chat
|
||||
```
|
||||
|
||||
!!! info "Inference backend required"
|
||||
The CLI requires a running inference backend (e.g., Ollama). See
|
||||
[Setting up an inference backend](#setting-up-an-inference-backend) below.
|
||||
|
||||
---
|
||||
|
||||
## Python SDK
|
||||
|
||||
For programmatic access, the `Jarvis` class provides a high-level sync API.
|
||||
|
||||
### Install
|
||||
|
||||
```bash
|
||||
pip install openjarvis
|
||||
```
|
||||
|
||||
### Quick example
|
||||
|
||||
```python
|
||||
from openjarvis import Jarvis
|
||||
|
||||
j = Jarvis()
|
||||
print(j.ask("Explain quicksort in two sentences."))
|
||||
j.close()
|
||||
```
|
||||
|
||||
### With agents and tools
|
||||
|
||||
```python
|
||||
result = j.ask_full(
|
||||
"What is the square root of 144?",
|
||||
agent="orchestrator",
|
||||
tools=["calculator", "think"],
|
||||
)
|
||||
print(result["content"]) # "12"
|
||||
print(result["tool_results"]) # tool invocations
|
||||
print(result["turns"]) # number of agent turns
|
||||
```
|
||||
|
||||
### Composition layer
|
||||
|
||||
For full control, use the `SystemBuilder`:
|
||||
|
||||
```python
|
||||
from openjarvis import SystemBuilder
|
||||
|
||||
system = (
|
||||
SystemBuilder()
|
||||
.engine("ollama")
|
||||
.model("qwen3:8b")
|
||||
.agent("orchestrator")
|
||||
.tools(["calculator", "web_search", "file_read"])
|
||||
.enable_telemetry()
|
||||
.enable_traces()
|
||||
.build()
|
||||
)
|
||||
|
||||
result = system.ask("Summarize the latest AI news.")
|
||||
system.close()
|
||||
```
|
||||
|
||||
See the [Python SDK guide](../user-guide/python-sdk.md) for the full API reference.
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
| Requirement | Version | Notes |
|
||||
|-------------|---------|-------|
|
||||
| Python | 3.10+ | Required |
|
||||
| Inference backend | Any | At least one of Ollama, vLLM, llama.cpp, SGLang, or a cloud API |
|
||||
| Node.js | 18+ | Required for the browser UI; 22+ for OpenClaw agent |
|
||||
|
||||
## Optional Extras
|
||||
|
||||
OpenJarvis uses optional extras to keep the base installation lightweight. Install only what you need.
|
||||
OpenJarvis uses optional extras to keep the base installation lightweight.
|
||||
|
||||
### Inference Backends
|
||||
|
||||
| Extra | Install Command | Dependencies | Description |
|
||||
|-------|----------------|--------------|-------------|
|
||||
| `inference-ollama` | `pip install 'openjarvis[inference-ollama]'` | None (HTTP-based) | Ollama backend. Communicates via HTTP API. |
|
||||
| `inference-vllm` | `pip install 'openjarvis[inference-vllm]'` | None (HTTP-based) | vLLM backend. Communicates via OpenAI-compatible API. |
|
||||
| `inference-llamacpp` | `pip install 'openjarvis[inference-llamacpp]'` | None (HTTP-based) | llama.cpp server backend. |
|
||||
| `inference-cloud` | `pip install 'openjarvis[inference-cloud]'` | `openai>=1.30`, `anthropic>=0.30` | Cloud inference via OpenAI and Anthropic APIs. |
|
||||
| `inference-google` | `pip install 'openjarvis[inference-google]'` | `google-genai>=1.0` | Google Gemini API backend. |
|
||||
| Extra | Install Command | Description |
|
||||
|-------|----------------|-------------|
|
||||
| `inference-cloud` | `pip install 'openjarvis[inference-cloud]'` | OpenAI and Anthropic APIs |
|
||||
| `inference-google` | `pip install 'openjarvis[inference-google]'` | Google Gemini API |
|
||||
|
||||
!!! note "Ollama, vLLM, and llama.cpp are HTTP-based"
|
||||
The `inference-ollama`, `inference-vllm`, and `inference-llamacpp` extras have no additional Python dependencies. OpenJarvis communicates with these engines over HTTP using the `httpx` library that is already a core dependency. You still need the actual engine software running on your machine or network.
|
||||
These engines have no additional Python dependencies — OpenJarvis communicates over HTTP. You still need the engine software running on your machine.
|
||||
|
||||
### Memory Backends
|
||||
|
||||
| Extra | Install Command | Dependencies | Description |
|
||||
|-------|----------------|--------------|-------------|
|
||||
| `memory-faiss` | `pip install 'openjarvis[memory-faiss]'` | `faiss-cpu>=1.7`, `sentence-transformers>=2.2`, `numpy>=1.24` | FAISS vector store with sentence-transformer embeddings. |
|
||||
| `memory-colbert` | `pip install 'openjarvis[memory-colbert]'` | `colbert-ai>=0.2`, `torch>=2.0` | ColBERTv2 late-interaction retrieval. |
|
||||
| `memory-bm25` | `pip install 'openjarvis[memory-bm25]'` | `rank-bm25>=0.2.2` | BM25 sparse retrieval backend. |
|
||||
| `memory-pdf` | `pip install 'openjarvis[memory-pdf]'` | `pdfplumber>=0.10` | PDF document ingestion support. |
|
||||
| Extra | Install Command | Description |
|
||||
|-------|----------------|-------------|
|
||||
| `memory-faiss` | `pip install 'openjarvis[memory-faiss]'` | FAISS vector store |
|
||||
| `memory-colbert` | `pip install 'openjarvis[memory-colbert]'` | ColBERTv2 late-interaction retrieval |
|
||||
| `memory-bm25` | `pip install 'openjarvis[memory-bm25]'` | BM25 sparse retrieval |
|
||||
|
||||
!!! tip "SQLite memory is always available"
|
||||
The default SQLite/FTS5 memory backend requires no additional dependencies. It is always available and suitable for most use cases.
|
||||
The default SQLite/FTS5 memory backend requires no additional dependencies.
|
||||
|
||||
### Tools
|
||||
### Server & Other
|
||||
|
||||
| Extra | Install Command | Dependencies | Description |
|
||||
|-------|----------------|--------------|-------------|
|
||||
| `tools-search` | `pip install 'openjarvis[tools-search]'` | `tavily-python>=0.3` | Web search tool via the Tavily API. |
|
||||
|
||||
### Server
|
||||
|
||||
| Extra | Install Command | Dependencies | Description |
|
||||
|-------|----------------|--------------|-------------|
|
||||
| `server` | `pip install 'openjarvis[server]'` | `fastapi>=0.110`, `uvicorn>=0.30`, `pydantic>=2.0` | OpenAI-compatible API server (`jarvis serve`). |
|
||||
|
||||
### Other Extras
|
||||
|
||||
| Extra | Install Command | Dependencies | Description |
|
||||
|-------|----------------|--------------|-------------|
|
||||
| `agents` | `pip install 'openjarvis[agents]'` | None | Agent infrastructure (included in base). |
|
||||
| `learning` | `pip install 'openjarvis[learning]'` | None | Learning/router policy system (included in base). |
|
||||
| `openclaw` | `pip install 'openjarvis[openclaw]'` | None | OpenClaw agent transport layer. Requires Node.js 22+ at runtime. |
|
||||
| `docs` | `pip install 'openjarvis[docs]'` | `mkdocs>=1.6`, `mkdocs-material>=9.5`, `mkdocstrings[python]>=0.25` | Documentation build tools. |
|
||||
| `dev` | `pip install 'openjarvis[dev]'` | `pytest>=8`, `pytest-asyncio>=0.24`, `pytest-cov>=5`, `respx>=0.22`, `ruff>=0.4` | Development and testing tools. |
|
||||
|
||||
### Installing Multiple Extras
|
||||
| Extra | Install Command | Description |
|
||||
|-------|----------------|-------------|
|
||||
| `server` | `pip install 'openjarvis[server]'` | OpenAI-compatible API server (`jarvis serve`) |
|
||||
| `dev` | `pip install 'openjarvis[dev]'` | Development and testing tools |
|
||||
| `docs` | `pip install 'openjarvis[docs]'` | Documentation build tools |
|
||||
|
||||
Combine extras with commas:
|
||||
|
||||
@@ -129,149 +280,52 @@ Combine extras with commas:
|
||||
pip install 'openjarvis[server,memory-faiss,inference-cloud]'
|
||||
```
|
||||
|
||||
Or with `uv`:
|
||||
|
||||
```bash
|
||||
uv pip install 'openjarvis[server,memory-faiss,inference-cloud]'
|
||||
```
|
||||
|
||||
## Verifying Installation
|
||||
|
||||
After installation, verify that the CLI is available:
|
||||
|
||||
```bash
|
||||
jarvis --version
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
jarvis, version 1.0.0
|
||||
```
|
||||
|
||||
View all available commands:
|
||||
|
||||
```bash
|
||||
jarvis --help
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
Usage: jarvis [OPTIONS] COMMAND [ARGS]...
|
||||
|
||||
OpenJarvis -- modular AI assistant backend
|
||||
|
||||
Options:
|
||||
--version Show the version and exit.
|
||||
--help Show this message and exit.
|
||||
|
||||
Commands:
|
||||
ask Ask Jarvis a question.
|
||||
bench Run inference benchmarks.
|
||||
init Detect hardware and generate ~/.openjarvis/config.toml.
|
||||
memory Manage the memory store.
|
||||
model Manage language models.
|
||||
serve Start the OpenAI-compatible API server.
|
||||
telemetry Query and manage inference telemetry data.
|
||||
```
|
||||
|
||||
## Setting Up an Inference Backend
|
||||
|
||||
OpenJarvis requires at least one inference backend to generate responses. Choose the backend that best matches your hardware.
|
||||
OpenJarvis requires at least one inference backend. Choose the one that matches your hardware.
|
||||
|
||||
### Ollama (Recommended for most users)
|
||||
### Ollama (Recommended)
|
||||
|
||||
Ollama is the easiest way to get started. It handles model downloading and serving automatically.
|
||||
The easiest way to get started. Handles model downloading and serving automatically.
|
||||
|
||||
1. Install Ollama from [ollama.com](https://ollama.com)
|
||||
2. Start the server:
|
||||
1. Install from [ollama.com](https://ollama.com)
|
||||
2. Start the server and pull a model:
|
||||
|
||||
```bash
|
||||
ollama serve
|
||||
ollama pull qwen3:0.6b
|
||||
```
|
||||
|
||||
3. Pull a model:
|
||||
|
||||
```bash
|
||||
ollama pull qwen3:8b
|
||||
```
|
||||
|
||||
Or pull directly via the Jarvis CLI:
|
||||
|
||||
```bash
|
||||
jarvis model pull qwen3:8b
|
||||
```
|
||||
|
||||
4. Verify the engine is detected:
|
||||
|
||||
```bash
|
||||
jarvis model list
|
||||
```
|
||||
3. Verify: `jarvis model list`
|
||||
|
||||
!!! tip "Best for: Apple Silicon Macs, consumer NVIDIA GPUs, CPU-only systems"
|
||||
|
||||
### vLLM (High-throughput serving)
|
||||
### vLLM
|
||||
|
||||
vLLM provides high-throughput serving optimized for datacenter GPUs.
|
||||
High-throughput serving optimized for datacenter GPUs.
|
||||
|
||||
1. Install vLLM following the [official guide](https://docs.vllm.ai)
|
||||
2. Start the server:
|
||||
1. Install following the [official guide](https://docs.vllm.ai)
|
||||
2. Start: `vllm serve Qwen/Qwen2.5-7B-Instruct`
|
||||
3. Auto-detected at `http://localhost:8000`
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-7B-Instruct
|
||||
```
|
||||
!!! tip "Best for: NVIDIA datacenter GPUs (A100, H100), AMD GPUs"
|
||||
|
||||
3. OpenJarvis will auto-detect it at `http://localhost:8000`
|
||||
### llama.cpp
|
||||
|
||||
!!! tip "Best for: NVIDIA datacenter GPUs (A100, H100, L40), AMD GPUs"
|
||||
Efficient CPU and GPU inference with GGUF quantized models.
|
||||
|
||||
### llama.cpp (Lightweight, CPU-friendly)
|
||||
1. Build from [github.com/ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp)
|
||||
2. Start: `llama-server -m /path/to/model.gguf --port 8080`
|
||||
3. Auto-detected at `http://localhost:8080`
|
||||
|
||||
llama.cpp provides efficient CPU and GPU inference with GGUF quantized models.
|
||||
|
||||
1. Build llama.cpp from [github.com/ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp)
|
||||
2. Start the server:
|
||||
|
||||
```bash
|
||||
llama-server -m /path/to/model.gguf --port 8080
|
||||
```
|
||||
|
||||
3. OpenJarvis will auto-detect it at `http://localhost:8080`
|
||||
|
||||
!!! tip "Best for: CPU-only machines, constrained environments, GGUF models"
|
||||
|
||||
### SGLang
|
||||
|
||||
SGLang provides structured generation and high-performance serving.
|
||||
|
||||
1. Install SGLang following the [official guide](https://github.com/sgl-project/sglang)
|
||||
2. Start the server:
|
||||
|
||||
```bash
|
||||
python -m sglang.launch_server --model Qwen/Qwen2.5-7B-Instruct --port 30000
|
||||
```
|
||||
|
||||
3. OpenJarvis will auto-detect it at `http://localhost:30000`
|
||||
|
||||
### Cloud APIs (OpenAI, Anthropic, Google)
|
||||
|
||||
For cloud-based inference, install the cloud extras and set your API keys:
|
||||
### Cloud APIs
|
||||
|
||||
```bash
|
||||
pip install 'openjarvis[inference-cloud,inference-google]'
|
||||
```
|
||||
|
||||
Set environment variables:
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
export ANTHROPIC_API_KEY="sk-ant-..."
|
||||
export GOOGLE_API_KEY="..."
|
||||
```
|
||||
|
||||
OpenJarvis will automatically detect available cloud providers.
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Quick Start](quickstart.md) — Run your first query
|
||||
|
||||
+18
-4
@@ -34,12 +34,26 @@ Everything runs on your hardware. Cloud APIs are optional.
|
||||
|
||||
=== "Desktop App"
|
||||
|
||||
Download the native desktop app — it bundles Ollama and the Python backend
|
||||
so everything works out of the box.
|
||||
The desktop app is a native window for the OpenJarvis UI.
|
||||
The backend (Ollama + inference) runs on your machine — start it first, then open the app.
|
||||
|
||||
[Download for macOS (Apple Silicon)](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_aarch64.dmg){ .md-button .md-button--primary }
|
||||
**Step 1.** Start the backend:
|
||||
|
||||
Also available for [macOS (Intel)](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_x64.dmg), [Windows](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_x64-setup.exe), [Linux (DEB)](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_amd64.deb), and [Linux (RPM)](https://github.com/HazyResearch/OpenJarvis/releases/latest/download/OpenJarvis_amd64.rpm). See the [Downloads](downloads.md) page for details.
|
||||
```bash
|
||||
git clone https://github.com/HazyResearch/OpenJarvis.git
|
||||
cd OpenJarvis
|
||||
./scripts/quickstart.sh
|
||||
```
|
||||
|
||||
**Step 2.** Download and open the desktop app:
|
||||
|
||||
[Download for macOS (Apple Silicon)](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_aarch64.dmg){ .md-button .md-button--primary }
|
||||
|
||||
Also available for [Windows](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_x64-setup.exe), [Linux (DEB)](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis_1.0.0_amd64.deb), and [Linux (RPM)](https://github.com/HazyResearch/OpenJarvis/releases/download/desktop-latest/OpenJarvis-1.0.0-1.x86_64.rpm). See the [Downloads](downloads.md) page for details.
|
||||
|
||||
The app connects to `http://localhost:8000` automatically.
|
||||
|
||||
!!! warning "macOS: run `xattr -cr /Applications/OpenJarvis.app` if the app shows as \"damaged\"."
|
||||
|
||||
=== "Python SDK"
|
||||
|
||||
|
||||
+6
-18
@@ -50,6 +50,11 @@ extra_css:
|
||||
|
||||
plugins:
|
||||
- search
|
||||
- gen-files:
|
||||
scripts:
|
||||
- docs/gen_ref_pages.py
|
||||
- literate-nav:
|
||||
nav_file: SUMMARY.md
|
||||
- mkdocstrings:
|
||||
default_handler: python
|
||||
handlers:
|
||||
@@ -124,7 +129,6 @@ extra:
|
||||
|
||||
nav:
|
||||
- Home: index.md
|
||||
- Downloads: downloads.md
|
||||
- Getting Started:
|
||||
- Installation: getting-started/installation.md
|
||||
- Quick Start: getting-started/quickstart.md
|
||||
@@ -152,23 +156,7 @@ nav:
|
||||
- Design Principles: architecture/design-principles.md
|
||||
- Security: architecture/security.md
|
||||
- Channels: architecture/channels.md
|
||||
- API Reference:
|
||||
- api/index.md
|
||||
- SDK (Jarvis): api/sdk.md
|
||||
- Core: api/core.md
|
||||
- Engine: api/engine.md
|
||||
- Agents: api/agents.md
|
||||
- Memory: api/memory.md
|
||||
- Tools: api/tools.md
|
||||
- Intelligence: api/intelligence.md
|
||||
- Learning: api/learning.md
|
||||
- Traces: api/traces.md
|
||||
- Telemetry: api/telemetry.md
|
||||
- Benchmarks: api/bench.md
|
||||
- Evals: api/evals.md
|
||||
- Server: api/server.md
|
||||
- Security: api/security.md
|
||||
- Channels: api/channels.md
|
||||
- API Reference: api-reference/
|
||||
- Deployment:
|
||||
- Docker: deployment/docker.md
|
||||
- systemd (Linux): deployment/systemd.md
|
||||
|
||||
@@ -85,6 +85,8 @@ docs = [
|
||||
"mkdocs>=1.6",
|
||||
"mkdocs-material>=9.5",
|
||||
"mkdocstrings[python]>=0.25",
|
||||
"mkdocs-gen-files>=0.5",
|
||||
"mkdocs-literate-nav>=0.6",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
|
||||
@@ -134,6 +134,75 @@ impl Default for LMStudioEngineConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ExoEngineConfig {
|
||||
#[serde(default = "default_exo_host")]
|
||||
pub host: String,
|
||||
}
|
||||
|
||||
fn default_exo_host() -> String {
|
||||
"http://localhost:52415".into()
|
||||
}
|
||||
|
||||
impl Default for ExoEngineConfig {
|
||||
fn default() -> Self {
|
||||
Self { host: default_exo_host() }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NexaEngineConfig {
|
||||
#[serde(default = "default_nexa_host")]
|
||||
pub host: String,
|
||||
#[serde(default)]
|
||||
pub device: String,
|
||||
}
|
||||
|
||||
fn default_nexa_host() -> String {
|
||||
"http://localhost:18181".into()
|
||||
}
|
||||
|
||||
impl Default for NexaEngineConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
host: default_nexa_host(),
|
||||
device: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct UzuEngineConfig {
|
||||
#[serde(default = "default_uzu_host")]
|
||||
pub host: String,
|
||||
}
|
||||
|
||||
fn default_uzu_host() -> String {
|
||||
"http://localhost:8080".into()
|
||||
}
|
||||
|
||||
impl Default for UzuEngineConfig {
|
||||
fn default() -> Self {
|
||||
Self { host: default_uzu_host() }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AppleFmEngineConfig {
|
||||
#[serde(default = "default_apple_fm_host")]
|
||||
pub host: String,
|
||||
}
|
||||
|
||||
fn default_apple_fm_host() -> String {
|
||||
"http://localhost:8079".into()
|
||||
}
|
||||
|
||||
impl Default for AppleFmEngineConfig {
|
||||
fn default() -> Self {
|
||||
Self { host: default_apple_fm_host() }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Engine config
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -154,6 +223,14 @@ pub struct EngineConfig {
|
||||
pub mlx: MLXEngineConfig,
|
||||
#[serde(default)]
|
||||
pub lmstudio: LMStudioEngineConfig,
|
||||
#[serde(default)]
|
||||
pub exo: ExoEngineConfig,
|
||||
#[serde(default)]
|
||||
pub nexa: NexaEngineConfig,
|
||||
#[serde(default)]
|
||||
pub uzu: UzuEngineConfig,
|
||||
#[serde(default)]
|
||||
pub apple_fm: AppleFmEngineConfig,
|
||||
}
|
||||
|
||||
fn default_engine_name() -> String {
|
||||
@@ -170,6 +247,10 @@ impl Default for EngineConfig {
|
||||
llamacpp: LlamaCppEngineConfig::default(),
|
||||
mlx: MLXEngineConfig::default(),
|
||||
lmstudio: LMStudioEngineConfig::default(),
|
||||
exo: ExoEngineConfig::default(),
|
||||
nexa: NexaEngineConfig::default(),
|
||||
uzu: UzuEngineConfig::default(),
|
||||
apple_fm: AppleFmEngineConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,10 @@ pub fn discover_engines(config: &JarvisConfig) -> Vec<EngineInfo> {
|
||||
("llamacpp", &config.engine.llamacpp.host),
|
||||
("mlx", &config.engine.mlx.host),
|
||||
("lmstudio", &config.engine.lmstudio.host),
|
||||
("exo", &config.engine.exo.host),
|
||||
("nexa", &config.engine.nexa.host),
|
||||
("uzu", &config.engine.uzu.host),
|
||||
("apple_fm", &config.engine.apple_fm.host),
|
||||
];
|
||||
|
||||
for (name, host) in compat_engines {
|
||||
@@ -95,6 +99,18 @@ pub fn get_engine_static(
|
||||
"lmstudio" => Ok(Engine::LmStudio(OpenAICompatEngine::lmstudio(
|
||||
&config.engine.lmstudio.host,
|
||||
))),
|
||||
"exo" => Ok(Engine::Exo(OpenAICompatEngine::exo(
|
||||
&config.engine.exo.host,
|
||||
))),
|
||||
"nexa" => Ok(Engine::Nexa(OpenAICompatEngine::nexa(
|
||||
&config.engine.nexa.host,
|
||||
))),
|
||||
"uzu" => Ok(Engine::Uzu(OpenAICompatEngine::uzu(
|
||||
&config.engine.uzu.host,
|
||||
))),
|
||||
"apple_fm" => Ok(Engine::AppleFm(OpenAICompatEngine::apple_fm(
|
||||
&config.engine.apple_fm.host,
|
||||
))),
|
||||
other => Err(OpenJarvisError::Engine(
|
||||
openjarvis_core::error::EngineError::ModelNotFound(format!(
|
||||
"Unknown engine: {}",
|
||||
|
||||
@@ -20,6 +20,10 @@ pub enum Engine {
|
||||
LlamaCpp(OpenAICompatEngine),
|
||||
Mlx(OpenAICompatEngine),
|
||||
LmStudio(OpenAICompatEngine),
|
||||
Exo(OpenAICompatEngine),
|
||||
Nexa(OpenAICompatEngine),
|
||||
Uzu(OpenAICompatEngine),
|
||||
AppleFm(OpenAICompatEngine),
|
||||
}
|
||||
|
||||
macro_rules! delegate_engine {
|
||||
@@ -31,6 +35,10 @@ macro_rules! delegate_engine {
|
||||
Engine::LlamaCpp(e) => e.$method($($arg),*),
|
||||
Engine::Mlx(e) => e.$method($($arg),*),
|
||||
Engine::LmStudio(e) => e.$method($($arg),*),
|
||||
Engine::Exo(e) => e.$method($($arg),*),
|
||||
Engine::Nexa(e) => e.$method($($arg),*),
|
||||
Engine::Uzu(e) => e.$method($($arg),*),
|
||||
Engine::AppleFm(e) => e.$method($($arg),*),
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -67,6 +75,10 @@ impl InferenceEngine for Engine {
|
||||
Engine::LlamaCpp(e) => e.stream(messages, model, temperature, max_tokens, extra).await,
|
||||
Engine::Mlx(e) => e.stream(messages, model, temperature, max_tokens, extra).await,
|
||||
Engine::LmStudio(e) => e.stream(messages, model, temperature, max_tokens, extra).await,
|
||||
Engine::Exo(e) => e.stream(messages, model, temperature, max_tokens, extra).await,
|
||||
Engine::Nexa(e) => e.stream(messages, model, temperature, max_tokens, extra).await,
|
||||
Engine::Uzu(e) => e.stream(messages, model, temperature, max_tokens, extra).await,
|
||||
Engine::AppleFm(e) => e.stream(messages, model, temperature, max_tokens, extra).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +109,10 @@ impl Engine {
|
||||
Engine::LlamaCpp(_) => "llamacpp",
|
||||
Engine::Mlx(_) => "mlx",
|
||||
Engine::LmStudio(_) => "lmstudio",
|
||||
Engine::Exo(_) => "exo",
|
||||
Engine::Nexa(_) => "nexa",
|
||||
Engine::Uzu(_) => "uzu",
|
||||
Engine::AppleFm(_) => "apple_fm",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,4 +134,32 @@ mod tests {
|
||||
assert_eq!(e.variant_key(), "vllm");
|
||||
assert_eq!(e.engine_id(), "vllm");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_engine_exo_variant() {
|
||||
let e = Engine::Exo(OpenAICompatEngine::exo("http://localhost:52415"));
|
||||
assert_eq!(e.variant_key(), "exo");
|
||||
assert_eq!(e.engine_id(), "exo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_engine_nexa_variant() {
|
||||
let e = Engine::Nexa(OpenAICompatEngine::nexa("http://localhost:18181"));
|
||||
assert_eq!(e.variant_key(), "nexa");
|
||||
assert_eq!(e.engine_id(), "nexa");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_engine_uzu_variant() {
|
||||
let e = Engine::Uzu(OpenAICompatEngine::uzu("http://localhost:8080"));
|
||||
assert_eq!(e.variant_key(), "uzu");
|
||||
assert_eq!(e.engine_id(), "uzu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_engine_apple_fm_variant() {
|
||||
let e = Engine::AppleFm(OpenAICompatEngine::apple_fm("http://localhost:8079"));
|
||||
assert_eq!(e.variant_key(), "apple_fm");
|
||||
assert_eq!(e.engine_id(), "apple_fm");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,22 @@ impl OpenAICompatEngine {
|
||||
Self::new("lmstudio", host, None, 120.0)
|
||||
}
|
||||
|
||||
pub fn exo(host: &str) -> Self {
|
||||
Self::new("exo", host, None, 120.0)
|
||||
}
|
||||
|
||||
pub fn nexa(host: &str) -> Self {
|
||||
Self::new("nexa", host, None, 120.0)
|
||||
}
|
||||
|
||||
pub fn uzu(host: &str) -> Self {
|
||||
Self::new("uzu", host, None, 120.0)
|
||||
}
|
||||
|
||||
pub fn apple_fm(host: &str) -> Self {
|
||||
Self::new("apple_fm", host, None, 120.0)
|
||||
}
|
||||
|
||||
fn build_headers(&self) -> reqwest::header::HeaderMap {
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
headers.insert(
|
||||
@@ -337,4 +353,28 @@ mod tests {
|
||||
let engine = OpenAICompatEngine::sglang("http://localhost:30000");
|
||||
assert_eq!(engine.engine_id(), "sglang");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exo_factory() {
|
||||
let engine = OpenAICompatEngine::exo("http://localhost:52415");
|
||||
assert_eq!(engine.engine_id(), "exo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nexa_factory() {
|
||||
let engine = OpenAICompatEngine::nexa("http://localhost:18181");
|
||||
assert_eq!(engine.engine_id(), "nexa");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_uzu_factory() {
|
||||
let engine = OpenAICompatEngine::uzu("http://localhost:8080");
|
||||
assert_eq!(engine.engine_id(), "uzu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_apple_fm_factory() {
|
||||
let engine = OpenAICompatEngine::apple_fm("http://localhost:8079");
|
||||
assert_eq!(engine.engine_id(), "apple_fm");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,8 @@ pub struct PyEngine {
|
||||
|
||||
#[pymethods]
|
||||
impl PyEngine {
|
||||
/// Create an engine by key (e.g. "ollama", "vllm", "sglang", "llamacpp", "mlx", "lmstudio").
|
||||
/// Create an engine by key (e.g. "ollama", "vllm", "sglang", "llamacpp",
|
||||
/// "mlx", "lmstudio", "exo", "nexa", "uzu", "apple_fm").
|
||||
#[new]
|
||||
#[pyo3(signature = (engine_key="ollama", host=None))]
|
||||
fn new(engine_key: &str, host: Option<&str>) -> PyResult<Self> {
|
||||
@@ -48,6 +49,26 @@ impl PyEngine {
|
||||
host.unwrap_or("http://localhost:1234"),
|
||||
),
|
||||
),
|
||||
"exo" => openjarvis_engine::Engine::Exo(
|
||||
openjarvis_engine::OpenAICompatEngine::exo(
|
||||
host.unwrap_or("http://localhost:52415"),
|
||||
),
|
||||
),
|
||||
"nexa" => openjarvis_engine::Engine::Nexa(
|
||||
openjarvis_engine::OpenAICompatEngine::nexa(
|
||||
host.unwrap_or("http://localhost:18181"),
|
||||
),
|
||||
),
|
||||
"uzu" => openjarvis_engine::Engine::Uzu(
|
||||
openjarvis_engine::OpenAICompatEngine::uzu(
|
||||
host.unwrap_or("http://localhost:8080"),
|
||||
),
|
||||
),
|
||||
"apple_fm" => openjarvis_engine::Engine::AppleFm(
|
||||
openjarvis_engine::OpenAICompatEngine::apple_fm(
|
||||
host.unwrap_or("http://localhost:8079"),
|
||||
),
|
||||
),
|
||||
other => {
|
||||
return Err(PyErr::new::<pyo3::exceptions::PyValueError, _>(
|
||||
format!("Unknown engine: {}", other),
|
||||
|
||||
@@ -55,6 +55,16 @@ try:
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import openjarvis.agents.monitor # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import openjarvis.agents.monitor_operative # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Registry alias: "react" -> NativeReActAgent (for backward compat)
|
||||
try:
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
"""MonitorOperativeAgent -- long-horizon agent with configurable strategies.
|
||||
|
||||
Extends ToolUsingAgent (not OperativeAgent) with four configurable strategy
|
||||
axes for long-horizon benchmark evaluation:
|
||||
|
||||
1. **memory_extraction** -- how findings are persisted to memory
|
||||
2. **observation_compression** -- how tool outputs are compressed
|
||||
3. **retrieval_strategy** -- how prior context is recalled
|
||||
4. **task_decomposition** -- how complex tasks are split
|
||||
|
||||
The agent also inherits cross-session state persistence from the
|
||||
OperativeAgent pattern (session_store, memory_backend, operator_id).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from openjarvis.agents._stubs import AgentContext, AgentResult, ToolUsingAgent
|
||||
from openjarvis.core.events import EventBus
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
from openjarvis.core.types import Message, Role, ToolCall, ToolResult
|
||||
from openjarvis.engine._stubs import InferenceEngine
|
||||
from openjarvis.tools._stubs import BaseTool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Valid strategy values
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_MEMORY_EXTRACTION = {"causality_graph", "scratchpad", "structured_json", "none"}
|
||||
VALID_OBSERVATION_COMPRESSION = {"summarize", "truncate", "none"}
|
||||
VALID_RETRIEVAL_STRATEGY = {"hybrid_with_self_eval", "keyword", "semantic", "none"}
|
||||
VALID_TASK_DECOMPOSITION = {"phased", "monolithic", "hierarchical"}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default system prompt
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MONITOR_OPERATIVE_SYSTEM_PROMPT = """\
|
||||
You are a Monitor Operative Agent designed for long-horizon tasks.
|
||||
|
||||
## Capabilities
|
||||
1. TOOLS: Call any available tool via function calling
|
||||
2. STATE: Your previous findings and state are automatically restored
|
||||
3. MEMORY: Store important findings for future recall
|
||||
|
||||
## Strategy
|
||||
- Memory extraction: {memory_extraction}
|
||||
- Observation compression: {observation_compression}
|
||||
- Retrieval strategy: {retrieval_strategy}
|
||||
- Task decomposition: {task_decomposition}
|
||||
|
||||
## Protocol
|
||||
- Break complex tasks into phases and track progress
|
||||
- Store causal relationships and key findings in memory
|
||||
- Compress long tool outputs before adding to context
|
||||
- Self-evaluate retrieved context for relevance
|
||||
- Always persist state before finishing
|
||||
|
||||
{tool_descriptions}"""
|
||||
|
||||
|
||||
@AgentRegistry.register("monitor_operative")
|
||||
class MonitorOperativeAgent(ToolUsingAgent):
|
||||
"""Long-horizon agent with configurable memory, compression, retrieval,
|
||||
and decomposition strategies.
|
||||
|
||||
The four strategy axes control how the agent manages information across
|
||||
turns and sessions:
|
||||
|
||||
- ``memory_extraction``: How findings are persisted (causality_graph,
|
||||
scratchpad, structured_json, none).
|
||||
- ``observation_compression``: How tool outputs are compressed before
|
||||
being added to context (summarize, truncate, none).
|
||||
- ``retrieval_strategy``: How prior context is recalled at the start
|
||||
of each run (hybrid_with_self_eval, keyword, semantic, none).
|
||||
- ``task_decomposition``: How complex tasks are broken down
|
||||
(phased, monolithic, hierarchical).
|
||||
"""
|
||||
|
||||
agent_id = "monitor_operative"
|
||||
accepts_tools = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: InferenceEngine,
|
||||
model: str,
|
||||
*,
|
||||
tools: Optional[List[BaseTool]] = None,
|
||||
bus: Optional[EventBus] = None,
|
||||
max_turns: int = 25,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 4096,
|
||||
system_prompt: Optional[str] = None,
|
||||
# Strategy parameters
|
||||
memory_extraction: str = "causality_graph",
|
||||
observation_compression: str = "summarize",
|
||||
retrieval_strategy: str = "hybrid_with_self_eval",
|
||||
task_decomposition: str = "phased",
|
||||
# State persistence (OperativeAgent pattern)
|
||||
operator_id: Optional[str] = None,
|
||||
session_store: Optional[Any] = None,
|
||||
memory_backend: Optional[Any] = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
engine, model, tools=tools, bus=bus,
|
||||
max_turns=max_turns, temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
# Validate strategies
|
||||
if memory_extraction not in VALID_MEMORY_EXTRACTION:
|
||||
raise ValueError(
|
||||
f"Invalid memory_extraction {memory_extraction!r}, "
|
||||
f"must be one of {VALID_MEMORY_EXTRACTION}"
|
||||
)
|
||||
if observation_compression not in VALID_OBSERVATION_COMPRESSION:
|
||||
raise ValueError(
|
||||
f"Invalid observation_compression {observation_compression!r}, "
|
||||
f"must be one of {VALID_OBSERVATION_COMPRESSION}"
|
||||
)
|
||||
if retrieval_strategy not in VALID_RETRIEVAL_STRATEGY:
|
||||
raise ValueError(
|
||||
f"Invalid retrieval_strategy {retrieval_strategy!r}, "
|
||||
f"must be one of {VALID_RETRIEVAL_STRATEGY}"
|
||||
)
|
||||
if task_decomposition not in VALID_TASK_DECOMPOSITION:
|
||||
raise ValueError(
|
||||
f"Invalid task_decomposition {task_decomposition!r}, "
|
||||
f"must be one of {VALID_TASK_DECOMPOSITION}"
|
||||
)
|
||||
|
||||
self._memory_extraction = memory_extraction
|
||||
self._observation_compression = observation_compression
|
||||
self._retrieval_strategy = retrieval_strategy
|
||||
self._task_decomposition = task_decomposition
|
||||
|
||||
self._system_prompt = system_prompt
|
||||
self._operator_id = operator_id
|
||||
self._session_store = session_store
|
||||
self._memory_backend = memory_backend
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Main run loop
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run(
|
||||
self,
|
||||
input: str,
|
||||
context: Optional[AgentContext] = None,
|
||||
**kwargs: Any,
|
||||
) -> AgentResult:
|
||||
"""Execute the agent on *input* with the configured strategies."""
|
||||
self._emit_turn_start(input)
|
||||
|
||||
# 1. Build system prompt with state context
|
||||
sys_parts: list[str] = []
|
||||
if self._system_prompt:
|
||||
sys_parts.append(self._system_prompt)
|
||||
else:
|
||||
tool_desc = self._build_tool_descriptions()
|
||||
try:
|
||||
sys_parts.append(
|
||||
MONITOR_OPERATIVE_SYSTEM_PROMPT.format(
|
||||
memory_extraction=self._memory_extraction,
|
||||
observation_compression=self._observation_compression,
|
||||
retrieval_strategy=self._retrieval_strategy,
|
||||
task_decomposition=self._task_decomposition,
|
||||
tool_descriptions=tool_desc,
|
||||
),
|
||||
)
|
||||
except KeyError:
|
||||
sys_parts.append(MONITOR_OPERATIVE_SYSTEM_PROMPT)
|
||||
|
||||
# 2. State recall from memory backend
|
||||
previous_state = self._recall_state()
|
||||
if previous_state:
|
||||
sys_parts.append(f"\n## Previous State\n{previous_state}")
|
||||
|
||||
system_prompt = "\n\n".join(sys_parts) if sys_parts else None
|
||||
|
||||
# 3. Load session history
|
||||
session_messages = self._load_session()
|
||||
|
||||
# 4. Build messages
|
||||
messages = self._build_operative_messages(
|
||||
input, context,
|
||||
system_prompt=system_prompt,
|
||||
session_messages=session_messages,
|
||||
)
|
||||
|
||||
# 5. Run function-calling tool loop
|
||||
openai_tools = self._executor.get_openai_tools() if self._tools else []
|
||||
all_tool_results: list[ToolResult] = []
|
||||
turns = 0
|
||||
content = ""
|
||||
state_stored_by_tool = False
|
||||
|
||||
for _turn in range(self._max_turns):
|
||||
turns += 1
|
||||
|
||||
gen_kwargs: dict[str, Any] = {}
|
||||
if openai_tools:
|
||||
gen_kwargs["tools"] = openai_tools
|
||||
|
||||
result = self._generate(messages, **gen_kwargs)
|
||||
content = result.get("content", "")
|
||||
raw_tool_calls = result.get("tool_calls", [])
|
||||
|
||||
# No tool calls -> check continuation, then final answer
|
||||
if not raw_tool_calls:
|
||||
content = self._check_continuation(result, messages)
|
||||
break
|
||||
|
||||
# Build ToolCall objects from raw dicts
|
||||
tool_calls = [
|
||||
ToolCall(
|
||||
id=tc.get("id", f"call_{i}"),
|
||||
name=tc.get("name", ""),
|
||||
arguments=tc.get("arguments", "{}"),
|
||||
)
|
||||
for i, tc in enumerate(raw_tool_calls)
|
||||
]
|
||||
|
||||
# Append assistant message with tool calls
|
||||
messages.append(Message(
|
||||
role=Role.ASSISTANT,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
))
|
||||
|
||||
# Execute each tool
|
||||
for tc in tool_calls:
|
||||
# Loop guard check
|
||||
if self._loop_guard:
|
||||
verdict = self._loop_guard.check_call(
|
||||
tc.name, tc.arguments,
|
||||
)
|
||||
if verdict.blocked:
|
||||
tool_result = ToolResult(
|
||||
tool_name=tc.name,
|
||||
content=f"Loop guard: {verdict.reason}",
|
||||
success=False,
|
||||
)
|
||||
all_tool_results.append(tool_result)
|
||||
messages.append(Message(
|
||||
role=Role.TOOL,
|
||||
content=tool_result.content,
|
||||
tool_call_id=tc.id,
|
||||
name=tc.name,
|
||||
))
|
||||
continue
|
||||
|
||||
tool_result = self._executor.execute(tc)
|
||||
all_tool_results.append(tool_result)
|
||||
|
||||
# Track explicit state storage
|
||||
if tc.name == "memory_store" and self._operator_id:
|
||||
try:
|
||||
args = json.loads(tc.arguments)
|
||||
state_key = f"monitor_operative:{self._operator_id}:state"
|
||||
if args.get("key", "") == state_key:
|
||||
state_stored_by_tool = True
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
# Compress observation if strategy requires it
|
||||
observation_content = self._compress_observation(tool_result.content)
|
||||
|
||||
messages.append(Message(
|
||||
role=Role.TOOL,
|
||||
content=observation_content,
|
||||
tool_call_id=tc.id,
|
||||
name=tc.name,
|
||||
))
|
||||
|
||||
# Extract and store findings based on memory strategy
|
||||
self._extract_and_store(tc.name, tool_result.content)
|
||||
else:
|
||||
# Max turns exceeded
|
||||
self._save_session(input, content)
|
||||
return self._max_turns_result(
|
||||
all_tool_results, turns, content=content,
|
||||
)
|
||||
|
||||
# 6. Save session
|
||||
self._save_session(input, content)
|
||||
|
||||
# 7. Auto-persist state if agent didn't do it explicitly
|
||||
if not state_stored_by_tool:
|
||||
self._auto_persist_state(content)
|
||||
|
||||
self._emit_turn_end(turns=turns, content_length=len(content))
|
||||
return AgentResult(
|
||||
content=content,
|
||||
tool_results=all_tool_results,
|
||||
turns=turns,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Message building
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _build_operative_messages(
|
||||
self,
|
||||
input: str,
|
||||
context: Optional[AgentContext],
|
||||
*,
|
||||
system_prompt: Optional[str] = None,
|
||||
session_messages: Optional[list[Message]] = None,
|
||||
) -> list[Message]:
|
||||
"""Build message list with system prompt, session history, and input."""
|
||||
messages: list[Message] = []
|
||||
if system_prompt:
|
||||
messages.append(Message(role=Role.SYSTEM, content=system_prompt))
|
||||
if session_messages:
|
||||
messages.extend(session_messages)
|
||||
if context and context.conversation.messages:
|
||||
messages.extend(context.conversation.messages)
|
||||
messages.append(Message(role=Role.USER, content=input))
|
||||
return messages
|
||||
|
||||
def _build_tool_descriptions(self) -> str:
|
||||
"""Build a text description of available tools for the system prompt."""
|
||||
if not self._tools:
|
||||
return ""
|
||||
from openjarvis.tools._stubs import build_tool_descriptions
|
||||
return build_tool_descriptions(self._tools)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Strategy methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _compress_observation(self, content: str) -> str:
|
||||
"""Compress a tool observation according to the compression strategy.
|
||||
|
||||
- ``summarize``: If content exceeds 2000 chars, ask the LLM to
|
||||
summarize. Falls back to truncation if generation fails.
|
||||
- ``truncate``: Hard-truncate at 2000 chars with an ellipsis.
|
||||
- ``none``: Return content unchanged.
|
||||
"""
|
||||
if self._observation_compression == "none":
|
||||
return content
|
||||
if self._observation_compression == "truncate":
|
||||
if len(content) > 2000:
|
||||
return content[:2000] + "\n... [truncated]"
|
||||
return content
|
||||
# "summarize"
|
||||
if len(content) <= 2000:
|
||||
return content
|
||||
try:
|
||||
summary_messages = [
|
||||
Message(
|
||||
role=Role.SYSTEM,
|
||||
content="Summarize the following tool output concisely, "
|
||||
"preserving all key facts and data points.",
|
||||
),
|
||||
Message(role=Role.USER, content=content[:8000]),
|
||||
]
|
||||
result = self._generate(summary_messages)
|
||||
summary = result.get("content", "")
|
||||
if summary:
|
||||
return summary
|
||||
except Exception:
|
||||
logger.debug("Observation summarization failed, falling back to truncation")
|
||||
# Fallback to truncation
|
||||
return content[:2000] + "\n... [truncated]"
|
||||
|
||||
def _extract_and_store(self, tool_name: str, content: str) -> None:
|
||||
"""Extract findings from a tool result and store them.
|
||||
|
||||
The extraction strategy depends on ``_memory_extraction``:
|
||||
|
||||
- ``causality_graph``: Extract causal relationships via
|
||||
:meth:`_extract_causality` and store as KG triples.
|
||||
- ``scratchpad``: Append raw content to a scratchpad key in
|
||||
memory.
|
||||
- ``structured_json``: Attempt to parse JSON from the content
|
||||
and store structured data.
|
||||
- ``none``: Do nothing.
|
||||
"""
|
||||
if self._memory_extraction == "none":
|
||||
return
|
||||
if not self._memory_backend:
|
||||
return
|
||||
|
||||
if self._memory_extraction == "causality_graph":
|
||||
self._extract_causality(tool_name, content)
|
||||
elif self._memory_extraction == "scratchpad":
|
||||
self._store_scratchpad(tool_name, content)
|
||||
elif self._memory_extraction == "structured_json":
|
||||
self._store_structured(tool_name, content)
|
||||
|
||||
def _extract_causality(self, tool_name: str, content: str) -> None:
|
||||
"""Extract causal relationships from tool output and store them.
|
||||
|
||||
Uses the LLM to identify cause-effect patterns, then stores
|
||||
them via the memory backend.
|
||||
"""
|
||||
if not self._memory_backend or not content.strip():
|
||||
return
|
||||
# Only attempt extraction for substantial outputs
|
||||
if len(content) < 50:
|
||||
return
|
||||
try:
|
||||
extract_messages = [
|
||||
Message(
|
||||
role=Role.SYSTEM,
|
||||
content=(
|
||||
"Extract causal relationships from the following tool "
|
||||
"output. Return a JSON array of objects with 'cause', "
|
||||
"'effect', and 'confidence' fields. If no causal "
|
||||
"relationships are found, return an empty array []."
|
||||
),
|
||||
),
|
||||
Message(role=Role.USER, content=content[:4000]),
|
||||
]
|
||||
result = self._generate(extract_messages)
|
||||
raw = result.get("content", "")
|
||||
# Try to parse JSON from the response
|
||||
raw = raw.strip()
|
||||
if raw.startswith("```"):
|
||||
# Strip code fences
|
||||
lines = raw.split("\n")
|
||||
raw = "\n".join(lines[1:-1] if len(lines) > 2 else lines)
|
||||
relations = json.loads(raw)
|
||||
if isinstance(relations, list):
|
||||
operator_prefix = (
|
||||
f"monitor_operative:{self._operator_id}"
|
||||
if self._operator_id
|
||||
else "monitor_operative"
|
||||
)
|
||||
for rel in relations[:10]: # Cap at 10 per extraction
|
||||
if isinstance(rel, dict) and "cause" in rel and "effect" in rel:
|
||||
key = f"{operator_prefix}:causality:{rel['cause'][:50]}"
|
||||
value = json.dumps(rel)
|
||||
try:
|
||||
self._memory_backend.store(key, value)
|
||||
except Exception:
|
||||
pass
|
||||
except (json.JSONDecodeError, Exception):
|
||||
logger.debug(
|
||||
"Causality extraction failed for tool %s output", tool_name,
|
||||
)
|
||||
|
||||
def _store_scratchpad(self, tool_name: str, content: str) -> None:
|
||||
"""Append content to a scratchpad entry in memory."""
|
||||
if not self._memory_backend:
|
||||
return
|
||||
operator_prefix = (
|
||||
f"monitor_operative:{self._operator_id}"
|
||||
if self._operator_id
|
||||
else "monitor_operative"
|
||||
)
|
||||
key = f"{operator_prefix}:scratchpad:{tool_name}"
|
||||
# Truncate long content
|
||||
snippet = content[:1000] if len(content) > 1000 else content
|
||||
try:
|
||||
self._memory_backend.store(key, snippet)
|
||||
except Exception:
|
||||
logger.debug("Could not store scratchpad for tool %s", tool_name)
|
||||
|
||||
def _store_structured(self, tool_name: str, content: str) -> None:
|
||||
"""Try to parse JSON from tool output and store structured data."""
|
||||
if not self._memory_backend:
|
||||
return
|
||||
operator_prefix = (
|
||||
f"monitor_operative:{self._operator_id}"
|
||||
if self._operator_id
|
||||
else "monitor_operative"
|
||||
)
|
||||
try:
|
||||
data = json.loads(content)
|
||||
key = f"{operator_prefix}:structured:{tool_name}"
|
||||
self._memory_backend.store(key, json.dumps(data))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
# Not JSON -- store as plain text truncated
|
||||
key = f"{operator_prefix}:structured:{tool_name}"
|
||||
try:
|
||||
self._memory_backend.store(key, content[:1000])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State persistence (OperativeAgent pattern)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _recall_state(self) -> str:
|
||||
"""Retrieve previous state from memory backend."""
|
||||
if not self._memory_backend or not self._operator_id:
|
||||
return ""
|
||||
state_key = f"monitor_operative:{self._operator_id}:state"
|
||||
try:
|
||||
result = self._memory_backend.retrieve(state_key)
|
||||
if result:
|
||||
return result if isinstance(result, str) else str(result)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"No previous state for monitor_operative %s",
|
||||
self._operator_id,
|
||||
)
|
||||
return ""
|
||||
|
||||
def _load_session(self) -> list[Message]:
|
||||
"""Load recent session history for this operator."""
|
||||
if not self._session_store or not self._operator_id:
|
||||
return []
|
||||
session_id = f"monitor_operative:{self._operator_id}"
|
||||
try:
|
||||
session = self._session_store.get_or_create(session_id)
|
||||
if hasattr(session, "messages") and session.messages:
|
||||
recent = session.messages[-10:]
|
||||
return [
|
||||
Message(
|
||||
role=Role(m.get("role", "user")),
|
||||
content=m.get("content", ""),
|
||||
)
|
||||
for m in recent
|
||||
if isinstance(m, dict)
|
||||
]
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not load session for monitor_operative %s",
|
||||
self._operator_id,
|
||||
)
|
||||
return []
|
||||
|
||||
def _save_session(self, input_text: str, response: str) -> None:
|
||||
"""Save the tick's prompt and response to the session store."""
|
||||
if not self._session_store or not self._operator_id:
|
||||
return
|
||||
session_id = f"monitor_operative:{self._operator_id}"
|
||||
try:
|
||||
self._session_store.save_message(
|
||||
session_id, {"role": "user", "content": input_text},
|
||||
)
|
||||
self._session_store.save_message(
|
||||
session_id, {"role": "assistant", "content": response},
|
||||
)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not save session for monitor_operative %s",
|
||||
self._operator_id,
|
||||
)
|
||||
|
||||
def _auto_persist_state(self, content: str) -> None:
|
||||
"""Auto-persist a state summary if agent didn't store explicitly."""
|
||||
if not self._memory_backend or not self._operator_id:
|
||||
return
|
||||
state_key = f"monitor_operative:{self._operator_id}:state"
|
||||
try:
|
||||
summary = content[:1000] if content else ""
|
||||
self._memory_backend.store(state_key, summary)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not auto-persist state for monitor_operative %s",
|
||||
self._operator_id,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["MonitorOperativeAgent", "MONITOR_OPERATIVE_SYSTEM_PROMPT"]
|
||||
@@ -257,6 +257,35 @@ class LMStudioEngineConfig:
|
||||
host: str = "http://localhost:1234"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ExoEngineConfig:
|
||||
"""Per-engine config for Exo."""
|
||||
|
||||
host: str = "http://localhost:52415"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class NexaEngineConfig:
|
||||
"""Per-engine config for Nexa."""
|
||||
|
||||
host: str = "http://localhost:18181"
|
||||
device: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class UzuEngineConfig:
|
||||
"""Per-engine config for Uzu."""
|
||||
|
||||
host: str = "http://localhost:8080"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AppleFmEngineConfig:
|
||||
"""Per-engine config for Apple Foundation Models."""
|
||||
|
||||
host: str = "http://localhost:8079"
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineConfig:
|
||||
"""Inference engine settings with nested per-engine configs."""
|
||||
@@ -268,6 +297,10 @@ class EngineConfig:
|
||||
llamacpp: LlamaCppEngineConfig = field(default_factory=LlamaCppEngineConfig)
|
||||
mlx: MLXEngineConfig = field(default_factory=MLXEngineConfig)
|
||||
lmstudio: LMStudioEngineConfig = field(default_factory=LMStudioEngineConfig)
|
||||
exo: ExoEngineConfig = field(default_factory=ExoEngineConfig)
|
||||
nexa: NexaEngineConfig = field(default_factory=NexaEngineConfig)
|
||||
uzu: UzuEngineConfig = field(default_factory=UzuEngineConfig)
|
||||
apple_fm: AppleFmEngineConfig = field(default_factory=AppleFmEngineConfig)
|
||||
|
||||
# Backward-compat properties for old flat attribute names
|
||||
@property
|
||||
@@ -333,6 +366,42 @@ class EngineConfig:
|
||||
def lmstudio_host(self, value: str) -> None:
|
||||
self.lmstudio.host = value
|
||||
|
||||
@property
|
||||
def exo_host(self) -> str:
|
||||
"""Deprecated: use ``engine.exo.host``."""
|
||||
return self.exo.host
|
||||
|
||||
@exo_host.setter
|
||||
def exo_host(self, value: str) -> None:
|
||||
self.exo.host = value
|
||||
|
||||
@property
|
||||
def nexa_host(self) -> str:
|
||||
"""Deprecated: use ``engine.nexa.host``."""
|
||||
return self.nexa.host
|
||||
|
||||
@nexa_host.setter
|
||||
def nexa_host(self, value: str) -> None:
|
||||
self.nexa.host = value
|
||||
|
||||
@property
|
||||
def uzu_host(self) -> str:
|
||||
"""Deprecated: use ``engine.uzu.host``."""
|
||||
return self.uzu.host
|
||||
|
||||
@uzu_host.setter
|
||||
def uzu_host(self, value: str) -> None:
|
||||
self.uzu.host = value
|
||||
|
||||
@property
|
||||
def apple_fm_host(self) -> str:
|
||||
"""Deprecated: use ``engine.apple_fm.host``."""
|
||||
return self.apple_fm.host
|
||||
|
||||
@apple_fm_host.setter
|
||||
def apple_fm_host(self, value: str) -> None:
|
||||
self.apple_fm.host = value
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class IntelligenceConfig:
|
||||
@@ -1028,6 +1097,19 @@ host = "http://localhost:8080"
|
||||
# [engine.lmstudio]
|
||||
# host = "http://localhost:1234"
|
||||
|
||||
# [engine.exo]
|
||||
# host = "http://localhost:52415"
|
||||
|
||||
# [engine.nexa]
|
||||
# host = "http://localhost:18181"
|
||||
# device = "" # cpu, gpu, npu
|
||||
|
||||
# [engine.uzu]
|
||||
# host = "http://localhost:8080"
|
||||
|
||||
# [engine.apple_fm]
|
||||
# host = "http://localhost:8079"
|
||||
|
||||
[intelligence]
|
||||
default_model = ""
|
||||
fallback_model = ""
|
||||
|
||||
@@ -16,6 +16,10 @@ _HOST_MAP: Dict[str, str | None] = {
|
||||
"sglang": "sglang_host",
|
||||
"mlx": "mlx_host",
|
||||
"lmstudio": "lmstudio_host",
|
||||
"exo": "exo_host",
|
||||
"nexa": "nexa_host",
|
||||
"uzu": "uzu_host",
|
||||
"apple_fm": "apple_fm_host",
|
||||
"cloud": None,
|
||||
"litellm": None,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Apple Foundation Models shim.
|
||||
|
||||
Thin FastAPI server exposing Apple FM SDK as OpenAI-compatible API.
|
||||
Only runs on macOS 15+ with Apple Silicon. Wraps python-apple-fm-sdk's
|
||||
LanguageModelSession as /v1/chat/completions and /v1/models endpoints.
|
||||
|
||||
Usage:
|
||||
uvicorn openjarvis.engine.apple_fm_shim:app \
|
||||
--host 127.0.0.1 --port 8079
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import sys
|
||||
|
||||
if platform.system() != "Darwin":
|
||||
print(
|
||||
"apple_fm_shim: only available on macOS",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import apple_fm # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
print(
|
||||
"apple_fm_shim: pip install python-apple-fm-sdk",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
app = FastAPI(title="Apple FM Shim")
|
||||
|
||||
MODEL_ID = "apple-fm"
|
||||
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
role: str
|
||||
content: str
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
model: str = MODEL_ID
|
||||
messages: list[ChatMessage]
|
||||
temperature: float = 0.7
|
||||
max_tokens: int = 1024
|
||||
stream: bool = False
|
||||
|
||||
|
||||
def _build_prompt(messages: list[ChatMessage]) -> str:
|
||||
parts: list[str] = []
|
||||
for m in messages:
|
||||
if m.role == "system":
|
||||
parts.append(f"[System] {m.content}")
|
||||
elif m.role in ("user", "assistant"):
|
||||
parts.append(m.content)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> JSONResponse:
|
||||
available = apple_fm.SystemLanguageModel.is_available()
|
||||
status = "ok" if available else "unavailable"
|
||||
code = 200 if available else 503
|
||||
return JSONResponse({"status": status}, status_code=code)
|
||||
|
||||
|
||||
@app.get("/v1/models")
|
||||
def list_models() -> JSONResponse:
|
||||
return JSONResponse({
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": MODEL_ID, "object": "model", "owned_by": "apple"},
|
||||
],
|
||||
})
|
||||
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
async def chat_completions(
|
||||
req: ChatRequest,
|
||||
) -> JSONResponse | StreamingResponse:
|
||||
prompt = _build_prompt(req.messages)
|
||||
session = apple_fm.LanguageModelSession()
|
||||
|
||||
if req.stream:
|
||||
async def generate():
|
||||
cid = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
stream = session.stream_response(
|
||||
prompt, max_tokens=req.max_tokens,
|
||||
)
|
||||
async for token in stream:
|
||||
chunk = {
|
||||
"id": cid,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": MODEL_ID,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": {"content": token},
|
||||
"finish_reason": None,
|
||||
}],
|
||||
}
|
||||
yield f"data: {json.dumps(chunk)}\n\n"
|
||||
final = {
|
||||
"id": cid,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": MODEL_ID,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": {},
|
||||
"finish_reason": "stop",
|
||||
}],
|
||||
}
|
||||
yield f"data: {json.dumps(final)}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
generate(), media_type="text/event-stream",
|
||||
)
|
||||
|
||||
text = session.respond(prompt, max_tokens=req.max_tokens)
|
||||
cid = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
return JSONResponse({
|
||||
"id": cid,
|
||||
"object": "chat.completion",
|
||||
"created": int(time.time()),
|
||||
"model": MODEL_ID,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": text},
|
||||
"finish_reason": "stop",
|
||||
}],
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
},
|
||||
})
|
||||
@@ -9,6 +9,10 @@ _ENGINES = {
|
||||
"llamacpp": ("LlamaCppEngine", "http://localhost:8080"),
|
||||
"mlx": ("MLXEngine", "http://localhost:8080"),
|
||||
"lmstudio": ("LMStudioEngine", "http://localhost:1234"),
|
||||
"exo": ("ExoEngine", "http://localhost:52415"),
|
||||
"nexa": ("NexaEngine", "http://localhost:18181"),
|
||||
"uzu": ("UzuEngine", "http://localhost:8080"),
|
||||
"apple_fm": ("AppleFmEngine", "http://localhost:8079"),
|
||||
}
|
||||
|
||||
for _key, (_cls_name, _default_host) in _ENGINES.items():
|
||||
|
||||
@@ -69,6 +69,34 @@ BENCHMARKS = {
|
||||
"category": "use-case",
|
||||
"description": "Function-level code generation",
|
||||
},
|
||||
"loghub": {
|
||||
"category": "agentic",
|
||||
"description": "LogHub log anomaly detection",
|
||||
},
|
||||
"ama-bench": {
|
||||
"category": "agentic",
|
||||
"description": "AMA-Bench agent memory assessment",
|
||||
},
|
||||
"lifelong-agent": {
|
||||
"category": "agentic",
|
||||
"description": "LifelongAgentBench sequential task learning",
|
||||
},
|
||||
"deepplanning": {
|
||||
"category": "agentic",
|
||||
"description": "DeepPlanning shopping constraints",
|
||||
},
|
||||
"paperarena": {
|
||||
"category": "agentic",
|
||||
"description": "PaperArena paper analysis",
|
||||
},
|
||||
"webchorearena": {
|
||||
"category": "agentic",
|
||||
"description": "WebChoreArena web chore tasks",
|
||||
},
|
||||
"workarena": {
|
||||
"category": "agentic",
|
||||
"description": "WorkArena++ enterprise workflows",
|
||||
},
|
||||
}
|
||||
|
||||
BACKENDS = {
|
||||
@@ -174,6 +202,27 @@ def _build_dataset(benchmark: str):
|
||||
elif benchmark == "coding_task":
|
||||
from openjarvis.evals.datasets.coding_task import CodingTaskDataset
|
||||
return CodingTaskDataset()
|
||||
elif benchmark == "loghub":
|
||||
from openjarvis.evals.datasets.loghub import LogHubDataset
|
||||
return LogHubDataset()
|
||||
elif benchmark == "ama-bench":
|
||||
from openjarvis.evals.datasets.ama_bench import AMABenchDataset
|
||||
return AMABenchDataset()
|
||||
elif benchmark == "lifelong-agent":
|
||||
from openjarvis.evals.datasets.lifelong_agent import LifelongAgentDataset
|
||||
return LifelongAgentDataset()
|
||||
elif benchmark == "deepplanning":
|
||||
from openjarvis.evals.datasets.deepplanning import DeepPlanningDataset
|
||||
return DeepPlanningDataset()
|
||||
elif benchmark == "paperarena":
|
||||
from openjarvis.evals.datasets.paperarena import PaperArenaDataset
|
||||
return PaperArenaDataset()
|
||||
elif benchmark == "webchorearena":
|
||||
from openjarvis.evals.datasets.webchorearena import WebChoreArenaDataset
|
||||
return WebChoreArenaDataset()
|
||||
elif benchmark == "workarena":
|
||||
from openjarvis.evals.datasets.workarena import WorkArenaDataset
|
||||
return WorkArenaDataset()
|
||||
else:
|
||||
raise click.ClickException(f"Unknown benchmark: {benchmark}")
|
||||
|
||||
@@ -239,6 +288,27 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str):
|
||||
elif benchmark == "coding_task":
|
||||
from openjarvis.evals.scorers.coding_task import CodingTaskScorer
|
||||
return CodingTaskScorer(judge_backend, judge_model)
|
||||
elif benchmark == "loghub":
|
||||
from openjarvis.evals.scorers.loghub_scorer import LogHubScorer
|
||||
return LogHubScorer(judge_backend, judge_model)
|
||||
elif benchmark == "ama-bench":
|
||||
from openjarvis.evals.scorers.ama_bench_judge import AMABenchScorer
|
||||
return AMABenchScorer(judge_backend, judge_model)
|
||||
elif benchmark == "lifelong-agent":
|
||||
from openjarvis.evals.scorers.lifelong_agent_scorer import LifelongAgentScorer
|
||||
return LifelongAgentScorer(judge_backend, judge_model)
|
||||
elif benchmark == "deepplanning":
|
||||
from openjarvis.evals.scorers.deepplanning_scorer import DeepPlanningScorer
|
||||
return DeepPlanningScorer(judge_backend, judge_model)
|
||||
elif benchmark == "paperarena":
|
||||
from openjarvis.evals.scorers.paperarena_judge import PaperArenaScorer
|
||||
return PaperArenaScorer(judge_backend, judge_model)
|
||||
elif benchmark == "webchorearena":
|
||||
from openjarvis.evals.scorers.webchorearena_scorer import WebChoreArenaScorer
|
||||
return WebChoreArenaScorer(judge_backend, judge_model)
|
||||
elif benchmark == "workarena":
|
||||
from openjarvis.evals.scorers.workarena_scorer import WorkArenaScorer
|
||||
return WorkArenaScorer(judge_backend, judge_model)
|
||||
else:
|
||||
raise click.ClickException(f"Unknown benchmark: {benchmark}")
|
||||
|
||||
|
||||
@@ -43,5 +43,15 @@ class DatasetProvider(ABC):
|
||||
"""Return list of unsatisfied requirements, or empty list."""
|
||||
return []
|
||||
|
||||
def iter_episodes(self) -> Iterable[List[EvalRecord]]:
|
||||
"""Iterate over episodes (groups of sequential records).
|
||||
|
||||
Default: each record is its own single-record episode.
|
||||
Override for benchmarks requiring sequential processing
|
||||
with shared agent state within an episode.
|
||||
"""
|
||||
for record in self.iter_records():
|
||||
yield [record]
|
||||
|
||||
|
||||
__all__ = ["DatasetProvider"]
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Environment provider ABC for benchmarks requiring external environments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
|
||||
class EnvironmentProvider(ABC):
|
||||
"""Manages an external environment for evaluation benchmarks.
|
||||
|
||||
Provides lifecycle management (setup/reset/teardown) and
|
||||
environment-state validation for benchmarks that need
|
||||
Docker containers, ServiceNow instances, or other live systems.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def setup(self) -> Dict[str, Any]:
|
||||
"""Start the environment and return connection info.
|
||||
|
||||
Returns a dict with environment-specific connection details
|
||||
(e.g., URLs, ports, credentials).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def reset(self) -> None:
|
||||
"""Reset environment state between tasks.
|
||||
|
||||
Called between records within an episode to restore
|
||||
the environment to a known state.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def validate(
|
||||
self, record: EvalRecord,
|
||||
) -> Tuple[bool, Dict[str, Any]]:
|
||||
"""Check environment state against expected outcome.
|
||||
|
||||
Args:
|
||||
record: The eval record containing the expected state in metadata.
|
||||
|
||||
Returns:
|
||||
(is_correct, metadata) where is_correct indicates whether
|
||||
the environment is in the expected state.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def teardown(self) -> None:
|
||||
"""Stop the environment and release resources."""
|
||||
@@ -78,6 +78,7 @@ class RunConfig:
|
||||
sheets_spreadsheet_id: str = ""
|
||||
sheets_worksheet: str = "Results"
|
||||
sheets_credentials_path: str = ""
|
||||
episode_mode: bool = False
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
"""AMA-Bench: Agent Memory Assessment benchmark.
|
||||
|
||||
Evaluates long-horizon agent memory across 4 capability types:
|
||||
recall, causal inference, state updating, and state abstraction.
|
||||
Source: https://github.com/AMA-Bench/AMA-Bench
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from openjarvis.evals.core.dataset import DatasetProvider
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are analyzing an agent's interaction trajectory. "
|
||||
"The trajectory shows a sequence of actions and observations "
|
||||
"from an agent completing a task. "
|
||||
"Answer the question about this trajectory accurately and concisely."
|
||||
)
|
||||
|
||||
|
||||
class AMABenchDataset(DatasetProvider):
|
||||
"""AMA-Bench agent memory assessment benchmark."""
|
||||
|
||||
dataset_id = "ama-bench"
|
||||
dataset_name = "AMA-Bench"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
subset: str = "real",
|
||||
cache_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
self._subset = subset # "real" or "synthetic"
|
||||
self._cache_dir = (
|
||||
Path(cache_dir) if cache_dir
|
||||
else Path.home() / ".cache" / "ama_bench"
|
||||
)
|
||||
self._records: List[EvalRecord] = []
|
||||
self._episodes: List[List[EvalRecord]] = []
|
||||
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
data_dir = self._cache_dir / self._subset
|
||||
|
||||
if not data_dir.exists():
|
||||
self._download(data_dir)
|
||||
|
||||
trajectories = self._load_trajectories(data_dir)
|
||||
|
||||
if seed is not None:
|
||||
random.Random(seed).shuffle(trajectories)
|
||||
if max_samples is not None:
|
||||
# max_samples applies to trajectories, not individual QA pairs
|
||||
trajectories = trajectories[:max_samples]
|
||||
|
||||
self._episodes = []
|
||||
self._records = []
|
||||
for traj in trajectories:
|
||||
episode = self._trajectory_to_episode(traj)
|
||||
self._episodes.append(episode)
|
||||
self._records.extend(episode)
|
||||
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
return iter(self._records)
|
||||
|
||||
def iter_episodes(self) -> Iterable[List[EvalRecord]]:
|
||||
"""Yield grouped QA pairs per trajectory for episode mode."""
|
||||
return iter(self._episodes)
|
||||
|
||||
def size(self) -> int:
|
||||
return len(self._records)
|
||||
|
||||
def _download(self, data_dir: Path) -> None:
|
||||
try:
|
||||
from huggingface_hub import snapshot_download
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"huggingface_hub required. Install with: pip install huggingface_hub"
|
||||
) from exc
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_download(
|
||||
repo_id="AMA-Bench/AMA-Bench",
|
||||
repo_type="dataset",
|
||||
local_dir=str(data_dir),
|
||||
)
|
||||
|
||||
def _load_trajectories(
|
||||
self, data_dir: Path,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Load trajectory + QA data from disk."""
|
||||
trajectories: List[Dict[str, Any]] = []
|
||||
# Look for JSON/JSONL files with trajectory data
|
||||
for p in sorted(data_dir.rglob("*.json")):
|
||||
try:
|
||||
with open(p) as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list):
|
||||
trajectories.extend(data)
|
||||
elif isinstance(data, dict):
|
||||
trajectories.append(data)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
logger.debug("Skipping non-JSON file: %s", p)
|
||||
|
||||
for p in sorted(data_dir.rglob("*.jsonl")):
|
||||
try:
|
||||
with open(p) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
trajectories.append(json.loads(line))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
logger.debug("Skipping non-JSONL file: %s", p)
|
||||
|
||||
return trajectories
|
||||
|
||||
def _trajectory_to_episode(
|
||||
self, traj: Dict[str, Any],
|
||||
) -> List[EvalRecord]:
|
||||
"""Convert a trajectory dict into a list of EvalRecords."""
|
||||
traj_id = traj.get("trajectory_id", traj.get("id", "unknown"))
|
||||
traj_text = traj.get("trajectory", traj.get("text", ""))
|
||||
domain = traj.get("domain", "general")
|
||||
qa_pairs = traj.get("qa_pairs", traj.get("questions", []))
|
||||
|
||||
# Truncate very long trajectories for the problem prompt
|
||||
if len(traj_text) > 100_000:
|
||||
traj_text = traj_text[:100_000] + "\n\n[Trajectory truncated]"
|
||||
|
||||
records: List[EvalRecord] = []
|
||||
for i, qa in enumerate(qa_pairs):
|
||||
question = qa.get("question", qa.get("q", ""))
|
||||
answer = qa.get("answer", qa.get("a", ""))
|
||||
capability = qa.get("capability", qa.get("type", "recall"))
|
||||
|
||||
problem = (
|
||||
f"{_SYSTEM_PROMPT}\n\n"
|
||||
f"## Trajectory\n{traj_text}\n\n"
|
||||
f"## Question\n{question}"
|
||||
)
|
||||
|
||||
records.append(EvalRecord(
|
||||
record_id=f"ama-{traj_id}-q{i}",
|
||||
problem=problem,
|
||||
reference=answer,
|
||||
category="agentic",
|
||||
subject=capability,
|
||||
metadata={
|
||||
"trajectory_id": traj_id,
|
||||
"domain": domain,
|
||||
"capability": capability,
|
||||
"question_index": i,
|
||||
"trajectory_length": len(traj_text),
|
||||
},
|
||||
))
|
||||
|
||||
return records
|
||||
|
||||
|
||||
__all__ = ["AMABenchDataset"]
|
||||
@@ -0,0 +1,174 @@
|
||||
"""LifelongAgentBench: Sequential task learning benchmark.
|
||||
|
||||
Evaluates agents on sequential tasks across DB, OS, and KG environments
|
||||
where knowledge from previous tasks is needed.
|
||||
Source: arXiv:2505.11942
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from openjarvis.evals.core.dataset import DatasetProvider
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are an agent operating in a live environment. "
|
||||
"Complete the following task using the tools available to you. "
|
||||
"Previous tasks in this session may have modified the environment."
|
||||
)
|
||||
|
||||
|
||||
class LifelongAgentDataset(DatasetProvider):
|
||||
"""LifelongAgentBench sequential task learning benchmark."""
|
||||
|
||||
dataset_id = "lifelong-agent"
|
||||
dataset_name = "LifelongAgentBench"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
subset: str = "database",
|
||||
cache_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
self._subset = subset # "database", "os", "knowledge_graph"
|
||||
self._cache_dir = (
|
||||
Path(cache_dir) if cache_dir
|
||||
else Path.home() / ".cache" / "lifelong_agent"
|
||||
)
|
||||
self._records: List[EvalRecord] = []
|
||||
self._episodes: List[List[EvalRecord]] = []
|
||||
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
data_dir = self._cache_dir / self._subset
|
||||
|
||||
if not data_dir.exists():
|
||||
self._download(data_dir)
|
||||
|
||||
task_sequences = self._load_task_sequences(data_dir)
|
||||
|
||||
if seed is not None:
|
||||
random.Random(seed).shuffle(task_sequences)
|
||||
if max_samples is not None:
|
||||
task_sequences = task_sequences[:max_samples]
|
||||
|
||||
self._episodes = []
|
||||
self._records = []
|
||||
for seq in task_sequences:
|
||||
episode = self._sequence_to_episode(seq)
|
||||
self._episodes.append(episode)
|
||||
self._records.extend(episode)
|
||||
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
return iter(self._records)
|
||||
|
||||
def iter_episodes(self) -> Iterable[List[EvalRecord]]:
|
||||
"""Yield task sequences for episode mode."""
|
||||
return iter(self._episodes)
|
||||
|
||||
def size(self) -> int:
|
||||
return len(self._records)
|
||||
|
||||
def _download(self, data_dir: Path) -> None:
|
||||
try:
|
||||
from huggingface_hub import snapshot_download
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"huggingface_hub required. Install with: pip install huggingface_hub"
|
||||
) from exc
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_download(
|
||||
repo_id="LifelongAgentBench/LifelongAgentBench",
|
||||
repo_type="dataset",
|
||||
local_dir=str(data_dir),
|
||||
)
|
||||
|
||||
def _load_task_sequences(
|
||||
self, data_dir: Path,
|
||||
) -> List[List[Dict[str, Any]]]:
|
||||
"""Load task sequences from disk."""
|
||||
sequences: List[List[Dict[str, Any]]] = []
|
||||
|
||||
for p in sorted(data_dir.rglob("*.json")):
|
||||
try:
|
||||
with open(p) as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list):
|
||||
# Could be a sequence of tasks or list of sequences
|
||||
if data and isinstance(data[0], list):
|
||||
sequences.extend(data)
|
||||
elif data and isinstance(data[0], dict):
|
||||
sequences.append(data)
|
||||
elif isinstance(data, dict):
|
||||
tasks = data.get("tasks", data.get("sequence", [data]))
|
||||
if tasks:
|
||||
sequences.append(tasks if isinstance(tasks, list) else [tasks])
|
||||
except (json.JSONDecodeError, OSError):
|
||||
logger.debug("Skipping file: %s", p)
|
||||
|
||||
for p in sorted(data_dir.rglob("*.jsonl")):
|
||||
try:
|
||||
seq: List[Dict[str, Any]] = []
|
||||
with open(p) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
seq.append(json.loads(line))
|
||||
if seq:
|
||||
sequences.append(seq)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
logger.debug("Skipping file: %s", p)
|
||||
|
||||
return sequences
|
||||
|
||||
def _sequence_to_episode(
|
||||
self, tasks: List[Dict[str, Any]],
|
||||
) -> List[EvalRecord]:
|
||||
"""Convert a task sequence into EvalRecords."""
|
||||
records: List[EvalRecord] = []
|
||||
seq_id = tasks[0].get("sequence_id", tasks[0].get("id", "unknown")) if tasks else "unknown"
|
||||
|
||||
for i, task in enumerate(tasks):
|
||||
task_id = task.get("task_id", task.get("id", f"task-{i}"))
|
||||
instruction = task.get("instruction", task.get("task", ""))
|
||||
expected = task.get("expected_output", task.get("answer", ""))
|
||||
env_type = task.get("environment", self._subset)
|
||||
dependencies = task.get("dependencies", [])
|
||||
|
||||
problem = (
|
||||
f"{_SYSTEM_PROMPT}\n\n"
|
||||
f"## Task {i + 1}\n{instruction}"
|
||||
)
|
||||
if dependencies:
|
||||
problem += f"\n\nThis task depends on previous tasks: {dependencies}"
|
||||
|
||||
records.append(EvalRecord(
|
||||
record_id=f"lifelong-{seq_id}-t{i}",
|
||||
problem=problem,
|
||||
reference=expected,
|
||||
category="agentic",
|
||||
subject=env_type,
|
||||
metadata={
|
||||
"sequence_id": seq_id,
|
||||
"task_index": i,
|
||||
"environment": env_type,
|
||||
"dependencies": dependencies,
|
||||
"task_id": task_id,
|
||||
},
|
||||
))
|
||||
|
||||
return records
|
||||
|
||||
|
||||
__all__ = ["LifelongAgentDataset"]
|
||||
@@ -0,0 +1,251 @@
|
||||
"""LogHub log anomaly detection dataset.
|
||||
|
||||
Supports HDFS, BGL, and Thunderbird log datasets from
|
||||
https://github.com/logpai/loghub for evaluating log analysis agents.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from openjarvis.evals.core.dataset import DatasetProvider
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are a log analysis expert. Analyze the following log session "
|
||||
"and determine if it indicates an anomaly or is normal behavior.\n\n"
|
||||
"Respond with exactly one of: ANOMALY or NORMAL\n"
|
||||
"Then provide a brief explanation of your reasoning."
|
||||
)
|
||||
|
||||
_DATASETS = {
|
||||
"hdfs": {
|
||||
"hf_path": "logpai/loghub-HDFS-v1",
|
||||
"log_file": "HDFS.log",
|
||||
"label_file": "anomaly_label.csv",
|
||||
"mode": "session", # group by block_id
|
||||
},
|
||||
"bgl": {
|
||||
"hf_path": "logpai/loghub-BGL",
|
||||
"log_file": "BGL.log",
|
||||
"mode": "window", # fixed-size windows
|
||||
"window_size": 100,
|
||||
},
|
||||
"thunderbird": {
|
||||
"hf_path": "logpai/loghub-Thunderbird",
|
||||
"log_file": "Thunderbird.log",
|
||||
"mode": "window",
|
||||
"window_size": 100,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class LogHubDataset(DatasetProvider):
|
||||
"""LogHub log anomaly detection benchmark."""
|
||||
|
||||
dataset_id = "loghub"
|
||||
dataset_name = "LogHub"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
subset: str = "hdfs",
|
||||
cache_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
if subset not in _DATASETS:
|
||||
raise ValueError(
|
||||
f"Unknown LogHub subset: {subset}. "
|
||||
f"Choose from: {list(_DATASETS.keys())}"
|
||||
)
|
||||
self._subset = subset
|
||||
self._cache_dir = (
|
||||
Path(cache_dir) if cache_dir
|
||||
else Path.home() / ".cache" / "loghub"
|
||||
)
|
||||
self._records: List[EvalRecord] = []
|
||||
|
||||
def load(
|
||||
self,
|
||||
*,
|
||||
max_samples: Optional[int] = None,
|
||||
split: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> None:
|
||||
meta = _DATASETS[self._subset]
|
||||
data_dir = self._cache_dir / self._subset
|
||||
|
||||
if not data_dir.exists():
|
||||
self._download(meta, data_dir)
|
||||
|
||||
if meta["mode"] == "session":
|
||||
records = self._load_session_mode(data_dir, meta)
|
||||
else:
|
||||
records = self._load_window_mode(data_dir, meta)
|
||||
|
||||
if seed is not None:
|
||||
random.Random(seed).shuffle(records)
|
||||
if max_samples is not None:
|
||||
records = records[:max_samples]
|
||||
|
||||
self._records = records
|
||||
|
||||
def iter_records(self) -> Iterable[EvalRecord]:
|
||||
return iter(self._records)
|
||||
|
||||
def size(self) -> int:
|
||||
return len(self._records)
|
||||
|
||||
def _download(self, meta: Dict[str, Any], data_dir: Path) -> None:
|
||||
"""Download dataset from HuggingFace."""
|
||||
try:
|
||||
from huggingface_hub import snapshot_download
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"huggingface_hub required for LogHub download. "
|
||||
"Install with: pip install huggingface_hub"
|
||||
)
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_download(
|
||||
repo_id=meta["hf_path"],
|
||||
repo_type="dataset",
|
||||
local_dir=str(data_dir),
|
||||
)
|
||||
|
||||
def _load_session_mode(
|
||||
self, data_dir: Path, meta: Dict[str, Any],
|
||||
) -> List[EvalRecord]:
|
||||
"""Load HDFS-style session-based records (group by block_id)."""
|
||||
log_path = data_dir / meta["log_file"]
|
||||
label_path = data_dir / meta["label_file"]
|
||||
|
||||
# Load labels: block_id -> "Anomaly" / "Normal"
|
||||
labels: Dict[str, str] = {}
|
||||
if label_path.exists():
|
||||
with open(label_path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
bid = row.get("BlockId", "")
|
||||
lbl = row.get("Label", "Normal")
|
||||
labels[bid] = lbl
|
||||
|
||||
# Group log lines by block_id
|
||||
block_pattern = re.compile(r"blk_[-]?\d+")
|
||||
sessions: Dict[str, List[str]] = {}
|
||||
|
||||
with open(log_path, errors="replace") as f:
|
||||
for line in f:
|
||||
match = block_pattern.search(line)
|
||||
if match:
|
||||
bid = match.group()
|
||||
sessions.setdefault(bid, []).append(line.rstrip())
|
||||
|
||||
records: List[EvalRecord] = []
|
||||
for bid, lines in sessions.items():
|
||||
label = labels.get(bid, "Normal")
|
||||
reference = "anomaly" if label == "Anomaly" else "normal"
|
||||
log_text = "\n".join(lines[:200]) # Cap at 200 lines per session
|
||||
|
||||
problem = (
|
||||
f"{_SYSTEM_PROMPT}\n\n"
|
||||
f"Log session for block {bid} "
|
||||
f"({len(lines)} lines):\n```\n{log_text}\n```"
|
||||
)
|
||||
|
||||
records.append(EvalRecord(
|
||||
record_id=f"loghub-{self._subset}-{bid}",
|
||||
problem=problem,
|
||||
reference=reference,
|
||||
category="agentic",
|
||||
subject=self._subset,
|
||||
metadata={
|
||||
"block_id": bid,
|
||||
"num_lines": len(lines),
|
||||
"dataset": self._subset,
|
||||
"label": label,
|
||||
},
|
||||
))
|
||||
|
||||
return records
|
||||
|
||||
def _load_window_mode(
|
||||
self, data_dir: Path, meta: Dict[str, Any],
|
||||
) -> List[EvalRecord]:
|
||||
"""Load BGL/Thunderbird-style windowed records."""
|
||||
log_path = data_dir / meta["log_file"]
|
||||
window_size = meta.get("window_size", 100)
|
||||
|
||||
records: List[EvalRecord] = []
|
||||
window: List[str] = []
|
||||
has_anomaly = False
|
||||
window_idx = 0
|
||||
|
||||
with open(log_path, errors="replace") as f:
|
||||
for line in f:
|
||||
stripped = line.rstrip()
|
||||
# BGL/Thunderbird: first token is "-" (normal) or fault category
|
||||
is_anomalous = not stripped.startswith("-")
|
||||
if is_anomalous:
|
||||
has_anomaly = True
|
||||
window.append(stripped)
|
||||
|
||||
if len(window) >= window_size:
|
||||
reference = "anomaly" if has_anomaly else "normal"
|
||||
log_text = "\n".join(window)
|
||||
|
||||
problem = (
|
||||
f"{_SYSTEM_PROMPT}\n\n"
|
||||
f"Log window {window_idx} "
|
||||
f"({len(window)} lines):\n```\n{log_text}\n```"
|
||||
)
|
||||
|
||||
records.append(EvalRecord(
|
||||
record_id=f"loghub-{self._subset}-w{window_idx}",
|
||||
problem=problem,
|
||||
reference=reference,
|
||||
category="agentic",
|
||||
subject=self._subset,
|
||||
metadata={
|
||||
"window_idx": window_idx,
|
||||
"num_lines": len(window),
|
||||
"dataset": self._subset,
|
||||
"has_anomaly": has_anomaly,
|
||||
},
|
||||
))
|
||||
|
||||
window = []
|
||||
has_anomaly = False
|
||||
window_idx += 1
|
||||
|
||||
# Flush remaining lines in partial window
|
||||
if window:
|
||||
reference = "anomaly" if has_anomaly else "normal"
|
||||
log_text = "\n".join(window)
|
||||
problem = (
|
||||
f"{_SYSTEM_PROMPT}\n\n"
|
||||
f"Log window {window_idx} "
|
||||
f"({len(window)} lines):\n```\n{log_text}\n```"
|
||||
)
|
||||
records.append(EvalRecord(
|
||||
record_id=f"loghub-{self._subset}-w{window_idx}",
|
||||
problem=problem,
|
||||
reference=reference,
|
||||
category="agentic",
|
||||
subject=self._subset,
|
||||
metadata={
|
||||
"window_idx": window_idx,
|
||||
"num_lines": len(window),
|
||||
"dataset": self._subset,
|
||||
"has_anomaly": has_anomaly,
|
||||
},
|
||||
))
|
||||
|
||||
return records
|
||||
|
||||
|
||||
__all__ = ["LogHubDataset"]
|
||||
@@ -0,0 +1,64 @@
|
||||
"""LLM-judge scorer for AMA-Bench agent memory assessment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from openjarvis.evals.core.scorer import LLMJudgeScorer
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
_JUDGE_PROMPT = """You are evaluating an agent memory assessment.
|
||||
|
||||
Question: {question}
|
||||
|
||||
Reference Answer: {reference}
|
||||
|
||||
Agent's Answer: {model_answer}
|
||||
|
||||
Is the agent's answer correct? Consider semantic equivalence, not exact wording.
|
||||
Respond with exactly: CORRECT or INCORRECT"""
|
||||
|
||||
|
||||
class AMABenchScorer(LLMJudgeScorer):
|
||||
"""Score AMA-Bench QA via LLM judge."""
|
||||
|
||||
scorer_id = "ama-bench"
|
||||
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
if not model_answer or not model_answer.strip():
|
||||
return False, {"reason": "empty_response"}
|
||||
|
||||
if not record.reference or not record.reference.strip():
|
||||
return None, {"reason": "no_ground_truth"}
|
||||
|
||||
# Extract just the question from the problem (after "## Question")
|
||||
question = record.problem
|
||||
if "## Question" in question:
|
||||
question = question.split("## Question")[-1].strip()
|
||||
|
||||
prompt = _JUDGE_PROMPT.format(
|
||||
question=question,
|
||||
reference=record.reference,
|
||||
model_answer=model_answer,
|
||||
)
|
||||
|
||||
try:
|
||||
raw = self._ask_judge(prompt, temperature=0.0, max_tokens=64)
|
||||
is_correct = bool(re.search(r"\bCORRECT\b", raw, re.IGNORECASE))
|
||||
# Check it's not "INCORRECT"
|
||||
if re.search(r"\bINCORRECT\b", raw, re.IGNORECASE):
|
||||
is_correct = False
|
||||
|
||||
return is_correct, {
|
||||
"match_type": "llm_judge",
|
||||
"raw_judge_output": raw,
|
||||
"capability": record.metadata.get("capability", ""),
|
||||
}
|
||||
except Exception as exc:
|
||||
return False, {
|
||||
"match_type": "llm_judge_error",
|
||||
"error": str(exc),
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
"""LLM-judge scorer for LifelongAgentBench."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from openjarvis.evals.core.scorer import LLMJudgeScorer
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
_JUDGE_PROMPT = """You are evaluating whether an agent completed a task correctly.
|
||||
|
||||
Task: {task}
|
||||
|
||||
Expected Outcome: {reference}
|
||||
|
||||
Agent's Response: {model_answer}
|
||||
|
||||
Did the agent produce the correct result? Consider semantic equivalence.
|
||||
Respond with exactly: CORRECT or INCORRECT"""
|
||||
|
||||
|
||||
class LifelongAgentScorer(LLMJudgeScorer):
|
||||
"""Score LifelongAgentBench tasks via LLM judge."""
|
||||
|
||||
scorer_id = "lifelong-agent"
|
||||
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
if not model_answer or not model_answer.strip():
|
||||
return False, {"reason": "empty_response"}
|
||||
|
||||
if not record.reference or not record.reference.strip():
|
||||
return None, {"reason": "no_ground_truth"}
|
||||
|
||||
task = record.problem
|
||||
if "## Task" in task:
|
||||
task = task.split("## Task")[-1].strip()
|
||||
|
||||
prompt = _JUDGE_PROMPT.format(
|
||||
task=task,
|
||||
reference=record.reference,
|
||||
model_answer=model_answer,
|
||||
)
|
||||
|
||||
try:
|
||||
raw = self._ask_judge(prompt, temperature=0.0, max_tokens=64)
|
||||
is_correct = bool(re.search(r"\bCORRECT\b", raw, re.IGNORECASE))
|
||||
if re.search(r"\bINCORRECT\b", raw, re.IGNORECASE):
|
||||
is_correct = False
|
||||
|
||||
return is_correct, {
|
||||
"match_type": "llm_judge",
|
||||
"raw_judge_output": raw,
|
||||
"environment": record.metadata.get("environment", ""),
|
||||
"task_index": record.metadata.get("task_index", -1),
|
||||
}
|
||||
except Exception as exc:
|
||||
return False, {
|
||||
"match_type": "llm_judge_error",
|
||||
"error": str(exc),
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Scorer for LogHub log anomaly detection benchmark."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from openjarvis.evals.core.scorer import LLMJudgeScorer
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
_ANOMALY_PATTERN = re.compile(r"\bANOMAL(?:Y|OUS)\b", re.IGNORECASE)
|
||||
_NORMAL_PATTERN = re.compile(r"\bNORMAL\b", re.IGNORECASE)
|
||||
|
||||
|
||||
class LogHubScorer(LLMJudgeScorer):
|
||||
"""Score log anomaly detection: extract ANOMALY/NORMAL classification."""
|
||||
|
||||
scorer_id = "loghub"
|
||||
|
||||
def score(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
if not model_answer or not model_answer.strip():
|
||||
return False, {"reason": "empty_response"}
|
||||
|
||||
reference = record.reference.lower().strip()
|
||||
|
||||
# Extract classification from response
|
||||
has_anomaly = bool(_ANOMALY_PATTERN.search(model_answer))
|
||||
has_normal = bool(_NORMAL_PATTERN.search(model_answer))
|
||||
|
||||
if has_anomaly and not has_normal:
|
||||
predicted = "anomaly"
|
||||
elif has_normal and not has_anomaly:
|
||||
predicted = "normal"
|
||||
elif has_anomaly and has_normal:
|
||||
# Ambiguous — check which appears first
|
||||
a_pos = _ANOMALY_PATTERN.search(model_answer).start()
|
||||
n_pos = _NORMAL_PATTERN.search(model_answer).start()
|
||||
predicted = "anomaly" if a_pos < n_pos else "normal"
|
||||
else:
|
||||
# Neither keyword found — use LLM judge fallback
|
||||
return self._llm_fallback(record, model_answer)
|
||||
|
||||
is_correct = predicted == reference
|
||||
return is_correct, {
|
||||
"match_type": "exact",
|
||||
"predicted": predicted,
|
||||
"reference": reference,
|
||||
}
|
||||
|
||||
def _llm_fallback(
|
||||
self, record: EvalRecord, model_answer: str,
|
||||
) -> Tuple[Optional[bool], Dict[str, Any]]:
|
||||
"""Use LLM judge when keyword extraction fails."""
|
||||
prompt = (
|
||||
f"A log analysis agent was asked to classify a log session.\n\n"
|
||||
f"The agent responded:\n{model_answer}\n\n"
|
||||
f"Does the agent's response indicate the logs are "
|
||||
f"ANOMALOUS or NORMAL?\n\n"
|
||||
f"Respond with exactly: ANOMALY or NORMAL"
|
||||
)
|
||||
try:
|
||||
raw = self._ask_judge(prompt, temperature=0.0, max_tokens=32)
|
||||
has_anomaly = bool(_ANOMALY_PATTERN.search(raw))
|
||||
predicted = "anomaly" if has_anomaly else "normal"
|
||||
reference = record.reference.lower().strip()
|
||||
is_correct = predicted == reference
|
||||
return is_correct, {
|
||||
"match_type": "llm_fallback",
|
||||
"predicted": predicted,
|
||||
"reference": reference,
|
||||
"raw_judge_output": raw,
|
||||
}
|
||||
except Exception as exc:
|
||||
return False, {
|
||||
"match_type": "llm_fallback_error",
|
||||
"error": str(exc),
|
||||
}
|
||||
@@ -124,7 +124,11 @@ DEFAULT_SEARCH_SPACE = SearchSpace(
|
||||
SearchDimension(
|
||||
name="engine.backend",
|
||||
dim_type="categorical",
|
||||
values=["ollama", "vllm", "sglang", "llamacpp", "mlx", "lmstudio"],
|
||||
values=[
|
||||
"ollama", "vllm", "sglang", "llamacpp",
|
||||
"mlx", "lmstudio", "exo", "nexa",
|
||||
"uzu", "apple_fm",
|
||||
],
|
||||
description="Inference engine backend",
|
||||
pillar="engine",
|
||||
),
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
[recipe]
|
||||
name = "monitor_assistant"
|
||||
description = "Evidence-driven monitoring with code execution, causality tracking, and cross-session memory"
|
||||
version = "1.0.0"
|
||||
|
||||
[intelligence]
|
||||
model = "qwen3:32b"
|
||||
|
||||
[engine]
|
||||
key = "vllm"
|
||||
|
||||
[agent]
|
||||
type = "monitor"
|
||||
max_turns = 25
|
||||
temperature = 0.3
|
||||
tools = [
|
||||
"think", "file_read", "shell_exec", "code_interpreter",
|
||||
"memory_store", "memory_search", "memory_index",
|
||||
"kg_add_entity", "kg_add_relation", "kg_query", "kg_neighbors",
|
||||
"database_query", "http_request", "web_search",
|
||||
]
|
||||
|
||||
[learning]
|
||||
routing = "grpo"
|
||||
agent = "icl_updater"
|
||||
@@ -0,0 +1,25 @@
|
||||
[template]
|
||||
name = "monitor"
|
||||
description = "Hybrid agent combining code execution with tool calling for monitoring, log analysis, and long-horizon observation tasks"
|
||||
|
||||
[agent]
|
||||
type = "monitor"
|
||||
max_turns = 25
|
||||
temperature = 0.3
|
||||
system_prompt = """You are a Monitor Agent with two modes: (1) TOOLS for calling available functions (memory, KG, grep, web), and (2) CODE for writing Python in code blocks for data processing and calculation. Use shell_exec for searching and filtering, memory_store and kg_add_entity for persisting findings, code_interpreter for calculations, and think for reasoning. Always store important findings before finishing and record causal patterns in the knowledge graph."""
|
||||
tools = [
|
||||
"think",
|
||||
"file_read",
|
||||
"shell_exec",
|
||||
"code_interpreter",
|
||||
"memory_store",
|
||||
"memory_search",
|
||||
"memory_index",
|
||||
"kg_add_entity",
|
||||
"kg_add_relation",
|
||||
"kg_query",
|
||||
"kg_neighbors",
|
||||
"database_query",
|
||||
"http_request",
|
||||
"web_search",
|
||||
]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Browser accessibility tree extraction tool.
|
||||
|
||||
Extracts the accessibility tree (AX tree) from the current browser page,
|
||||
providing a structured text representation of the DOM with element IDs,
|
||||
roles, names, and states. Used by top-performing agents on WebArena-family
|
||||
benchmarks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.core.registry import ToolRegistry
|
||||
from openjarvis.core.types import ToolResult
|
||||
from openjarvis.tools._stubs import BaseTool, ToolSpec
|
||||
|
||||
# Re-use the shared browser session from the browser module.
|
||||
# This is imported at module level so tests can patch
|
||||
# ``openjarvis.tools.browser_axtree._session``.
|
||||
try:
|
||||
from openjarvis.tools.browser import _session
|
||||
except Exception: # pragma: no cover — optional dependency
|
||||
_session = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@ToolRegistry.register("browser_axtree")
|
||||
class BrowserAXTreeTool(BaseTool):
|
||||
"""Extract the accessibility tree from the current browser page."""
|
||||
|
||||
tool_id = "browser_axtree"
|
||||
|
||||
@property
|
||||
def spec(self) -> ToolSpec:
|
||||
return ToolSpec(
|
||||
name="browser_axtree",
|
||||
description=(
|
||||
"Extract the accessibility tree from the current browser page. "
|
||||
"Returns a structured text representation with element roles, "
|
||||
"names, values, and states. More structured than raw HTML."
|
||||
),
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"max_depth": {
|
||||
"type": "integer",
|
||||
"description": "Maximum tree depth to traverse. Default: 10.",
|
||||
},
|
||||
},
|
||||
},
|
||||
category="browser",
|
||||
required_capabilities=["network:fetch"],
|
||||
)
|
||||
|
||||
def execute(self, **params: Any) -> ToolResult:
|
||||
max_depth = params.get("max_depth", 10)
|
||||
|
||||
try:
|
||||
page = _session.page # type: ignore[union-attr]
|
||||
except ImportError as exc:
|
||||
return ToolResult(
|
||||
tool_name="browser_axtree",
|
||||
content=f"Playwright not installed: {exc}",
|
||||
success=False,
|
||||
)
|
||||
except AttributeError:
|
||||
return ToolResult(
|
||||
tool_name="browser_axtree",
|
||||
content="Browser session not available.",
|
||||
success=False,
|
||||
)
|
||||
|
||||
try:
|
||||
snapshot = page.accessibility.snapshot()
|
||||
if not snapshot:
|
||||
return ToolResult(
|
||||
tool_name="browser_axtree",
|
||||
content="No accessibility tree available.",
|
||||
success=False,
|
||||
)
|
||||
|
||||
text = _format_axtree(snapshot, max_depth=max_depth)
|
||||
|
||||
return ToolResult(
|
||||
tool_name="browser_axtree",
|
||||
content=text,
|
||||
success=True,
|
||||
metadata={"node_count": _count_nodes(snapshot)},
|
||||
)
|
||||
except Exception as exc:
|
||||
return ToolResult(
|
||||
tool_name="browser_axtree",
|
||||
content=f"AX tree extraction error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
|
||||
|
||||
def _format_axtree(
|
||||
node: dict,
|
||||
depth: int = 0,
|
||||
max_depth: int = 10,
|
||||
) -> str:
|
||||
"""Format an accessibility tree node as indented text."""
|
||||
if depth >= max_depth:
|
||||
return ""
|
||||
|
||||
indent = " " * depth
|
||||
role = node.get("role", "unknown")
|
||||
name = node.get("name", "")
|
||||
value = node.get("value", "")
|
||||
|
||||
parts = [f"{indent}[{role}]"]
|
||||
if name:
|
||||
parts.append(f' "{name}"')
|
||||
if value:
|
||||
parts.append(f" value={value}")
|
||||
|
||||
line = "".join(parts)
|
||||
lines = [line]
|
||||
|
||||
for child in node.get("children", []):
|
||||
child_text = _format_axtree(child, depth + 1, max_depth)
|
||||
if child_text:
|
||||
lines.append(child_text)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _count_nodes(node: dict) -> int:
|
||||
"""Count total nodes in the accessibility tree."""
|
||||
count = 1
|
||||
for child in node.get("children", []):
|
||||
count += _count_nodes(child)
|
||||
return count
|
||||
|
||||
|
||||
__all__ = ["BrowserAXTreeTool"]
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Tests for MonitorOperativeAgent."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from openjarvis.agents.monitor_operative import MonitorOperativeAgent
|
||||
from openjarvis.core.registry import AgentRegistry
|
||||
|
||||
|
||||
def _make_engine(content: str = "Hello") -> MagicMock:
|
||||
engine = MagicMock()
|
||||
engine.engine_id = "mock"
|
||||
engine.generate.return_value = {
|
||||
"content": content,
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
"model": "test-model",
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
return engine
|
||||
|
||||
|
||||
class TestMonitorOperativeAgent:
|
||||
def test_registration(self) -> None:
|
||||
# Import triggers registration; re-register after autouse fixture
|
||||
# clears the registry (same pattern as test_monitor.py)
|
||||
import openjarvis.agents.monitor_operative # noqa: F401
|
||||
if not AgentRegistry.contains("monitor_operative"):
|
||||
AgentRegistry.register_value("monitor_operative", MonitorOperativeAgent)
|
||||
assert AgentRegistry.contains("monitor_operative")
|
||||
cls = AgentRegistry.get("monitor_operative")
|
||||
assert cls is MonitorOperativeAgent
|
||||
|
||||
def test_instantiation(self) -> None:
|
||||
engine = _make_engine()
|
||||
agent = MonitorOperativeAgent(engine, "test-model")
|
||||
assert agent.agent_id == "monitor_operative"
|
||||
assert agent.accepts_tools is True
|
||||
|
||||
def test_default_strategies(self) -> None:
|
||||
engine = _make_engine()
|
||||
agent = MonitorOperativeAgent(engine, "test-model")
|
||||
assert agent._memory_extraction == "causality_graph"
|
||||
assert agent._observation_compression == "summarize"
|
||||
assert agent._retrieval_strategy == "hybrid_with_self_eval"
|
||||
assert agent._task_decomposition == "phased"
|
||||
|
||||
def test_custom_strategies(self) -> None:
|
||||
engine = _make_engine()
|
||||
agent = MonitorOperativeAgent(
|
||||
engine, "test-model",
|
||||
memory_extraction="scratchpad",
|
||||
observation_compression="none",
|
||||
retrieval_strategy="keyword",
|
||||
task_decomposition="monolithic",
|
||||
)
|
||||
assert agent._memory_extraction == "scratchpad"
|
||||
assert agent._observation_compression == "none"
|
||||
|
||||
def test_simple_run(self) -> None:
|
||||
engine = _make_engine("The answer is 42.")
|
||||
agent = MonitorOperativeAgent(engine, "test-model")
|
||||
result = agent.run("What is the answer?")
|
||||
assert result.content == "The answer is 42."
|
||||
assert result.turns >= 1
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Tests for AMA-Bench dataset provider."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
from openjarvis.evals.datasets.ama_bench import AMABenchDataset
|
||||
from openjarvis.evals.scorers.ama_bench_judge import AMABenchScorer
|
||||
|
||||
|
||||
class TestAMABenchDataset:
|
||||
def test_instantiation(self) -> None:
|
||||
ds = AMABenchDataset()
|
||||
assert ds.dataset_id == "ama-bench"
|
||||
assert ds.dataset_name == "AMA-Bench"
|
||||
|
||||
def test_has_required_methods(self) -> None:
|
||||
ds = AMABenchDataset()
|
||||
assert hasattr(ds, "load")
|
||||
assert hasattr(ds, "iter_records")
|
||||
assert hasattr(ds, "size")
|
||||
assert hasattr(ds, "iter_episodes")
|
||||
|
||||
|
||||
def _mock_backend() -> MagicMock:
|
||||
backend = MagicMock()
|
||||
backend.generate.return_value = "CORRECT"
|
||||
return backend
|
||||
|
||||
|
||||
class TestAMABenchScorer:
|
||||
def test_instantiation(self) -> None:
|
||||
s = AMABenchScorer(_mock_backend(), "test-model")
|
||||
assert s.scorer_id == "ama-bench"
|
||||
|
||||
def test_empty_response(self) -> None:
|
||||
s = AMABenchScorer(_mock_backend(), "test-model")
|
||||
record = EvalRecord(
|
||||
record_id="test-1", problem="question",
|
||||
reference="answer", category="agentic",
|
||||
)
|
||||
is_correct, meta = s.score(record, "")
|
||||
assert is_correct is False
|
||||
assert meta["reason"] == "empty_response"
|
||||
|
||||
|
||||
class TestAMABenchCLI:
|
||||
def test_in_benchmarks_dict(self) -> None:
|
||||
from openjarvis.evals.cli import BENCHMARKS
|
||||
assert "ama-bench" in BENCHMARKS
|
||||
|
||||
def test_build_dataset(self) -> None:
|
||||
from openjarvis.evals.cli import _build_dataset
|
||||
ds = _build_dataset("ama-bench")
|
||||
assert ds.dataset_id == "ama-bench"
|
||||
|
||||
def test_build_scorer(self) -> None:
|
||||
from openjarvis.evals.cli import _build_scorer
|
||||
s = _build_scorer("ama-bench", _mock_backend(), "test-model")
|
||||
assert s.scorer_id == "ama-bench"
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Tests for EnvironmentProvider ABC."""
|
||||
|
||||
from openjarvis.evals.core.environment import EnvironmentProvider
|
||||
|
||||
|
||||
class _MockEnv(EnvironmentProvider):
|
||||
"""Concrete implementation for testing."""
|
||||
|
||||
def setup(self):
|
||||
return {"url": "http://localhost:8080"}
|
||||
|
||||
def reset(self):
|
||||
pass
|
||||
|
||||
def validate(self, record):
|
||||
return True, {"status": "ok"}
|
||||
|
||||
def teardown(self):
|
||||
pass
|
||||
|
||||
|
||||
class TestEnvironmentProvider:
|
||||
def test_concrete_implementation(self) -> None:
|
||||
env = _MockEnv()
|
||||
info = env.setup()
|
||||
assert info["url"] == "http://localhost:8080"
|
||||
|
||||
def test_validate_returns_tuple(self) -> None:
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
env = _MockEnv()
|
||||
record = EvalRecord("r1", "problem", "ref", "agentic")
|
||||
is_correct, meta = env.validate(record)
|
||||
assert is_correct is True
|
||||
assert meta["status"] == "ok"
|
||||
|
||||
def test_lifecycle(self) -> None:
|
||||
env = _MockEnv()
|
||||
env.setup()
|
||||
env.reset()
|
||||
env.teardown()
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Tests for EvalRunner episode mode."""
|
||||
|
||||
from openjarvis.evals.core.dataset import DatasetProvider
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
|
||||
class TestDatasetProviderEpisodes:
|
||||
def test_default_iter_episodes(self) -> None:
|
||||
"""Default iter_episodes wraps each record in its own episode."""
|
||||
|
||||
class SimpleDataset(DatasetProvider):
|
||||
dataset_id = "test"
|
||||
dataset_name = "Test"
|
||||
|
||||
def __init__(self):
|
||||
self._records = [
|
||||
EvalRecord("r1", "q1", "a1", "chat"),
|
||||
EvalRecord("r2", "q2", "a2", "chat"),
|
||||
]
|
||||
|
||||
def load(self, **kw):
|
||||
pass
|
||||
|
||||
def iter_records(self):
|
||||
return iter(self._records)
|
||||
|
||||
def size(self):
|
||||
return len(self._records)
|
||||
|
||||
ds = SimpleDataset()
|
||||
episodes = list(ds.iter_episodes())
|
||||
assert len(episodes) == 2
|
||||
assert len(episodes[0]) == 1
|
||||
assert episodes[0][0].record_id == "r1"
|
||||
|
||||
|
||||
class TestRunConfigEpisodeMode:
|
||||
def test_episode_mode_field(self) -> None:
|
||||
from openjarvis.evals.core.types import RunConfig
|
||||
cfg = RunConfig(
|
||||
benchmark="test", backend="test", model="test",
|
||||
episode_mode=True,
|
||||
)
|
||||
assert cfg.episode_mode is True
|
||||
|
||||
def test_episode_mode_default_false(self) -> None:
|
||||
from openjarvis.evals.core.types import RunConfig
|
||||
cfg = RunConfig(benchmark="test", backend="test", model="test")
|
||||
assert cfg.episode_mode is False
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Tests for LifelongAgentBench benchmark."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
from openjarvis.evals.datasets.lifelong_agent import LifelongAgentDataset
|
||||
from openjarvis.evals.scorers.lifelong_agent_scorer import LifelongAgentScorer
|
||||
|
||||
|
||||
def _mock_backend() -> MagicMock:
|
||||
backend = MagicMock()
|
||||
backend.generate.return_value = "CORRECT"
|
||||
return backend
|
||||
|
||||
|
||||
class TestLifelongAgentDataset:
|
||||
def test_instantiation(self) -> None:
|
||||
ds = LifelongAgentDataset()
|
||||
assert ds.dataset_id == "lifelong-agent"
|
||||
assert ds.dataset_name == "LifelongAgentBench"
|
||||
|
||||
def test_has_episode_support(self) -> None:
|
||||
ds = LifelongAgentDataset()
|
||||
assert hasattr(ds, "iter_episodes")
|
||||
|
||||
|
||||
class TestLifelongAgentScorer:
|
||||
def test_instantiation(self) -> None:
|
||||
s = LifelongAgentScorer(_mock_backend(), "test-model")
|
||||
assert s.scorer_id == "lifelong-agent"
|
||||
|
||||
def test_empty_response(self) -> None:
|
||||
s = LifelongAgentScorer(_mock_backend(), "test-model")
|
||||
record = EvalRecord("t-1", "task", "expected", "agentic")
|
||||
is_correct, meta = s.score(record, "")
|
||||
assert is_correct is False
|
||||
|
||||
|
||||
class TestLifelongAgentCLI:
|
||||
def test_in_benchmarks(self) -> None:
|
||||
from openjarvis.evals.cli import BENCHMARKS
|
||||
assert "lifelong-agent" in BENCHMARKS
|
||||
|
||||
def test_build_dataset(self) -> None:
|
||||
from openjarvis.evals.cli import _build_dataset
|
||||
ds = _build_dataset("lifelong-agent")
|
||||
assert ds.dataset_id == "lifelong-agent"
|
||||
|
||||
def test_build_scorer(self) -> None:
|
||||
from openjarvis.evals.cli import _build_scorer
|
||||
s = _build_scorer("lifelong-agent", _mock_backend(), "test-model")
|
||||
assert s.scorer_id == "lifelong-agent"
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Tests for LogHub dataset provider."""
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
from openjarvis.evals.datasets.loghub import LogHubDataset
|
||||
from openjarvis.evals.scorers.loghub_scorer import LogHubScorer
|
||||
|
||||
|
||||
class TestLogHubDataset:
|
||||
def test_instantiation(self) -> None:
|
||||
ds = LogHubDataset()
|
||||
assert ds.dataset_id == "loghub"
|
||||
assert ds.dataset_name == "LogHub"
|
||||
|
||||
def test_has_required_methods(self) -> None:
|
||||
ds = LogHubDataset()
|
||||
assert hasattr(ds, "load")
|
||||
assert hasattr(ds, "iter_records")
|
||||
assert hasattr(ds, "size")
|
||||
|
||||
|
||||
class TestLogHubDatasetDetails:
|
||||
def test_invalid_subset_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="Unknown LogHub subset"):
|
||||
LogHubDataset(subset="nonexistent")
|
||||
|
||||
def test_session_mode_parsing(self, tmp_path: Path) -> None:
|
||||
log_file = tmp_path / "HDFS.log"
|
||||
label_file = tmp_path / "anomaly_label.csv"
|
||||
|
||||
log_file.write_text(
|
||||
"081109 event blk_123 info\n"
|
||||
"081109 event blk_123 detail\n"
|
||||
"081109 event blk_456 info\n"
|
||||
)
|
||||
with open(label_file, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=["BlockId", "Label"])
|
||||
writer.writeheader()
|
||||
writer.writerow({"BlockId": "blk_123", "Label": "Anomaly"})
|
||||
writer.writerow({"BlockId": "blk_456", "Label": "Normal"})
|
||||
|
||||
ds = LogHubDataset()
|
||||
meta = {
|
||||
"log_file": "HDFS.log",
|
||||
"label_file": "anomaly_label.csv",
|
||||
"mode": "session",
|
||||
}
|
||||
records = ds._load_session_mode(tmp_path, meta)
|
||||
|
||||
assert len(records) == 2
|
||||
by_id = {r.metadata["block_id"]: r for r in records}
|
||||
assert by_id["blk_123"].reference == "anomaly"
|
||||
assert by_id["blk_456"].reference == "normal"
|
||||
assert by_id["blk_123"].category == "agentic"
|
||||
|
||||
def test_window_mode_parsing(self, tmp_path: Path) -> None:
|
||||
log_file = tmp_path / "BGL.log"
|
||||
# 5 lines: 3 normal (start with -), 2 anomalous.
|
||||
# Window size 3 = 1 full + 1 partial
|
||||
lines = [
|
||||
"- normal line 1\n",
|
||||
"- normal line 2\n",
|
||||
"FATAL error line\n",
|
||||
"- normal line 3\n",
|
||||
"WARN warning line\n",
|
||||
]
|
||||
log_file.write_text("".join(lines))
|
||||
|
||||
ds = LogHubDataset(subset="bgl")
|
||||
meta = {"log_file": "BGL.log", "mode": "window", "window_size": 3}
|
||||
records = ds._load_window_mode(tmp_path, meta)
|
||||
|
||||
assert len(records) == 2 # 1 full window + 1 partial
|
||||
assert records[0].reference == "anomaly" # window 0 has "FATAL" line
|
||||
assert records[0].metadata["window_idx"] == 0
|
||||
# Window 1 has "WARN" line
|
||||
assert records[1].reference == "anomaly"
|
||||
assert records[1].metadata["num_lines"] == 2
|
||||
|
||||
def test_size_before_load(self) -> None:
|
||||
ds = LogHubDataset()
|
||||
assert ds.size() == 0
|
||||
|
||||
|
||||
def _mock_backend() -> MagicMock:
|
||||
backend = MagicMock()
|
||||
backend.generate.return_value = "A"
|
||||
return backend
|
||||
|
||||
|
||||
class TestLogHubScorer:
|
||||
def test_instantiation(self) -> None:
|
||||
s = LogHubScorer(_mock_backend(), "test-model")
|
||||
assert s.scorer_id == "loghub"
|
||||
|
||||
def test_exact_match_anomaly(self) -> None:
|
||||
s = LogHubScorer(_mock_backend(), "test-model")
|
||||
record = EvalRecord(
|
||||
record_id="test-1", problem="analyze logs",
|
||||
reference="anomaly", category="agentic",
|
||||
)
|
||||
is_correct, meta = s.score(record, "ANOMALY\nThe logs show errors.")
|
||||
assert is_correct is True
|
||||
assert meta["match_type"] == "exact"
|
||||
|
||||
def test_exact_match_normal(self) -> None:
|
||||
s = LogHubScorer(_mock_backend(), "test-model")
|
||||
record = EvalRecord(
|
||||
record_id="test-2", problem="analyze logs",
|
||||
reference="normal", category="agentic",
|
||||
)
|
||||
is_correct, meta = s.score(record, "NORMAL - no issues detected")
|
||||
assert is_correct is True
|
||||
|
||||
def test_empty_response(self) -> None:
|
||||
s = LogHubScorer(_mock_backend(), "test-model")
|
||||
record = EvalRecord(
|
||||
record_id="test-3", problem="analyze logs",
|
||||
reference="anomaly", category="agentic",
|
||||
)
|
||||
is_correct, meta = s.score(record, "")
|
||||
assert is_correct is False
|
||||
assert meta["reason"] == "empty_response"
|
||||
|
||||
def test_wrong_classification(self) -> None:
|
||||
s = LogHubScorer(_mock_backend(), "test-model")
|
||||
record = EvalRecord(
|
||||
record_id="test-4", problem="analyze logs",
|
||||
reference="anomaly", category="agentic",
|
||||
)
|
||||
is_correct, meta = s.score(record, "NORMAL - everything looks fine")
|
||||
assert is_correct is False
|
||||
|
||||
|
||||
class TestLogHubCLI:
|
||||
def test_in_benchmarks_dict(self) -> None:
|
||||
from openjarvis.evals.cli import BENCHMARKS
|
||||
assert "loghub" in BENCHMARKS
|
||||
assert BENCHMARKS["loghub"]["category"] == "agentic"
|
||||
|
||||
def test_build_dataset(self) -> None:
|
||||
from openjarvis.evals.cli import _build_dataset
|
||||
ds = _build_dataset("loghub")
|
||||
assert ds is not None
|
||||
assert ds.dataset_id == "loghub"
|
||||
|
||||
def test_build_scorer(self) -> None:
|
||||
from openjarvis.evals.cli import _build_scorer
|
||||
s = _build_scorer("loghub", _mock_backend(), "test-model")
|
||||
assert s is not None
|
||||
assert s.scorer_id == "loghub"
|
||||
@@ -20,7 +20,7 @@ TEMPLATES_DIR = (
|
||||
/ "data"
|
||||
)
|
||||
|
||||
VALID_AGENT_TYPES = {"simple", "orchestrator", "native_react"}
|
||||
VALID_AGENT_TYPES = {"simple", "orchestrator", "native_react", "monitor"}
|
||||
|
||||
|
||||
def test_load_single_template() -> None:
|
||||
|
||||
@@ -352,6 +352,7 @@ class TestDefaultSearchSpace:
|
||||
expected = {
|
||||
"ollama", "vllm", "sglang",
|
||||
"llamacpp", "mlx", "lmstudio",
|
||||
"exo", "nexa", "uzu", "apple_fm",
|
||||
}
|
||||
assert set(dim.values) == expected
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Tests for browser_axtree tool."""
|
||||
|
||||
from unittest.mock import MagicMock, PropertyMock, patch
|
||||
|
||||
from openjarvis.tools.browser_axtree import BrowserAXTreeTool
|
||||
|
||||
|
||||
def _make_mock_page():
|
||||
page = MagicMock()
|
||||
page.accessibility.snapshot.return_value = {
|
||||
"role": "WebArea",
|
||||
"name": "Test Page",
|
||||
"children": [
|
||||
{"role": "heading", "name": "Welcome", "level": 1},
|
||||
{"role": "link", "name": "Click me", "url": "https://example.com"},
|
||||
{"role": "textbox", "name": "Search", "value": ""},
|
||||
],
|
||||
}
|
||||
return page
|
||||
|
||||
|
||||
def _make_mock_session(page=None):
|
||||
if page is None:
|
||||
page = _make_mock_page()
|
||||
session = MagicMock()
|
||||
type(session).page = PropertyMock(return_value=page)
|
||||
return session
|
||||
|
||||
|
||||
class TestBrowserAXTreeTool:
|
||||
def test_instantiation(self) -> None:
|
||||
tool = BrowserAXTreeTool()
|
||||
assert tool.tool_id == "browser_axtree"
|
||||
assert tool.spec.name == "browser_axtree"
|
||||
|
||||
def test_execute_returns_tree(self) -> None:
|
||||
session = _make_mock_session()
|
||||
with patch("openjarvis.tools.browser_axtree._session", session):
|
||||
tool = BrowserAXTreeTool()
|
||||
result = tool.execute()
|
||||
assert result.success is True
|
||||
assert "heading" in result.content
|
||||
assert "Welcome" in result.content
|
||||
|
||||
def test_execute_includes_all_roles(self) -> None:
|
||||
session = _make_mock_session()
|
||||
with patch("openjarvis.tools.browser_axtree._session", session):
|
||||
tool = BrowserAXTreeTool()
|
||||
result = tool.execute()
|
||||
assert result.success is True
|
||||
assert "WebArea" in result.content
|
||||
assert "link" in result.content
|
||||
assert "textbox" in result.content
|
||||
|
||||
def test_execute_includes_node_count_metadata(self) -> None:
|
||||
session = _make_mock_session()
|
||||
with patch("openjarvis.tools.browser_axtree._session", session):
|
||||
tool = BrowserAXTreeTool()
|
||||
result = tool.execute()
|
||||
assert result.success is True
|
||||
# 1 root + 3 children = 4 nodes
|
||||
assert result.metadata["node_count"] == 4
|
||||
|
||||
def test_execute_max_depth(self) -> None:
|
||||
"""When max_depth=1 only the root node should appear."""
|
||||
session = _make_mock_session()
|
||||
with patch("openjarvis.tools.browser_axtree._session", session):
|
||||
tool = BrowserAXTreeTool()
|
||||
result = tool.execute(max_depth=1)
|
||||
assert result.success is True
|
||||
assert "WebArea" in result.content
|
||||
# Children at depth 1 should not be present
|
||||
assert "heading" not in result.content
|
||||
|
||||
def test_execute_empty_snapshot(self) -> None:
|
||||
page = MagicMock()
|
||||
page.accessibility.snapshot.return_value = None
|
||||
session = _make_mock_session(page)
|
||||
with patch("openjarvis.tools.browser_axtree._session", session):
|
||||
tool = BrowserAXTreeTool()
|
||||
result = tool.execute()
|
||||
assert result.success is False
|
||||
assert "No accessibility tree" in result.content
|
||||
|
||||
def test_playwright_not_installed(self) -> None:
|
||||
session = MagicMock()
|
||||
type(session).page = PropertyMock(
|
||||
side_effect=ImportError("playwright not installed")
|
||||
)
|
||||
with patch("openjarvis.tools.browser_axtree._session", session):
|
||||
tool = BrowserAXTreeTool()
|
||||
result = tool.execute()
|
||||
assert result.success is False
|
||||
assert "playwright" in result.content.lower()
|
||||
|
||||
def test_spec_category_and_capabilities(self) -> None:
|
||||
tool = BrowserAXTreeTool()
|
||||
assert tool.spec.category == "browser"
|
||||
assert "network:fetch" in tool.spec.required_capabilities
|
||||
|
||||
def test_spec_has_max_depth_parameter(self) -> None:
|
||||
tool = BrowserAXTreeTool()
|
||||
props = tool.spec.parameters.get("properties", {})
|
||||
assert "max_depth" in props
|
||||
assert props["max_depth"]["type"] == "integer"
|
||||
|
||||
def test_to_openai_function(self) -> None:
|
||||
tool = BrowserAXTreeTool()
|
||||
fn = tool.to_openai_function()
|
||||
assert fn["type"] == "function"
|
||||
assert fn["function"]["name"] == "browser_axtree"
|
||||
|
||||
def test_execute_snapshot_error(self) -> None:
|
||||
page = MagicMock()
|
||||
page.accessibility.snapshot.side_effect = Exception("Browser crashed")
|
||||
session = _make_mock_session(page)
|
||||
with patch("openjarvis.tools.browser_axtree._session", session):
|
||||
tool = BrowserAXTreeTool()
|
||||
result = tool.execute()
|
||||
assert result.success is False
|
||||
assert "AX tree extraction error" in result.content
|
||||
@@ -2535,6 +2535,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089", size = 25530, upload-time = "2026-02-10T15:23:53.817Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mkdocs-gen-files"
|
||||
version = "0.6.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "mkdocs" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/35/f26349f7fa18414eb2e25d75a6fa9c7e3186c36e1d227c0b2d785a7bd5c4/mkdocs_gen_files-0.6.0.tar.gz", hash = "sha256:52022dc14dcc0451e05e54a8f5d5e7760351b6701eff816d1e9739577ec5635e", size = 8642, upload-time = "2025-11-23T12:13:22.124Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/ec/72417415563c60ae01b36f0d497f1f4c803972f447ef4fb7f7746d6e07db/mkdocs_gen_files-0.6.0-py3-none-any.whl", hash = "sha256:815af15f3e2dbfda379629c1b95c02c8e6f232edf2a901186ea3b204ab1135b2", size = 8182, upload-time = "2025-11-23T12:13:20.756Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mkdocs-get-deps"
|
||||
version = "0.2.0"
|
||||
@@ -2549,6 +2561,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/d4/029f984e8d3f3b6b726bd33cafc473b75e9e44c0f7e80a5b29abc466bdea/mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134", size = 9521, upload-time = "2023-11-20T17:51:08.587Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mkdocs-literate-nav"
|
||||
version = "0.6.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "mkdocs" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f6/5f/99aa379b305cd1c2084d42db3d26f6de0ea9bf2cc1d10ed17f61aff35b9a/mkdocs_literate_nav-0.6.2.tar.gz", hash = "sha256:760e1708aa4be86af81a2b56e82c739d5a8388a0eab1517ecfd8e5aa40810a75", size = 17419, upload-time = "2025-03-18T21:53:09.711Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/84/b5b14d2745e4dd1a90115186284e9ee1b4d0863104011ab46abb7355a1c3/mkdocs_literate_nav-0.6.2-py3-none-any.whl", hash = "sha256:0a6489a26ec7598477b56fa112056a5e3a6c15729f0214bea8a4dbc55bd5f630", size = 13261, upload-time = "2025-03-18T21:53:08.1Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mkdocs-material"
|
||||
version = "9.7.2"
|
||||
@@ -3391,6 +3415,8 @@ dev = [
|
||||
]
|
||||
docs = [
|
||||
{ name = "mkdocs" },
|
||||
{ name = "mkdocs-gen-files" },
|
||||
{ name = "mkdocs-literate-nav" },
|
||||
{ name = "mkdocs-material" },
|
||||
{ name = "mkdocstrings", extra = ["python"] },
|
||||
]
|
||||
@@ -3503,6 +3529,8 @@ requires-dist = [
|
||||
{ name = "litellm", marker = "extra == 'inference-litellm'", specifier = ">=1.40" },
|
||||
{ name = "mastodon-py", marker = "extra == 'channel-mastodon'", specifier = ">=1.8" },
|
||||
{ name = "mkdocs", marker = "extra == 'docs'", specifier = ">=1.6" },
|
||||
{ name = "mkdocs-gen-files", marker = "extra == 'docs'", specifier = ">=0.5" },
|
||||
{ name = "mkdocs-literate-nav", marker = "extra == 'docs'", specifier = ">=0.6" },
|
||||
{ name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.5" },
|
||||
{ name = "mkdocstrings", extras = ["python"], marker = "extra == 'docs'", specifier = ">=0.25" },
|
||||
{ name = "mlx-lm", marker = "sys_platform == 'darwin' and extra == 'inference-mlx'", specifier = ">=0.19" },
|
||||
|
||||
Reference in New Issue
Block a user