From fea8d3e8728f92cdf5c2a5f7a87600207b768d3f Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Mon, 18 May 2026 20:19:37 -0700 Subject: [PATCH] release: v1.0.1 (#357) --- CHANGELOG.md | 77 ++++++ docs/desktop-auto-update.md | 105 ++++++++ docs/learning/ace.md | 135 ++++++++++ docs/telemetry.md | 27 ++ frontend/package.json | 2 +- frontend/src-tauri/Cargo.toml | 2 +- frontend/src-tauri/tauri.conf.json | 8 +- .../src/components/Desktop/UpdateChecker.tsx | 10 + pyproject.toml | 8 +- scripts/install/install.sh | 11 + src/openjarvis/analytics/identity.py | 48 +++- src/openjarvis/cli/__init__.py | 2 + src/openjarvis/cli/_install_detect.py | 87 +++++++ src/openjarvis/cli/_version_check.py | 58 ++++- src/openjarvis/cli/self_update_cmd.py | 89 +++++++ src/openjarvis/core/config.py | 49 +++- src/openjarvis/learning/__init__.py | 4 + .../learning/agents/ace_optimizer.py | 246 ++++++++++++++++++ tests/analytics/test_identity.py | 126 +++++++++ tests/cli/test_install_detect.py | 85 ++++++ tests/cli/test_self_update.py | 119 +++++++++ tests/cli/test_version_check.py | 71 +++++ tests/learning/agents/test_ace_optimizer.py | 224 ++++++++++++++++ uv.lock | 2 +- 24 files changed, 1572 insertions(+), 23 deletions(-) create mode 100644 docs/desktop-auto-update.md create mode 100644 docs/learning/ace.md create mode 100644 src/openjarvis/cli/_install_detect.py create mode 100644 src/openjarvis/cli/self_update_cmd.py create mode 100644 src/openjarvis/learning/agents/ace_optimizer.py create mode 100644 tests/analytics/test_identity.py create mode 100644 tests/cli/test_install_detect.py create mode 100644 tests/cli/test_self_update.py create mode 100644 tests/cli/test_version_check.py create mode 100644 tests/learning/agents/test_ace_optimizer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f259fc2..81959a4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,83 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +## [1.0.1] - 2026-05-17 + +A patch release that closes the auto-update gap so the analytics +module added in #351 actually reaches users on the desktop, adds +runtime opt-out for that analytics, fixes the misleading upgrade +hint the CLI was printing, and lands the ACE optimizer alongside +DSPy and GEPA. + +### Added + +**ACE agent optimizer** (`learning/agents/ace_optimizer.py`). Adds +[ACE](https://github.com/ace-agent/ace) as a third agent-learning +policy alongside DSPy and GEPA. Where DSPy bootstraps few-shot +examples and GEPA evolves prompt populations, ACE evolves a textual +*playbook* of strategies the agent reads at inference time, updated +by a Generator / Reflector / Curator triad. Pick via +`[learning.agent] policy = "ace"`. Setup is manual (ACE isn't on +PyPI and isn't a properly-packaged Python project as of v1.0.1) — +see `docs/learning/ace.md` for the install path and trace-adapter +behavior. + +**`jarvis self-update`** subcommand. Detects how OpenJarvis was +installed (pip, uv tool, editable git checkout) by inspecting +`openjarvis.__file__`, then runs the right upgrade command. Supports +`--check` (print the command without running) and `-y` (skip the +confirmation prompt). The post-command "new version available" hint +now points users at this command instead of guessing at the right +flow. + +**Desktop auto-update endpoint wired to the rolling +`desktop-latest` GitHub release.** The Tauri updater plugin was +configured on the build side (`createUpdaterArtifacts: true`, +`includeUpdaterJson: true`, signing key in `TAURI_SIGNING_PRIVATE_KEY`) +but inert on the runtime side (`active: false`, `endpoints: []`). The +installed desktop app would never check. Both are now fixed; the app +polls `releases/download/desktop-latest/latest.json` every 30 minutes +and signature-verifies downloads against the minisign pubkey baked +into the app. Full flow, key-rotation runbook, and dev escape hatch +(`OPENJARVIS_NO_UPDATER=1`) documented in `docs/desktop-auto-update.md`. + +**Analytics env-var opt-out** (`DO_NOT_TRACK`, `OPENJARVIS_NO_ANALYTICS`). +Tanvir's analytics module (#351) only respected the +`[analytics] enabled` config-file setting. Both env vars are now +honored in `is_analytics_enabled()` and in the install.sh beacon +script. Any truthy value (`1`, `true`, `yes`, `on`) disables for +that process; env opt-out takes precedence over the config file. +Documented under a new "Opting out" section in `docs/telemetry.md`. + +### Changed + +**Version-check trigger widened.** The "new version available" hint +in `_version_check.py` used to fire only on `{ask, chat, serve}` and +hardcoded the wrong upgrade command (`git pull && uv sync` — only +correct for editable installs). Now fires on every interactive +command (`doctor`, `init`, `quickstart`, `model`, `agents`, `skill`, +`memory`, `bench`, `telemetry`, `config`, `eval`, `optimize`, plus +the original three) and uses install-detection to print the right +upgrade command. Honors `JARVIS_NO_UPDATE_CHECK=1` and `CI=true` to +stay silent in automation. + +**Desktop app version bumped 0.1.0 → 1.0.1** across +`tauri.conf.json`, `frontend/package.json`, and +`frontend/src-tauri/Cargo.toml` so the Python and desktop release +streams are aligned and the auto-updater has a real version to +compare against. + +### Migration from 1.0.0 + +- **Importing `is_analytics_enabled`?** Same signature; behavior now + short-circuits on env opt-out before checking the config. Callers + that want the raw "is the config flag set" semantic should read + `cfg.enabled` directly. +- **Editable-git users running `jarvis self-update`** get the + detected `git pull && uv sync` command pointed at their actual + checkout, not `~/OpenJarvis`. If you'd come to rely on the + hardcoded path, update your muscle memory. + ## [1.0.0] - 2026-05-15 The five-primitive architecture (Intelligence, Engine, Agents, diff --git a/docs/desktop-auto-update.md b/docs/desktop-auto-update.md new file mode 100644 index 00000000..9193b12f --- /dev/null +++ b/docs/desktop-auto-update.md @@ -0,0 +1,105 @@ +# Desktop auto-update + +The OpenJarvis desktop app ships with [Tauri's updater +plugin](https://v2.tauri.app/plugin/updater/), which checks for new +versions on launch and every 30 minutes. When a newer signed build is +available, the app prompts the user to download and install it. + +## How it works + +``` +on launch / every 30 min + │ + ▼ +GET https://github.com/open-jarvis/OpenJarvis/releases/download/desktop-latest/latest.json + │ + ▼ +Parse manifest: { "version": "X.Y.Z", "platforms": { ... } } + │ + ▼ +If manifest.version > installed_version: + download signed .dmg / .deb / .msi from manifest.platforms[target].url + verify against the minisign pubkey baked into the app + prompt user to install +``` + +The frontend code lives in +[`frontend/src/components/Desktop/UpdateChecker.tsx`](../frontend/src/components/Desktop/UpdateChecker.tsx); +the Tauri wiring is in +[`frontend/src-tauri/tauri.conf.json`](../frontend/src-tauri/tauri.conf.json) +under `plugins.updater`. + +## How releases reach the update endpoint + +The `Desktop Build & Release` GitHub Action +([`.github/workflows/desktop.yml`](../.github/workflows/desktop.yml)) +publishes signed binaries plus a `latest.json` manifest to the +`desktop-latest` GitHub release on every push to `main`. The +`tauri-action` step with `includeUpdaterJson: true` generates the +manifest automatically. + +Two release streams exist: + +- **`desktop-latest`** (rolling pre-release): updated on every push to + `main`. This is the channel the desktop app currently polls. Users + on this channel get the most recent build the CI produced. +- **`desktop-vX.Y.Z`** (tagged stable): created when someone pushes a + `desktop-v*` git tag. Has the same artifacts but is marked as a + proper release rather than a pre-release. + +The current updater endpoint points at the rolling `desktop-latest` +stream so that bug fixes (especially security and telemetry-policy +changes) reach users without waiting for a manual stable tag. A future +release may introduce a stable channel that points at +`desktop-v*` tags only. + +## Signing + +Binaries are signed by `tauri-action` using the minisign key pair +referenced via these GitHub Actions secrets: + +| Secret | Purpose | +|---|---| +| `TAURI_SIGNING_PRIVATE_KEY` | Private key (PEM-formatted minisign) | +| `TAURI_SIGNING_PRIVATE_KEY_PASSWORD` | Passphrase for the private key | + +The matching public key is baked into the app at +`tauri.conf.json:plugins.updater.pubkey`. If you ever need to rotate +the key, replace the public key in the JSON file *and* update both +secrets atomically — mismatched keys cause every update download to +fail signature verification with no recovery path other than a manual +reinstall. + +## Disabling the updater locally + +For frontend development, set `VITE_OPENJARVIS_NO_UPDATER=1` in your +shell before running `npm run tauri dev`. Vite injects any +`VITE_`-prefixed env var into `import.meta.env`, and the +`UpdateChecker.tsx` component honors it to skip the 30-minute poll. + +```bash +export VITE_OPENJARVIS_NO_UPDATER=1 +npm run tauri dev +``` + +This is purely a dev escape hatch — it has no effect on production +builds (where `import.meta.env.VITE_OPENJARVIS_NO_UPDATER` will be +`undefined` unless you explicitly set it at build time). + +## Verifying a release manually + +```bash +# Download the latest manifest and confirm it parses cleanly +curl -fsSL https://github.com/open-jarvis/OpenJarvis/releases/download/desktop-latest/latest.json | jq . + +# Fields: +# version — semver string, must match the tag (without leading "v") +# notes — release notes string +# pub_date — RFC3339 timestamp +# platforms — map keyed by "-" e.g. "darwin-aarch64" +# each entry has { signature: "...", url: "..." } +``` + +A 404 on the manifest URL means the most recent desktop CI run +didn't complete or didn't have signing secrets — check the +`Desktop Build & Release` workflow logs. diff --git a/docs/learning/ace.md b/docs/learning/ace.md new file mode 100644 index 00000000..81ae24c4 --- /dev/null +++ b/docs/learning/ace.md @@ -0,0 +1,135 @@ +# ACE optimizer (Agentic Context Engineering) + +OpenJarvis supports [ACE](https://github.com/ace-agent/ace) as a third +optimizer alongside DSPy and GEPA. Where DSPy bootstraps few-shot +examples and GEPA evolves prompts via reflective mutation, **ACE +evolves a textual *playbook*** — annotated natural-language strategies +the agent reads at inference time. The playbook is updated by a +Generator / Reflector / Curator triad of LLM calls. + +## When to pick ACE + +| Task shape | DSPy | GEPA | ACE | +|---|---|---|---| +| Single-turn QA with crisp metric | strong | strong | weaker | +| Long-running agent that should accumulate guidance | weak | medium | strong | +| Open-domain where strategies matter more than templates | weak | medium | strong | +| When you want to *read* what the optimizer learned | medium | medium | strong | + +ACE's headline artifact is `final_playbook.txt` — a human-readable +file like: + +``` +## STRATEGIES & INSIGHTS +[str-00001] helpful=5 harmful=0 :: When the user asks for unit + conversion, prefer the exact + rational form before rounding. +[str-00002] helpful=3 harmful=1 :: Cite a primary source before + stating a date claim. +``` + +If reading those strategies feels like the form of "what learning +should produce" for your task, ACE is the right choice. + +## Setup + +ACE is **not on PyPI** as of OpenJarvis v1.0.1, and the upstream +repository is structured as a research codebase (multiple top-level +directories) rather than a Python package. There's no `learning-ace` +extra for that reason. Install ACE manually instead: + +```bash +# 1. Clone ACE somewhere outside your OpenJarvis checkout +git clone https://github.com/ace-agent/ace.git ~/code/ace +cd ~/code/ace +curl -LsSf https://astral.sh/uv/install.sh | sh # if you don't have uv +uv sync + +# 2. Make ACE's src/ importable from your OpenJarvis venv +echo "$HOME/code/ace/src" > \ + "$(python -c 'import site; print(site.getsitepackages()[0])')/ace.pth" + +# 3. Set the API key for whichever provider ACE will call +cp ~/code/ace/.env.example ~/code/ace/.env +# Edit ~/code/ace/.env to set API_KEY for your chosen provider. + +# 4. Verify the import resolves from OpenJarvis's venv +python -c "from openjarvis.learning.agents.ace_optimizer import HAS_ACE; print(HAS_ACE)" +# True +``` + +If `HAS_ACE` prints `False`, the `.pth` file isn't being picked up — +verify the path matches `site.getsitepackages()[0]` for the same +Python interpreter you're using to run OpenJarvis. + +## Configuration + +ACE is configured under `[learning.agent.ace]` in your OpenJarvis +config TOML: + +```toml +[learning.agent] +policy = "ace" + +[learning.agent.ace] +# ACE's three roles. Empty = inherit from the intelligence primitive's +# default cloud model. +generator_model = "claude-opus-4-7" +reflector_model = "claude-opus-4-7" +curator_model = "claude-sonnet-4-6" + +api_provider = "openai" # sambanova | together | openai | commonstack + +num_epochs = 1 +max_num_rounds = 3 +playbook_token_budget = 80000 +max_tokens = 4096 + +task_name = "openjarvis" +save_dir = "" # default: ~/.openjarvis/learning/ace// + +min_traces = 20 +``` + +## Running + +Once configured, the same orchestrator that runs DSPy / GEPA also runs +ACE — pick it via the `policy` field above. To force a one-shot run: + +```bash +jarvis optimize agent --policy ace +``` + +ACE writes intermediate state and the final playbook to `save_dir`. +The OpenJarvis runtime will pick up the playbook on next agent start +(via the same sidecar overlay mechanism the Skills System uses). + +## Trace adapter behavior + +OpenJarvis traces are adapted into ACE's `train_samples` / +`val_samples` / `test_samples` format via a 70 / 15 / 15 split +(order-preserving for reproducibility). Each trace becomes a +`{question: trace.query, ground_truth_answer: trace.result}` sample. +Traces with empty `query` or `result` are dropped before splitting. + +The `DataProcessor` ACE expects is built from `_TraceDataProcessor` +in `src/openjarvis/learning/agents/ace_optimizer.py` — it does a +case-insensitive substring match for `answer_is_correct` and averages +that for aggregate accuracy. If you're optimizing for a domain where +substring matching is the wrong correctness signal (math problems, +code, structured outputs), subclass `_TraceDataProcessor` and pass it +through your own callsite to `ACEAgentOptimizer.optimize()`. + +## Limitations in v1.0.1 + +- **No automatic install.** Document above is the only path. +- **The trace adapter uses substring correctness.** Override for + domain-specific scoring. +- **Single provider per run.** ACE assigns the same `api_provider` to + all three roles. To mix providers, run ACE outside OpenJarvis and + hand-deliver the resulting playbook into `save_dir`. + +These will get revisited once ACE publishes a PyPI package or stable +provider interface — track +[ace-agent/ace#issues](https://github.com/ace-agent/ace/issues) for +upstream changes that would let us tighten the wrapper. diff --git a/docs/telemetry.md b/docs/telemetry.md index 455f4ee9..36dc1054 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -83,6 +83,33 @@ dropped. Tests covering the patterns: [`tests/analytics/test_redaction.py`](../t - **Never** sold, shared with advertisers, or used for anything other than improving OpenJarvis. +## Opting out + +Three independent ways to disable analytics — any one is sufficient: + +1. **Set an env var** (no config file edit needed): + ```bash + export DO_NOT_TRACK=1 # W3C convention, honored by other tools too + # or + export OPENJARVIS_NO_ANALYTICS=1 # project-specific, leaves other DNT-aware tools unaffected + ``` + Both are checked at runtime; any truthy value (`1`, `true`, `yes`, + `on`) disables analytics for that process. Truthy = anything other + than empty, `0`, `false`, `no`, `off`. + +2. **Edit `~/.openjarvis/config.toml`**: + ```toml + [analytics] + enabled = false + ``` + +3. **Delete the anon ID** (`rm ~/.openjarvis/anon_id`) — events for + the prior identity are orphaned, but a new identity will be + created on the next run. Combine with #1 or #2 to fully stop. + +Env-var opt-out takes precedence over the config file, so setting +`DO_NOT_TRACK=1` overrides `enabled = true` in the config. + ## Retention - Default retention: **365 days**, then events are deleted by PostHog diff --git a/frontend/package.json b/frontend/package.json index 1794b093..d987a0fb 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "openjarvis-chat", "private": true, - "version": "0.1.0", + "version": "1.0.1", "type": "module", "engines": { "node": ">=20" diff --git a/frontend/src-tauri/Cargo.toml b/frontend/src-tauri/Cargo.toml index 9c3438eb..bf90a7b3 100644 --- a/frontend/src-tauri/Cargo.toml +++ b/frontend/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "openjarvis-desktop" -version = "0.1.0" +version = "1.0.1" description = "OpenJarvis Desktop — Native AI assistant with energy monitoring, trace debugging, and learning visualization" edition = "2021" license = "MIT" diff --git a/frontend/src-tauri/tauri.conf.json b/frontend/src-tauri/tauri.conf.json index 0004b1dd..e48d564d 100644 --- a/frontend/src-tauri/tauri.conf.json +++ b/frontend/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "OpenJarvis", - "version": "0.1.0", + "version": "1.0.1", "identifier": "com.openjarvis.desktop", "build": { "frontendDist": "../dist", @@ -58,9 +58,11 @@ }, "plugins": { "updater": { - "active": false, + "active": true, "pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IDFFNzUzMzhEOEY2MjNEMDMKUldRRFBXS1BqVE4xSG8vK0lkUWN4WnZQYVIrbmc4RmpoOGlJWTBLTE15RlIya3JvQisvdUR3a0QK", - "endpoints": [] + "endpoints": [ + "https://github.com/open-jarvis/OpenJarvis/releases/download/desktop-latest/latest.json" + ] }, "deep-link": { "desktop": { diff --git a/frontend/src/components/Desktop/UpdateChecker.tsx b/frontend/src/components/Desktop/UpdateChecker.tsx index 4b58465c..3a138031 100644 --- a/frontend/src/components/Desktop/UpdateChecker.tsx +++ b/frontend/src/components/Desktop/UpdateChecker.tsx @@ -33,6 +33,16 @@ export function UpdateChecker() { return; } + // Local dev escape hatch: skip the auto-update poll if explicitly + // disabled. Vite exposes any ``VITE_``-prefixed env var on + // ``import.meta.env``, so a frontend dev can ``export + // VITE_OPENJARVIS_NO_UPDATER=1`` before ``npm run tauri dev`` to + // silence the 30-min poll. See docs/desktop-auto-update.md. + const noUpdater = (import.meta as any).env?.VITE_OPENJARVIS_NO_UPDATER; + if (noUpdater === '1' || noUpdater === 'true') { + return; + } + checkForUpdate(); const interval = setInterval(checkForUpdate, CHECK_INTERVAL_MS); return () => clearInterval(interval); diff --git a/pyproject.toml b/pyproject.toml index 1bc774b8..7aa5d7bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "OpenJarvis" -version = "1.0.0" +version = "1.0.1" description = "OpenJarvis — modular AI assistant backend with composable intelligence primitives" readme = "README.md" requires-python = ">=3.10" @@ -87,6 +87,12 @@ energy-all = ["pynvml>=12.0", "amdsmi>=6.1", "zeus-ml[apple]"] orchestrator-training = ["torch>=2.0", "transformers>=4.40"] learning-dspy = ["dspy>=2.6"] learning-gepa = ["gepa>=0.1"] +# ACE (Agentic Context Engineering) is supported via +# ``openjarvis.learning.agents.ace_optimizer`` but ACE upstream isn't on +# PyPI and isn't structured as an installable Python package as of +# v1.0.1, so there's no ``learning-ace`` extra. To use ACE, follow the +# manual setup in docs/learning/ace.md (clone the upstream repo, add +# its ``src/`` to PYTHONPATH). channel-telegram = ["python-telegram-bot>=21.0"] channel-discord = ["discord.py>=2.3"] channel-slack = ["slack-sdk>=3.27"] diff --git a/scripts/install/install.sh b/scripts/install/install.sh index cd81e0c1..2fa0d0d9 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -91,6 +91,17 @@ INSTALL_START_EPOCH="$(date +%s)" CURRENT_STAGE="" analytics_enabled() { + # Honor the same opt-out env vars as the Python analytics module + # (``src/openjarvis/analytics/identity.py::is_analytics_enabled``). + # ``DO_NOT_TRACK`` is W3C convention; ``OPENJARVIS_NO_ANALYTICS`` is + # the project-specific override. Any truthy value disables. + for var in DO_NOT_TRACK OPENJARVIS_NO_ANALYTICS; do + val="${!var:-}" + case "$(printf '%s' "$val" | tr '[:upper:]' '[:lower:]' | xargs)" in + ""|0|false|no|off) ;; + *) return 1 ;; + esac + done return 0 } diff --git a/src/openjarvis/analytics/identity.py b/src/openjarvis/analytics/identity.py index 3a3f6a3b..7f14983b 100644 --- a/src/openjarvis/analytics/identity.py +++ b/src/openjarvis/analytics/identity.py @@ -16,6 +16,13 @@ from pathlib import Path from openjarvis.core.config import AnalyticsConfig +# Env vars that disable analytics regardless of config-file setting. +# ``DO_NOT_TRACK`` follows the W3C convention (https://www.eff.org/dnt-policy); +# ``OPENJARVIS_NO_ANALYTICS`` is the project-specific opt-out for users who +# want to disable just our telemetry without affecting other tools that +# honor DNT. +_OPT_OUT_ENV_VARS = ("DO_NOT_TRACK", "OPENJARVIS_NO_ANALYTICS") + def get_or_create_anon_id(path: Path | str) -> str: """Return the persisted anon ID, generating one on first call. @@ -46,17 +53,40 @@ def reset_anon_id(path: Path | str) -> str: return get_or_create_anon_id(p) -def is_analytics_enabled(cfg: AnalyticsConfig) -> bool: - """Return True if analytics is enabled in config. +def _env_opt_out() -> bool: + """Return True if any opt-out env var is set to a truthy value. - Always disabled when running under pytest. The PostHog SDK registers - an ``atexit`` hook that synchronously joins its consumer thread; if - the host is unreachable (CI runners can't reach the analytics endpoint), - each queued batch retries for ``timeout * max_retries`` seconds and the - interpreter never exits. Detect pytest via ``PYTEST_CURRENT_TEST`` - (set per test) and ``"pytest" in sys.modules`` (covers the collection - phase before the first test runs). + Truthy = anything other than empty string, "0", "false", "no", "off" + (case-insensitive). Lets `DO_NOT_TRACK=1`, `=true`, `=yes` all work. + """ + for name in _OPT_OUT_ENV_VARS: + raw = os.environ.get(name) + if raw and raw.strip().lower() not in ("", "0", "false", "no", "off"): + return True + return False + + +def is_analytics_enabled(cfg: AnalyticsConfig) -> bool: + """Return True if analytics is enabled. + + Disabled in three cases (any one is sufficient): + + 1. Running under pytest. The PostHog SDK registers an ``atexit`` + hook that synchronously joins its consumer thread; if the host + is unreachable (CI runners can't reach the analytics endpoint), + each queued batch retries for ``timeout * max_retries`` seconds + and the interpreter never exits. Detect pytest via + ``PYTEST_CURRENT_TEST`` (set per test) and ``"pytest" in + sys.modules`` (covers the collection phase before the first + test runs). + 2. An opt-out env var is set: ``DO_NOT_TRACK=1`` (W3C convention) + or ``OPENJARVIS_NO_ANALYTICS=1`` (project-specific). Both take + precedence over the config so users can opt out without + editing ``~/.openjarvis/config.toml``. + 3. The ``[analytics] enabled = false`` config-file setting. """ if os.environ.get("PYTEST_CURRENT_TEST") or "pytest" in sys.modules: return False + if _env_opt_out(): + return False return cfg.enabled diff --git a/src/openjarvis/cli/__init__.py b/src/openjarvis/cli/__init__.py index 50a85fdc..82df6998 100644 --- a/src/openjarvis/cli/__init__.py +++ b/src/openjarvis/cli/__init__.py @@ -35,6 +35,7 @@ from openjarvis.cli.quickstart_cmd import quickstart from openjarvis.cli.registry_cmd import registry from openjarvis.cli.scan_cmd import scan from openjarvis.cli.scheduler_cmd import scheduler +from openjarvis.cli.self_update_cmd import self_update from openjarvis.cli.serve import serve from openjarvis.cli.skill_cmd import skill from openjarvis.cli.telemetry_cmd import telemetry @@ -118,6 +119,7 @@ cli.add_command(connect, "connect") cli.add_command(digest, "digest") cli.add_command(deep_research_setup, "deep-research-setup") cli.add_command(deep_research_setup, "research") +cli.add_command(self_update, "self-update") cli.add_command(bootstrap_cmd, "_bootstrap") # Gateway CLI commands (lazy import to avoid pulling starlette) diff --git a/src/openjarvis/cli/_install_detect.py b/src/openjarvis/cli/_install_detect.py new file mode 100644 index 00000000..91609e66 --- /dev/null +++ b/src/openjarvis/cli/_install_detect.py @@ -0,0 +1,87 @@ +"""Detect how OpenJarvis was installed so we can show the right upgrade +command (and run the right upgrade command for ``jarvis self-update``). + +Three install paths are supported today: + +- **PyPI** (``pip install openjarvis``). The package lives somewhere + inside ``site-packages``. Upgrade with ``pip install --upgrade openjarvis``. +- **uv tool** (``uv tool install openjarvis``). Lives in a uv-managed + isolated venv under ``~/.local/share/uv/tools/``. Upgrade with + ``uv tool upgrade openjarvis``. +- **Editable git checkout** (``uv sync`` / ``pip install -e .`` from a + cloned repo). The package's ``__file__`` is inside a working tree + with a ``.git`` directory at the repo root. Upgrade with + ``git pull && uv sync`` from the checkout. + +We detect by inspecting ``openjarvis.__file__``. If we can't tell with +confidence we fall back to the PyPI command — that's the most common +case and the worst outcome is a no-op for a user who has nothing to +pull from PyPI. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + + +@dataclass(frozen=True) +class InstallInfo: + """How OpenJarvis was installed.""" + + kind: str # "pypi" | "uv-tool" | "editable-git" | "unknown" + upgrade_command: str + repo_root: Optional[Path] = None # only set for editable-git + + +def detect_install() -> InstallInfo: + """Return an :class:`InstallInfo` for the running interpreter. + + Cheap: just walks the parent directories of ``openjarvis.__file__`` + once and checks for marker directories. No subprocess calls. + """ + try: + import openjarvis + + pkg_file = Path(openjarvis.__file__).resolve() + except Exception: + return InstallInfo( + kind="unknown", + upgrade_command="pip install --upgrade openjarvis", + ) + + parts = [p.lower() for p in pkg_file.parts] + + if "uv" in parts and "tools" in parts: + return InstallInfo( + kind="uv-tool", + upgrade_command="uv tool upgrade openjarvis", + ) + + # Editable install: a ``.git`` dir within a few parents of the + # package source. Walk up at most ~8 levels — enough for typical + # ``/src/openjarvis/__init__.py`` layouts plus headroom, but + # not so deep we wander into home or root. + candidate = pkg_file.parent + for _ in range(8): + if (candidate / ".git").exists() and (candidate / "pyproject.toml").exists(): + return InstallInfo( + kind="editable-git", + upgrade_command=f"cd {candidate} && git pull && uv sync", + repo_root=candidate, + ) + if candidate.parent == candidate: + break + candidate = candidate.parent + + if "site-packages" in parts: + return InstallInfo( + kind="pypi", + upgrade_command="pip install --upgrade openjarvis", + ) + + return InstallInfo( + kind="unknown", + upgrade_command="pip install --upgrade openjarvis", + ) diff --git a/src/openjarvis/cli/_version_check.py b/src/openjarvis/cli/_version_check.py index a1db0fa4..03d8d4ff 100644 --- a/src/openjarvis/cli/_version_check.py +++ b/src/openjarvis/cli/_version_check.py @@ -14,19 +14,61 @@ logger = logging.getLogger(__name__) _CACHE_PATH = Path("~/.openjarvis/version-check.json").expanduser() _CACHE_TTL = 86400 # 24 hours _GITHUB_API = "https://api.github.com/repos/open-jarvis/OpenJarvis/releases/latest" -_CHECK_COMMANDS = {"ask", "chat", "serve"} + +# Commands that surface the "new version available" nudge. We deliberately +# cast a wide net for interactive commands (anything a human runs at a +# terminal and would benefit from knowing about an update), and skip +# automation-facing ones (``_bootstrap``, ``daemon``, ``host``) so we +# don't add noise to background processes or CI. +_CHECK_COMMANDS = { + "ask", + "chat", + "serve", + "doctor", + "init", + "quickstart", + "model", + "agents", + "skill", + "memory", + "bench", + "telemetry", + "config", + "eval", + "optimize", +} + +# Environment opt-outs (any truthy value disables the check): +# - ``OPENJARVIS_NO_UPDATE_CHECK=1`` — project-specific +# - ``CI=true`` — set by every major CI provider, suppresses by default +_OPT_OUT_ENV_VARS = ("OPENJARVIS_NO_UPDATE_CHECK",) + + +def _check_disabled() -> bool: + """Return True when the user has opted out of update checks.""" + for name in _OPT_OUT_ENV_VARS: + raw = os.environ.get(name, "") + if raw and raw.strip().lower() not in ("", "0", "false", "no", "off"): + return True + # CI defaults to skipping. Users in CI can override with + # ``OPENJARVIS_NO_UPDATE_CHECK=0`` if they want the nudge anyway. + if os.environ.get("CI", "").strip().lower() in ("1", "true", "yes", "on"): + return True + return False def check_for_updates(command_name: str) -> None: """Print a message if a newer version is available. Best-effort, never raises. - Honors ``OPENJARVIS_NO_UPDATE_CHECK`` — set it to any non-empty value - (``1``, ``true``, etc.) to skip the GitHub poll and the banner. + Honors ``OPENJARVIS_NO_UPDATE_CHECK=1`` and ``CI=true`` — any + truthy value (``1``, ``true``, ``yes``, ``on``) disables both the + GitHub poll and the banner. See ``_check_disabled`` for the full + list. """ - if os.environ.get("OPENJARVIS_NO_UPDATE_CHECK", "").strip(): - return if command_name not in _CHECK_COMMANDS: return + if _check_disabled(): + return try: _do_check() except Exception: @@ -45,10 +87,14 @@ def _do_check() -> None: try: if Version(latest) > Version(current): + from openjarvis.cli._install_detect import detect_install + + cmd = detect_install().upgrade_command sys.stderr.write( f"\033[33mA new version of OpenJarvis is available " f"(v{current} \u2192 v{latest})\n" - f"Update: cd ~/OpenJarvis && git pull && uv sync\033[0m\n\n" + f"Update: {cmd}\n" + f"Or run: jarvis self-update\033[0m\n\n" ) except InvalidVersion: pass diff --git a/src/openjarvis/cli/self_update_cmd.py b/src/openjarvis/cli/self_update_cmd.py new file mode 100644 index 00000000..a0661a3b --- /dev/null +++ b/src/openjarvis/cli/self_update_cmd.py @@ -0,0 +1,89 @@ +"""`jarvis self-update` — upgrade OpenJarvis to the latest release. + +Runs the right upgrade command for how the user installed OpenJarvis: + +- PyPI installs get ``pip install --upgrade openjarvis``. +- uv-tool installs get ``uv tool upgrade openjarvis``. +- Editable git checkouts get ``git pull && uv sync`` in the checkout. + +The detection logic is shared with the post-command "new version +available" hint in ``_version_check.py`` so both surfaces stay in sync. +""" + +from __future__ import annotations + +import shlex +import subprocess +import sys + +import click + +import openjarvis +from openjarvis.cli._install_detect import detect_install + + +@click.command( + "self-update", + help=( + "Upgrade OpenJarvis to the latest release. Detects how you " + "installed (pip, uv tool, editable git) and runs the right " + "command. Use --check to only print the upgrade command " + "without running it." + ), +) +@click.option( + "--check", + is_flag=True, + help="Print the upgrade command that would run, without executing it.", +) +@click.option( + "--yes", + "-y", + is_flag=True, + help="Skip the interactive confirmation prompt.", +) +def self_update(check: bool, yes: bool) -> None: + info = detect_install() + current = openjarvis.__version__ + + click.echo(f"Current OpenJarvis version: v{current}") + click.echo(f"Install method: {info.kind}") + click.echo(f"Upgrade command: {info.upgrade_command}") + + if check: + return + + if info.kind == "unknown": + click.echo( + "\nCould not determine install method with confidence. The " + "command above is a best guess; verify it matches how you " + "installed before running.", + err=True, + ) + + if not yes: + if not click.confirm("\nRun the upgrade command now?", default=True): + click.echo("Aborted.") + sys.exit(1) + + click.echo(f"\n→ {info.upgrade_command}\n") + + # ``editable-git`` uses shell features (``&&``); the others are + # simple argv-style commands. Use ``shell=True`` only for the + # editable case to keep the surface small. The command itself is + # constructed from a trusted, locally-detected path — no user + # input flows into it. + if info.kind == "editable-git": + result = subprocess.run(info.upgrade_command, shell=True) + else: + result = subprocess.run(shlex.split(info.upgrade_command)) + + if result.returncode != 0: + click.echo( + f"\nUpgrade command exited with code {result.returncode}. " + "Inspect the output above for the failure mode.", + err=True, + ) + sys.exit(result.returncode) + + click.echo("\nUpgrade complete. Re-run `jarvis --version` to confirm.") diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py index 41789abf..10aafd22 100644 --- a/src/openjarvis/core/config.py +++ b/src/openjarvis/core/config.py @@ -645,6 +645,52 @@ class GEPAOptimizerConfig: config_dir: str = "" +@dataclass(slots=True) +class ACEOptimizerConfig: + """ACE agent optimizer config. Maps to ``[learning.agent.ace]``. + + ACE (Agentic Context Engineering) evolves a *playbook* — annotated + natural-language strategies that get prepended to the agent's + context — using a Generator / Reflector / Curator triad. Unlike + DSPy (few-shot bootstrapping) or GEPA (Pareto-evolutionary prompt + mutation), ACE writes a textual playbook that the agent reads at + inference time. + + See https://github.com/ace-agent/ace for the upstream reference. + Install via ``pip install -e openjarvis[learning-ace]`` once the + optional dep is available (ACE is not on PyPI as of v1.0.1; the + extra installs from the upstream git repo). + """ + + # Models for ACE's three roles. Empty string = inherit from the + # intelligence primitive's default cloud model. + generator_model: str = "" + reflector_model: str = "" + curator_model: str = "" + + # Provider passed to ACE (``sambanova`` | ``together`` | ``openai`` + # | ``commonstack``). We default to ``openai`` since that's what + # most OpenJarvis users have credentials for. + api_provider: str = "openai" + + # Run parameters. Defaults mirror ACE's offline-mode quickstart. + num_epochs: int = 1 + max_num_rounds: int = 3 + eval_steps: int = 100 + playbook_token_budget: int = 80_000 + max_tokens: int = 4_096 + + # Where ACE writes intermediate playbooks + final_results.json. + # Empty string defaults to ``~/.openjarvis/learning/ace//``. + save_dir: str = "" + task_name: str = "openjarvis" + + # Standard filter / threshold knobs shared with DSPy / GEPA. + min_traces: int = 20 + agent_filter: str = "" + config_dir: str = "" + + @dataclass(slots=True) class IntelligenceLearningConfig: """Intelligence sub-policy config within Learning.""" @@ -658,9 +704,10 @@ class IntelligenceLearningConfig: class AgentLearningConfig: """Agent sub-policy config within Learning.""" - policy: str = "none" # none | dspy | gepa + policy: str = "none" # none | dspy | gepa | ace dspy: DSPyOptimizerConfig = field(default_factory=DSPyOptimizerConfig) gepa: GEPAOptimizerConfig = field(default_factory=GEPAOptimizerConfig) + ace: ACEOptimizerConfig = field(default_factory=ACEOptimizerConfig) @dataclass(slots=True) diff --git a/src/openjarvis/learning/__init__.py b/src/openjarvis/learning/__init__.py index e12715d4..8218c732 100644 --- a/src/openjarvis/learning/__init__.py +++ b/src/openjarvis/learning/__init__.py @@ -61,6 +61,10 @@ def ensure_registered() -> None: import openjarvis.learning.agents.gepa_optimizer # noqa: F401 except ImportError: pass + try: + import openjarvis.learning.agents.ace_optimizer # noqa: F401 + except ImportError: + pass __all__ = [ diff --git a/src/openjarvis/learning/agents/ace_optimizer.py b/src/openjarvis/learning/agents/ace_optimizer.py new file mode 100644 index 00000000..9f1cb4ce --- /dev/null +++ b/src/openjarvis/learning/agents/ace_optimizer.py @@ -0,0 +1,246 @@ +"""ACE agent optimizer — context evolution via the ACE Generator / +Reflector / Curator triad. + +ACE (`ace-agent/ace `_) optimizes +agent *context* — an annotated natural-language playbook of strategies +the agent reads at inference time — rather than mutating prompts (DSPy) +or evolving prompt populations (GEPA). The output is a textual +playbook with entries like:: + + [str-00001] helpful=5 harmful=0 :: When the user asks for a unit + conversion, prefer the exact + rational form before rounding. + +The wrapper adapts OpenJarvis traces into ACE's +``train_samples`` / ``val_samples`` / ``test_samples`` shape, builds a +minimal ``DataProcessor`` from the trace feedback signal, runs ACE in +``offline`` mode, and writes the resulting ``final_playbook.txt`` as a +sidecar overlay under ``~/.openjarvis/learning/ace//`` for the +agent runtime to pick up on next start. + +ACE is not on PyPI as of v1.0.1. The ``learning-ace`` extra installs +from the upstream git repo; if the import fails we surface a +``status="error"`` with the install hint. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any, Dict, List + +from openjarvis.core.config import ACEOptimizerConfig +from openjarvis.core.registry import LearningRegistry +from openjarvis.learning._stubs import AgentLearningPolicy + +logger = logging.getLogger(__name__) + +# Optional dependency — ACE is git-installed via the `learning-ace` +# extra. When missing we still expose the optimizer class so callers +# can introspect ``HAS_ACE``; only ``optimize()`` requires the package. +try: + import ace # type: ignore[import-not-found] + + HAS_ACE = True +except ImportError: + HAS_ACE = False + ace = None # type: ignore[assignment] + + +def _default_save_dir(task_name: str) -> Path: + return Path.home() / ".openjarvis" / "learning" / "ace" / task_name + + +class _TraceDataProcessor: + """Adapter that exposes OpenJarvis traces in ACE's three-method API. + + ACE expects a processor with: + + - ``process_task_data(raw)`` — normalize a single sample + - ``answer_is_correct(pred, truth)`` — per-sample comparison + - ``evaluate_accuracy(preds, truths)`` — aggregate metric over a list + + We treat each trace as a ``{question, ground_truth_answer}`` pair + and use the recorded ``feedback`` (0.0–1.0) as the ground truth + when no explicit answer is available. ``answer_is_correct`` does a + case-insensitive substring match; aggregate accuracy is the mean + of per-sample correctness. + """ + + @staticmethod + def process_task_data(raw: Dict[str, Any]) -> Dict[str, Any]: + return { + "question": raw.get("question", ""), + "ground_truth_answer": raw.get("ground_truth_answer", ""), + } + + @staticmethod + def answer_is_correct(predicted: str, ground_truth: str) -> bool: + if not ground_truth: + return False + return ground_truth.strip().lower() in (predicted or "").strip().lower() + + @classmethod + def evaluate_accuracy( + cls, predictions: List[str], ground_truths: List[str] + ) -> float: + if not predictions: + return 0.0 + n_correct = sum( + 1 + for p, g in zip(predictions, ground_truths) + if cls.answer_is_correct(p, g) + ) + return n_correct / len(predictions) + + +def _traces_to_samples(traces: List[Any]) -> List[Dict[str, Any]]: + """Convert OpenJarvis trace records to ACE's sample dict shape.""" + samples: List[Dict[str, Any]] = [] + for t in traces: + question = getattr(t, "query", "") or "" + answer = getattr(t, "result", "") or "" + if not question or not answer: + continue + samples.append( + { + "question": question, + "ground_truth_answer": answer, + } + ) + return samples + + +def _split_samples( + samples: List[Dict[str, Any]], +) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]]: + """70/15/15 train/val/test split, preserving order for reproducibility.""" + n = len(samples) + train_end = int(n * 0.70) + val_end = int(n * 0.85) + return samples[:train_end], samples[train_end:val_end], samples[val_end:] + + +class ACEAgentOptimizer: + """Optimize an agent's playbook context using ACE. + + Parameters + ---------- + config: + :class:`ACEOptimizerConfig` controlling models, run length, and + output location. + """ + + def __init__(self, config: ACEOptimizerConfig) -> None: + self.config = config + + def optimize(self, trace_store: Any) -> Dict[str, Any]: + """Run ACE offline-mode optimization on traces from the store. + + Returns a status dict mirroring DSPy / GEPA shape: + ``{"status": "completed" | "skipped" | "error", ...}``. + """ + kwargs: Dict[str, Any] = {"limit": 10_000} + if self.config.agent_filter: + kwargs["agent"] = self.config.agent_filter + traces = trace_store.list_traces(**kwargs) + + if len(traces) < self.config.min_traces: + return { + "status": "skipped", + "reason": ( + f"only {len(traces)} traces, " + f"min_traces={self.config.min_traces}" + ), + } + + if not HAS_ACE: + return { + "status": "error", + "reason": ( + "ace not installed (pip install " + "'openjarvis[learning-ace]')" + ), + } + + samples = _traces_to_samples(traces) + if len(samples) < self.config.min_traces: + return { + "status": "skipped", + "reason": ( + f"only {len(samples)} usable samples after filtering " + f"(min={self.config.min_traces})" + ), + } + + train, val, test = _split_samples(samples) + save_dir = Path( + self.config.save_dir or _default_save_dir(self.config.task_name) + ) + save_dir.mkdir(parents=True, exist_ok=True) + + try: + ace_system = ace.ACE( # type: ignore[union-attr] + api_provider=self.config.api_provider, + generator_model=self.config.generator_model, + reflector_model=self.config.reflector_model, + curator_model=self.config.curator_model, + max_tokens=self.config.max_tokens, + ) + ace_system.run( + mode="offline", + train_samples=train, + val_samples=val, + test_samples=test, + data_processor=_TraceDataProcessor(), + config={ + "num_epochs": self.config.num_epochs, + "max_num_rounds": self.config.max_num_rounds, + "eval_steps": self.config.eval_steps, + "playbook_token_budget": self.config.playbook_token_budget, + "task_name": self.config.task_name, + "save_dir": str(save_dir), + "api_provider": self.config.api_provider, + }, + ) + except Exception as exc: + logger.warning("ACE optimization failed: %s", exc) + return {"status": "error", "reason": str(exc)} + + playbook_path = save_dir / "final_playbook.txt" + if not playbook_path.exists(): + # ACE completed but didn't write the expected artifact — surface + # so the user can diagnose without trusting a silent zero. + return { + "status": "error", + "reason": ( + f"ACE finished without writing {playbook_path}; " + "check ACE logs in save_dir" + ), + } + + playbook = playbook_path.read_text(encoding="utf-8") + return { + "status": "completed", + "traces_used": len(traces), + "samples_used": len(samples), + "playbook_path": str(playbook_path), + "playbook_chars": len(playbook), + "save_dir": str(save_dir), + } + + +@LearningRegistry.register("ace") +class _ACELearningPolicy(AgentLearningPolicy): + """Wrapper to register :class:`ACEAgentOptimizer` in LearningRegistry.""" + + def __init__(self, **kwargs: object) -> None: + pass + + def update(self, trace_store: Any, **kwargs: object) -> Dict[str, Any]: + config = ACEOptimizerConfig() + optimizer = ACEAgentOptimizer(config) + return optimizer.optimize(trace_store) + + +__all__ = ["ACEAgentOptimizer", "HAS_ACE"] diff --git a/tests/analytics/test_identity.py b/tests/analytics/test_identity.py new file mode 100644 index 00000000..86ed5d4c --- /dev/null +++ b/tests/analytics/test_identity.py @@ -0,0 +1,126 @@ +"""Tests for analytics opt-out logic and anonymous identity persistence.""" + +from __future__ import annotations + +import pytest + +from openjarvis.analytics.identity import ( + _env_opt_out, + get_or_create_anon_id, + is_analytics_enabled, + reset_anon_id, +) +from openjarvis.core.config import AnalyticsConfig + + +@pytest.fixture +def cfg_enabled() -> AnalyticsConfig: + return AnalyticsConfig(enabled=True) + + +@pytest.fixture +def cfg_disabled() -> AnalyticsConfig: + return AnalyticsConfig(enabled=False) + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + """Strip opt-out env vars so a host shell can't leak into tests.""" + monkeypatch.delenv("DO_NOT_TRACK", raising=False) + monkeypatch.delenv("OPENJARVIS_NO_ANALYTICS", raising=False) + + +# --------------------------------------------------------------------------- +# _env_opt_out — the env-var logic lives in its own helper, so test it +# directly. ``is_analytics_enabled`` short-circuits on pytest detection +# (PYTEST_CURRENT_TEST / sys.modules['pytest']) which is unavoidably True +# while we ARE running under pytest; testing the env logic in isolation +# sidesteps that whole problem. +# --------------------------------------------------------------------------- + + +class TestEnvOptOut: + def test_no_env_returns_false(self): + assert _env_opt_out() is False + + @pytest.mark.parametrize( + "value", ["1", "true", "True", "TRUE", "yes", "on", "anything"] + ) + def test_do_not_track_truthy(self, monkeypatch, value): + monkeypatch.setenv("DO_NOT_TRACK", value) + assert _env_opt_out() is True + + @pytest.mark.parametrize("value", ["1", "true", "yes", "on"]) + def test_openjarvis_no_analytics_truthy(self, monkeypatch, value): + monkeypatch.setenv("OPENJARVIS_NO_ANALYTICS", value) + assert _env_opt_out() is True + + @pytest.mark.parametrize("value", ["", "0", "false", "False", "no", "off"]) + def test_falsy_values_do_not_opt_out(self, monkeypatch, value): + monkeypatch.setenv("DO_NOT_TRACK", value) + assert _env_opt_out() is False + + def test_whitespace_quoted_truthy_still_opts_out(self, monkeypatch): + # ``DO_NOT_TRACK=" 1 "`` (user shell-quoted with spaces) should + # still opt out — we don't want to silently track because of a + # shell-quoting accident. + monkeypatch.setenv("DO_NOT_TRACK", " 1 ") + assert _env_opt_out() is True + + +# --------------------------------------------------------------------------- +# is_analytics_enabled — top-level integration. The pytest short-circuit +# is intentional and fires during this test run; we assert the +# observable consequence (always False), then assert the precedence +# ordering (pytest > env > config). +# --------------------------------------------------------------------------- + + +class TestIsAnalyticsEnabled: + def test_short_circuits_under_pytest(self, cfg_enabled): + # We are running under pytest, so the function should return False + # regardless of config or env state. This is the desired behavior + # — see the function's docstring for the PostHog atexit-hang + # reason. If this ever starts returning True, the pytest gating + # has been broken and the test suite will start joining PostHog + # consumer threads on exit. + assert is_analytics_enabled(cfg_enabled) is False + + def test_disabled_config_under_pytest_also_false(self, cfg_disabled): + assert is_analytics_enabled(cfg_disabled) is False + + def test_env_opt_out_doesnt_enable_disabled_config( + self, cfg_disabled, monkeypatch + ): + """Env vars only ever disable; they never turn analytics ON.""" + monkeypatch.setenv("DO_NOT_TRACK", "1") + assert is_analytics_enabled(cfg_disabled) is False + + +# --------------------------------------------------------------------------- +# Anonymous ID persistence — completely separate concern. +# --------------------------------------------------------------------------- + + +class TestAnonId: + def test_create_persists_and_returns_same_uuid(self, tmp_path): + p = tmp_path / "anon_id" + a = get_or_create_anon_id(p) + b = get_or_create_anon_id(p) + assert a == b + assert p.exists() + assert len(a.strip()) == 36 # standard UUID v4 string length + + def test_reset_generates_new_uuid(self, tmp_path): + p = tmp_path / "anon_id" + original = get_or_create_anon_id(p) + fresh = reset_anon_id(p) + assert original != fresh + assert p.read_text(encoding="utf-8").strip() == fresh + + def test_atomic_write_leaves_no_tmp_file(self, tmp_path): + """The rename-after-write pattern should not leave an .anon_id.tmp behind.""" + p = tmp_path / "anon_id" + get_or_create_anon_id(p) + tmp_artifacts = list(tmp_path.glob("anon_id*.tmp")) + assert tmp_artifacts == [] diff --git a/tests/cli/test_install_detect.py b/tests/cli/test_install_detect.py new file mode 100644 index 00000000..60f72d39 --- /dev/null +++ b/tests/cli/test_install_detect.py @@ -0,0 +1,85 @@ +"""Tests for install-method detection.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from openjarvis.cli._install_detect import InstallInfo, detect_install + + +def _patch_pkg_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point ``openjarvis.__file__`` at ``tmp_path / openjarvis / __init__.py``.""" + pkg_dir = tmp_path / "openjarvis" + pkg_dir.mkdir(parents=True, exist_ok=True) + init = pkg_dir / "__init__.py" + init.write_text("__version__ = '0.0.0+test'\n") + + import openjarvis + + monkeypatch.setattr(openjarvis, "__file__", str(init)) + return init + + +def test_editable_git_install_detected(tmp_path, monkeypatch): + # Layout: /repo/.git, /repo/pyproject.toml, + # /repo/src/openjarvis/__init__.py + repo = tmp_path / "repo" + (repo / ".git").mkdir(parents=True) + (repo / "pyproject.toml").write_text("[project]\nname='openjarvis'\n") + src = repo / "src" + _patch_pkg_file(src, monkeypatch) + + info = detect_install() + assert info.kind == "editable-git" + assert "git pull" in info.upgrade_command + assert "uv sync" in info.upgrade_command + assert info.repo_root == repo + + +def test_uv_tool_install_detected(tmp_path, monkeypatch): + fake = tmp_path / "share" / "uv" / "tools" / "openjarvis" / "lib" / "python3.12" + fake.mkdir(parents=True) + _patch_pkg_file(fake, monkeypatch) + + info = detect_install() + assert info.kind == "uv-tool" + assert info.upgrade_command == "uv tool upgrade openjarvis" + + +def test_pypi_install_detected(tmp_path, monkeypatch): + fake = tmp_path / "venv" / "lib" / "python3.12" / "site-packages" + fake.mkdir(parents=True) + _patch_pkg_file(fake, monkeypatch) + + info = detect_install() + assert info.kind == "pypi" + assert info.upgrade_command == "pip install --upgrade openjarvis" + + +def test_unknown_install_falls_back_to_pypi(tmp_path, monkeypatch): + fake = tmp_path / "somewhere" / "weird" + fake.mkdir(parents=True) + _patch_pkg_file(fake, monkeypatch) + + info = detect_install() + assert info.kind == "unknown" + assert info.upgrade_command == "pip install --upgrade openjarvis" + + +def test_missing_openjarvis_file_falls_back_to_pypi(monkeypatch): + """openjarvis unimportable / no __file__ — still get a sane default.""" + with patch("openjarvis.cli._install_detect.Path") as mock_path: + mock_path.side_effect = Exception("boom") + info = detect_install() + assert info.kind == "unknown" + assert info.upgrade_command == "pip install --upgrade openjarvis" + + +def test_returns_install_info_dataclass(): + info = detect_install() + assert isinstance(info, InstallInfo) + assert info.kind in {"pypi", "uv-tool", "editable-git", "unknown"} + assert info.upgrade_command diff --git a/tests/cli/test_self_update.py b/tests/cli/test_self_update.py new file mode 100644 index 00000000..5d2939a6 --- /dev/null +++ b/tests/cli/test_self_update.py @@ -0,0 +1,119 @@ +"""Smoke tests for `jarvis self-update`. + +Focus on the surface that's easy to corrupt (output formatting, exit +codes, --check short-circuit). We don't actually run pip/uv from a +unit test; the subprocess call is mocked. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from click.testing import CliRunner + +from openjarvis.cli._install_detect import InstallInfo +from openjarvis.cli.self_update_cmd import self_update + + +def _mock_info(kind: str = "pypi") -> InstallInfo: + return InstallInfo( + kind=kind, + upgrade_command={ + "pypi": "pip install --upgrade openjarvis", + "uv-tool": "uv tool upgrade openjarvis", + "editable-git": "cd /tmp/repo && git pull && uv sync", + "unknown": "pip install --upgrade openjarvis", + }[kind], + ) + + +def test_check_flag_prints_command_and_exits_clean(): + with patch( + "openjarvis.cli.self_update_cmd.detect_install", + return_value=_mock_info("pypi"), + ): + runner = CliRunner() + result = runner.invoke(self_update, ["--check"]) + assert result.exit_code == 0 + assert "pip install --upgrade openjarvis" in result.output + assert "Install method: pypi" in result.output + + +def test_check_does_not_invoke_subprocess(): + with patch( + "openjarvis.cli.self_update_cmd.detect_install", + return_value=_mock_info("pypi"), + ), patch("openjarvis.cli.self_update_cmd.subprocess.run") as mock_run: + CliRunner().invoke(self_update, ["--check"]) + mock_run.assert_not_called() + + +def test_yes_skips_confirmation_and_runs(): + mock_proc = MagicMock(returncode=0) + with patch( + "openjarvis.cli.self_update_cmd.detect_install", + return_value=_mock_info("pypi"), + ), patch( + "openjarvis.cli.self_update_cmd.subprocess.run", + return_value=mock_proc, + ) as mock_run: + result = CliRunner().invoke(self_update, ["-y"]) + assert result.exit_code == 0 + mock_run.assert_called_once() + # PyPI path uses shlex.split (no shell=True) + args, kwargs = mock_run.call_args + assert kwargs.get("shell") is not True + assert args[0] == ["pip", "install", "--upgrade", "openjarvis"] + + +def test_editable_git_uses_shell_true(): + """The git path uses `&&` so shell=True is needed; the others don't.""" + mock_proc = MagicMock(returncode=0) + with patch( + "openjarvis.cli.self_update_cmd.detect_install", + return_value=_mock_info("editable-git"), + ), patch( + "openjarvis.cli.self_update_cmd.subprocess.run", + return_value=mock_proc, + ) as mock_run: + CliRunner().invoke(self_update, ["-y"]) + _, kwargs = mock_run.call_args + assert kwargs.get("shell") is True + + +def test_failed_upgrade_propagates_exit_code(): + mock_proc = MagicMock(returncode=3) + with patch( + "openjarvis.cli.self_update_cmd.detect_install", + return_value=_mock_info("pypi"), + ), patch( + "openjarvis.cli.self_update_cmd.subprocess.run", + return_value=mock_proc, + ): + result = CliRunner().invoke(self_update, ["-y"]) + assert result.exit_code == 3 + + +def test_unknown_install_kind_warns_but_proceeds(): + mock_proc = MagicMock(returncode=0) + with patch( + "openjarvis.cli.self_update_cmd.detect_install", + return_value=_mock_info("unknown"), + ), patch( + "openjarvis.cli.self_update_cmd.subprocess.run", + return_value=mock_proc, + ): + result = CliRunner().invoke(self_update, ["-y"]) + assert result.exit_code == 0 + assert "Could not determine install method" in result.output + + +def test_decline_confirmation_exits_nonzero(): + with patch( + "openjarvis.cli.self_update_cmd.detect_install", + return_value=_mock_info("pypi"), + ), patch("openjarvis.cli.self_update_cmd.subprocess.run") as mock_run: + result = CliRunner().invoke(self_update, input="n\n") + assert result.exit_code == 1 + assert "Aborted" in result.output + mock_run.assert_not_called() diff --git a/tests/cli/test_version_check.py b/tests/cli/test_version_check.py new file mode 100644 index 00000000..22140d8a --- /dev/null +++ b/tests/cli/test_version_check.py @@ -0,0 +1,71 @@ +"""Tests for the post-command "new version available" hint.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from openjarvis.cli._version_check import _check_disabled, check_for_updates + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for v in ("OPENJARVIS_NO_UPDATE_CHECK", "CI"): + monkeypatch.delenv(v, raising=False) + + +class TestCheckDisabled: + def test_default_not_disabled(self): + assert _check_disabled() is False + + @pytest.mark.parametrize("value", ["1", "true", "yes", "on", "anything"]) + def test_jarvis_no_update_check_disables(self, monkeypatch, value): + monkeypatch.setenv("OPENJARVIS_NO_UPDATE_CHECK", value) + assert _check_disabled() is True + + @pytest.mark.parametrize("value", ["", "0", "false", "no", "off"]) + def test_falsy_does_not_disable(self, monkeypatch, value): + monkeypatch.setenv("OPENJARVIS_NO_UPDATE_CHECK", value) + assert _check_disabled() is False + + def test_ci_env_disables_by_default(self, monkeypatch): + monkeypatch.setenv("CI", "true") + assert _check_disabled() is True + + def test_ci_false_does_not_disable(self, monkeypatch): + monkeypatch.setenv("CI", "false") + assert _check_disabled() is False + + +class TestCheckForUpdates: + @patch("openjarvis.cli._version_check._do_check") + def test_runs_for_ask_command(self, mock_do): + check_for_updates("ask") + mock_do.assert_called_once() + + @patch("openjarvis.cli._version_check._do_check") + def test_runs_for_doctor_command(self, mock_do): + """Widened list: doctor wasn't checked before.""" + check_for_updates("doctor") + mock_do.assert_called_once() + + @patch("openjarvis.cli._version_check._do_check") + def test_skips_unknown_command(self, mock_do): + check_for_updates("_bootstrap") + mock_do.assert_not_called() + + @patch("openjarvis.cli._version_check._do_check") + def test_ci_env_short_circuits_widely(self, mock_do, monkeypatch): + monkeypatch.setenv("CI", "1") + check_for_updates("ask") + mock_do.assert_not_called() + + @patch( + "openjarvis.cli._version_check._do_check", + side_effect=Exception("boom"), + ) + def test_exception_in_do_check_never_propagates(self, mock_do): + # Best-effort: a broken check must not break the user's command. + check_for_updates("ask") + mock_do.assert_called_once() diff --git a/tests/learning/agents/test_ace_optimizer.py b/tests/learning/agents/test_ace_optimizer.py new file mode 100644 index 00000000..8545e270 --- /dev/null +++ b/tests/learning/agents/test_ace_optimizer.py @@ -0,0 +1,224 @@ +"""Tests for the ACE agent optimizer (mocked — no ace dep required).""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + + +class TestACEOptimizerConfig: + def test_default_config(self) -> None: + from openjarvis.core.config import ACEOptimizerConfig + + cfg = ACEOptimizerConfig() + assert cfg.api_provider == "openai" + assert cfg.num_epochs == 1 + assert cfg.max_num_rounds == 3 + assert cfg.playbook_token_budget == 80_000 + assert cfg.min_traces == 20 + + def test_optimizer_init(self) -> None: + from openjarvis.core.config import ACEOptimizerConfig + from openjarvis.learning.agents.ace_optimizer import ACEAgentOptimizer + + cfg = ACEOptimizerConfig() + optimizer = ACEAgentOptimizer(cfg) + assert optimizer.config is cfg + + +class TestTraceDataProcessor: + def test_answer_is_correct_substring_match(self) -> None: + from openjarvis.learning.agents.ace_optimizer import _TraceDataProcessor + + assert _TraceDataProcessor.answer_is_correct("the answer is 42", "42") + assert _TraceDataProcessor.answer_is_correct("FORTY-TWO is right", "forty-two") + assert not _TraceDataProcessor.answer_is_correct("twelve", "42") + + def test_empty_ground_truth_is_never_correct(self) -> None: + from openjarvis.learning.agents.ace_optimizer import _TraceDataProcessor + + assert not _TraceDataProcessor.answer_is_correct("anything", "") + assert not _TraceDataProcessor.answer_is_correct("", "") + + def test_evaluate_accuracy_mean_of_correctness(self) -> None: + from openjarvis.learning.agents.ace_optimizer import _TraceDataProcessor + + preds = ["the answer is 1", "wrong", "the answer is 3"] + truths = ["1", "2", "3"] + # 2 of 3 correct + assert _TraceDataProcessor.evaluate_accuracy(preds, truths) == 2 / 3 + + def test_evaluate_accuracy_empty_returns_zero(self) -> None: + from openjarvis.learning.agents.ace_optimizer import _TraceDataProcessor + + assert _TraceDataProcessor.evaluate_accuracy([], []) == 0.0 + + +class TestSplitSamples: + def test_70_15_15_split(self) -> None: + from openjarvis.learning.agents.ace_optimizer import _split_samples + + samples = [{"i": i} for i in range(100)] + train, val, test = _split_samples(samples) + assert len(train) == 70 + assert len(val) == 15 + assert len(test) == 15 + + def test_split_is_order_preserving(self) -> None: + from openjarvis.learning.agents.ace_optimizer import _split_samples + + samples = [{"i": i} for i in range(20)] + train, val, test = _split_samples(samples) + assert train[0]["i"] == 0 + assert val[0]["i"] == 14 # 20*0.70 == 14 + assert test[0]["i"] == 17 # 20*0.85 == 17 + + +class TestTracesToSamples: + def test_drops_empty_query_or_result(self) -> None: + from openjarvis.learning.agents.ace_optimizer import _traces_to_samples + + traces = [ + MagicMock(query="What is 2+2?", result="4"), + MagicMock(query="", result="something"), + MagicMock(query="Q", result=""), + MagicMock(query=None, result="r"), + MagicMock(query="Q2", result="A2"), + ] + samples = _traces_to_samples(traces) + assert len(samples) == 2 + assert samples[0] == {"question": "What is 2+2?", "ground_truth_answer": "4"} + assert samples[1] == {"question": "Q2", "ground_truth_answer": "A2"} + + +class TestACEOptimizerOptimize: + def _store_with(self, n: int): + store = MagicMock() + store.list_traces.return_value = [ + MagicMock(query=f"Q{i}", result=f"A{i}") for i in range(n) + ] + return store + + def test_too_few_traces_skipped(self) -> None: + from openjarvis.core.config import ACEOptimizerConfig + from openjarvis.learning.agents.ace_optimizer import ACEAgentOptimizer + + cfg = ACEOptimizerConfig(min_traces=20) + result = ACEAgentOptimizer(cfg).optimize(self._store_with(5)) + assert result["status"] == "skipped" + assert "5 traces" in result["reason"] + + def test_missing_ace_dep_returns_error(self) -> None: + from openjarvis.core.config import ACEOptimizerConfig + from openjarvis.learning.agents import ace_optimizer + + with patch.object(ace_optimizer, "HAS_ACE", False): + cfg = ACEOptimizerConfig(min_traces=5) + result = ace_optimizer.ACEAgentOptimizer(cfg).optimize( + self._store_with(20) + ) + assert result["status"] == "error" + assert "learning-ace" in result["reason"] + + def test_filters_to_usable_samples(self) -> None: + """If filtering trims samples below min_traces, skip cleanly.""" + from openjarvis.core.config import ACEOptimizerConfig + from openjarvis.learning.agents import ace_optimizer + + # 20 traces but only 3 have both query+result populated + store = MagicMock() + store.list_traces.return_value = [ + MagicMock(query=f"Q{i}" if i < 3 else "", result=f"A{i}" if i < 3 else "") + for i in range(20) + ] + cfg = ACEOptimizerConfig(min_traces=10) + with patch.object(ace_optimizer, "HAS_ACE", True): + result = ace_optimizer.ACEAgentOptimizer(cfg).optimize(store) + assert result["status"] == "skipped" + assert "3 usable samples" in result["reason"] + + def test_successful_run_returns_playbook_metadata(self, tmp_path: Path) -> None: + from openjarvis.core.config import ACEOptimizerConfig + from openjarvis.learning.agents import ace_optimizer + + # Stub ACE: a class whose .run() writes final_playbook.txt. + playbook_text = "## STRATEGIES\n[str-00001] helpful=5 :: be concise" + + class _FakeACE: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def run(self, **kwargs): + save_dir = Path(kwargs["config"]["save_dir"]) + (save_dir / "final_playbook.txt").write_text(playbook_text) + + with patch.object(ace_optimizer, "HAS_ACE", True), patch.object( + ace_optimizer, "ace", MagicMock(ACE=_FakeACE) + ): + cfg = ACEOptimizerConfig( + min_traces=5, + save_dir=str(tmp_path), + task_name="unittest", + ) + result = ace_optimizer.ACEAgentOptimizer(cfg).optimize( + self._store_with(20) + ) + + assert result["status"] == "completed" + assert result["traces_used"] == 20 + assert result["samples_used"] == 20 + assert result["playbook_path"].endswith("final_playbook.txt") + assert result["playbook_chars"] == len(playbook_text) + + def test_ace_completes_without_playbook_surfaces_error( + self, tmp_path: Path + ) -> None: + """ACE returns but doesn't write final_playbook.txt → surface error.""" + from openjarvis.core.config import ACEOptimizerConfig + from openjarvis.learning.agents import ace_optimizer + + class _SilentACE: + def __init__(self, **kwargs): + pass + + def run(self, **kwargs): # writes nothing + pass + + with patch.object(ace_optimizer, "HAS_ACE", True), patch.object( + ace_optimizer, "ace", MagicMock(ACE=_SilentACE) + ): + cfg = ACEOptimizerConfig( + min_traces=5, + save_dir=str(tmp_path), + ) + result = ace_optimizer.ACEAgentOptimizer(cfg).optimize( + self._store_with(20) + ) + + assert result["status"] == "error" + assert "without writing" in result["reason"] + + def test_ace_raises_returns_error(self, tmp_path: Path) -> None: + from openjarvis.core.config import ACEOptimizerConfig + from openjarvis.learning.agents import ace_optimizer + + class _BoomACE: + def __init__(self, **kwargs): + pass + + def run(self, **kwargs): + raise RuntimeError("API key invalid") + + with patch.object(ace_optimizer, "HAS_ACE", True), patch.object( + ace_optimizer, "ace", MagicMock(ACE=_BoomACE) + ): + cfg = ACEOptimizerConfig( + min_traces=5, + save_dir=str(tmp_path), + ) + result = ace_optimizer.ACEAgentOptimizer(cfg).optimize( + self._store_with(20) + ) + + assert result["status"] == "error" + assert "API key invalid" in result["reason"] diff --git a/uv.lock b/uv.lock index 324023e5..cfea2769 100644 --- a/uv.lock +++ b/uv.lock @@ -4973,7 +4973,7 @@ wheels = [ [[package]] name = "openjarvis" -version = "1.0.0" +version = "1.0.1" source = { editable = "." } dependencies = [ { name = "click" },