diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 7fb64f044..ade505179 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -3430,6 +3430,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + [[package]] name = "hmac" version = "0.12.1" @@ -5384,6 +5393,7 @@ dependencies = [ "futures-util", "glob", "hex", + "hkdf", "hmac 0.12.1", "hostname", "hound", diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 7df029b44..b52442db3 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -2822,7 +2822,16 @@ pub fn run() { let run = req.run; input_app .run_on_main_thread(move || { - let _ = tx.send((run)()); + // Catch an enigo FFI panic so it can't unwind + // across the app main thread (which would be + // UB / abort). Convert it to a clean Err. + let result = std::panic::catch_unwind( + std::panic::AssertUnwindSafe(run), + ) + .unwrap_or_else(|_| { + Err("synthetic input panicked on the main thread".to_string()) + }); + let _ = tx.send(result); }) .map_err(|e| format!("run_on_main_thread dispatch failed: {e}"))?; rx.await diff --git a/app/src/utils/toolDefinitions.ts b/app/src/utils/toolDefinitions.ts index 01213cd1f..4dfa74ab0 100644 --- a/app/src/utils/toolDefinitions.ts +++ b/app/src/utils/toolDefinitions.ts @@ -45,6 +45,15 @@ export const TOOL_CATALOG: ToolDefinition[] = [ defaultEnabled: true, rustToolNames: ['ax_interact'], }, + { + id: 'automate', + displayName: 'App Automation', + description: + 'Accomplish a multi-step goal in an app in one go (e.g. "play a song in Music", "message someone in Slack") β€” the agent drives the UI step by step.', + category: 'System', + defaultEnabled: true, + rustToolNames: ['automate'], + }, { id: 'git_operations', displayName: 'Git Operations', diff --git a/docs/voice-system-actions.md b/docs/voice-system-actions.md index 92133e37c..f5ffa593e 100644 --- a/docs/voice-system-actions.md +++ b/docs/voice-system-actions.md @@ -268,6 +268,112 @@ test ... ok --- +### Change 1.14 β€” `automate(app, goal)`: Rust-driven multi-step automation πŸ”¨ In progress (M1 done) + +**Status:** πŸ”¨ In progress β€” **M1 + M2 + M3 shipped and M3 proven live on macOS**; M4–M6 pending. See **Phase 1.5** below and [`voice-automate-plan.md`](voice-automate-plan.md). + +**Agent-in-the-loop fixes (2026-06-03, from two live chat sessions):** +- **Mutations were off** β€” the agent correctly called `automate` but it (and `ax_interact`) refused because `computer_control.ax_interact_mutations=false`. Enabled it; also rewrote both refusal messages to point at **Settings β†’ Agent Access** instead of a config key (the agent had relayed "controls are locked down"). +- **Query mis-parse** β€” orchestrator goal `…search for "Highway to Hell" by AC/DC, and play it` made the after-"play" parser extract `"it"`. `extract_play_query` now prefers a **quoted title + `by `** and rejects bare pronouns. (Unit-tested with the exact failing goal.) +- **General loop spun** β€” pressed "Search" 11Γ— to budget exhaustion. Added a **no-progress guard**: 3 identical actions in a row β†’ abort. +- **Search-results timing** β€” the fast-path's retry burned out before catalog results rendered (`settle` reports count-stable while the network fetch is pending). Added a real, mockable `wait` between attempts (6 Γ— ~800ms). + +**M5 finding β€” AXEnabled is unreliable:** plumbed an `enabled` field end-to-end (Swift `axEnabled` β†’ `AXElement.enabled` β†’ Windows stub), but Apple Music reports its **pressable** search-result rows as `enabled=Some(false)`. Gating `pick_row` on it broke playback. So `enabled` is kept **informational only** (documented on the struct); matchers never skip on it. The better future actionability signal is AXPress-action support, not AXEnabled. + +**M4 β€” live progress in the notch (2026-06-03):** the notch indicator (originally PR #3166) was cherry-picked onto this branch (`feat(notch)` + fmt commits β†’ `notch_window.rs` NSPanel + `notch/NotchApp.tsx`, auto-shown on startup, transparent when idle). The `automate` loop and Music fast-path now call `overlay::publish_attention(...)` at each step (`Opening …`, `Searching Music for …`, `Pressing …`, `Typing …`, `Playing …`, plus done/fail), which the existing Socket.IO bridge emits as `overlay:attention` and the notch renders as a pill β€” so the user sees the automation happening live. Verified: app boots with `[notch-window] panel shown at top-center`; Tauri shell + frontend compile; 31 automate unit tests green. + +**M3 live proof (2026-06-03):** `music_fastpath_live` drives real Apple Music end-to-end and **hard-asserts `player state == playing`** β€” confirmed: pre-state `paused` β†’ post-state `playing`. Three bugs the live runs surfaced, all fixed + tested: +1. **Perceive filter was the whole multi-word query** β€” a substring filter can't match a full title β†’ now filters by the first strong token and `pick_row` does the token match. +2. **Search results render late (Β§1.13 race)** β€” retry perceive across up to 4 settles; `AC/DC` now percent-encodes correctly. +3. **False success: pressed the toolbar Play, not the song's** β€” the first run reported success but *nothing played*. AX probing showed the search screen has only the toolbar transport Play (empty queue β†’ silence); pressing the song row navigates to a detail page where a **second** Play appears (23β†’24 controls). Fix: capture the baseline Play count, **wait for the detail Play to render**, press it, then **verify real playback** via `osascript … player state` and retry (≀3Γ—). Added `verify_playing()` to `AutomateBackend` (macOS osascript; `None` elsewhere = best-effort). `automate` now only reports a play success when audio is actually playing β€” the false-success class (Β§1.11) is closed. + +**M3 β€” shipped (Music fast-path):** +- `src/openhuman/accessibility/app_fastpaths/{mod.rs,music.rs}` (new) β€” deterministic accelerators consulted by `run()` **before** the general loop. Music encodes the Β§1.11 proven sequence: launch β†’ open `music://…/search?term=…` β†’ settle β†’ press the song row (navigate) β†’ settle β†’ press the detail-page **Play**. Pure helpers `matches` / `extract_play_query` (handles "play X by Y", "launch Music and play …", "play X in Apple Music"). +- **Structurally different from the removed `play_music` tool (Β§1.13):** this is *internal* to `automate`, not a tool the LLM selects, and on any failure/`None` the loop **falls through** to the general model-driven path β€” so it can only help. Added `open_url` to the `AutomateBackend` trait (cross-platform opener; fast-path only). +- **Tests:** 9 unit (parser cases, full scripted sequence, no-row fallthrough, dispatch) + 1 `#[ignore]` macOS live test. **Live proof on a Mac:** `cargo test --lib music_fastpath_live -- --ignored --nocapture` (needs Music + Accessibility permission). + +**M2 β€” shipped (real settle):** +- `src/openhuman/accessibility/ax_interact.rs` β€” new `ax_wait_settled(app, stable_ms, timeout_ms)`: polls the app's interactive-element count and returns once it holds steady for `stable_ms` (or `timeout_ms` elapses). Portable β€” rides on `ax_list_elements`, which already cfg-dispatches (macOS AX / Windows UIA). Pure decision core `counts_settled(history, n)` extracted and unit-tested (5 non-OS-gated tests). +- `automate.rs` β€” `RealBackend::settle` now calls `ax_wait_settled` (240ms stable / 2s cap) via `spawn_blocking` instead of the M1 blind 450ms wait. This is the piece that removes the timing-race failure class (Β§1.11/Β§1.13): the next perceive always sees a settled tree. An AXObserver-driven settle can later sit behind the same signature. + +**M1 β€” shipped:** +- `src/openhuman/accessibility/automate.rs` (new) β€” the perceiveβ†’decideβ†’actβ†’settle loop, generic over an injectable `AutomateBackend` (so the model + AX + launcher are all mockable). Strict JSON action schema (`launch`/`list`/`press`/`set_value`/`done`/`fail`) with a one-shot repair retry on unparseable output (never acts on a hallucinated guess), a step budget (default 12), and a snapshot cap (40 elements) mirroring `ax_interact`'s anti-truncation guard. `RealBackend` calls the existing AX primitives + `launch_platform`, and routes decisions through the **fast tier** (`create_chat_provider("memory", …)` for now; a dedicated `automation_provider` knob is a follow-up). Settle is a short fixed wait in M1 (M2 makes it AXObserver-driven). +- `src/openhuman/tools/impl/computer/automate.rs` (new) β€” `AutomateTool { app, goal }`. Always `Dangerous` + `external_effect` (routes through the ApprovalGate); reuses `ax_interact`'s mutations opt-in (`computer_control.ax_interact_mutations`) and the shared `is_sensitive_app` denylist. +- Registered everywhere: `tools/ops.rs`, `tools/user_filter.rs` (`automate` family), `orchestrator/agent.toml` (`named`), `app/src/utils/toolDefinitions.ts` (Settings β†’ "App Automation"). +- **Tests:** 18 passing β€” loop happy path, navigate-then-activate, app override, budget exhaustion, repair retry (1 ok / 2 fail), explicit fail, non-fatal press failure, JSON parse (plain/fenced/garbage), snapshot cap/empty-hint; tool gating (missing args, mutations-off, sensitive-app refusal, schema). + +**Problem (the real-time bar):** The user's target is *"whatever I say happens, live, in front of me"* β€” e.g. *"Launch Music and play Numb by Linkin Park"* or *"open Slack and message Steven 'hi'"*. Today every UI step (`list` β†’ `set_value` β†’ `list` β†’ `press` …) is a **separate chat-LLM turn**. A Slack message is ~7 turns; at 1–3 s each that's 15–25 s, and each turn is a fresh chance to hit a timing race (1.13) or hallucinate. The heavy chat model is sitting *inside* the click loop β€” the wrong place for it. + +**Root causes (all four documented earlier in Phase 1):** +1. **Timing races** β€” `list`/`press` do a single AX walk with no settle/wait; the UI hasn't rendered yet (1.11/1.13). +2. **Navigate-then-activate is re-reasoned every call** β€” pressing a row selects; you must then press the action control. That logic lives in prose, so it's re-derived (often wrongly) each turn (1.10/1.11). +3. **Round-trip explosion** β€” N full chat turns per task = latency + cost + N chances to fail. +4. **Weak element model + no verification** β€” `list` returns flat `[role, label]`; `press` reports success on `AXAction == .success` even when nothing changed. + +**Design β€” take the chat model out of the click loop:** +- **New tool `automate { app, goal }`** β€” one call from the orchestrator. Rust then runs a tight **perceive β†’ act β†’ verify** loop internally: read a *filtered* AX snapshot β†’ pick the next action β†’ act β†’ **wait for the UI to settle (AXObserver, not fixed sleeps)** β†’ verify it took effect β†’ repeat until the goal is met or a step budget is hit. +- **A fast model drives the inner loop** (Haiku-class) with a *tiny* context: just the goal, the current small AX snapshot, and the last result β€” not the whole conversation. Each inner step is ~0.5–1 s and self-corrects, instead of one 3 s chat turn that falsely reports success. +- **Settle + verify in Rust** between steps β€” deterministic, kills the timing-race class in one place. +- **Native fast-paths for high-value apps** (skip the UI entirely where possible): + - **Music** β€” `music://` search URL β†’ AX play (already explored in 1.11), or AppleScript for library. + - **Spotify** β€” Web API search β†’ `spotify:track:…` URI + AppleScript `play`. Fully deterministic, no UI poking. + - **Slack** β€” deep link `slack://channel?…` to open the DM, then AX to type + send. + The general AX loop is the fallback for everything else. +- **Vision fallback for Electron/Chromium apps** (Slack, Discord, VS Code, Spotify-desktop) whose AX/UIA tree is partial (documented limitation). Slack needs accessibility enabled (`defaults write com.tinyspeck.slackmacgap AccessibilityEnabled -bool true`, relaunch). Where AX returns empty, fall back to **screenshot β†’ vision-locate β†’ guarded click**. This is the reverted CGEventPost path (1.8) β€” but it crashed only when events hit *OpenHuman's own focused CEF window*; a guarded click into a *different, foregrounded* app does not have that failure mode. +- **Stream progress events** to the UI / notch pill (PR #3166) so the user sees each step happen live. + +**Why a generic `automate`, not per-app tools:** Change 1.13 already established that app-specific tools (`play_music`) are the wrong abstraction. The abstraction that *is* generic is the **navigate-then-activate sequence itself** β€” `automate(app, goal)` encapsulates it once, in Rust, for every app, instead of asking the chat model to re-orchestrate fragile primitives every time. + +--- + +## Phase 1.5 β€” Reliable, real-time multi-step automation ⏳ Not Started + +> The bridge between today's `ax_interact` primitives and the always-on voice work. **Prerequisite for Phase 3** β€” fast voice routing into a slow/fragile action loop still feels slow. This is where "whatever I say happens, live" actually gets delivered. + +**Detailed implementation plan:** [`voice-automate-plan.md`](voice-automate-plan.md) β€” decided approach: **Rust inner loop + fast model**, first proof target **Music**. + +**Planned files:** +- `src/openhuman/accessibility/automate.rs` (new) β€” the perceiveβ†’actβ†’verify loop + settle/verify primitives, reusing `ax_interact` helpers. +- `src/openhuman/accessibility/app_fastpaths/` (new) β€” per-app deterministic paths (`music.rs`, `spotify.rs`, `slack.rs`), behind a generic dispatch. +- `src/openhuman/tools/impl/computer/automate.rs` (new) β€” `AutomateTool { app, goal }`, gated like `ax_interact` (mutations opt-in, sensitive-app denylist reused). +- macOS helper (`accessibility/helper.rs`) β€” AXObserver-based settle (`ax_wait_settled`) + post-action verify; richer element model (enabled/onscreen/actions). +- Vision fallback β€” screenshot via `accessibility/capture.rs` β†’ locate β†’ guarded click (only when AX tree is empty, target app foregrounded, never OpenHuman's own window). + +**Acceptance criteria:** +- [ ] One `automate{app, goal}` call performs a multi-step flow end-to-end (no per-step chat turns) +- [ ] Settle/verify removes the timing-race + false-success failure classes (1.11/1.13 do not recur) +- [ ] Music flow ("play ") works end-to-end via the inner loop +- [ ] Spotify + Slack fast-paths land their action deterministically +- [ ] Electron/partial-AX apps fall back to vision+guarded-click without the CEF crash +- [ ] Step-by-step progress streamed to the UI / notch indicator + +--- + +### Change 1.15 β€” Full computer control (mouse/keyboard/screenshot) βœ… Crash fixed (main-thread dispatch) + +**Status:** βœ… Keyboard/mouse now run on the app main thread β†’ no CEF crash. Screenshot downscales for inline view. Live: `[computer] registered main-thread synthetic-input executor` on boot. + +**The fix:** the crash was enigo's `TSMGetInputSourceProperty` running on a tokio worker (`_dispatch_assert_queue_fail`/SIGTRAP). macOS TSM must run on the main thread. New `tools/impl/computer/main_thread.rs` (`MainThreadInputOp` + `run_input_on_main`) dispatches each enigo op over the native registry to a handler the Tauri shell registers at startup, which runs it via `AppHandle::run_on_main_thread`. Keyboard + mouse tools no longer `spawn_blocking` enigo on a worker. Headless/CLI (no executor) returns a clear error instead of crashing. 66 keyboard/mouse tests green. + +**Goal:** make the agent fully autonomous β€” when the accessibility tree is empty (Electron apps: Slack/Discord/VS Code), fall back to vision + synthetic input. Enabled `computer_control.enabled`, added `mouse`/`keyboard`/`screenshot` to the orchestrator `named` list + `autonomy.auto_approve`, and taught `prompt.md` a keyboard-first ladder (foreground via `launch_app` β†’ `keyboard type` + Enter; Slack `Cmd+K` recipe). + +**Foreground-first:** `automate::run` now `open -a`s the target app at the very start, always, so AX/input hit the right window. + +**Screenshot fix:** oversized Retina captures were returned as "too large to base64-encode inline" (the model was blind). Now downscaled to a viewable JPEG (`downscale_to_jpeg`) with reported dimensions. + +**Root cause (now fixed) β€” `OpenHuman-2026-06-03-170058.ips`:** `EXC_BREAKPOINT/SIGTRAP` on a **`tokio-rt-worker`** thread: +```text +enigo::macos::get_layoutdependent_keycode β†’ TSMGetInputSourceProperty +β†’ dispatch_assert_queue β†’ _dispatch_assert_queue_fail β†’ SIGTRAP +``` +enigo's keyboard-layout lookup (`TSMGetInputSourceProperty`) **must run on the app's main thread**; the keyboard tool ran on a tokio worker β†’ macOS trapped. **Not** a focus issue (same Β§1.8 root cause); a frontmost-app guard would not have fixed it. + +**Fix applied:** enigo now runs on the Tauri **main thread** via `AppHandle::run_on_main_thread`, bridged to the core through a native-registry handler (see *The fix* above) and wrapped in `catch_unwind` so an FFI panic can't unwind across the main thread. (Alternative considered but not needed: TSM-free primitives β€” `CGEventKeyboardSetUnicodeString` for text + raw virtual keycodes for keys/hotkeys.) + +**Tests:** voice-actions + autonomy suite is exhaustive β€” 220 feature unit tests + a JSON-RPC E2E (`json_rpc_voice_server_settings_roundtrip_always_on_and_wake_word`). The E2E caught + fixed real gaps (`wake_word` missing from the get output and the update RPC path). Screenshot downscale unit-tested. + +--- + ## Windows port β€” app interaction πŸͺŸ βœ… Implemented Phase 1's app-interaction layer is now ported to Windows. The macOS path uses the @@ -416,19 +522,30 @@ Shipped on the Windows machine (2026-06-02): --- -## Phase 2 β€” Always-On Listening ⏳ Not Started +## Phase 2 β€” Always-On Listening βœ… Implemented > Continuous microphone listening without requiring a hotkey press. -**Planned files:** -- `src/openhuman/voice/always_on.rs` (new) β€” dedicated tokio task holding the mic open, running VAD, emitting utterances to the STT pipeline -- `src/openhuman/config/schema/voice_server.rs` β€” add `always_on_enabled: bool` config flag -- Privacy hook: pause always-on when screen is locked +**Shipped:** +- `src/openhuman/voice/always_on.rs` β€” pure `VadSegmenter` (onset / silence-hangover / min-speech / max-utterance, 7 unit tests) **plus** the continuous capture loop: a dedicated cpal thread streams 16 kHz mono frames β†’ segmenter β†’ each utterance is encoded (`encode_wav_16k`) β†’ `voice_transcribe_bytes` β†’ `publish_transcription` (so it reaches the agent's auto-send and the notch, exactly like hotkey dictation). Started at boot in `credentials::ops`. +- `src/openhuman/config/schema/voice_server.rs` β€” `always_on_enabled` flag + VAD tuning (`vad_onset_threshold`, `vad_hangover_ms`, `vad_min_speech_ms`, `vad_max_utterance_secs`), opt-in/off by default. +- **Settings toggle** β€” "Always-on listening" in the Voice debug panel, wired through `get/update_voice_server_settings` (RPC patch β†’ apply β†’ snapshot); i18n in en + all 13 locales. +- **Privacy hook** β€” `spawn_lock_watcher` pauses capture + resets the segmenter while the screen is locked (macOS via `CGSessionCopyCurrentDictionary`, null/type-safe FFI; other platforms never pause yet). +- Reused `audio_capture` helpers (`to_mono`/`resample`/`chunk_rms` made `pub(crate)` + new `encode_wav_16k`). **Acceptance criteria:** -- [ ] User can speak without pressing any hotkey -- [ ] VAD detects end of utterance and sends to agent -- [ ] Toggle in Settings β†’ Voice +- [x] User can speak without pressing any hotkey +- [x] VAD detects end of utterance and sends to agent +- [x] Toggle in Settings β†’ Voice + +**Wake word "Hey Tiny" (live-fix, 2026-06-03):** always-on now only delivers an utterance to the agent when its transcript contains the wake word (`config.voice_server.wake_word`, default "Hey Tiny"); the phrase is stripped and the remainder is sent. Tolerant match (case/punctuation/leading-filler), empty wake word = deliver everything. This is a **text-based** wake word (transcribe-then-gate) β€” a first cut of Phase 3's trigger phrase; it fixes the "sends every utterance" spam but still runs STT on all speech (an on-device audio wake-word model for efficiency is the Phase 3 follow-up). + +**Live-fixes found by running it:** +- **Toggle did nothing** β€” `always_on_enabled` wasn't in the `update_voice_server_settings` RPC *param schema*, so validation rejected it before the handler. Added it; the config RPC now also calls `always_on::start_if_enabled` so the toggle starts/idles capture **live** (runtime `ENABLED` gate, no restart). +- **`transcription failed: local ai is disabled`** β€” always-on used `voice_transcribe_bytes` (local whisper only). Now routes through `effective_stt_provider` + `create_stt_provider` (same factory dispatch as `voice.stt_dispatch`), honoring cloud STT. +- Toggle surfaced in the reachable **VoicePanel** (Settings β†’ Advanced β†’ AI β†’ Voice), not the hidden debug panel. + +**Pending live validation (mic-dependent, can't be CI-tested):** say "Hey Tiny, " and confirm the command reaches the agent; tune `vad_onset_threshold`/`vad_hangover_ms` to the user's mic + room. Windows/Linux screen-lock pause is a follow-up (no signal wired). --- @@ -457,6 +574,24 @@ Shipped on the Windows machine (2026-06-02): --- +## Fine-tuning backlog ⏳ (deferred until all phases complete) + +From live agent-in-the-loop testing on 2026-06-03 (grounded in `~/.openhuman/logs/openhuman.2026-06-03.log`, `session_raw/*.jsonl`, and the dev run: **keyboard=69 / mouse=0 / screenshot=10** tool calls; **26 wake matches vs 93 misses**; emit=true utterances ranged 0.7s–28s). The feature works but needs tuning. **Do not implement until Phases 3–4 land.** + +### F1 β€” Listening window too short for long commands +- **Observed:** `vad_hangover_ms = 800` closes an utterance on any pause > 0.8s, so multi-clause commands ("Hey Tiny, open Slack and message the team channel saying …") split across utterances β€” the tail lacks the wake word and is dropped. Compounded by the notch "Listening" pill TTL (2500ms) expiring mid-speech, so it *looks* like it stopped listening. +- **Resolve:** (a) raise `vad_hangover_ms` to ~1500ms; (b) **two-stage capture** β€” once the wake word is detected, open a dedicated longer command window (until a longer silence / N-second cap) instead of relying on a single VAD utterance; (c) keep the "Listening" pill alive for the whole utterance (extend/re-emit on each voiced frame, clear on `SpeechEnd`) so the notch reflects real mic state. + +### F2 β€” Agent uses keyboard only, never the mouse +- **Observed:** keyboard=69, mouse=0. Two causes: the orchestrator prompt is deliberately *keyboard-first*, **and** the downscaled screenshot's coordinates don't map to screen pixels β€” the capture is shrunk to ≀1568px while `mouse` expects absolute screen pixels (and Retina is 2Γ— points), so any coordinate read from the image clicks the wrong spot. Vision-driven clicking is therefore currently unsafe and the agent (correctly) avoids it. +- **Resolve:** (a) make `screenshot` emit a coordinate transform (shown WxH + real screen WxH + backing scale) **or** have `mouse` accept image-relative coordinates and convert internally; (b) once coordinates are trustworthy, soften the prompt so the agent uses screenshotβ†’mouse to click specific elements, not just keyboard. + +### F3 β€” No periodic screenshot/verify + foreground re-check +- **Observed:** the agent screenshots ad-hoc (0 in the last session); `automate` only foregrounds at the start. +- **Resolve:** in the `automate` loop **and** the orchestrator prompt β€” screenshot + verify at **start, after every ~3 actions, and at the end**; before each action confirm the frontmost app is the target and re-`launch_app` (foreground) it if not, then proceed. Fold the actual-vs-expected check into the loop's `verify` step. + +--- + ## Summary | Phase | Item | Status | @@ -472,9 +607,18 @@ Shipped on the Windows machine (2026-06-02): | 1 | AXUIElement app UI interaction (`ax_interact`) | βœ… Done | | 1 | Multi-step UI workflow guidance | βœ… Done | | 1 | Apple Music two-step play (navigateβ†’play) | βœ… Done (playback best-effort) | -| 2 | Always-on microphone loop | ⏳ Not started | -| 2 | `always_on_enabled` config flag | ⏳ Not started | -| 2 | Privacy hook (screen lock pause) | ⏳ Not started | +| 1 | `automate(app, goal)` Rust-driven loop (Change 1.14) | πŸ”¨ M1+M2+M3 done (37 tests; live proof pending) | +| 1.5 | M1: automate loop skeleton + tool | βœ… Done | +| 1.5 | M2: poll-until-stable settle | βœ… Done | +| 1.5 | M3: Music fast-path | βœ… Done (proven live on macOS) | +| 1.5 | Robustness: quoted-query parse + no-progress guard | βœ… Done (from live agent failures) | +| 1.5 | M4: progress streaming to notch | βœ… Done β€” notch cherry-picked in; automate streams live steps | +| 1.5 | M5: richer element model (`enabled`) | βœ… Plumbed; AXEnabled found unreliable β†’ informational only | +| 1.5 | Native fast-paths (Music/Spotify/Slack) | ⏳ Not started | +| 1.5 | Vision fallback for Electron apps | ⏳ Not started | +| 2 | Always-on microphone loop | βœ… Done (cpal β†’ VAD β†’ STT β†’ agent) | +| 2 | `always_on_enabled` config flag + Settings toggle | βœ… Done (RPC + UI + i18n) | +| 2 | Privacy hook (screen lock pause) | βœ… Done (macOS; other OSes follow-up) | | 3 | Wake-word detection | ⏳ Not started | | 3 | Local command router | ⏳ Not started | | 4 | Voice confirmation loop | ⏳ Not started | diff --git a/src/openhuman/agent_registry/agents/orchestrator/agent.toml b/src/openhuman/agent_registry/agents/orchestrator/agent.toml index 2ee6e58ae..80a7cb7f7 100644 --- a/src/openhuman/agent_registry/agents/orchestrator/agent.toml +++ b/src/openhuman/agent_registry/agents/orchestrator/agent.toml @@ -130,6 +130,23 @@ named = [ # `computer_control.ax_interact_mutations`, and refuse a sensitive-app # denylist (password managers, Keychain, System Settings, terminals). "ax_interact", + # Multi-step UI automation in one call (e.g. "play in Music", + # "message in Slack"). Prefer over many individual ax_interact + # calls when the task needs several UI steps β€” a Rust perceiveβ†’actβ†’verify + # loop runs the flow with a fast model. Same opt-in + sensitive-app denylist + # as ax_interact; `Dangerous`, gates through the ApprovalGate. + "automate", + # Full computer control (autonomy). Fallback for apps the accessibility API + # can't drive β€” notably Electron apps (Slack, Discord, VS Code) whose AX/UIA + # tree is empty. `screenshot` to see the screen, then `mouse` (move/click/ + # drag/scroll) + `keyboard` (type/press/hotkey) to act by pixel coordinates. + # All `Dangerous` and gate through the ApprovalGate. mouse/keyboard require + # `computer_control.enabled = true`. Prefer `automate`/`ax_interact` first; + # use these when the AX tree comes back empty. NB: foreground the target app + # before typing/clicking (synthetic input goes to the focused window). + "screenshot", + "mouse", + "keyboard", # Time + scheduling β€” lets the orchestrator answer "what time is it", # "remind me in 10 minutes", "every morning at 8" directly rather than # delegating or telling the user it can't. `current_time` grounds diff --git a/src/openhuman/agent_registry/agents/orchestrator/prompt.md b/src/openhuman/agent_registry/agents/orchestrator/prompt.md index e1b0d21c9..70d9873a9 100644 --- a/src/openhuman/agent_registry/agents/orchestrator/prompt.md +++ b/src/openhuman/agent_registry/agents/orchestrator/prompt.md @@ -38,6 +38,25 @@ Follow this sequence for every user message: Default bias: **do not spawn a sub-agent when a direct response or direct tool call is sufficient** β€” but a live external-service request is *not* something to answer from memory, it requires the integration. Use `spawn_worker_thread` for long tasks that need their own thread. +## Controlling desktop apps (full autonomy) + +You can open and operate native apps on this machine. **Never tell the user you "can't control the app" or "don't have mouse/keyboard" β€” you do.** + +**Rule 0 β€” foreground first, every time.** Before *any* keyboard/mouse action, call `launch_app ""` for the target. `open -a` both opens and **brings it to the front**, so your typing/clicks land on it (not on OpenHuman's own window β€” injecting there can crash the app). Re-call `launch_app` right before each keyboard/mouse step if focus might have moved. + +**The reliable path is the keyboard, not the mouse.** When a channel/chat/doc is open, its text box is already focused β€” you usually do **not** need coordinates. Prefer this: + +1. `launch_app ""` (foreground). +2. `automate {app, goal}` for multi-step UI (it foregrounds + runs a perceiveβ†’actβ†’verify loop). Good for native apps (Music, Mail, Notes). +3. **If `automate`/`ax_interact` come back empty / "stuck" / only menu-bar items** β€” that's an **Electron/Chromium app (Slack, Discord, VS Code, Spotify desktop)**; its content isn't in the accessibility tree. Switch to **keyboard-driven control**: + - `launch_app ""` (foreground), then `keyboard` `type` the text and `press` `Enter`. The focused input receives it. Use app **hotkeys** to navigate (no mouse needed). +4. **Only if you must click a specific spot that isn't focused:** `screenshot` β†’ `mouse` click. (Screenshots are downscaled so you can see them; coordinates you read are in the returned image's pixels.) + +**Worked example β€” "message hi on Slack" (keyboard-only, no vision):** +`launch_app "Slack"` β†’ `keyboard hotkey "cmd+k"` (Slack quick switcher) β†’ `keyboard type ""` β†’ `keyboard press "Enter"` (opens the chat, focuses the message box) β†’ `keyboard type "hi"` β†’ `keyboard press "Enter"` (sends). If no recipient was given and a channel is already open, skip the switcher and just `keyboard type "hi"` β†’ `press "Enter"`. + +`screenshot`/`mouse`/`keyboard` run without an approval prompt (they're on your auto-approve list) β€” just proceed. + ## Rules - **You are the chat tier.** You run on a fast UX-focused model (TTFT > deep reasoning). When a task needs sustained multi-step thinking β€” planning across many steps, comparing several non-obvious options, untangling ambiguous requirements β€” **delegate to the reasoning tier (`delegate_plan`)** rather than reasoning through it yourself. Your job at that point is to brief the planner well and synthesise its output back to the user. diff --git a/src/openhuman/tools/impl/computer/automate.rs b/src/openhuman/tools/impl/computer/automate.rs new file mode 100644 index 000000000..638ab6567 --- /dev/null +++ b/src/openhuman/tools/impl/computer/automate.rs @@ -0,0 +1,223 @@ +//! Tool: `automate` β€” accomplish a multi-step UI goal in one call. +//! +//! The orchestrator calls `automate{app, goal}` once; the Rust loop in +//! `accessibility::automate` then perceives β†’ decides (fast model) β†’ acts β†’ +//! settles β†’ verifies until the goal is met or a step budget is hit. This keeps +//! the heavy chat model out of the click loop (latency + reliability β€” see +//! `docs/voice-automate-plan.md`). +//! +//! Safety mirrors `ax_interact`: it actuates real controls, so it is a mutating +//! tool β€” opt-in via `computer_control.ax_interact_mutations`, routed through the +//! ApprovalGate, and it refuses the sensitive-app denylist (password managers, +//! Keychain, System Settings, terminals) even on auto-approved turns. + +use super::ax_interact::is_sensitive_app; +use crate::openhuman::accessibility::automate::{self, AutomateOptions, RealBackend}; +use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCallOptions, ToolResult}; +use async_trait::async_trait; +use serde_json::json; + +pub struct AutomateTool { + /// When false the tool refuses to run (it is inherently mutating). Mirrors + /// `AxInteractTool::allow_mutations` so one opt-in governs both. + allow_mutations: bool, +} + +impl AutomateTool { + pub fn new(allow_mutations: bool) -> Self { + Self { allow_mutations } + } +} + +impl Default for AutomateTool { + fn default() -> Self { + Self::new(false) + } +} + +#[async_trait] +impl Tool for AutomateTool { + fn name(&self) -> &str { + "automate" + } + + fn description(&self) -> &str { + "Accomplish a MULTI-STEP goal inside a desktop app in a single call β€” e.g. \ + 'play in Music', 'message in Slack'. Give the app \ + name and a plain-English goal; the system drives the app's UI step by step \ + (find elements β†’ press/type β†’ verify) using the platform accessibility API, \ + no screen coordinates. Prefer this over issuing many individual \ + `ax_interact` calls when the task needs several UI steps. The app should \ + usually be launched first (or include 'launch' in the goal). Refuses \ + password managers, Keychain, System Settings, and terminals." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "properties": { + "app": { + "type": "string", + "description": "Display name of the target application (e.g. 'Music', 'Slack')." + }, + "goal": { + "type": "string", + "description": "Plain-English description of the multi-step outcome to achieve." + } + }, + "required": ["app", "goal"] + }) + } + + fn permission_level(&self) -> PermissionLevel { + // Always mutating β€” it actuates controls. Kept as the base level so the + // approval gate fires regardless of args. + PermissionLevel::Dangerous + } + + fn external_effect(&self) -> bool { + true + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.execute_with_options(args, ToolCallOptions::default()) + .await + } + + async fn execute_with_options( + &self, + args: serde_json::Value, + _options: ToolCallOptions, + ) -> anyhow::Result { + let app = args + .get("app") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + let goal = args + .get("goal") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + + log::info!("[automate] β–Ά tool execute app={app:?} goal={goal:?}"); + + if app.is_empty() { + return Ok(ToolResult::error("app is required")); + } + if goal.is_empty() { + return Ok(ToolResult::error("goal is required")); + } + + // Hard safety boundary β€” identical to ax_interact's denylist. + if is_sensitive_app(&app) { + log::warn!("[automate] refused: sensitive app '{app}'"); + return Ok(ToolResult::error(format!( + "Refusing to automate '{app}': it is on the sensitive-app denylist \ + (password managers, Keychain, System Settings, terminals). This is a \ + hard safety boundary." + ))); + } + + if !self.allow_mutations { + log::warn!("[automate] refused: mutations disabled"); + return Ok(ToolResult::error( + "App control isn't enabled yet. Turn on App Automation in \ + Settings β†’ Agent Access (it grants permission to control apps), \ + then ask again. (Sets computer_control.ax_interact_mutations = true.)", + )); + } + + let config = match crate::openhuman::config::rpc::load_config_with_timeout().await { + Ok(c) => c, + Err(e) => return Ok(ToolResult::error(format!("could not load config: {e}"))), + }; + + let backend = RealBackend::new(config); + let outcome = automate::run(&app, &goal, &backend, AutomateOptions::default()).await; + + let mut body = format!("{}\n\nSteps:", outcome.summary); + if outcome.steps.is_empty() { + body.push_str("\n (no steps executed)"); + } else { + for s in &outcome.steps { + body.push_str(&format!("\n - {s}")); + } + } + + if outcome.success { + Ok(ToolResult::success(body)) + } else { + Ok(ToolResult::error(body)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn name_and_permission() { + let t = AutomateTool::new(true); + assert_eq!(t.name(), "automate"); + assert_eq!(t.permission_level(), PermissionLevel::Dangerous); + assert!(t.external_effect()); + } + + #[test] + fn schema_requires_app_and_goal() { + let schema = AutomateTool::new(true).parameters_schema(); + let req = schema["required"].as_array().unwrap(); + assert!(req.iter().any(|v| v == "app")); + assert!(req.iter().any(|v| v == "goal")); + } + + #[tokio::test] + async fn rejects_missing_app_or_goal() { + let t = AutomateTool::new(true); + assert!( + t.execute(json!({"app": "", "goal": "x"})) + .await + .unwrap() + .is_error + ); + assert!( + t.execute(json!({"app": "Music", "goal": ""})) + .await + .unwrap() + .is_error + ); + } + + #[tokio::test] + async fn refuses_when_mutations_disabled() { + let t = AutomateTool::new(false); + let r = t + .execute(json!({"app": "Music", "goal": "play a song"})) + .await + .unwrap(); + assert!(r.is_error); + assert!(r.output().contains("ax_interact_mutations")); + } + + #[tokio::test] + async fn refuses_sensitive_app() { + let t = AutomateTool::new(true); + for app in [ + "Keychain Access", + "1Password", + "Terminal", + "System Settings", + ] { + let r = t + .execute(json!({"app": app, "goal": "do something"})) + .await + .unwrap(); + assert!(r.is_error, "expected refusal for {app}"); + assert!(r.output().to_lowercase().contains("denylist")); + } + } +} diff --git a/src/openhuman/tools/impl/computer/ax_interact.rs b/src/openhuman/tools/impl/computer/ax_interact.rs index 358d9179c..8cceb8a3b 100644 --- a/src/openhuman/tools/impl/computer/ax_interact.rs +++ b/src/openhuman/tools/impl/computer/ax_interact.rs @@ -48,7 +48,9 @@ const SENSITIVE_APPS: &[&str] = &[ "rio", ]; -fn is_sensitive_app(app_name: &str) -> bool { +/// True when `app_name` is on the never-actuate denylist. `pub(crate)` so the +/// `automate` tool shares the exact same boundary as `ax_interact`. +pub(crate) fn is_sensitive_app(app_name: &str) -> bool { let lower = app_name.to_lowercase(); SENSITIVE_APPS.iter().any(|s| lower.contains(s)) } @@ -56,8 +58,10 @@ fn is_sensitive_app(app_name: &str) -> bool { pub struct AxInteractTool { /// When false, the mutating actions (`press` / `set_value`) are refused /// with guidance to enable `computer_control.ax_interact_mutations`. The - /// read-only `list` action is always available. Mirrors the opt-in posture - /// of the mouse/keyboard tools (`computer_control.enabled`). + /// read-only `list` action is always available. Like the mouse/keyboard + /// tools (`computer_control.enabled`), this is opt-in **and** approval-gated: + /// the mutating actions return `external_effect_with_args == true` so they + /// route through the ApprovalGate. allow_mutations: bool, } @@ -229,10 +233,10 @@ impl Tool for AxInteractTool { if mutating && !self.allow_mutations { log::warn!("[ax_interact] refused: mutations disabled (action={action})"); return Ok(ToolResult::error( - "ax_interact mutations (press/set_value) are disabled. They actuate arbitrary \ - app controls and type into arbitrary fields, so they require explicit opt-in: \ - set `computer_control.ax_interact_mutations = true`. The read-only 'list' \ - action remains available.", + "App control isn't enabled yet, so I can't press buttons or type into \ + this app. Turn on App UI Control / App Automation in Settings β†’ Agent \ + Access, then ask again. (Reading the UI still works without it; sets \ + computer_control.ax_interact_mutations = true.)", )); } diff --git a/src/openhuman/tools/impl/computer/keyboard.rs b/src/openhuman/tools/impl/computer/keyboard.rs index 9e9a9a629..b2a9a581e 100644 --- a/src/openhuman/tools/impl/computer/keyboard.rs +++ b/src/openhuman/tools/impl/computer/keyboard.rs @@ -115,6 +115,16 @@ impl Tool for KeyboardTool { PermissionLevel::Dangerous } + /// Route every call through the ApprovalGate. Arbitrary keystrokes can land + /// in a focused sudo/password field or Terminal, and there's no sensitive-app + /// denylist for raw input, so the gate is the only boundary β€” and it fires on + /// `external_effect_with_args`, NOT on `PermissionLevel::Dangerous`. Without + /// this, keystrokes could run unattended on an auto-approved turn once + /// `computer_control.enabled`. + fn external_effect(&self) -> bool { + true + } + fn parameters_schema(&self) -> Value { json!({ "type": "object", diff --git a/src/openhuman/tools/impl/computer/keyboard_tests.rs b/src/openhuman/tools/impl/computer/keyboard_tests.rs index 0b0ef90f7..2a455df5d 100644 --- a/src/openhuman/tools/impl/computer/keyboard_tests.rs +++ b/src/openhuman/tools/impl/computer/keyboard_tests.rs @@ -27,6 +27,20 @@ fn permission_is_dangerous() { assert_eq!(make_tool().permission_level(), PermissionLevel::Dangerous); } +#[test] +fn routes_through_approval_gate() { + // The gate keys off external_effect_with_args, NOT PermissionLevel::Dangerous. + let tool = make_tool(); + assert!( + tool.external_effect(), + "keyboard must declare an external effect" + ); + assert!( + tool.external_effect_with_args(&json!({"action": "type", "text": "x"})), + "every keyboard action must route through the ApprovalGate" + ); +} + #[test] fn name_is_keyboard() { assert_eq!(make_tool().name(), "keyboard"); diff --git a/src/openhuman/tools/impl/computer/mod.rs b/src/openhuman/tools/impl/computer/mod.rs index 6603105d9..379c8833f 100644 --- a/src/openhuman/tools/impl/computer/mod.rs +++ b/src/openhuman/tools/impl/computer/mod.rs @@ -1,9 +1,11 @@ +mod automate; mod ax_interact; mod human_path; mod keyboard; mod main_thread; mod mouse; +pub use automate::AutomateTool; pub use ax_interact::AxInteractTool; pub use keyboard::KeyboardTool; pub use main_thread::{run_input_on_main, MainThreadInputOp, INPUT_ON_MAIN_THREAD_METHOD}; diff --git a/src/openhuman/tools/impl/computer/mouse.rs b/src/openhuman/tools/impl/computer/mouse.rs index 40f016e5b..1236622c6 100644 --- a/src/openhuman/tools/impl/computer/mouse.rs +++ b/src/openhuman/tools/impl/computer/mouse.rs @@ -152,6 +152,16 @@ impl Tool for MouseTool { PermissionLevel::Dangerous } + /// Route every call through the ApprovalGate. Raw coordinate input has no + /// app/element scoping (nothing to denylist), so the gate is the only real + /// boundary β€” and the gate fires on `external_effect_with_args`, NOT on + /// `PermissionLevel::Dangerous` (which is just a static channel-capability + /// filter). Without this, blind clicks could run unattended on an + /// auto-approved turn once `computer_control.enabled`. + fn external_effect(&self) -> bool { + true + } + fn parameters_schema(&self) -> Value { json!({ "type": "object", diff --git a/src/openhuman/tools/impl/computer/mouse_tests.rs b/src/openhuman/tools/impl/computer/mouse_tests.rs index cb1932c3b..2d33e6d8e 100644 --- a/src/openhuman/tools/impl/computer/mouse_tests.rs +++ b/src/openhuman/tools/impl/computer/mouse_tests.rs @@ -40,6 +40,20 @@ fn permission_is_dangerous() { assert_eq!(tool.permission_level(), PermissionLevel::Dangerous); } +#[test] +fn routes_through_approval_gate() { + // The gate keys off external_effect_with_args, NOT PermissionLevel::Dangerous. + let tool = make_tool(); + assert!( + tool.external_effect(), + "mouse must declare an external effect" + ); + assert!( + tool.external_effect_with_args(&json!({"action": "click", "x": 1, "y": 1})), + "every mouse action must route through the ApprovalGate" + ); +} + #[test] fn name_is_mouse() { assert_eq!(make_tool().name(), "mouse"); diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index 8c9f2e810..8ce5445a2 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -177,6 +177,12 @@ pub fn all_tools_with_runtime( Box::new(AxInteractTool::new( root_config.computer_control.ax_interact_mutations, )), + // Multi-step UI automation in one call. Shares the ax_interact opt-in + // (mutations) and sensitive-app denylist; runs a Rust perceiveβ†’actβ†’verify + // loop with a fast model so the chat model stays out of the click loop. + Box::new(AutomateTool::new( + root_config.computer_control.ax_interact_mutations, + )), Box::new(CodegraphIndexTool::new( config.clone(), action_dir.to_path_buf(), diff --git a/src/openhuman/tools/user_filter.rs b/src/openhuman/tools/user_filter.rs index 43c029932..835824fe5 100644 --- a/src/openhuman/tools/user_filter.rs +++ b/src/openhuman/tools/user_filter.rs @@ -41,9 +41,18 @@ const TOOL_FAMILIES: &[ToolFamily] = &[ rust_names: &["ax_interact"], default_enabled: true, }, + // Multi-step UI automation (one call β†’ whole flow). Same opt-in as + // ax_interact; surfaced as its own catalog toggle. + ToolFamily { + id: "automate", + rust_names: &["automate"], + default_enabled: true, + }, // Computer control β€” mouse and keyboard. Gated by computer_control.enabled - // in config (tools only register when that flag is true). PermissionLevel::Dangerous - // so the approval gate fires per-action; user opts in explicitly. + // in config (tools only register when that flag is true). Each tool also + // overrides `external_effect` β†’ true so the ApprovalGate fires per-action β€” + // `PermissionLevel::Dangerous` alone does NOT trigger the gate (it's only a + // static channel-capability filter); the gate keys off `external_effect_with_args`. ToolFamily { id: "computer_control", rust_names: &["mouse", "keyboard"],