diff --git a/Cargo.lock b/Cargo.lock index 70a7384a7..3a9e8660b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5234,6 +5234,7 @@ dependencies = [ "tracing-appender", "tracing-log", "tracing-subscriber", + "uiautomation", "unicode-normalization", "unicode-segmentation", "unicode-width", @@ -8388,6 +8389,29 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e36a83ea2b3c704935a01b4642946aadd445cea40b10935e3f8bd8052b8193d6" +[[package]] +name = "uiautomation" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68495a701b9f2f21f29353ac446f0d27dd0d7ce97aa9ccf9061bca0446cd744" +dependencies = [ + "chrono", + "uiautomation_derive", + "windows 0.62.2", + "windows-core 0.62.2", +] + +[[package]] +name = "uiautomation_derive" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffcc4d404aa1c03a848f95cf5feadc3e63946d7f095bf388770b85550093d388" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "uint" version = "0.9.5" @@ -9287,6 +9311,27 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core 0.62.2", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.54.0" @@ -9335,6 +9380,17 @@ dependencies = [ "windows-strings 0.5.1", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.57.0" @@ -9407,6 +9463,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link", +] + [[package]] name = "windows-registry" version = "0.6.1" @@ -9572,6 +9638,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" diff --git a/Cargo.toml b/Cargo.toml index db43743e9..67c14712a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -207,16 +207,23 @@ tokio-tungstenite = { version = "0.24", default-features = false, features = ["c # AppContainer / process-jail backend in `openhuman::cwd_jail`. # Feature list mirrors the Win32 surface used by cwd_jail/windows.rs: # AppContainer profile APIs, ACL editing, STARTUPINFOEXW process spawn, -# and the GENERIC_* file access masks. +# and the GENERIC_* file access masks. `Win32_System_Com` is used by the +# UIA accessibility backend (`accessibility::uia_interact`) to initialise COM +# on the worker thread before creating the UI Automation client. windows-sys = { version = "0.61", features = [ "Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_Security_Isolation", "Win32_Storage_FileSystem", + "Win32_System_Com", "Win32_System_Memory", "Win32_System_Threading", ] } +# Microsoft UI Automation (UIA) bindings — the Windows backend for the +# `ax_interact` tool (`accessibility::uia_interact`). Safe Rust wrappers over +# the UIA COM API; the Windows analogue of the macOS AXUIElement Swift helper. +uiautomation = "0.25" [target.'cfg(not(windows))'.dependencies] # macOS / Linux: keep rustls + Mozilla webpki-roots — the historical diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 358660a74..bf0c7d64d 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -5443,6 +5443,7 @@ dependencies = [ "tracing-appender", "tracing-log", "tracing-subscriber", + "uiautomation", "unicode-normalization", "unicode-segmentation", "unicode-width", @@ -9286,6 +9287,29 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "uiautomation" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68495a701b9f2f21f29353ac446f0d27dd0d7ce97aa9ccf9061bca0446cd744" +dependencies = [ + "chrono", + "uiautomation_derive", + "windows 0.62.2", + "windows-core 0.62.2", +] + +[[package]] +name = "uiautomation_derive" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffcc4d404aa1c03a848f95cf5feadc3e63946d7f095bf388770b85550093d388" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "uint" version = "0.9.5" @@ -10087,11 +10111,23 @@ version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-collections", + "windows-collections 0.2.0", "windows-core 0.61.2", - "windows-future", + "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -10103,6 +10139,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.54.0" @@ -10172,7 +10217,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -10263,6 +10319,16 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + [[package]] name = "windows-registry" version = "0.5.3" @@ -10464,6 +10530,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-version" version = "0.1.7" diff --git a/app/src/hooks/__tests__/useDictationHotkey.test.tsx b/app/src/hooks/__tests__/useDictationHotkey.test.tsx index 6ab3d3211..bed2ea3ce 100644 --- a/app/src/hooks/__tests__/useDictationHotkey.test.tsx +++ b/app/src/hooks/__tests__/useDictationHotkey.test.tsx @@ -79,4 +79,43 @@ describe('useDictationHotkey', () => { }); expect(hoisted.mockSocket.on).not.toHaveBeenCalled(); }); + + it('dispatches an autoSend insert-text event on transcription', async () => { + renderHook(() => useDictationHotkey()); + await waitFor(() => { + expect(hoisted.handlers['dictation:transcription']).toBeDefined(); + }); + + const received: CustomEvent<{ text?: string; autoSend?: boolean }>[] = []; + const listener = (e: Event) => + received.push(e as CustomEvent<{ text?: string; autoSend?: boolean }>); + window.addEventListener('dictation://insert-text', listener); + try { + hoisted.handlers['dictation:transcription']({ text: ' hello world ' }); + } finally { + window.removeEventListener('dictation://insert-text', listener); + } + + expect(received).toHaveLength(1); + // Trimmed text, and autoSend flag set so Conversations sends it straight to the agent. + expect(received[0].detail).toEqual({ text: 'hello world', autoSend: true }); + }); + + it('ignores blank transcription text (no event dispatched)', async () => { + renderHook(() => useDictationHotkey()); + await waitFor(() => { + expect(hoisted.handlers['dictation:transcription']).toBeDefined(); + }); + + const received: Event[] = []; + const listener = (e: Event) => received.push(e); + window.addEventListener('dictation://insert-text', listener); + try { + hoisted.handlers['dictation:transcription']({ text: ' ' }); + } finally { + window.removeEventListener('dictation://insert-text', listener); + } + + expect(received).toHaveLength(0); + }); }); diff --git a/app/src/hooks/useDictationHotkey.ts b/app/src/hooks/useDictationHotkey.ts index 5da7d1569..c430e5845 100644 --- a/app/src/hooks/useDictationHotkey.ts +++ b/app/src/hooks/useDictationHotkey.ts @@ -150,7 +150,9 @@ export function useDictationHotkey(): DictationHotkeyState { if (!text) return; console.debug(`[dictation] transcription received: ${text.length} chars — "${text}"`); - window.dispatchEvent(new CustomEvent('dictation://insert-text', { detail: { text } })); + window.dispatchEvent( + new CustomEvent('dictation://insert-text', { detail: { text, autoSend: true } }) + ); }); socket.connect(); diff --git a/app/src/pages/Conversations.tsx b/app/src/pages/Conversations.tsx index 7fd61fcc4..50efb4f29 100644 --- a/app/src/pages/Conversations.tsx +++ b/app/src/pages/Conversations.tsx @@ -302,6 +302,8 @@ const Conversations = ({ // from `selectedThreadId` so switching threads mid-turn doesn't move the // timer's reference point. const sendingThreadIdRef = useRef(null); + // Ref so the mount-time dictation event handler can call the latest send fn. + const handleSendMessageRef = useRef<((text?: string) => Promise) | null>(null); // Previous inference status for the sending thread; lets the rearm effect // distinguish "status was just cleared (chat_done / chat_error)" from // "status was never set yet (in-flight turn pre-status)". @@ -462,11 +464,19 @@ const Conversations = ({ useEffect(() => { const onDictationInsert = (event: Event) => { - const customEvent = event as CustomEvent<{ text?: string }>; + const customEvent = event as CustomEvent<{ text?: string; autoSend?: boolean }>; const text = customEvent.detail?.text?.trim(); if (!text) return; customEvent.preventDefault(); + + // When autoSend is set (hotkey dictation), dispatch the transcript directly + // to the agent without going through the text composer. + if (customEvent.detail?.autoSend) { + void handleSendMessageRef.current?.(text); + return; + } + setInputMode('text'); setInputValue(prev => { const base = prev.trim(); @@ -844,6 +854,8 @@ const Conversations = ({ } }; + handleSendMessageRef.current = handleSendMessage; + const transcribeAndSendAudio = async (mimeType: string) => { setIsRecording(false); mediaRecorderRef.current = null; diff --git a/app/src/pages/__tests__/Conversations.render.test.tsx b/app/src/pages/__tests__/Conversations.render.test.tsx index f356edf4c..c1c0e2e54 100644 --- a/app/src/pages/__tests__/Conversations.render.test.tsx +++ b/app/src/pages/__tests__/Conversations.render.test.tsx @@ -675,6 +675,44 @@ describe('Conversations — smoke render (#1123 welcome-lock removal)', () => { }); }); + it('auto-sends a dictation transcript (autoSend) straight to chat without the composer', async () => { + const { thread } = await renderSelectedConversation(); + + // Hotkey dictation dispatches this event with autoSend:true (see + // useDictationHotkey). Conversations must route it directly to chatSend, + // bypassing the text composer. + await act(async () => { + window.dispatchEvent( + new CustomEvent('dictation://insert-text', { + detail: { text: ' play highway to hell ', autoSend: true }, + }) + ); + }); + + await waitFor(() => { + expect(chatSend).toHaveBeenCalledWith({ + threadId: thread.id, + message: 'play highway to hell', + model: 'reasoning-v1', + profileId: 'default', + locale: 'en', + }); + }); + }); + + it('ignores a blank autoSend dictation event (no send)', async () => { + await renderSelectedConversation(); + vi.mocked(chatSend).mockClear(); + + await act(async () => { + window.dispatchEvent( + new CustomEvent('dictation://insert-text', { detail: { text: ' ', autoSend: true } }) + ); + }); + + expect(chatSend).not.toHaveBeenCalled(); + }); + it('blocks duplicate sends while the first send is still pending', async () => { let resolveSend: (() => void) | undefined; vi.mocked(chatSend).mockImplementationOnce( diff --git a/app/src/utils/toolDefinitions.ts b/app/src/utils/toolDefinitions.ts index 5aec2e72b..01213cd1f 100644 --- a/app/src/utils/toolDefinitions.ts +++ b/app/src/utils/toolDefinitions.ts @@ -28,6 +28,23 @@ export const TOOL_CATALOG: ToolDefinition[] = [ defaultEnabled: true, rustToolNames: ['shell'], }, + { + id: 'launch_app', + displayName: 'Launch Applications', + description: 'Open apps on your desktop by name (e.g. Music, Spotify, Safari).', + category: 'System', + defaultEnabled: true, + rustToolNames: ['launch_app'], + }, + { + id: 'ax_interact', + displayName: 'App UI Control', + description: + 'Interact with desktop app UI by element label via the platform accessibility API — click buttons, type in fields, without needing screen coordinates.', + category: 'System', + defaultEnabled: true, + rustToolNames: ['ax_interact'], + }, { id: 'git_operations', displayName: 'Git Operations', diff --git a/docs/voice-system-actions.md b/docs/voice-system-actions.md new file mode 100644 index 000000000..92133e37c --- /dev/null +++ b/docs/voice-system-actions.md @@ -0,0 +1,481 @@ +# Voice → System Action Feature Tracker + +**GitHub Issue:** [#3148](https://github.com/tinyhumansai/openhuman/issues/3148) +**Branch:** `feat/voice-always-on` +**PR:** [#3168](https://github.com/tinyhumansai/openhuman/pull/3168) +**Started:** 2026-06-02 + +--- + +## Goal + +Enable the app to continuously listen to the user, understand spoken commands, and perform system actions on the laptop — e.g., saying *"open my Music player"* causes the app to open it, without any hotkey press or manual send. + +--- + +## Companion Feature (Separate PR) + +**Notch Live Activity Indicator** — [PR #3166](https://github.com/tinyhumansai/openhuman/pull/3166) +A transparent NSPanel pill at the top of the primary screen (the macOS notch area) that shows live voice/agent status. Built alongside this feature; will light up automatically once always-on listening is implemented. + +--- + +## Phase 1 — Quick Wins ✅ Complete + +> Low-effort changes that make the existing hotkey-triggered dictation flow work end-to-end without manual sends or approval prompts. + +--- + +### Change 1.1 — Auto-send after transcription + +**Status:** ✅ Done +**Commit:** `7269f4373` + +**Problem:** After speaking via the dictation hotkey, the transcript appeared in the chat composer but the user had to press Enter manually to send it. + +**Fix:** +- `app/src/hooks/useDictationHotkey.ts` — added `autoSend: true` to the `dictation://insert-text` event detail +- `app/src/pages/Conversations.tsx` — `onDictationInsert` now checks the flag; when set, calls `handleSendMessage(text)` directly instead of inserting into the textarea. Added `handleSendMessageRef` (updated every render) so the mount-time effect can access the latest send function + +**Result:** Press hotkey → speak → message auto-sends to agent. No Enter key needed. + +--- + +### Change 1.2 — Shell allowlist for app-launching — ⚠️ REVERTED / SUPERSEDED + +**Status:** ❌ Reverted — superseded by Change 1.4 (`launch_app`) and the security review. + +**What was tried (commit `7269f4373`):** added `"open"` / `"xdg-open"` to `READ_ONLY_BASES` so `open -a Music` would run without an approval prompt. + +**Why reverted:** base-command classification can't see args, and `open`/`xdg-open`/`start` can open arbitrary `https://` URLs and custom URI handlers — too broad for the `Read` (no-approval) path (maintainer security review). They were **removed** from `READ_ONLY_BASES`; the current code (`policy_command.rs:514-520`) deliberately keeps them out, with a comment. App launching now goes through the dedicated, gated `launch_app` tool (Change 1.4), which is scoped to named applications only. + +--- + +### Change 1.3 — Shell tool description fix + +**Status:** ✅ Done +**Commit:** `ec8f5be2e` + +**Problem:** Shell tool description said *"Execute a shell command in the workspace directory"* — the LLM was reasoning that it could only run workspace commands, not launch apps. + +**Fix:** +- `src/openhuman/tools/impl/system/shell.rs` — updated description to explicitly mention system actions and app launching examples + +**Result:** Agent now understands the shell tool can perform system actions, not just workspace file operations. + +--- + +### Change 1.4 — Dedicated `launch_app` tool + +**Status:** ✅ Done +**Commit:** `802fbca76` + +**Problem:** Using the `shell` tool for app launching requires loosening `workspace_only` and expanding `allowed_commands` — a security regression. The `shell` tool also couldn't be used because the orchestrator's strict `named` tool list excluded it. + +**Fix (production approach):** +- `src/openhuman/tools/impl/system/launch_app.rs` — **new tool** with `PermissionLevel::ReadOnly` (never triggers approval gate) + - macOS: `open -a ""` via `tokio::process::Command` + - Linux: `gtk-launch`, fallback `xdg-open` + - Windows: `Start-Process` via PowerShell + - Input validation: rejects paths, metacharacters, empty names + - Unit tests: name, permission, schema, validation, error cases +- `src/openhuman/tools/impl/system/mod.rs` — registered module + pub use +- `src/openhuman/tools/ops.rs` — added `LaunchAppTool` to `all_tools_with_runtime` +- `src/openhuman/tools/user_filter.rs` — added `"launch_app"` family, `default_enabled = true` +- `app/src/utils/toolDefinitions.ts` — added to frontend tool catalog (Settings → Agent Access toggle) + +**Result:** Agent has a purpose-built, always-allow tool for launching apps. No shell exposure, no path security concerns. + +--- + +### Change 1.5 — Orchestrator agent tool scope + +**Status:** ✅ Done +**Commit:** `7d04fc4bc` + +**Problem:** Even though `launch_app` was registered, it was invisible to the agent. The orchestrator (`src/openhuman/agent_registry/agents/orchestrator/agent.toml`) has a strict `named = [...]` allowlist. `launch_app` was not in it, so it was filtered out. Confirmed via logs: `visible=24, names=[...no launch_app...]`. + +**Fix:** +- `src/openhuman/agent_registry/agents/orchestrator/agent.toml` — added `"launch_app"` to the `[tools] named` list, alongside `"current_time"` (same pattern: direct answer without delegation) + +**Confirmed working via logs:** +```text +visible=25, names=[..., launch_app, ...] +[launch_app] ▶ execute called app_name="Music" +[launch_app] macOS: running `open -a "Music"` +[launch_app] macOS: `open -a` exit=exit status: 0 stderr= +[launch_app] ✓ launch succeeded msg="Opened 'Music'." +``` + +**Result:** Saying "open my Music app" now opens Music directly. No approval prompt, no delegation, no refusal. + +--- + +### Change 1.6 — SOUL.md capability hint + +**Status:** ✅ Done +**Commit:** `cdd3bb4a4` + +**Problem:** Even with the tool available, the agent was refusing ("I can't open apps on your device") because its training overrides the function-calling schema. + +**Fix:** +- `src/openhuman/agent/prompts/SOUL.md` — added explicit *"What you can do on the user's machine"* section listing `launch_app`, `shell`, `file_read`/`file_write` with the instruction: *"Never say 'I can't open apps' when you have a tool to do it. Use the tool."* + +**Result:** Agent now knows it has these capabilities and is instructed to use them. + +--- + +### Change 1.7 — Diagnostic logging + +**Status:** ✅ Done +**Commit:** `cdd3bb4a4` + +**Added logging to:** +- `src/openhuman/tools/impl/system/launch_app.rs` — logs every step: `▶ execute`, validation pass/fail, platform dispatch, `open -a` exit code + stderr, fallback result +- `src/openhuman/agent/harness/session/builder.rs` — logs the **full list** of visible tool names at session build time (previously only logged count) + +**Result:** Can now confirm at a glance whether `launch_app` is in the tool list and trace every step of its execution. + +--- + +--- + +### Change 1.8 — Computer control (mouse + keyboard) — ⚠️ REVERTED + +**Status:** ❌ Reverted (commits `50ca434b7` add, `bi0rd96sa` revert) + +**Problem:** Agent could open apps but couldn't interact with their UI. + +**What was tried:** Enabled the existing `mouse` + `keyboard` tools (enigo / `CGEventPost`), wired into the orchestrator, user filter, and frontend catalog. + +**Why reverted:** `CGEventPost` injects synthetic events to the **currently focused window**. When the focused window was OpenHuman's own CEF renderer (the chat UI), a Space keypress crashed the app — `EXC_BREAKPOINT / SIGTRAP` in `CFRelease → NSKeyValueWillChangeWithPerThreadPendingNotifications → -[NSApplication stop:]`. CEF can't handle arbitrary key injection. Confirmed via crash report `OpenHuman-2026-06-02-035139.ips`. + +**Replaced by:** Change 1.9 (`ax_interact`) — AXUIElement targets elements directly by label with no synthetic events and no CEF crash risk. + +--- + +### Change 1.9 — AXUIElement app UI interaction (`ax_interact`) + +**Status:** ✅ Done +**Commits:** `4f9ca1cad` (feature), `2c32b59c9` (exact-match fix), `betuerj11`/test commits + +**Problem:** Need to interact with desktop app UIs reliably, without the CEF crash from synthetic events. + +**Fix — uses the macOS Accessibility API (AXUIElement) instead of CGEventPost:** +- `src/openhuman/accessibility/helper.rs` — extended the unified Swift helper with three commands: + - `ax_list` → walk the AX tree, return interactive elements (buttons, fields, cells) + - `ax_press` → `AXUIElementPerformAction(kAXPressAction)` by label, **exact match preferred over contains** (so "Play" beats "Playlist") + - `ax_set_value` → `AXUIElementSetAttributeValue(kAXValueAttribute)` by label +- `src/openhuman/accessibility/ax_interact.rs` (new) — Rust wrappers `ax_list_elements`, `ax_press_element`, `ax_set_field_value` +- `src/openhuman/tools/impl/computer/ax_interact.rs` (new) — `AxInteractTool` with actions `list` / `press` / `set_value`, `PermissionLevel::ReadOnly` +- `src/openhuman/accessibility/ax_interact_tests.rs` (new) — integration tests (open Music → search AC/DC → find row → press) +- Wired into `tools/ops.rs`, `tools/user_filter.rs`, `toolDefinitions.ts` (App UI Control), `orchestrator/agent.toml`, `SOUL.md` + +**Why it's better than mouse/keyboard:** + +| | mouse/keyboard (reverted) | ax_interact | +|---|---|---| +| Mechanism | `CGEventPost` synthetic events | `AXUIElementPerformAction` direct API | +| CEF crash risk | Yes | None | +| Coordinates | Required (needs screenshot) | None — finds by label | +| Works when app unfocused | No | Yes | + +**Verified working:** Direct AX test against Music listed 256 elements including `Bollywood Hits`, `Play`, etc.; pressing `Bollywood Hits` then `Play` both returned `exact=true` and acted correctly. + +--- + +### Change 1.10 — Multi-step UI workflow guidance + +**Status:** ✅ Done + +**Problem:** When asked to "play Highway to Hell by AC/DC", the agent ran: launch → list → press Library → press Songs → press "Show Filter Field" → set_value "Highway to Hell" → **press "Play"**. The final press hit the **global playback bar Play button** (plays last queue item), not the specific song row. Result: app navigated correctly but the wrong/no track played. + +**Fix:** +- `src/openhuman/agent/prompts/SOUL.md` — added explicit multi-step workflow: + 1. `list` → discover elements + 2. `set_value` → type in filter/search + 3. `list` **again** → see filtered results + 4. `press` the **specific item** (song row), not the generic Play button +- Added Apple Music guidance: use `shell` to open `music://music.apple.com/search?term=...`, then `ax_interact list` to see song rows as AXCells, then press the specific row. More reliable than the Library filter field. + +**Result:** Agent is directed to select the specific item before pressing playback, instead of pressing the global Play button after filtering. + +--- + +### Change 1.11 — Apple Music two-step play (navigate then play) + +**Status:** ✅ Done + +**Problem:** When asked to "play Highway to Hell by AC/DC", the agent navigated to the right screen but **nothing played**. Pressing a search-result row in Apple Music only *selects/navigates* — it does not start playback. The agent then pressed the global transport Play button, but nothing was queued. + +**Investigation (empirical AX probing against live Music):** +- Every "Highway to Hell" element (AXCell, AXGroup, AXButton) exposes only the `AXPress` action — which selects/navigates, never plays. +- Double `AXPress`, a real CGEvent double-click on the Top-Results card, and AX-select + Return key **all left player state `stopped`**. +- **Working sequence found:** AXPress the search-result card to **navigate into the song's detail page**, then AXPress the **Play button on that detail page** → `player state: playing` ✅ + +**Fix:** +- `src/openhuman/agent/prompts/SOUL.md` — replaced the Apple Music guidance with the exact 5-step sequence: URL-scheme search → list → press song row (navigates in) → list detail page → press detail-page Play. Explicitly warns that pressing a search result only navigates, and the second Play press is mandatory. +- `src/openhuman/accessibility/ax_interact_tests.rs` — `test_full_flow_search_and_play_acdc` exercises the full navigate-then-play flow and **logs** the `osascript ... get player state` outcome. Playback is **best-effort, not hard-asserted** (Apple Music's UI is nondeterministic — see change 1.13); the test hard-asserts only the tool-level press/list successes. + +**Verified:** +```text +[step 4] navigate into song: Ok("Pressed 'Highway to Hell' in 'Music'.") +[step 5] press detail Play: Ok("Pressed 'Play' in 'Music'.") +[step 6] player state: playing +test ... ok +``` + +--- + +### Change 1.12 — One-shot `play_music` tool (root-cause fix) + +**Status:** ✅ Done + +**Problem:** Even after change 1.11, the agent still used the broken filter-field approach and didn't play. Transcript analysis (`~/.openhuman/users//workspace/session_raw/*.jsonl`) revealed two real root causes: + +1. **The orchestrator has no `shell` tool.** Change 1.11 put the play guidance in `SOUL.md` — but the orchestrator runs with `omit_identity = true` and **never sees SOUL.md**. Change 1.11b moved it to the `ax_interact` description, which told the agent to "use the shell tool to open `music://...`" — but the orchestrator can't run shell (it delegates). The agent wrapped the command in a `prompt` arg to a delegation tool; it never executed, and it fell back to the filter approach. +2. **Cross-chat memory contamination.** The user message was prefixed with `[Cross-chat context — historical]` containing prior filter-approach "Progress Checkpoint" steps, biasing the agent back to the wrong method. + +**Fix — stop relying on the LLM to orchestrate a fragile multi-step flow with a tool it lacks. Encapsulate the whole proven sequence in native Rust:** +- `src/openhuman/accessibility/ax_interact.rs` — `play_apple_music(query)`: open search URL → AX-find + press song cell (navigate) → press detail-page Play → verify `player state == playing` +- `src/openhuman/tools/impl/computer/play_music.rs` (new) — `PlayMusicTool`, single call `play_music{query}`, `PermissionLevel::ReadOnly`, runs the blocking flow via `spawn_blocking` +- Registered in `ops.rs`, `user_filter.rs`, `orchestrator/agent.toml`, `toolDefinitions.ts` + +**Result:** Agent calls `play_music{query:'Highway to Hell AC/DC'}` **once**; Rust does search→navigate→play. No shell dependency, no multi-step LLM orchestration, no filter-field fallback. Unit tests pass; the underlying flow is exercised by `test_full_flow_search_and_play_acdc` (tool-level success hard-asserted, playback best-effort). **Note:** `play_music` was later removed in change 1.13 in favour of the generic `ax_interact` tool — this entry documents the investigation that led there. + +**Key learning:** The orchestrator (chat agent) only reads **tool descriptions + agent.toml** — NOT SOUL.md (omit_identity=true). Behavior guidance for the chat agent must live in tool descriptions or be encapsulated in the tool itself. + +--- + +### Change 1.13 — Generic any-app tool + filtered list (remove play_music) + +**Status:** ✅ Done + +**Problem:** "Play Numb by Linkin Park" still failed, and the agent **hallucinated**. Transcript (`session_raw/*.jsonl`) showed: +1. `play_music` hit a 4s timing race — results hadn't rendered, so it returned "No matching song found. Top result cells: [empty]". +2. The agent fell back to `ax_interact list`, which dumped **273 elements**. The tool result was **truncated mid-list**, so the model reasoned over a partial view and hallucinated a wrong result ("Numb - Single by Marshmello"). + +**Feedback:** A music-specific tool is the wrong abstraction. Build a generic tool that interacts with **any** app. + +**Fix:** +- **Removed** `play_music` tool + `play_apple_music` helper and all registrations. +- **`ax_interact` is now a robust generic any-app tool:** + - `ax_list_elements_filtered(app, filter)` — Rust-side label filter so `list` returns only relevant elements (fixes the truncation→hallucination root cause). + - `list` action takes a new `filter` param; output capped at 60 elements with a "narrow your filter" hint; empty-match returns a "UI may still be loading" hint instead of failing hard. + - Description rewritten to be app-agnostic and document the general **navigate-then-activate** pattern (pressing a list row/search result selects/opens it; press the action button afterward) — no hardcoded Apple Music steps. + +**Key learning:** Dumping a full AX tree (hundreds of elements) overflows the tool-result budget; the truncated view makes the model hallucinate. Always filter list results to keep them small and accurate. + +--- + +## Windows port — app interaction 🪟 ✅ Implemented + +Phase 1's app-interaction layer is now ported to Windows. The macOS path uses the +Accessibility API via a Swift helper; the Windows path uses **Microsoft UI +Automation (UIA)** called directly from Rust (no helper process). The +agent-facing tool is a single `ax_interact` tool on both platforms — only the +backend differs, via cfg-dispatch. The sections below were the design plan; see +**"Windows port — implementation status"** at the end of this part for what +shipped and the test evidence. + +### What already works cross-platform + +| Capability | macOS (done) | Windows status | +|---|---|---| +| Auto-send dictation transcript | TS (`useDictationHotkey`/`Conversations`) | ✅ Already cross-platform (frontend) | +| App launching | `launch_app` / `policy_command.rs` | ✅ Launchers (`open`/`xdg-open`/`start`) stay OUT of `READ_ONLY_BASES` (can open arbitrary URLs/handlers); app launching goes through the gated `launch_app` tool. | +| `launch_app` | `open -a` | ⚠️ Already has a Windows branch (`Start-Process`) — verify it resolves app display names | +| `ax_interact` (list/press/set_value) | AXUIElement Swift helper | ❌ Needs a UI Automation (UIA) backend | + +### 1. Launching apps on Windows (`launch_app`) + +`launch_app.rs` already has a `#[cfg(target_os = "windows")]` branch using PowerShell `Start-Process ""`. Caveats to verify on the Windows machine: +- `Start-Process "Spotify"` works for apps on PATH or registered App Paths, but **Store/UWP apps** (e.g. the Windows "Media Player", "Spotify" from the Store) need their AUMID: `Start-Process "shell:AppsFolder\"`. Consider enumerating Store apps via `Get-StartApps` (returns Name + AppID) and matching by display name. +- For URIs (e.g. `spotify:`, `mailto:`), `Start-Process ""` works the same as macOS `open`. + +### 2. App UI interaction (`ax_interact` → UI Automation) + +**The Windows analog of macOS AXUIElement is Microsoft UI Automation (UIA)** — the OS-level accessibility tree. It exposes the same concepts: + +| macOS AX concept | Windows UIA equivalent | +|---|---| +| `AXUIElement` | `IUIAutomationElement` | +| `kAXRoleAttribute` (AXButton, AXCell…) | `ControlType` (Button, ListItem, Edit, Text…) | +| `kAXTitleAttribute` / `kAXDescriptionAttribute` | `Name` property (+ `AutomationId`, `HelpText`) | +| `AXUIElementPerformAction(kAXPressAction)` | `InvokePattern.Invoke()` (buttons) / `SelectionItemPattern.Select()` (list rows) | +| `AXUIElementSetAttributeValue(kAXValueAttribute)` | `ValuePattern.SetValue()` (text fields) | +| `AXUIElementCopyAttributeValue(kAXChildrenAttribute)` | `TreeWalker` / `FindAll(TreeScope.Descendants, …)` | +| Walk tree from app PID | `IUIAutomation.ElementFromHandle(hwnd)` or root + `ProcessId` condition | + +**Recommended implementation path (Rust-native, no helper process needed):** +- Use the [`uiautomation`](https://crates.io/crates/uiautomation) crate (safe Rust bindings over the UIA COM API). This is cleaner than macOS, where we had to shell out to a Swift helper — on Windows the COM API is callable directly from Rust. +- Mirror the existing `accessibility::ax_interact` surface so the **tool stays identical**: + - `list(app, filter)` → `CreateTreeWalker` over the app's window, collect elements whose `Name` matches `filter`, return `[{control_type, name}]`. + - `press(app, label)` → find element by `Name` (exact-first), then call `InvokePattern` if supported, else `SelectionItemPattern.Select()`, else `LegacyIAccessiblePattern.DoDefaultAction()`. + - `set_value(app, label, value)` → find `Edit`/`ComboBox`, call `ValuePattern.SetValue()`. +- **Key win over macOS:** UIA Invoke is generally a real "activate" (it triggers the control's default action), so the navigate-then-activate two-step that plagued Apple Music is less likely. A list-item Invoke on most Windows media apps plays directly. Still expect per-app quirks. + +**Suggested module layout (parallel to macOS):** +```text +src/openhuman/accessibility/ + ax_interact.rs # macOS (existing) + uia_interact.rs # NEW — Windows UIA backend, same fn signatures + mod.rs # cfg-dispatch: pub use the right backend per-OS +``` +Then `tools/impl/computer/ax_interact.rs` calls a thin cfg-gated facade so the **agent-facing tool is one tool on both platforms** (same name `ax_interact`, same actions). Update its description to be OS-neutral ("uses the platform accessibility API"). + +### 3. Permissions + +- macOS needs the Accessibility permission. **Windows UIA needs no special permission** for same-session, same-integrity-level apps — a big simplification. Caveat: a non-elevated process can't drive an **elevated** app's UI (UIPI). If the agent must control an elevated app, OpenHuman would need to run elevated too (avoid unless necessary). + +### 4. Diagnostics + +Keep the same `[ax_interact]`/`[uia_interact]` log prefixes and the verbose step logging (`▶ action`, found-count, press result) — they were essential for diagnosing the macOS flow and will be just as useful on Windows. + +### 5. Testing + +Port the integration tests using a built-in Windows app that's always present and UIA-friendly: +- **Calculator** (`calc.exe`) — press digit/operator buttons by Name, read the result `Text` element. Deterministic, no network, ideal smoke test. +- **Notepad** — `set_value` into the `Edit` control, verify via `ValuePattern.Value`. +- Media: **Windows Media Player** or the Store **Media Player** for a play test, but expect the same nondeterminism caveat as Apple Music — assert tool-level success, log playback as best-effort. + +### 6. Known-different behaviors to watch for + +- **Element naming:** Windows apps often populate `AutomationId` (stable) where macOS only had a visible title. Prefer matching `Name`, fall back to `AutomationId`. +- **Chromium/Electron apps** (Slack, Discord, VS Code, Spotify desktop): on Windows these expose a partial UIA tree by default; some require the app to have accessibility enabled. Same class of limitation as the macOS `chromiumAppPatterns` special-casing already in `helper.rs`. +- **Focus/foreground:** UIA generally doesn't require foregrounding to read/invoke, like macOS AX. No CGEventPost-style CEF crash risk because UIA Invoke is not synthetic input injection. + +### Quick start for the Windows machine + +1. `launch_app` should already work — test `"open notepad"` / `"open calculator"` first. +2. Do NOT add launchers to `READ_ONLY_BASES` — `launch_app` (gated, URI-rejecting) is the Windows app-launch path. `Start-Process` lives inside that tool, not the shell allowlist. +3. Build `uia_interact.rs` against the `uiautomation` crate, mirroring the three `ax_interact` fns. +4. cfg-dispatch in `accessibility/mod.rs` so `ax_interact` the tool resolves to UIA on Windows. +5. Smoke-test with Calculator (deterministic), then a media app (best-effort). + +### Cross-platform compatibility audit (current state) + +Every Phase 1 change was written to **compile on all platforms** — nothing here breaks the Windows build. macOS-specific native code is `#[cfg(target_os = "macos")]`-gated and the non-macOS branches return a clean `"…macOS-only"` error at runtime rather than failing to build. + +| Change | Cross-platform status | Notes for Windows | +|---|---|---| +| Auto-send dictation transcript (TS) | ✅ Fully portable | Pure frontend; no OS code. Works as-is. | +| `launch_app` | ✅ macOS / Linux / Windows branches | Windows branch now uses `Start-Process` with a **Store/UWP (`Get-StartApps` AUMID) fallback** and injection-safe env passing (§1). | +| `ax_interact` **tool** (`tools/impl/computer/ax_interact.rs`) | ✅ Functional on Windows | Delegates to `accessibility::ax_interact` fns, which now cfg-dispatch to the UIA backend on Windows. Description made OS-neutral. | +| `accessibility::ax_interact` helpers | ✅ cfg-dispatched | macOS → Swift helper; Windows → `uia_interact.rs`; other → clean runtime error. | +| `accessibility::uia_interact` (NEW) | ✅ Windows backend | UIA `list`/`press`/`set_value` via the `uiautomation` crate; same fn signatures as the macOS path. | +| Swift unified helper (`accessibility/helper.rs`) | ⚠️ macOS-only by design | Windows needs no helper process — UIA COM API is called directly from Rust. | +| App launching | ✅ Done | Launchers stay out of `READ_ONLY_BASES`; `launch_app` (gated) handles Windows `Start-Process`. | +| Notch indicator (separate PR #3166) | ⚠️ macOS NSPanel | A Windows equivalent would be a borderless always-on-top WebView2 window or a tray flyout — out of scope for this branch. | + +**Before merging the Windows port, confirm the whole branch still builds and runs on macOS too** (`cargo check` on both `Cargo.toml` and `app/src-tauri/Cargo.toml`) so the cfg-dispatch doesn't regress the working macOS path. + +### ⚠️ Mandatory: extensive E2E testing on Windows + +The macOS path was hardened only through repeated real-app runs (each bug — CEF crash, select-vs-play, list truncation/hallucination — surfaced only by actually driving live apps, not by unit tests). **Do the same on Windows before considering it done.** Treat the following as the required E2E matrix: + +1. **App launch** — `launch_app` for: a Win32 app (Notepad), a Store/UWP app (Media Player / Spotify from Store), and a URI (`spotify:`). Confirm each actually opens. +2. **Deterministic UI control** — Calculator: `list filter='5'` → `press '5'`, `press '+'`, `press '='`, then read the result element. Assert the computed value. This is the Windows analog of the AC/DC test and should be a **hard-asserted** automated test (Calculator is deterministic). +3. **Text entry** — Notepad: `set_value` into the Edit control, verify via `ValuePattern.Value`. +4. **Filtered list correctness** — confirm `list` with a `filter` returns a small, accurate set (the truncation→hallucination bug must not recur; verify the 60-element cap + filter behaves on a busy app like Settings or a browser). +5. **Real-world app** — drive a media app end-to-end (open → search → play). Assert tool-level success; treat playback state as **best-effort** (same nondeterminism caveat as Apple Music). +6. **Chromium/Electron app** — Slack/Discord/VS Code: confirm whether their UIA tree is exposed; document any app that needs accessibility explicitly enabled. +7. **Permissions/elevation** — verify behavior against a normal app vs an elevated one (UIPI boundary); document what fails and why. +8. **Agent-in-the-loop run** — the real test: ask the running agent (chat) to perform each action and confirm it picks `launch_app` / `ax_interact` and the action lands. Watch `[ax_interact]`/`[launch_app]` logs exactly as we did on macOS. +9. **Regression** — re-run the macOS E2E suite after the Windows changes land to prove cfg-dispatch didn't break the Mac path. + +Add the deterministic ones (Calculator, Notepad, launch) as `#[cfg(target_os = "windows")]` `#[ignore]` integration tests mirroring `ax_interact_tests.rs`, runnable with `cargo test ... -- --include-ignored` on the Windows machine. + +### Windows port — implementation status ✅ + +Shipped on the Windows machine (2026-06-02): + +**Code** +- `Cargo.toml` — `uiautomation = "0.25"` under `[target.'cfg(windows)'.dependencies]`; `Win32_System_Com` feature added to `windows-sys` for COM init. +- `src/openhuman/accessibility/uia_interact.rs` (**new**) — UIA backend. `list` / `press` / `set_value` over the UIA COM tree via the `uiautomation` crate, mirroring the macOS `ax_interact` fn signatures. `press` activates via UIA patterns in order — `Invoke` → `SelectionItem.Select` → `LegacyIAccessible.DoDefaultAction` — never injecting synthetic input. `set_value` finds an editable field, preferring `Edit`, then `ComboBox`, then `Document` (so the Win11 RichEdit Notepad, whose editor is a `Document`, works). Exact-label match preferred over substring. Per-thread COM init via `CoInitializeEx(MTA)`. +- `src/openhuman/accessibility/ax_interact.rs` — the three public helpers now cfg-dispatch: macOS → Swift helper, Windows → `uia_interact`, else → clean runtime error. Module + tool docs made OS-neutral. +- `src/openhuman/accessibility/mod.rs` — declares `uia_interact` (cfg-gated to Windows). +- `src/openhuman/tools/impl/computer/ax_interact.rs` — description rewritten to be platform-neutral ("platform accessibility API (macOS AXUIElement / Windows UI Automation)"). +- `src/openhuman/tools/impl/system/launch_app.rs` — Windows launcher hardened: app name passed via env var (no string interpolation → no injection), `Start-Process` first, then Store/UWP fallback by display name via `Get-StartApps` → AUMID (`shell:AppsFolder\`); stderr surfaced on failure. +- `src/openhuman/security/policy_command.rs` — launchers (`open`/`xdg-open`/`start`) deliberately kept OUT of `READ_ONLY_BASES`; `launch_app` is the gated launch path. +- `src/openhuman/accessibility/uia_interact_tests.rs` (**new**) — `#[cfg(all(test, target_os = "windows"))]` integration tests, wired into `ax_interact.rs`. + +**Test evidence (real apps on Windows 11)** +- `test_uia_calculator_five_plus_five` ✅ — drove the live Calculator entirely by element label: `list` → 41 interactive elements; pressed `Five`/`Plus`/`Five`/`Equals`; **hard-asserted** the readout `[Text] "Display is 10"` (5 + 5 = 10). Deterministic — the Windows analogue of the macOS AC/DC test. +- `test_uia_notepad_set_value` ✅ — `set_value` wrote into the live Win11 Notepad's `Document` "Text editor" (`Ok("Set 'Text editor' in 'Notepad' to the provided value.")`). The `Document` fallback is what makes the redesigned Notepad work. +- `test_uia_list_nonexistent_app` ✅ (non-ignored) — exercises COM init + window walk + error path deterministically. +- `launch_app` (×8) and `ax_interact` tool (×4) unit tests ✅. +- Full `cargo test --lib --no-run` compiles clean on Windows (warnings only, all pre-existing). + +**Environment gotcha (this machine):** Norton real-time protection blocks `link.exe` from writing the freshly-linked ~150 MB test `.exe` (LNK1104, "Access denied" creating the file). Fix: exclude the repo's `target` dir under Norton's **Auto-Protect / SONAR / Download Intelligence** exclusion list (not the separate "Scans" list), and restore the file from Quarantine if already flagged. + +**Follow-ups / not done here** +- **macOS regression check** — the cfg-dispatch is additive (the `#[cfg(target_os="macos")]` arms are untouched; only the non-macOS catch-all message changed), but per the branch note, re-run `cargo check` + the macOS E2E suite on a Mac before merge to prove the Mac path didn't regress (can't be done from the Windows machine). +- **Agent-in-the-loop run** (E2E item 8) — the full Tauri desktop app was built and run on Windows (`pnpm dev:app:win`) and the in-process core booted fine (verbose `[launch_app]`/`[ax_interact]`/`[uia_interact]` logging wired via `RUST_LOG`). The first chat attempt couldn't complete because the configured **local AI model was still downloading** (`kind="empty_provider_response"` — the agent returned an empty response, so it never reached a tool call). **Still pending:** a working model (finish the local download or select a configured cloud model), then ask the agent "open Calculator" / "press 5 in Calculator" and confirm it picks `launch_app`/`ax_interact` and the action lands. +- Chromium/Electron UIA coverage, elevation/UIPI behavior, and a busy-app filtered-list check (E2E items 4/6/7) remain to be spot-checked manually. + +--- + +## Phase 2 — Always-On Listening ⏳ Not Started + +> Continuous microphone listening without requiring a hotkey press. + +**Planned files:** +- `src/openhuman/voice/always_on.rs` (new) — dedicated tokio task holding the mic open, running VAD, emitting utterances to the STT pipeline +- `src/openhuman/config/schema/voice_server.rs` — add `always_on_enabled: bool` config flag +- Privacy hook: pause always-on when screen is locked + +**Acceptance criteria:** +- [ ] User can speak without pressing any hotkey +- [ ] VAD detects end of utterance and sends to agent +- [ ] Toggle in Settings → Voice + +--- + +## Phase 3 — Wake-Word + Fast Routing ⏳ Not Started + +> Activate only on a trigger phrase; route simple commands locally without a full LLM turn. + +**Planned files:** +- `src/openhuman/inference/voice/wake_word.rs` (new) — lightweight always-on model (Porcupine or custom ONNX) +- `src/openhuman/voice/command_router.rs` (new) — intent→tool mapping for high-confidence commands, LLM fallback for ambiguous input + +**Acceptance criteria:** +- [ ] Wake-word detection runs fully on-device +- [ ] Latency from end-of-utterance to action start ≤ 500ms for local-routed commands + +--- + +## Phase 4 — Polish ⏳ Not Started + +> Voice confirmation loop, UI indicator, computer control onboarding. + +**Planned:** +- TTS confirmation before executing sensitive actions ("Opening Music — confirm?") +- Always-on status indicator (notch pill from PR #3166 will handle this automatically) +- Computer control (`mouse`/`keyboard` tools) toggle in Settings onboarding + +--- + +## Summary + +| Phase | Item | Status | +|---|---|---| +| 1 | Auto-send after transcription | ✅ Done | +| 1 | Shell allowlist for `open`/`xdg-open` | ✅ Done | +| 1 | Shell tool description clarification | ✅ Done | +| 1 | Dedicated `launch_app` tool | ✅ Done | +| 1 | Orchestrator tool scope | ✅ Done | +| 1 | SOUL.md capability hint | ✅ Done | +| 1 | Diagnostic logging | ✅ Done | +| 1 | Computer control (mouse/keyboard) | ❌ Reverted (CEF crash) | +| 1 | AXUIElement app UI interaction (`ax_interact`) | ✅ Done | +| 1 | Multi-step UI workflow guidance | ✅ Done | +| 1 | Apple Music two-step play (navigate→play) | ✅ Done (playback best-effort) | +| 2 | Always-on microphone loop | ⏳ Not started | +| 2 | `always_on_enabled` config flag | ⏳ Not started | +| 2 | Privacy hook (screen lock pause) | ⏳ Not started | +| 3 | Wake-word detection | ⏳ Not started | +| 3 | Local command router | ⏳ Not started | +| 4 | Voice confirmation loop | ⏳ Not started | +| 4 | Always-on UI indicator | ✅ Done (notch PR #3166) | diff --git a/src/openhuman/accessibility/ax_interact.rs b/src/openhuman/accessibility/ax_interact.rs new file mode 100644 index 000000000..dda9724e0 --- /dev/null +++ b/src/openhuman/accessibility/ax_interact.rs @@ -0,0 +1,152 @@ +//! Accessibility interaction helpers — list, press, and set-value for named apps. +//! +//! Cross-platform facade over the OS accessibility tree. Each public fn +//! cfg-dispatches to the right backend: +//! - macOS: the unified Swift helper (`helper.rs`), which walks the AX tree +//! without injecting synthetic events (unlike enigo/CGEventPost). +//! Works even when OpenHuman is not focused, and never crashes CEF. +//! - Windows: the UI Automation backend (`uia_interact.rs`), which drives the +//! UIA COM tree directly — same "no synthetic input" guarantee. +//! +//! Other platforms return a clean runtime error. The agent-facing `ax_interact` +//! tool is a single tool on every platform; only the backend differs. + +use serde::Deserialize; + +#[cfg(test)] +#[path = "ax_interact_tests.rs"] +mod tests; + +#[cfg(all(test, target_os = "windows"))] +#[path = "uia_interact_tests.rs"] +mod uia_tests; + +#[derive(Debug, Clone, Deserialize)] +pub struct AXElement { + pub role: String, + pub label: String, +} + +/// List interactive UI elements (buttons, text fields, checkboxes, …) in `app_name`. +pub fn ax_list_elements(app_name: &str) -> Result, String> { + ax_list_elements_filtered(app_name, "") +} + +/// List interactive UI elements in `app_name`, optionally keeping only those +/// whose label contains `filter` (case-insensitive). An empty `filter` returns +/// everything. Filtering happens on the Rust side so the tool result stays +/// small — dumping every element (apps expose hundreds) overflows the result +/// budget and causes the model to hallucinate from a truncated view. +pub fn ax_list_elements_filtered(app_name: &str, filter: &str) -> Result, String> { + #[cfg(target_os = "macos")] + { + let req = serde_json::json!({ "type": "ax_list", "app_name": app_name }); + let resp = super::helper::helper_send_receive(&req)?; + if resp.get("ok").and_then(|v| v.as_bool()) == Some(false) { + let err = resp + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(err.to_string()); + } + let mut elements: Vec = resp + .get("elements") + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or_default(); + let needle = filter.trim().to_lowercase(); + if !needle.is_empty() { + elements.retain(|e| e.label.to_lowercase().contains(&needle)); + } + return Ok(elements); + } + #[cfg(target_os = "windows")] + { + return super::uia_interact::list(app_name, filter); + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + let _ = (app_name, filter); + Err("ax_interact is supported on macOS and Windows only".into()) + } +} + +/// Press the first UI element in `app_name` whose label contains `label` (case-insensitive). +/// +/// Rejects a blank `label`: with an empty needle the helper's `contains` +/// match degenerates to match-all and would press the first named control it +/// finds. Guard here rather than trusting every caller to pre-validate. +pub fn ax_press_element(app_name: &str, label: &str) -> Result { + if label.trim().is_empty() { + return Err("label must not be empty for press".into()); + } + #[cfg(target_os = "macos")] + { + let req = serde_json::json!({ + "type": "ax_press", + "app_name": app_name, + "label": label, + }); + let resp = super::helper::helper_send_receive(&req)?; + if resp.get("ok").and_then(|v| v.as_bool()) == Some(false) { + let err = resp + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(err.to_string()); + } + let pressed = resp + .get("pressed") + .and_then(|v| v.as_str()) + .unwrap_or(label) + .to_string(); + return Ok(format!("Pressed '{pressed}' in '{app_name}'.")); + } + #[cfg(target_os = "windows")] + { + return super::uia_interact::press(app_name, label); + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + let _ = (app_name, label); + Err("ax_interact is supported on macOS and Windows only".into()) + } +} + +/// Set the value of the first text field in `app_name` whose label contains `label`. +/// Pass an empty `label` to target the first available text field. +pub fn ax_set_field_value(app_name: &str, label: &str, value: &str) -> Result { + #[cfg(target_os = "macos")] + { + let req = serde_json::json!({ + "type": "ax_set_value", + "app_name": app_name, + "label": label, + "value": value, + }); + let resp = super::helper::helper_send_receive(&req)?; + if resp.get("ok").and_then(|v| v.as_bool()) == Some(false) { + let err = resp + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(err.to_string()); + } + let field = resp + .get("field") + .and_then(|v| v.as_str()) + .unwrap_or(label) + .to_string(); + return Ok(format!( + "Set '{field}' in '{app_name}' to the provided value." + )); + } + #[cfg(target_os = "windows")] + { + return super::uia_interact::set_value(app_name, label, value); + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + let _ = (app_name, label, value); + Err("ax_interact is supported on macOS and Windows only".into()) + } +} diff --git a/src/openhuman/accessibility/ax_interact_tests.rs b/src/openhuman/accessibility/ax_interact_tests.rs new file mode 100644 index 000000000..89f57906e --- /dev/null +++ b/src/openhuman/accessibility/ax_interact_tests.rs @@ -0,0 +1,167 @@ +//! Integration tests for AXUIElement-based app interaction. +//! +//! These tests require: +//! 1. macOS with Accessibility permission granted to the test runner +//! 2. Apple Music to be running (or openable) +//! +//! Run with: cargo test ax_interact -- --nocapture --include-ignored + +#![cfg(all(test, target_os = "macos"))] + +use super::{ax_list_elements, ax_press_element, ax_set_field_value}; +use std::process::Command; +use std::thread::sleep; +use std::time::Duration; + +fn ensure_music_open() -> bool { + let ok = Command::new("open") + .arg("-a") + .arg("Music") + .status() + .map(|s| s.success()) + .unwrap_or(false); + if ok { + sleep(Duration::from_secs(2)); + } + ok +} + +fn open_acdc_search() { + Command::new("open") + .arg("music://music.apple.com/search?term=Highway+to+Hell+ACDC") + .status() + .ok(); + sleep(Duration::from_secs(3)); +} + +#[test] +#[ignore = "requires macOS Accessibility permission and Apple Music"] +fn test_ax_list_returns_elements() { + assert!(ensure_music_open(), "Could not open Music"); + let elements = ax_list_elements("Music").expect("ax_list_elements failed"); + assert!( + !elements.is_empty(), + "Expected interactive elements in Music" + ); + println!("Found {} elements:", elements.len()); + for el in &elements { + println!(" [{}] {}", el.role, el.label); + } +} + +#[test] +#[ignore = "requires macOS Accessibility permission and Apple Music"] +fn test_ax_press_play_button() { + assert!(ensure_music_open(), "Could not open Music"); + let result = ax_press_element("Music", "Play"); + println!("press Play: {:?}", result); + assert!( + result.is_ok(), + "Expected Play button to be pressable: {:?}", + result + ); +} + +#[test] +#[ignore = "requires macOS Accessibility permission and Apple Music"] +fn test_full_flow_search_and_play_acdc() { + assert!(ensure_music_open(), "Could not open Music"); + + let elements = ax_list_elements("Music").expect("ax_list failed"); + assert!( + !elements.is_empty(), + "Music AX tree empty — check Accessibility permission" + ); + println!("[step 1] AX tree: {} elements", elements.len()); + + open_acdc_search(); + println!("[step 2] search URL opened"); + + let after_search = ax_list_elements("Music").expect("ax_list post-search failed"); + let highway = after_search + .iter() + .find(|e| e.label.contains("Highway to Hell")); + println!( + "[step 3] 'Highway to Hell' element: {:?}", + highway.map(|e| &e.label) + ); + assert!( + highway.is_some(), + "Expected 'Highway to Hell' in results. Found:\n{}", + after_search + .iter() + .map(|e| format!(" [{}] {}", e.role, e.label)) + .collect::>() + .join("\n") + ); + + // Pressing the search result NAVIGATES into the song detail page + // (it does not start playback — Apple Music only selects/opens on press). + let nav_result = ax_press_element("Music", "Highway to Hell"); + println!("[step 4] navigate into song: {:?}", nav_result); + assert!( + nav_result.is_ok(), + "Could not navigate into song: {:?}", + nav_result + ); + + sleep(Duration::from_secs(2)); + + // On the detail page, press the prominent Play button to actually play. + let play_result = ax_press_element("Music", "Play"); + println!("[step 5] press detail Play: {:?}", play_result); + assert!( + play_result.is_ok(), + "Could not press detail Play: {:?}", + play_result + ); + + sleep(Duration::from_secs(2)); + + // Playback outcome is best-effort and NOT asserted: Apple Music's UI is + // nondeterministic here (detail-page render timing varies, and there are + // multiple "Play" elements — detail-page vs transport-bar — that AX can't + // reliably disambiguate). What this test verifies is that the generic + // ax_interact primitives (list / press) work against a real app; the + // player state is logged for diagnosis only. + let state = Command::new("osascript") + .args(["-e", "tell application \"Music\" to get player state"]) + .output() + .expect("osascript player state failed"); + let state_str = String::from_utf8_lossy(&state.stdout); + println!( + "[step 6] player state (best-effort, not asserted): {}", + state_str.trim() + ); +} + +#[test] +#[ignore = "requires macOS Accessibility permission and Apple Music"] +fn test_ax_set_search_field() { + assert!(ensure_music_open(), "Could not open Music"); + Command::new("open") + .arg("music://music.apple.com/search") + .status() + .ok(); + sleep(Duration::from_secs(2)); + let result = ax_set_field_value("Music", "Search", "Bollywood"); + println!("set_value Search=Bollywood: {:?}", result); + assert!( + result.is_ok(), + "Expected the Search field to accept text: {:?}", + result + ); +} + +#[test] +fn test_ax_list_nonexistent_app() { + let result = ax_list_elements("NonExistentApp12345"); + assert!(result.is_err(), "Expected error for non-existent app"); + println!("Error (expected): {:?}", result.unwrap_err()); +} + +#[test] +fn test_ax_press_nonexistent_app() { + let result = ax_press_element("NonExistentApp12345", "Play"); + assert!(result.is_err()); +} diff --git a/src/openhuman/accessibility/helper.rs b/src/openhuman/accessibility/helper.rs index 123d67962..c271915ae 100644 --- a/src/openhuman/accessibility/helper.rs +++ b/src/openhuman/accessibility/helper.rs @@ -80,7 +80,7 @@ const HELPER_RECV_TIMEOUT: Duration = Duration::from_secs(8); /// `UNIFIED_HELPER` before blocking on the channel recv, so fire-and-forget /// callers (`show`/`hide`) are never blocked by an in-flight focus query. #[cfg(target_os = "macos")] -pub(super) fn helper_send_receive( +pub(crate) fn helper_send_receive( request: &serde_json::Value, ) -> Result { // Serialise request/response pairs — prevents interleaved reads. @@ -645,6 +645,138 @@ func pasteText(id: String?, text: String) -> [String: Any] { return result } +// MARK: - AXUIElement App Interaction + +/// Find a running application by display name (exact, prefix, or contains match). +func findRunningApp(named appName: String) -> NSRunningApplication? { + let apps = NSWorkspace.shared.runningApplications + let lower = appName.lowercased() + if let app = apps.first(where: { $0.localizedName?.lowercased() == lower }) { return app } + if let app = apps.first(where: { $0.localizedName?.lowercased().hasPrefix(lower) ?? false }) { return app } + return apps.first(where: { $0.localizedName?.lowercased().contains(lower) ?? false }) +} + +/// Walk the AX element tree depth-first. Visitor returns true to stop early. +func axWalk(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = 12, + visitor: (AXUIElement, String, String) -> Bool) -> Bool { + if depth > maxDepth { return false } + var roleRef: AnyObject? + AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &roleRef) + let role = (roleRef as? String) ?? "" + var labelRef: AnyObject? + AXUIElementCopyAttributeValue(element, kAXTitleAttribute as CFString, &labelRef) + var label = (labelRef as? String) ?? "" + if label.isEmpty { + var descRef: AnyObject? + AXUIElementCopyAttributeValue(element, kAXDescriptionAttribute as CFString, &descRef) + label = (descRef as? String) ?? "" + } + if visitor(element, role, label) { return true } + var childrenRef: AnyObject? + guard AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &childrenRef) == .success, + let children = childrenRef as? [AXUIElement] else { return false } + for child in children { + if axWalk(child, depth: depth + 1, maxDepth: maxDepth, visitor: visitor) { return true } + } + return false +} + +/// List interactive UI elements in the named app. +func axListElements(appName: String, id: String?) -> [String: Any] { + guard let app = findRunningApp(named: appName) else { + return ["type": "ax_list", "id": id ?? "", "ok": false, + "error": "App '\(appName)' not found or not running", "elements": [] as [Any]] + } + let axApp = AXUIElementCreateApplication(app.processIdentifier) + let interactiveRoles: Set = [ + "AXButton", "AXMenuItem", "AXMenuBarItem", "AXTextField", "AXTextArea", + "AXCheckBox", "AXRadioButton", "AXSlider", "AXPopUpButton", + "AXComboBox", "AXLink", "AXTab" + ] + var elements: [[String: String]] = [] + axWalk(axApp, maxDepth: 10) { _, role, label in + if interactiveRoles.contains(role) && !label.isEmpty { + elements.append(["role": role, "label": label]) + } + return false + } + return ["type": "ax_list", "id": id ?? "", "ok": true, "error": NSNull(), "elements": elements] +} + +/// Collect all AX elements whose label contains `label` (case-insensitive). +/// Returns matches sorted exact-first so "Play" beats "Playlist". +struct AXCandidate { + var element: AXUIElement + var label: String + var exact: Bool +} + +func axCollectMatches(_ root: AXUIElement, label: String, depth: Int = 0, maxDepth: Int = 12) -> [AXCandidate] { + if depth > maxDepth { return [] } + var roleRef: AnyObject?; AXUIElementCopyAttributeValue(root, kAXRoleAttribute as CFString, &roleRef) + var titleRef: AnyObject?; AXUIElementCopyAttributeValue(root, kAXTitleAttribute as CFString, &titleRef) + var elemLabel = (titleRef as? String) ?? "" + if elemLabel.isEmpty { var d: AnyObject?; AXUIElementCopyAttributeValue(root, kAXDescriptionAttribute as CFString, &d); elemLabel = (d as? String) ?? "" } + let lower = label.lowercased(); let elLower = elemLabel.lowercased() + var results: [AXCandidate] = [] + if !elemLabel.isEmpty, elLower.contains(lower) { + results.append(AXCandidate(element: root, label: elemLabel, exact: elLower == lower)) + } + var childrenRef: AnyObject? + guard AXUIElementCopyAttributeValue(root, kAXChildrenAttribute as CFString, &childrenRef) == .success, + let children = childrenRef as? [AXUIElement] else { return results } + for child in children { results += axCollectMatches(child, label: label, depth: depth+1, maxDepth: maxDepth) } + return results +} + +/// Press a UI element by label — exact match preferred over contains match. +func axPress(appName: String, label: String, id: String?) -> [String: Any] { + guard let app = findRunningApp(named: appName) else { + return ["type": "ax_press", "id": id ?? "", "ok": false, + "error": "App '\(appName)' not found or not running"] + } + let axApp = AXUIElementCreateApplication(app.processIdentifier) + var matches = axCollectMatches(axApp, label: label) + // Exact matches first so "Play" beats "Playlist" + matches.sort { $0.exact && !$1.exact } + for match in matches { + if AXUIElementPerformAction(match.element, kAXPressAction as CFString) == .success { + return ["type": "ax_press", "id": id ?? "", "ok": true, "error": NSNull(), + "pressed": match.label] + } + } + return ["type": "ax_press", "id": id ?? "", "ok": false, + "error": "No pressable element matching '\(label)' found in '\(appName)' (\(matches.count) label match(es) not actionable)"] +} + +/// Set the value of a text field by partial label match (or first text field if label is empty). +func axSetValue(appName: String, label: String, value: String, id: String?) -> [String: Any] { + guard let app = findRunningApp(named: appName) else { + return ["type": "ax_set_value", "id": id ?? "", "ok": false, + "error": "App '\(appName)' not found or not running"] + } + let axApp = AXUIElementCreateApplication(app.processIdentifier) + let textRoles: Set = ["AXTextField", "AXTextArea", "AXSearchField", "AXComboBox"] + let lower = label.lowercased() + var setLabel = "" + let found = axWalk(axApp) { element, role, elemLabel in + guard textRoles.contains(role) else { return false } + let matchLabel = lower.isEmpty || elemLabel.lowercased().contains(lower) + guard matchLabel else { return false } + if AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFTypeRef) == .success { + setLabel = elemLabel + return true + } + return false + } + if found { + return ["type": "ax_set_value", "id": id ?? "", "ok": true, "error": NSNull(), + "field": setLabel] + } + return ["type": "ax_set_value", "id": id ?? "", "ok": false, + "error": "No text field matching '\(label)' found in '\(appName)'"] +} + // MARK: - Overlay Controller final class OverlayController { @@ -807,6 +939,21 @@ DispatchQueue.global(qos: .userInitiated).async { let response = pasteText(id: id, text: text) writeResponse(response) + case "ax_list": + let appName = (payload["app_name"] as? String) ?? "" + writeResponse(axListElements(appName: appName, id: id)) + + case "ax_press": + let appName = (payload["app_name"] as? String) ?? "" + let label = (payload["label"] as? String) ?? "" + writeResponse(axPress(appName: appName, label: label, id: id)) + + case "ax_set_value": + let appName = (payload["app_name"] as? String) ?? "" + let label = (payload["label"] as? String) ?? "" + let value = (payload["value"] as? String) ?? "" + writeResponse(axSetValue(appName: appName, label: label, value: value, id: id)) + case "show": let x = CGFloat((payload["x"] as? NSNumber)?.doubleValue ?? 0) let y = CGFloat((payload["y"] as? NSNumber)?.doubleValue ?? 0) diff --git a/src/openhuman/accessibility/mod.rs b/src/openhuman/accessibility/mod.rs index 5673b3544..ccd1dd184 100644 --- a/src/openhuman/accessibility/mod.rs +++ b/src/openhuman/accessibility/mod.rs @@ -6,6 +6,7 @@ //! instead of owning platform-specific code directly. mod automation_state; +pub mod ax_interact; mod capture; mod focus; mod globe; @@ -17,6 +18,10 @@ mod permissions; mod terminal; mod text_util; mod types; +// Windows accessibility backend for `ax_interact` (UI Automation). Sibling of +// the macOS Swift-helper path; selected via cfg-dispatch in `ax_interact.rs`. +#[cfg(target_os = "windows")] +mod uia_interact; #[cfg(test)] pub(crate) use automation_state::test_lock as automation_state_test_lock; diff --git a/src/openhuman/accessibility/uia_interact.rs b/src/openhuman/accessibility/uia_interact.rs new file mode 100644 index 000000000..4ec106023 --- /dev/null +++ b/src/openhuman/accessibility/uia_interact.rs @@ -0,0 +1,295 @@ +//! Windows UI Automation (UIA) backend for the `ax_interact` tool. +//! +//! The Windows analogue of the macOS AXUIElement helper (`ax_interact.rs`): +//! it provides the same three primitives — `list` / `press` / `set_value` — +//! addressing UI elements by their semantic label, using Microsoft UI +//! Automation (the OS-level accessibility tree) via the `uiautomation` crate +//! (safe Rust wrappers over the UIA COM API). +//! +//! Why this is cleaner than the macOS path: +//! - No helper process. macOS shells out to a Swift helper; on Windows the +//! UIA COM API is callable directly from Rust. +//! - No synthetic input. `press` activates controls through UIA patterns +//! (`Invoke` / `SelectionItem.Select` / `LegacyIAccessible.DoDefaultAction`), +//! never injecting mouse/keyboard events — so there is no CEF-crash risk +//! (the bug that forced the macOS `mouse`/`keyboard` revert) and it works +//! regardless of which window is focused. +//! - No special permission for same-integrity-level apps. (UIPI still blocks +//! a non-elevated process from driving an *elevated* app's UI.) +//! +//! Windows only. Reached through cfg-dispatch in `ax_interact.rs`; the +//! agent-facing tool stays a single `ax_interact` tool on every platform. + +use super::ax_interact::AXElement; +use uiautomation::controls::ControlType; +use uiautomation::core::{UIAutomation, UIElement}; +use uiautomation::patterns::{ + UIInvokePattern, UILegacyIAccessiblePattern, UISelectionItemPattern, UIValuePattern, +}; + +/// Matcher retry-wait budget. UIA windows/elements can lag behind a launch or a +/// navigation; the matcher polls until this deadline before giving up. +const FIND_TIMEOUT_MS: u64 = 2000; + +/// How deep to walk an app's UI subtree. Deep enough for nested panes/lists, +/// shallow enough to stay fast; the tool layer caps and filters the output. +const TREE_DEPTH: u32 = 40; + +/// Initialise COM on the calling (tokio blocking-pool) thread before creating a +/// UIA client. Idempotent — `S_OK` the first time, `S_FALSE` if COM is already +/// up on this thread, `RPC_E_CHANGED_MODE` if a different apartment was already +/// chosen (all acceptable). We never `CoUninitialize`: the worker thread keeps +/// COM live for the process lifetime, which is how `uiautomation` expects to be +/// driven. +fn ensure_com() { + use windows_sys::Win32::System::Com::{CoInitializeEx, COINIT_MULTITHREADED}; + let _ = unsafe { CoInitializeEx(std::ptr::null(), COINIT_MULTITHREADED as u32) }; +} + +/// Control types worth surfacing as "interactive" in a `list`. Mirrors the +/// macOS helper's button/field/cell focus. `Text` is included because read-only +/// readouts (e.g. a Calculator result, a status label) are often the thing a +/// caller wants to inspect; everything is still narrowed by the caller's filter +/// and capped by the tool layer. +fn is_interactive(ct: ControlType) -> bool { + matches!( + ct, + ControlType::Button + | ControlType::Edit + | ControlType::ListItem + | ControlType::MenuItem + | ControlType::CheckBox + | ControlType::RadioButton + | ControlType::ComboBox + | ControlType::Hyperlink + | ControlType::TabItem + | ControlType::TreeItem + | ControlType::SplitButton + | ControlType::Text + ) +} + +/// Locate the top-level window for `app_name`. Matches a `Window` element whose +/// `Name` equals (preferred) or contains `app_name`, case-insensitively. UWP +/// apps nest under `ApplicationFrameWindow`, so we allow a few levels of depth. +fn find_window(automation: &UIAutomation, app_name: &str) -> Result { + let root = automation + .get_root_element() + .map_err(|e| format!("UIA root element unavailable: {e}"))?; + let matcher = automation + .create_matcher() + .from(root) + .control_type(ControlType::Window) + .depth(6) + .timeout(FIND_TIMEOUT_MS); + let windows = matcher.find_all().unwrap_or_default(); + + let needle = app_name.trim().to_lowercase(); + let mut contains: Option = None; + for w in windows { + let Ok(name) = w.get_name() else { continue }; + let nl = name.trim().to_lowercase(); + if nl.is_empty() { + continue; + } + if nl == needle { + return Ok(w); // exact title match wins + } + if contains.is_none() && !needle.is_empty() && nl.contains(&needle) { + contains = Some(w); + } + } + contains.ok_or_else(|| { + format!( + "No open window matches app '{app_name}'. Make sure it is running \ + (try launch_app first), then retry." + ) + }) +} + +/// Find an element under `window` by label. Exact (case-insensitive) match is +/// preferred over a substring match — so "Play" beats "Playlist", mirroring the +/// macOS exact-match-preferred behaviour. Returns the element plus its resolved +/// `Name` for messaging. +fn find_by_label( + automation: &UIAutomation, + window: &UIElement, + label: &str, +) -> Result<(UIElement, String), String> { + let matcher = automation + .create_matcher() + .from(window.clone()) + .depth(TREE_DEPTH) + .timeout(FIND_TIMEOUT_MS); + let elements = matcher.find_all().unwrap_or_default(); + + let needle = label.trim().to_lowercase(); + let mut contains: Option<(UIElement, String)> = None; + for el in elements { + let Ok(name) = el.get_name() else { continue }; + let nl = name.trim().to_lowercase(); + if nl.is_empty() { + continue; + } + if nl == needle { + return Ok((el, name)); // exact preferred + } + if contains.is_none() && nl.contains(&needle) { + contains = Some((el, name)); + } + } + contains.ok_or_else(|| { + format!( + "No element labelled '{label}' found. Try action='list' with a \ + filter to see available labels." + ) + }) +} + +/// Find the first editable text field under `window`. Prefers a plain `Edit` +/// control (classic text boxes, WordPad, classic Notepad), then falls back to a +/// `ComboBox` (editable dropdowns) and finally a `Document` control (rich-text +/// editors such as the redesigned Windows 11 Notepad, which exposes its editor +/// as a `Document` rather than an `Edit`). The caller still requires the chosen +/// element to expose the UIA `Value` pattern before writing to it. +fn find_first_edit(automation: &UIAutomation, window: &UIElement) -> Result { + let matcher = automation + .create_matcher() + .from(window.clone()) + .depth(TREE_DEPTH) + .timeout(FIND_TIMEOUT_MS); + let elements = matcher.find_all().unwrap_or_default(); + + let mut combo: Option = None; + let mut document: Option = None; + for el in elements { + match el.get_control_type() { + Ok(ControlType::Edit) => return Ok(el), // best match — return immediately + Ok(ControlType::ComboBox) if combo.is_none() => combo = Some(el), + Ok(ControlType::Document) if document.is_none() => document = Some(el), + _ => {} + } + } + combo.or(document).ok_or_else(|| { + "no editable text field (Edit / ComboBox / Document) found in the app".to_string() + }) +} + +/// List interactive UI elements in `app_name`, keeping only those whose label +/// contains `filter` (case-insensitive; empty = all). Filtering happens here so +/// the tool result stays small — dumping a full UIA tree (apps expose hundreds +/// of elements) overflows the result budget and makes the model hallucinate +/// from a truncated view. +pub fn list(app_name: &str, filter: &str) -> Result, String> { + ensure_com(); + let automation = UIAutomation::new().map_err(|e| format!("UIA init failed: {e}"))?; + let window = find_window(&automation, app_name)?; + + let matcher = automation + .create_matcher() + .from(window) + .depth(TREE_DEPTH) + .timeout(FIND_TIMEOUT_MS); + let elements = match matcher.find_all() { + Ok(v) => v, + Err(e) => { + log::debug!("[uia_interact] list: tree walk returned empty for '{app_name}': {e}"); + Vec::new() + } + }; + + let needle = filter.trim().to_lowercase(); + let mut out = Vec::new(); + for el in elements { + let Ok(ct) = el.get_control_type() else { + continue; + }; + if !is_interactive(ct) { + continue; + } + let label = el.get_name().unwrap_or_default().trim().to_string(); + if label.is_empty() { + continue; + } + if !needle.is_empty() && !label.to_lowercase().contains(&needle) { + continue; + } + out.push(AXElement { + role: format!("{ct:?}"), + label, + }); + } + + log::info!( + "[uia_interact] list app={app_name:?} filter={filter:?} -> {} elements", + out.len() + ); + Ok(out) +} + +/// Activate the element in `app_name` whose label matches `label`. Uses UIA +/// patterns in order of preference — `Invoke` (buttons/links/menu items), then +/// `SelectionItem.Select` (list rows/tabs), then the `LegacyIAccessible` +/// default action — and never injects synthetic mouse/keyboard input. +pub fn press(app_name: &str, label: &str) -> Result { + ensure_com(); + let automation = UIAutomation::new().map_err(|e| format!("UIA init failed: {e}"))?; + let window = find_window(&automation, app_name)?; + let (element, matched) = find_by_label(&automation, &window, label)?; + + log::info!("[uia_interact] press app={app_name:?} label={label:?} matched={matched:?}"); + + if let Ok(p) = element.get_pattern::() { + p.invoke() + .map_err(|e| format!("invoke '{matched}' failed: {e}"))?; + return Ok(format!("Pressed '{matched}' in '{app_name}'.")); + } + if let Ok(p) = element.get_pattern::() { + p.select() + .map_err(|e| format!("select '{matched}' failed: {e}"))?; + return Ok(format!("Selected '{matched}' in '{app_name}'.")); + } + if let Ok(p) = element.get_pattern::() { + p.do_default_action() + .map_err(|e| format!("default action on '{matched}' failed: {e}"))?; + return Ok(format!("Activated '{matched}' in '{app_name}'.")); + } + + Err(format!( + "Element '{matched}' in '{app_name}' exposes no Invoke / Select / default \ + action — it may not be activatable. Try action='list' to find the real control." + )) +} + +/// Set the value of a text field in `app_name`. With an empty `label`, targets +/// the first editable field; otherwise finds the field by label. Requires the +/// element to expose the UIA `Value` pattern. +pub fn set_value(app_name: &str, label: &str, value: &str) -> Result { + ensure_com(); + let automation = UIAutomation::new().map_err(|e| format!("UIA init failed: {e}"))?; + let window = find_window(&automation, app_name)?; + + let (element, matched) = if label.trim().is_empty() { + let el = find_first_edit(&automation, &window)?; + let name = el.get_name().unwrap_or_default(); + let name = if name.trim().is_empty() { + "text field".to_string() + } else { + name + }; + (el, name) + } else { + find_by_label(&automation, &window, label)? + }; + + log::info!("[uia_interact] set_value app={app_name:?} field={matched:?}"); + + let vp = element + .get_pattern::() + .map_err(|e| format!("'{matched}' is not a settable text field (no Value pattern): {e}"))?; + vp.set_value(value) + .map_err(|e| format!("set_value on '{matched}' failed: {e}"))?; + Ok(format!( + "Set '{matched}' in '{app_name}' to the provided value." + )) +} diff --git a/src/openhuman/accessibility/uia_interact_tests.rs b/src/openhuman/accessibility/uia_interact_tests.rs new file mode 100644 index 000000000..4ae3dbc75 --- /dev/null +++ b/src/openhuman/accessibility/uia_interact_tests.rs @@ -0,0 +1,114 @@ +//! Integration tests for the Windows UI Automation backend of `ax_interact`. +//! +//! These exercise the SAME public helpers the tool uses (`ax_list_elements`, +//! `ax_press_element`, `ax_set_field_value`) — which cfg-dispatch to +//! `uia_interact` on Windows — so they validate the real agent-facing path. +//! +//! Most need a live desktop session and a real app, so they are `#[ignore]` by +//! default. Run them on a Windows machine with: +//! +//! cargo test --lib uia_interact -- --nocapture --include-ignored +//! +//! `test_uia_list_nonexistent_app` is deterministic (asserts an error) and runs +//! in the normal suite. + +#![cfg(all(test, target_os = "windows"))] + +use super::{ax_list_elements, ax_list_elements_filtered, ax_press_element, ax_set_field_value}; +use std::process::Command; +use std::thread::sleep; +use std::time::Duration; + +/// Spawn a launcher exe (e.g. `calc.exe`, `notepad.exe`). Returns whether the +/// spawn itself succeeded; the launched app may appear a moment later. +fn launch(exe: &str) -> bool { + Command::new(exe).spawn().is_ok() +} + +/// Deterministic UI control test (the Windows analogue of the macOS AC/DC test): +/// drive Calculator through `5 + 5 =` purely by element label, then hard-assert +/// the readout shows 10. Calculator is deterministic and always present, so this +/// is a real assertion, not best-effort. +#[test] +#[ignore = "requires a desktop session; launches the Calculator app"] +fn test_uia_calculator_five_plus_five() { + assert!(launch("calc.exe"), "could not spawn calc.exe"); + sleep(Duration::from_secs(3)); + + // 1. List — Calculator should expose its buttons via UIA. + let elements = ax_list_elements("Calculator").expect("ax_list_elements(Calculator) failed"); + assert!( + !elements.is_empty(), + "Calculator exposed no interactive elements — UIA tree empty?" + ); + println!("[calc] {} interactive elements", elements.len()); + + // 2. Press 5, +, 5, = by their (English) UIA Names. + for label in ["Five", "Plus", "Five", "Equals"] { + let r = ax_press_element("Calculator", label); + println!("[calc] press {label}: {r:?}"); + assert!(r.is_ok(), "press '{label}' failed: {r:?}"); + sleep(Duration::from_millis(300)); + } + + // 3. Assert the result readout computed 10. + sleep(Duration::from_millis(500)); + let readout = ax_list_elements_filtered("Calculator", "Display").unwrap_or_default(); + println!("[calc] readout (filter='Display'): {readout:?}"); + let shows_ten = readout.iter().any(|e| e.label.contains("10")) + || ax_list_elements("Calculator") + .unwrap_or_default() + .iter() + .any(|e| e.label.contains("10")); + assert!( + shows_ten, + "expected a result element showing 10 after 5 + 5 =; readout={readout:?}" + ); +} + +/// Text entry via `set_value` into Notepad's edit control. +/// +/// Win11's redesigned Notepad uses a RichEdit control that may not expose the +/// UIA `Value` pattern, whereas classic Notepad does. A `Value`-pattern absence +/// is treated as a documented limitation (best-effort); any other failure is a +/// real test failure. +#[test] +#[ignore = "requires a desktop session; launches Notepad"] +fn test_uia_notepad_set_value() { + assert!(launch("notepad.exe"), "could not spawn notepad.exe"); + sleep(Duration::from_secs(2)); + + let r = ax_set_field_value("Notepad", "", "OpenHuman UIA test"); + println!("[notepad] set_value: {r:?}"); + if let Err(e) = &r { + // The redesigned Win11 Notepad exposes its editor as a Document/RichEdit + // that may not support the UIA Value pattern (or any settable text + // field). That is a documented platform limitation, not a code bug, so + // treat it as a best-effort skip; classic Notepad / WordPad / ordinary + // Edit controls still exercise the real write path. + if e.contains("Value pattern") + || e.contains("settable") + || e.contains("no editable text field") + { + println!( + "[notepad] set_value unsupported on this Notepad build \ + (expected on Win11 RichEdit Notepad): {e}" + ); + return; + } + } + assert!(r.is_ok(), "set_value failed unexpectedly: {r:?}"); +} + +/// Deterministic, no-app-needed: a non-existent app must surface an error +/// (either "no window matches" or, in a session-less environment, a UIA-init +/// error — both are `Err` from our wrapper). +#[test] +fn test_uia_list_nonexistent_app() { + let r = ax_list_elements("OpenHuman_NoSuchApp_ZZZ123"); + assert!(r.is_err(), "expected error for non-existent app, got {r:?}"); + println!( + "[uia] nonexistent app error (expected): {:?}", + r.unwrap_err() + ); +} diff --git a/src/openhuman/agent/harness/session/builder.rs b/src/openhuman/agent/harness/session/builder.rs index 2f811bec3..3ee332f01 100644 --- a/src/openhuman/agent/harness/session/builder.rs +++ b/src/openhuman/agent/harness/session/builder.rs @@ -454,12 +454,15 @@ impl AgentBuilder { let visible_tool_specs: Vec = dedup_visible_tool_specs(visible_tool_specs_unfiltered); + let visible_names_list: Vec<&str> = + visible_tool_specs.iter().map(|s| s.name.as_str()).collect(); log::info!( - "[agent] tool spec filter: total={} visible={} (filter_active={} policy_restricted={})", + "[agent] tool spec filter: total={} visible={} (filter_active={} policy_restricted={}) names=[{}]", tool_specs.len(), visible_tool_specs.len(), !visible_names.is_empty(), - tool_policy_session.has_restrictions() + tool_policy_session.has_restrictions(), + visible_names_list.join(", ") ); // Pull the provider out of the builder once. We store it on diff --git a/src/openhuman/agent/prompts/SOUL.md b/src/openhuman/agent/prompts/SOUL.md index 6c632a6d0..916ccdbb5 100644 --- a/src/openhuman/agent/prompts/SOUL.md +++ b/src/openhuman/agent/prompts/SOUL.md @@ -17,6 +17,35 @@ You are OpenHuman — the user's AI teammate for productivity, research, and tea - Present alternatives and trade-offs when the call isn't obvious — then let the user pick. - Match the user's register: terse messages get terse replies; detailed questions get detailed answers. +## What you can do on the user's machine + +You run on the user's own desktop. You have tools that let you act on their behalf: + +- **`launch_app`** — open any application by name (e.g. Music, Spotify, Safari, Calculator, VS Code). When the user asks you to open an app, **always use this tool** — do not tell them to open it themselves. +- **`ax_interact`** — interact with a running app's UI via the platform accessibility API (macOS Accessibility / Windows UI Automation). Finds buttons, text fields, and controls by their label — no screen coordinates needed. Always call `action='list'` first to discover available elements, then `action='press'` to click or `action='set_value'` to type. +- **`shell`** — run shell commands in the workspace (git, npm, cargo, file operations, etc.). +- **`file_read` / `file_write`** — read and edit files in the workspace. + +Never say "I can't open apps" or "that's outside what I can do" when you have a tool to do it. Use the tool. + +**Workflow for interacting with an app's UI:** +1. `action='list'` — discover what buttons/fields/rows exist +2. `action='set_value'` to type in a filter or search field +3. `action='list'` again — see the updated/filtered results that appeared +4. `action='press'` — press the specific item (song row, playlist, etc.), NOT the generic Play button +5. Only press the playback-bar "Play" button after the right item is selected/playing + +**For playing a specific song in Apple Music (macOS) — use this EXACT sequence:** +1. `shell`: `open "music://music.apple.com/search?term=Song+Name+Artist"` (URL-encode the query) +2. Wait ~3s for results to load, then `ax_interact action='list' app_name='Music'` +3. `ax_interact action='press' app_name='Music' label=''` — this **navigates into** the song's detail page (it does NOT start playback yet — pressing a search-result row only opens it) +4. Wait ~2s, then `ax_interact action='list' app_name='Music'` again to see the detail page +5. `ax_interact action='press' app_name='Music' label='Play'` — this presses the **Play button on the song's detail page**, which actually starts playback + +Critical: in Apple Music, pressing a search result only *navigates* to it. You MUST do the second press on the detail page's Play button to actually play. Do not stop after step 3. Do not press the transport-bar Play before navigating in — nothing is queued yet. + +The example above is macOS-specific (the `open`/`music://` scheme and Apple Music). On Windows the same **list → press** pattern applies via UI Automation, but `ax_interact action='press'` usually *activates* a control directly (a list-row Invoke often plays/opens in one step), so the second navigate-then-play press is frequently unnecessary. Use `launch_app` to open the player, then `list` with a `filter` and `press` the specific item; re-`list` if a press only navigated. + ## When things go wrong - **Tool failure:** try a different approach before escalating. If you're stuck, name what failed and what you'd need to proceed. diff --git a/src/openhuman/agent_registry/agents/orchestrator/agent.toml b/src/openhuman/agent_registry/agents/orchestrator/agent.toml index b5dcebf12..0e354a913 100644 --- a/src/openhuman/agent_registry/agents/orchestrator/agent.toml +++ b/src/openhuman/agent_registry/agents/orchestrator/agent.toml @@ -117,6 +117,19 @@ named = [ "spawn_worker_thread", "spawn_parallel_agents", "composio_list_connections", + # App launching — lets the orchestrator open desktop applications + # directly when asked ("open Music", "launch Spotify", etc.) without + # delegating or telling the user it can't. Routes through the ApprovalGate + # (external effect), so launching prompts for confirmation per autonomy tier. + "launch_app", + # AXUIElement / UIA interaction — find UI by semantic label. No CGEventPost, + # no coordinate dependency, no CEF crash risk. Always call action='list' + # first to discover elements. Only the read-only `list` action is + # ReadOnly/unprompted; the mutating `press` / `set_value` actions are + # `Dangerous`, gate interactively through the ApprovalGate, are opt-in via + # `computer_control.ax_interact_mutations`, and refuse a sensitive-app + # denylist (password managers, Keychain, System Settings, terminals). + "ax_interact", # Time + scheduling — lets the orchestrator answer "what time is it", # "remind me in 10 minutes", "every morning at 8" directly rather than # delegating or telling the user it can't. `current_time` grounds diff --git a/src/openhuman/config/schema/tools.rs b/src/openhuman/config/schema/tools.rs index 4809c5085..76370a032 100644 --- a/src/openhuman/config/schema/tools.rs +++ b/src/openhuman/config/schema/tools.rs @@ -972,6 +972,12 @@ pub struct ComputerControlConfig { /// the user must explicitly opt in. #[serde(default)] pub enabled: bool, + /// Opt-in for the mutating `ax_interact` actions (`press` / `set_value`). + /// Disabled by default: the read-only `list` action is always available, + /// but actuating arbitrary app controls / typing into arbitrary fields + /// requires explicit user opt-in (mirrors `enabled` for mouse/keyboard). + #[serde(default)] + pub ax_interact_mutations: bool, } // ── Agent integration tools (backend-proxied) ─────────────────────── diff --git a/src/openhuman/security/policy_command.rs b/src/openhuman/security/policy_command.rs index 738809c99..f255652a9 100644 --- a/src/openhuman/security/policy_command.rs +++ b/src/openhuman/security/policy_command.rs @@ -511,6 +511,13 @@ const READ_ONLY_BASES: &[&str] = &[ "lsblk", "lscpu", "cut", + // NOTE: OS-native launchers (`open`, `xdg-open`, `start`) are deliberately + // NOT in the read-only set. `classify_command` only sees the base command, + // not its args, and these launchers can open arbitrary `https://` URLs and + // custom URI handlers — i.e. trigger outbound network / system actions — so + // treating them as `Read` (no approval) is too broad. App launching now + // goes through the dedicated `launch_app` tool, which is scoped to named + // applications only and carries no shell-arg ambiguity. // Windows cmd / PowerShell read verbs + common aliases "dir", "type", diff --git a/src/openhuman/tools/impl/computer/ax_interact.rs b/src/openhuman/tools/impl/computer/ax_interact.rs new file mode 100644 index 000000000..358d9179c --- /dev/null +++ b/src/openhuman/tools/impl/computer/ax_interact.rs @@ -0,0 +1,442 @@ +//! Tool: ax_interact — interact with desktop app UI via the OS accessibility API. +//! +//! Cross-platform: macOS uses AXUIElement (Swift helper), Windows uses UI +//! Automation (UIA COM API). Both back-ends: +//! - Never crash CEF (no synthetic key/mouse events injected system-wide) +//! - Work regardless of which app is focused +//! - Find elements by semantic label, not pixel coordinates +//! +//! Three actions: +//! list — enumerate interactive elements in a running app +//! press — activate a button/control by label +//! set_value — type text into a field by label +//! +//! Requires: macOS Accessibility permission granted to OpenHuman. On Windows no +//! special permission is needed for same-integrity-level apps (UIPI blocks +//! driving an elevated app from a non-elevated process). + +use crate::openhuman::accessibility::ax_interact as ax; +use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCallOptions, ToolResult}; +use async_trait::async_trait; +use serde_json::json; + +/// Apps whose UI must never be actuated by the agent. `press` / `set_value` +/// are refused when `app_name` matches any of these (case-insensitive +/// substring) — defense-in-depth that holds even on background/auto-approved +/// turns where the ApprovalGate may not prompt. `list` is also refused so the +/// agent can't enumerate, e.g., a password manager's fields. Matched by display +/// name; broad substrings ("keychain", "1password") cover localized variants. +const SENSITIVE_APPS: &[&str] = &[ + "keychain", + "1password", + "bitwarden", + "lastpass", + "dashlane", + "system settings", + "system preferences", + "console", // macOS Console (logs) + // Terminal emulators — mirror the set helper.rs treats as terminals + // (helper.rs ~:557) so "terminals are denied" actually holds. + "terminal", + "iterm", + "wezterm", + "warp", + "alacritty", + "kitty", + "ghostty", + "hyper", + "rio", +]; + +fn is_sensitive_app(app_name: &str) -> bool { + let lower = app_name.to_lowercase(); + SENSITIVE_APPS.iter().any(|s| lower.contains(s)) +} + +pub struct AxInteractTool { + /// When false, the mutating actions (`press` / `set_value`) are refused + /// with guidance to enable `computer_control.ax_interact_mutations`. The + /// read-only `list` action is always available. Mirrors the opt-in posture + /// of the mouse/keyboard tools (`computer_control.enabled`). + allow_mutations: bool, +} + +impl AxInteractTool { + pub fn new(allow_mutations: bool) -> Self { + Self { allow_mutations } + } +} + +impl Default for AxInteractTool { + fn default() -> Self { + // Default to read-only (mutations opt-in) — safe baseline. + Self::new(false) + } +} + +#[async_trait] +impl Tool for AxInteractTool { + fn name(&self) -> &str { + "ax_interact" + } + + fn description(&self) -> &str { + "Interact with ANY desktop application's UI using the platform accessibility API \ + (macOS AXUIElement / Windows UI Automation). Finds buttons, text fields, list rows, \ + and controls by their label — no screen coordinates, no synthetic key/mouse events. \ + Works for any app: a music player, browser, mail, notes, Slack, system settings, etc.\n\ + \n\ + Actions:\n\ + • 'list' → show interactive elements. ALWAYS pass a `filter` substring to narrow \ + results (apps expose hundreds of elements; an unfiltered list is huge and unreliable). \ + e.g. filter='Play', filter='Send', filter='Highway'.\n\ + • 'press' → activate a button/control/row by label (exact match preferred). \ + e.g. label='Play', label='Send', label='OK'.\n\ + • 'set_value' → type text into a field by label (omit label for the first text field).\n\ + \n\ + General pattern: (1) `list` with a `filter` to find the element, (2) `press` it. \ + Note that in many apps, pressing a LIST ROW or SEARCH RESULT only selects/opens it — \ + to trigger an action you then press the relevant action button (e.g. after opening a \ + song's page, press its 'Play' button). If a press doesn't have the intended effect, \ + `list` again to see the new screen and press the actual action control.\n\ + \n\ + On macOS this requires Accessibility permission for OpenHuman; on Windows no special \ + permission is needed for normal (non-elevated) apps." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": ["list", "press", "set_value"], + "description": "'list' = show interactive elements (use with filter); 'press' = activate a control by label; 'set_value' = type into a text field." + }, + "app_name": { + "type": "string", + "description": "Display name of the running application (e.g. 'Music', 'Safari', 'Telegram')." + }, + "filter": { + "type": "string", + "description": "For 'list': only return elements whose label contains this substring (case-insensitive). Strongly recommended — keeps results small and accurate." + }, + "label": { + "type": "string", + "description": "For 'press'/'set_value': label of the element to target (case-insensitive, exact match preferred). For 'set_value', omit to target the first text field." + }, + "value": { + "type": "string", + "description": "Text to enter (required for 'set_value')." + } + }, + "required": ["action", "app_name"] + }) + } + + fn permission_level(&self) -> PermissionLevel { + // Minimum across actions: `list` is read-only. The per-call level + // (Dangerous for press/set_value) is enforced by + // `permission_level_with_args`. Returning the minimum here keeps the + // tool available on channels that can run the read-only `list`. + PermissionLevel::ReadOnly + } + + fn permission_level_with_args(&self, args: &serde_json::Value) -> PermissionLevel { + match args.get("action").and_then(|v| v.as_str()) { + // `list` only reads the AX tree — no state change. + Some("list") | None => PermissionLevel::ReadOnly, + // `press` / `set_value` actuate real controls (click buttons, + // type into fields) and change application state, so they must + // not ride on the read-only path. + _ => PermissionLevel::Dangerous, + } + } + + fn external_effect_with_args(&self, args: &serde_json::Value) -> bool { + // Route mutating actions through the ApprovalGate before execute(); + // `list` is a pure read and flows through unprompted. + !matches!( + args.get("action").and_then(|v| v.as_str()), + Some("list") | None + ) + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.execute_with_options(args, ToolCallOptions::default()) + .await + } + + async fn execute_with_options( + &self, + args: serde_json::Value, + _options: ToolCallOptions, + ) -> anyhow::Result { + let action = args + .get("action") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + let app_name = args + .get("app_name") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + let label = args + .get("label") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + let value = args + .get("value") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let filter = args + .get("filter") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + + log::info!( + "[ax_interact] ▶ action={action:?} app={app_name:?} label={label:?} filter={filter:?}" + ); + + if app_name.is_empty() { + return Ok(ToolResult::error("app_name is required")); + } + + let mutating = matches!(action.as_str(), "press" | "set_value"); + + // Denylist: never actuate or enumerate sensitive apps (password + // managers, Keychain, System Settings, terminals). Defense-in-depth + // that holds even when the ApprovalGate doesn't prompt (background / + // auto-approved turns). + if is_sensitive_app(&app_name) { + log::warn!("[ax_interact] refused: sensitive app '{app_name}' (action={action})"); + return Ok(ToolResult::error(format!( + "Refusing to interact with '{app_name}': it is on the sensitive-app denylist \ + (password managers, Keychain, System Settings, terminals). This is a hard \ + safety boundary." + ))); + } + + // Mutating actions are opt-in. Read-only `list` is always allowed. + if mutating && !self.allow_mutations { + log::warn!("[ax_interact] refused: mutations disabled (action={action})"); + return Ok(ToolResult::error( + "ax_interact mutations (press/set_value) are disabled. They actuate arbitrary \ + app controls and type into arbitrary fields, so they require explicit opt-in: \ + set `computer_control.ax_interact_mutations = true`. The read-only 'list' \ + action remains available.", + )); + } + + // Cap how many elements we render so a broad/empty filter can't overflow + // the tool-result budget and cause the model to reason over a truncated view. + const MAX_LISTED: usize = 60; + + let result = match action.as_str() { + "list" => match ax::ax_list_elements_filtered(&app_name, &filter) { + Ok(elements) if elements.is_empty() => { + log::info!( + "[ax_interact] list: no elements in '{app_name}' (filter={filter:?})" + ); + let hint = if filter.is_empty() { + format!( + "No interactive elements found in '{app_name}'. \ + The app may not expose its UI tree via Accessibility API, \ + or OpenHuman may need Accessibility permission." + ) + } else { + format!( + "No elements in '{app_name}' match filter '{filter}'. \ + The UI may still be loading — wait and try again, or call \ + 'list' with a shorter/different filter." + ) + }; + ToolResult::success(hint) + } + Ok(elements) => { + let total = elements.len(); + log::info!( + "[ax_interact] list: {total} elements in '{app_name}' (filter={filter:?})" + ); + let shown = total.min(MAX_LISTED); + let lines: Vec = elements + .iter() + .take(MAX_LISTED) + .map(|e| format!(" [{role}] {label}", role = e.role, label = e.label)) + .collect(); + let mut out = if filter.is_empty() { + format!("Elements in '{app_name}' (showing {shown} of {total}):\n") + } else { + format!( + "Elements in '{app_name}' matching '{filter}' (showing {shown} of {total}):\n" + ) + }; + out.push_str(&lines.join("\n")); + if total > MAX_LISTED { + out.push_str(&format!( + "\n… {} more — narrow with a more specific `filter`.", + total - MAX_LISTED + )); + } + ToolResult::success(out) + } + Err(e) => { + log::warn!("[ax_interact] list failed: {e}"); + ToolResult::error(e) + } + }, + + "press" => { + if label.is_empty() { + return Ok(ToolResult::error( + "'label' is required for action='press'. Use action='list' first to discover element labels.", + )); + } + match ax::ax_press_element(&app_name, &label) { + Ok(msg) => { + log::info!("[ax_interact] press succeeded: {msg}"); + ToolResult::success(msg) + } + Err(e) => { + log::warn!("[ax_interact] press failed: {e}"); + ToolResult::error(format!( + "{e}. Try action='list' to see available element labels." + )) + } + } + } + + "set_value" => { + if value.is_empty() { + return Ok(ToolResult::error( + "'value' is required for action='set_value'", + )); + } + match ax::ax_set_field_value(&app_name, &label, &value) { + Ok(msg) => { + log::info!("[ax_interact] set_value succeeded: {msg}"); + ToolResult::success(msg) + } + Err(e) => { + log::warn!("[ax_interact] set_value failed: {e}"); + ToolResult::error(format!( + "{e}. Try action='list' to see available text field labels." + )) + } + } + } + + other => ToolResult::error(format!( + "Unknown action '{other}'. Valid actions: 'list', 'press', 'set_value'." + )), + }; + + Ok(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn name_and_permission() { + let tool = AxInteractTool::new(true); + assert_eq!(tool.name(), "ax_interact"); + assert_eq!(tool.permission_level(), PermissionLevel::ReadOnly); + // Mutating actions gate per-call. + assert_eq!( + tool.permission_level_with_args(&json!({"action": "press"})), + PermissionLevel::Dangerous + ); + assert!(tool.external_effect_with_args(&json!({"action": "press"}))); + assert!(!tool.external_effect_with_args(&json!({"action": "list"}))); + } + + #[test] + fn schema_requires_action_and_app_name() { + let schema = AxInteractTool::new(true).parameters_schema(); + let required = schema["required"].as_array().unwrap(); + assert!(required.iter().any(|v| v == "action")); + assert!(required.iter().any(|v| v == "app_name")); + } + + #[test] + fn sensitive_apps_detected() { + assert!(is_sensitive_app("Keychain Access")); + assert!(is_sensitive_app("1Password 7")); + assert!(is_sensitive_app("System Settings")); + assert!(is_sensitive_app("Terminal")); + // All terminal emulators helper.rs recognizes must also be denied. + for t in [ + "iTerm", + "WezTerm", + "Warp", + "Alacritty", + "kitty", + "Ghostty", + "Hyper", + "Rio", + ] { + assert!(is_sensitive_app(t), "expected '{t}' to be denied"); + } + assert!(!is_sensitive_app("Music")); + assert!(!is_sensitive_app("Safari")); + } + + #[tokio::test] + async fn rejects_missing_app_name() { + let result = AxInteractTool::new(true) + .execute(json!({"action": "list", "app_name": ""})) + .await + .unwrap(); + assert!(result.is_error); + } + + #[tokio::test] + async fn rejects_press_without_label() { + let result = AxInteractTool::new(true) + .execute(json!({"action": "press", "app_name": "Music"})) + .await + .unwrap(); + assert!(result.is_error); + } + + #[tokio::test] + async fn refuses_mutations_when_disabled() { + // mutations off → press/set_value blocked, but list still allowed past this guard. + let tool = AxInteractTool::new(false); + let press = tool + .execute(json!({"action": "press", "app_name": "Music", "label": "Play"})) + .await + .unwrap(); + assert!(press.is_error); + assert!(press.output().contains("ax_interact_mutations")); + } + + #[tokio::test] + async fn refuses_sensitive_app_even_with_mutations() { + let tool = AxInteractTool::new(true); + for app in [ + "Keychain Access", + "1Password", + "Terminal", + "System Settings", + ] { + let r = tool + .execute(json!({"action": "press", "app_name": app, "label": "OK"})) + .await + .unwrap(); + assert!(r.is_error, "expected refusal for {app}"); + assert!(r.output().to_lowercase().contains("denylist")); + } + } +} diff --git a/src/openhuman/tools/impl/computer/mod.rs b/src/openhuman/tools/impl/computer/mod.rs index ce9fde911..ec8363c0f 100644 --- a/src/openhuman/tools/impl/computer/mod.rs +++ b/src/openhuman/tools/impl/computer/mod.rs @@ -1,6 +1,8 @@ +mod ax_interact; mod human_path; mod keyboard; mod mouse; +pub use ax_interact::AxInteractTool; pub use keyboard::KeyboardTool; pub use mouse::MouseTool; diff --git a/src/openhuman/tools/impl/system/launch_app.rs b/src/openhuman/tools/impl/system/launch_app.rs new file mode 100644 index 000000000..c423a3444 --- /dev/null +++ b/src/openhuman/tools/impl/system/launch_app.rs @@ -0,0 +1,430 @@ +//! Tool: launch_app — open a named application on the user's desktop. +//! +//! A dedicated, narrow-scope alternative to using the `shell` tool with +//! `open -a ` / `xdg-open` / `Start-Process`. It carries no shell +//! injection risk and accepts **named applications only** (URI schemes like +//! `spotify:` / `mailto:` are rejected — see `validate_app_name`). +//! +//! Being injection-safe does NOT make it side-effect-free: opening an app +//! window (and, on Linux/Windows, potentially firing a registered URI handler) +//! is an externally-observable action on the user's machine. So the tool is an +//! external-effect tool (`external_effect() == true`) and routes through the +//! `ApprovalGate` before executing, like `shell` — it is NOT always-allow. +//! +//! Platform dispatch: +//! macOS — `open -a ""` (falls back to `open ""`) +//! Linux — `gtk-launch ""`, fallback `xdg-open ""` +//! Windows — `Start-Process ""` + +use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCallOptions, ToolResult}; +use async_trait::async_trait; +use serde_json::json; +use std::process::Stdio; + +pub struct LaunchAppTool; + +impl LaunchAppTool { + pub fn new() -> Self { + Self + } +} + +impl Default for LaunchAppTool { + fn default() -> Self { + Self::new() + } +} + +/// Reject names that look like path traversal or contain shell metacharacters. +fn validate_app_name(name: &str) -> Result<(), String> { + let trimmed = name.trim(); + if trimmed.is_empty() { + return Err("app_name must not be empty".into()); + } + if trimmed.len() > 128 { + return Err("app_name too long (max 128 chars)".into()); + } + // No path separators or traversal sequences. + if trimmed.contains('/') || trimmed.contains('\\') || trimmed.contains("..") { + return Err(format!( + "app_name '{trimmed}' looks like a path; supply a bare application name instead \ + (e.g. 'Music', 'Spotify')" + )); + } + // Reject shell metacharacters — not needed here since we bypass the shell, + // but guard against accidental misuse of the API. + for ch in ['$', '`', '|', '&', ';', '>', '<', '!', '(', ')', '\n', '\r'] { + if trimmed.contains(ch) { + return Err(format!("app_name contains disallowed character '{ch}'")); + } + } + // Reject URI schemes (e.g. `spotify:`, `mailto:`, `slack:`, `https:`). On + // Linux/Windows the launcher fallbacks (`xdg-open`/`Start-Process`) would + // fire an arbitrary registered URI handler — exactly the ungated + // network/system reach that `open`/`xdg-open`/`start` were kept out of + // READ_ONLY_BASES to avoid. This tool is "named applications only". + if is_uri_scheme(trimmed) { + return Err(format!( + "app_name '{trimmed}' looks like a URI scheme; this tool launches named \ + applications only, not URIs/handlers" + )); + } + Ok(()) +} + +/// True if `s` begins with a URI scheme per RFC 3986: `ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) ":"`. +fn is_uri_scheme(s: &str) -> bool { + let Some(colon) = s.find(':') else { + return false; + }; + if colon == 0 { + return false; + } + let scheme = &s[..colon]; + let mut chars = scheme.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() => {} + _ => return false, + } + scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) +} + +#[async_trait] +impl Tool for LaunchAppTool { + fn name(&self) -> &str { + "launch_app" + } + + fn description(&self) -> &str { + "Open a named application on the user's desktop. Supply the app's display name \ + (e.g. 'Music', 'Spotify', 'Safari', 'Calculator', 'VS Code'). \ + Works on macOS, Linux, and Windows. \ + Use this instead of the shell tool whenever the goal is simply to open an app." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "properties": { + "app_name": { + "type": "string", + "description": "Display name of the application to open \ + (e.g. 'Music', 'Spotify', 'Google Chrome'). \ + Do not supply a file path — use the bare name." + } + }, + "required": ["app_name"] + }) + } + + fn permission_level(&self) -> PermissionLevel { + // Launching an app executes a process / opens a window on the user's + // machine — an execution-class action, not a read. + PermissionLevel::Execute + } + + fn external_effect(&self) -> bool { + // Opening an application is an externally-observable side effect, so the + // harness routes this through the ApprovalGate before execute() — same + // contract as `shell`. Not always-allow. + true + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.execute_with_options(args, ToolCallOptions::default()) + .await + } + + async fn execute_with_options( + &self, + args: serde_json::Value, + _options: ToolCallOptions, + ) -> anyhow::Result { + let app_name = args + .get("app_name") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + + log::info!("[launch_app] ▶ execute called app_name={app_name:?} raw_args={args}"); + + if let Err(reason) = validate_app_name(&app_name) { + log::warn!("[launch_app] ✗ validation failed app_name={app_name:?} reason={reason}"); + return Ok(ToolResult::error(reason)); + } + + log::info!("[launch_app] ✓ validation passed — dispatching to platform launcher"); + + let result = launch_platform(&app_name).await; + + match result { + Ok(msg) => { + log::info!("[launch_app] ✓ launch succeeded app_name={app_name:?} msg={msg:?}"); + Ok(ToolResult::success(msg)) + } + Err(err) => { + log::warn!("[launch_app] ✗ launch failed app_name={app_name:?} error={err}"); + Ok(ToolResult::error(format!( + "Could not open '{app_name}': {err}" + ))) + } + } + } +} + +/// Platform-specific launch dispatch. Returns a human-readable success message. +async fn launch_platform(app_name: &str) -> Result { + log::info!( + "[launch_app] platform={} dispatching launch for app_name={app_name:?}", + std::env::consts::OS + ); + + #[cfg(target_os = "macos")] + return launch_macos(app_name).await; + + #[cfg(target_os = "linux")] + return launch_linux(app_name).await; + + #[cfg(target_os = "windows")] + return launch_windows(app_name).await; + + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] + Err("launch_app is not supported on this platform".into()) +} + +#[cfg(target_os = "macos")] +async fn launch_macos(app_name: &str) -> Result { + log::info!("[launch_app] macOS: running `open -a {app_name:?}`"); + + // `open -a "App Name"` resolves by display name via LaunchServices. + let output = tokio::process::Command::new("open") + .arg("-a") + .arg(app_name) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|e| format!("failed to invoke `open`: {e}"))?; + + log::info!( + "[launch_app] macOS: `open -a` exit={} stderr={}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + + if output.status.success() { + return Ok(format!("Opened '{app_name}'.")); + } + + log::info!("[launch_app] macOS: primary failed — trying fallback `open {app_name:?}`"); + + // Fallback: `open ""` — works for bundle names and some URIs. + let fallback = tokio::process::Command::new("open") + .arg(app_name) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .await + .map_err(|e| format!("failed to invoke `open` (fallback): {e}"))?; + + log::info!("[launch_app] macOS: fallback exit={fallback}"); + + if fallback.success() { + Ok(format!("Opened '{app_name}'.")) + } else { + Err(format!( + "`open -a \"{app_name}\"` failed (exit {}) — check the app name matches its title in /Applications", + output.status + )) + } +} + +#[cfg(target_os = "linux")] +async fn launch_linux(app_name: &str) -> Result { + // `gtk-launch` takes a .desktop file ID (e.g. "google-chrome"), NOT a + // human-readable display name ("Google Chrome"). Try the name as given + // first, then a best-effort desktop-id derived from the display name + // (lowercased, spaces → hyphens). `xdg-open` does NOT launch apps by + // name — it only opens URIs/paths in the default handler — so it's a + // last resort for app_name values that happen to be a URI. + let desktop_id = app_name.to_lowercase().replace(' ', "-"); + let mut candidates = vec![app_name.to_string()]; + if desktop_id != app_name { + candidates.push(desktop_id); + } + + for candidate in &candidates { + let gtk = tokio::process::Command::new("gtk-launch") + .arg(candidate) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .await; + if let Ok(s) = gtk { + if s.success() { + return Ok(format!("Opened '{app_name}'.")); + } + } + } + + // Fallback for URI-shaped inputs only (xdg-open won't resolve app names). + let xdg = tokio::process::Command::new("xdg-open") + .arg(app_name) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .await + .map_err(|e| format!("failed to invoke `xdg-open`: {e}"))?; + + if xdg.success() { + Ok(format!("Opened '{app_name}'.")) + } else { + Err(format!( + "Could not launch '{app_name}' on Linux. gtk-launch needs a .desktop \ + ID (e.g. 'google-chrome'); xdg-open only opens URIs/paths, not app names. \ + Try the .desktop ID, or supply a URI." + )) + } +} + +#[cfg(target_os = "windows")] +async fn launch_windows(app_name: &str) -> Result { + // The app name is passed through an env var (`OH_LAUNCH_APP`) and never + // string-interpolated into the script, so a name containing a quote cannot + // break out of the command. `validate_app_name` already blocks shell + // metacharacters; the static script + env passing is belt-and-suspenders. + // + // Resolution order: + // 1. `Start-Process -FilePath ` — resolves PATH executables, + // registered App Paths (e.g. "Spotify" desktop), and URIs ("spotify:"). + // 2. Fallback for Store/UWP apps that have no plain exe: match by display + // name via `Get-StartApps` and launch by AUMID + // (`shell:AppsFolder\`), e.g. the Store "Media Player". + const PS_LAUNCH: &str = "\ + $ErrorActionPreference='Stop'; \ + $n=$env:OH_LAUNCH_APP; \ + try { Start-Process -FilePath $n } \ + catch { \ + $app = Get-StartApps | Where-Object { $_.Name -like \"*$n*\" } | Select-Object -First 1; \ + if ($app) { Start-Process -FilePath ('shell:AppsFolder\\' + $app.AppID) } \ + else { throw } \ + }"; + + log::info!( + "[launch_app] windows: launching app_name={app_name:?} (Start-Process, Store fallback)" + ); + + let output = tokio::process::Command::new("powershell") + .args(["-NoProfile", "-NonInteractive", "-Command", PS_LAUNCH]) + .env("OH_LAUNCH_APP", app_name) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|e| format!("failed to invoke PowerShell: {e}"))?; + + let stderr = String::from_utf8_lossy(&output.stderr); + log::info!( + "[launch_app] windows: exit={} stderr={}", + output.status, + stderr.trim() + ); + + if output.status.success() { + Ok(format!("Opened '{app_name}'.")) + } else if stderr.trim().is_empty() { + Err(format!( + "could not open '{app_name}' (Start-Process and Store-app lookup both failed)" + )) + } else { + Err(format!("could not open '{app_name}': {}", stderr.trim())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn name_and_permission() { + let tool = LaunchAppTool::new(); + assert_eq!(tool.name(), "launch_app"); + // Execution-class + external effect so it routes through the ApprovalGate. + assert_eq!(tool.permission_level(), PermissionLevel::Execute); + assert!(tool.external_effect()); + } + + #[test] + fn validate_rejects_uri_schemes() { + // URI schemes would fire arbitrary registered handlers via the + // Linux/Windows launcher fallbacks — reject them (named apps only). + assert!(validate_app_name("spotify:track/123").is_err()); + assert!(validate_app_name("mailto:a@b.com").is_err()); + assert!(validate_app_name("slack:").is_err()); + assert!(validate_app_name("https://evil.example").is_err()); + assert!(validate_app_name("x-custom-scheme:payload").is_err()); + } + + #[test] + fn schema_requires_app_name() { + let schema = LaunchAppTool::new().parameters_schema(); + assert_eq!(schema["type"], "object"); + assert!(schema["required"] + .as_array() + .unwrap() + .iter() + .any(|v| v == "app_name")); + } + + #[test] + fn validate_rejects_empty() { + assert!(validate_app_name("").is_err()); + assert!(validate_app_name(" ").is_err()); + } + + #[test] + fn validate_rejects_paths() { + assert!(validate_app_name("/Applications/Music.app").is_err()); + assert!(validate_app_name("../etc/passwd").is_err()); + } + + #[test] + fn validate_rejects_metacharacters() { + assert!(validate_app_name("Music; rm -rf /").is_err()); + assert!(validate_app_name("$(evil)").is_err()); + } + + #[test] + fn validate_accepts_normal_names() { + assert!(validate_app_name("Music").is_ok()); + assert!(validate_app_name("Google Chrome").is_ok()); + assert!(validate_app_name("VS Code").is_ok()); + assert!(validate_app_name("Spotify").is_ok()); + } + + #[tokio::test] + async fn returns_error_for_empty_app_name() { + let result = LaunchAppTool::new() + .execute(json!({"app_name": ""})) + .await + .unwrap(); + assert!(result.is_error); + } + + #[tokio::test] + async fn returns_error_for_path_traversal() { + let result = LaunchAppTool::new() + .execute(json!({"app_name": "/Applications/Music.app"})) + .await + .unwrap(); + assert!(result.is_error); + } +} diff --git a/src/openhuman/tools/impl/system/mod.rs b/src/openhuman/tools/impl/system/mod.rs index cd10b9156..118a567b3 100644 --- a/src/openhuman/tools/impl/system/mod.rs +++ b/src/openhuman/tools/impl/system/mod.rs @@ -2,6 +2,7 @@ mod current_time; mod detect_tools; mod insert_sql_record; mod install_tool; +mod launch_app; mod lsp; mod node_exec; mod npm_exec; @@ -18,6 +19,7 @@ pub use current_time::CurrentTimeTool; pub use detect_tools::DetectToolsTool; pub use insert_sql_record::InsertSqlRecordTool; pub use install_tool::InstallToolTool; +pub use launch_app::LaunchAppTool; pub use lsp::{lsp_capability_enabled, LspTool, LSP_ENABLED_ENV}; pub use node_exec::NodeExecTool; pub use npm_exec::NpmExecTool; diff --git a/src/openhuman/tools/impl/system/shell.rs b/src/openhuman/tools/impl/system/shell.rs index 6b0413d54..51cb9ba58 100644 --- a/src/openhuman/tools/impl/system/shell.rs +++ b/src/openhuman/tools/impl/system/shell.rs @@ -120,7 +120,9 @@ impl Tool for ShellTool { } fn description(&self) -> &str { - "Execute a shell command in the workspace directory" + "Execute a shell command. Use this to run code, manipulate files in the workspace, \ + or perform system actions on the user's machine — including launching applications \ + (e.g. `open -a Music` on macOS, `xdg-open music://` on Linux)." } fn parameters_schema(&self) -> serde_json::Value { diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index ada742452..5b15828d6 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -168,6 +168,10 @@ pub fn all_tools_with_runtime( // and tool callers share one spawn path. Box::new(RunSkillTool::new()), Box::new(CurrentTimeTool::new()), + Box::new(LaunchAppTool::new()), + Box::new(AxInteractTool::new( + root_config.computer_control.ax_interact_mutations, + )), Box::new(CodegraphIndexTool::new( config.clone(), action_dir.to_path_buf(), diff --git a/src/openhuman/tools/user_filter.rs b/src/openhuman/tools/user_filter.rs index 6af19393a..43c029932 100644 --- a/src/openhuman/tools/user_filter.rs +++ b/src/openhuman/tools/user_filter.rs @@ -28,6 +28,27 @@ const TOOL_FAMILIES: &[ToolFamily] = &[ rust_names: &["shell"], default_enabled: true, }, + // Dedicated app-launcher: always-allow, no shell exposure, no workspace_only concern. + ToolFamily { + id: "launch_app", + rust_names: &["launch_app"], + default_enabled: true, + }, + // AXUIElement interaction: semantic UI control via macOS Accessibility API. + // No CGEventPost, no coordinate dependency, no CEF crash risk. + ToolFamily { + id: "ax_interact", + rust_names: &["ax_interact"], + default_enabled: true, + }, + // Computer control — mouse and keyboard. Gated by computer_control.enabled + // in config (tools only register when that flag is true). PermissionLevel::Dangerous + // so the approval gate fires per-action; user opts in explicitly. + ToolFamily { + id: "computer_control", + rust_names: &["mouse", "keyboard"], + default_enabled: true, + }, // detect_tools / install_tool are filterable but not surfaced in the // default-ON catalog, so they stay opt-in (default-OFF). ToolFamily {