diff --git a/app/src/components/settings/panels/VoicePanel.tsx b/app/src/components/settings/panels/VoicePanel.tsx index 4e7a4d23b..1b8a35070 100644 --- a/app/src/components/settings/panels/VoicePanel.tsx +++ b/app/src/components/settings/panels/VoicePanel.tsx @@ -18,6 +18,7 @@ import { type VoiceProviderView, type VoiceSettings, } from '../../../services/api/voiceSettingsApi'; +import { IS_DEV_LIKE } from '../../../utils/config'; import { openhumanGetVoiceServerSettings, openhumanUpdateVoiceServerSettings, @@ -87,10 +88,10 @@ interface VoicePanelProps { embedded?: boolean; } -/** Temporarily hide the always-on listening toggle. Set back to `true` to - * restore the control (the backend engine is unchanged). See - * docs/voice-system-actions.md. */ -const SHOW_ALWAYS_ON_TOGGLE = false; +/** Always-on listening toggle is hidden in production for now, but shown in + * dev/debug builds so the feature can be exercised. Set unconditionally to + * `true` to expose it everywhere. See docs/voice-system-actions.md. */ +const SHOW_ALWAYS_ON_TOGGLE = IS_DEV_LIKE; const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => { const { t } = useT(); @@ -494,7 +495,7 @@ const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => {
{/* ─── Always-on listening (Phase 2) ──────────────────────────── */} - {/* Temporarily hidden — gated on SHOW_ALWAYS_ON_TOGGLE (set to false). */} + {/* Gated on SHOW_ALWAYS_ON_TOGGLE — shown in dev/debug builds, hidden in prod. */} {SHOW_ALWAYS_ON_TOGGLE && settings && (
diff --git a/app/src/components/settings/panels/__tests__/VoicePanel.test.tsx b/app/src/components/settings/panels/__tests__/VoicePanel.test.tsx index 8ec11a25f..415ac6aa4 100644 --- a/app/src/components/settings/panels/__tests__/VoicePanel.test.tsx +++ b/app/src/components/settings/panels/__tests__/VoicePanel.test.tsx @@ -511,8 +511,7 @@ describe('VoicePanel', () => { // ─── Always-on listening toggle ↔ notch indicator ─────────────────────── - // Temporarily hidden in the UI (always-on toggle disabled for now in VoicePanel). - it.skip('shows the notch when always-on listening is enabled and hides it when disabled', async () => { + it('shows the notch when always-on listening is enabled and hides it when disabled', async () => { renderWithProviders(, { initialEntries: ['/settings/voice'] }); const toggle = await screen.findByTestId('voice-always-on-toggle'); @@ -537,8 +536,7 @@ describe('VoicePanel', () => { await waitFor(() => expect(vi.mocked(syncNotchVisibility)).toHaveBeenCalledWith(false)); }); - // Temporarily hidden in the UI (always-on toggle disabled for now in VoicePanel). - it.skip('does not touch the notch and reverts the toggle when the update RPC fails', async () => { + it('does not touch the notch and reverts the toggle when the update RPC fails', async () => { vi.mocked(openhumanUpdateVoiceServerSettings).mockRejectedValueOnce(new Error('rpc down')); renderWithProviders(, { initialEntries: ['/settings/voice'] }); diff --git a/docs/voice-system-actions.md b/docs/voice-system-actions.md index 988c945bc..736da098c 100644 --- a/docs/voice-system-actions.md +++ b/docs/voice-system-actions.md @@ -3,7 +3,7 @@ **GitHub Issue:** [#3148](https://github.com/tinyhumansai/openhuman/issues/3148) **Branch:** `feat/voice-always-on-all` (cumulative) — **all merged to `main`.** Phase 1 landed via [#3168](https://github.com/tinyhumansai/openhuman/pull/3168); the full feature was split from the mega-PR [#3307](https://github.com/tinyhumansai/openhuman/pull/3307) (now **closed** in favour of the stack) into an 8-PR stack — **all 8 merged 2026-06-04**: [#3340](https://github.com/tinyhumansai/openhuman/pull/3340) (main-thread input + CEF fix), [#3341](https://github.com/tinyhumansai/openhuman/pull/3341) (AX/UIA perception + automate engine), [#3342](https://github.com/tinyhumansai/openhuman/pull/3342) (wire automate/ax_interact tools), [#3343](https://github.com/tinyhumansai/openhuman/pull/3343) (Phase 2 always-on engine + RPC), [#3344](https://github.com/tinyhumansai/openhuman/pull/3344) (always-on Settings toggle + i18n), [#3345](https://github.com/tinyhumansai/openhuman/pull/3345) (notch status pill — supersedes the closed [#3166](https://github.com/tinyhumansai/openhuman/pull/3166)), [#3346](https://github.com/tinyhumansai/openhuman/pull/3346) (Phase 3 fast command router), [#3362](https://github.com/tinyhumansai/openhuman/pull/3362) (Phase 1.5 vision-click fallback) **Started:** 2026-06-02 -**Last updated:** 2026-06-09 — verified all 8 stack PRs merged to `main`; code paths confirmed present post-refactor [#3424](https://github.com/tinyhumansai/openhuman/pull/3424). Related: global push-to-talk ([#3090](https://github.com/tinyhumansai/openhuman/issues/3090) via [#3349](https://github.com/tinyhumansai/openhuman/pull/3349)) landed separately as an alternative manual trigger. **Only open item across all phases: the on-device audio wake-word model (Phase 3) — not started; text-based "Hey Tiny" match remains the interim.** +**Last updated:** 2026-06-09 — added **Change 1.17** (PR [#3558](https://github.com/tinyhumansai/openhuman/pull/3558)): browser & app (Spotify/Apple Music/Slack) keyboard-shortcut fast-paths, cross-platform shortcut tables, the `hotkey` verb, artist-aware Music verification, and actionable `automate` failure responses — all from live-transcript analysis. Earlier this day: verified all 8 always-on stack PRs merged to `main` (code paths confirmed post-refactor [#3424](https://github.com/tinyhumansai/openhuman/pull/3424)); global push-to-talk ([#3090](https://github.com/tinyhumansai/openhuman/issues/3090) via [#3349](https://github.com/tinyhumansai/openhuman/pull/3349)) landed separately. **Only open item across all phases: the on-device audio wake-word model (Phase 3) — not started; text-based "Hey Tiny" match remains the interim.** --- @@ -330,7 +330,7 @@ test ... ok > The bridge between today's `ax_interact` primitives and the always-on voice work. **Prerequisite for Phase 3** — fast voice routing into a slow/fragile action loop still feels slow. This is where "whatever I say happens, live" actually gets delivered. > -> **Status:** the Rust inner loop (M1), poll-until-stable settle (M2), Music fast-path (M3, proven live), notch progress streaming (M4), the richer element model (M5), and the **vision fallback for Electron/partial-AX apps** (Change 1.16) are all shipped. (Additional per-app native fast-paths beyond Music — e.g. Spotify/Slack — are **descoped**: the general model-driven loop covers them, so a deterministic accelerator isn't worth the per-app maintenance right now.) +> **Status:** the Rust inner loop (M1), poll-until-stable settle (M2), Music fast-path (M3, proven live), notch progress streaming (M4), the richer element model (M5), the **vision fallback for Electron/partial-AX apps** (Change 1.16), and **Change 1.17** (browser & app fast-paths + keyboard-shortcut driving + artist-aware Music + actionable responses) are all shipped. Per-app fast-paths now cover **Music, browsers, Spotify, Apple Music, and Slack** (Spotify/Slack — previously descoped — are now handled by the `app_shortcuts.rs` keyboard-shortcut path). **Detailed implementation plan:** [`voice-automate-plan.md`](voice-automate-plan.md) — decided approach: **Rust inner loop + fast model**, first proof target **Music**. @@ -392,6 +392,49 @@ enigo's keyboard-layout lookup (`TSMGetInputSourceProperty`) **must run on the a --- +### Change 1.17 — browser & app fast-paths, keyboard-shortcut driving, artist-aware Music, actionable responses ✅ Done + +**Status:** ✅ Shipped (PR [#3558](https://github.com/tinyhumansai/openhuman/pull/3558)). One body of work driven by **live-transcript analysis of the `automate` agent** across browsers, Apple Music, Spotify, and Slack. The transcripts surfaced four recurring failures the agent kept hitting: + +1. **Browsers** — for *"open my Brave browser, go to youtube.com and play a music video"* the `desktop_control_agent` was re-delegated **6×** (each sub re-launched Brave + re-navigated), ran `ax_interact` on the **wrong app names** (`Google Chrome`/`Safari`, never the real `Brave Browser`, because Chromium exposes no AX tree), then "played" via a **blind hardcoded mouse click at (400,350)** with no verification. +2. **Wrong track** — *"play Numb by Linkin Park"* played a **same-titled song by the wrong artist** ("Numb" by Marshmello & Khalid) and the agent **claimed success**, because the AX row label is title-only and verification only checked *that* something played. +3. **Inert responses** — the `automate` tool's replies were *honest but gave no next move*, so the agent **re-ran the same failing search 6×** instead of changing tactics. +4. **No keyboard-shortcut lever** — the loop hunted AX labels even where an app's own shortcut would be instant and reliable. + +The six parts below address all four. All fast-paths are consulted by `automate::run` **before** the model loop (`try_fastpath` order: `music` → `browser` → `app_shortcuts`); on any failure they fall through, so they can only *help*. + +**(a) Browser fast-path** — `app_fastpaths/browser.rs` (new). Pure parsers + `run(app, goal, backend)`: + - `resolve_browser(app, goal)` — aliases → macOS display names (`brave`→`Brave Browser`, `chrome`→`Google Chrome`, `edge`→`Microsoft Edge`, `safari`/`firefox`/`arc`). Checks the `app` arg first, then the goal; a generic "browser" with no named product → **`None`** so it never guesses the wrong app (the original bug). + - `extract_destination(goal)` — YouTube → `youtube.com/results?search_query=…` (percent-encoded); Google/"search for X" → `google.com/search?q=…`; a bare domain/URL → normalized to `https://…`. + - **Navigation is one deterministic step:** `backend.open_url_in_app(browser, url)` → `open -a "" ""` launches/foregrounds + navigates — no address-bar typing, no AX. Pure-navigation goals **complete here**; **play goals** navigate to the results URL then return non-success so the loop does the single "click first result" via `vision_click` (no native shortcut *selects* a result — we don't fake it). In-page media control ("pause/next the video") sends the YouTube shortcut directly. + +**(b) Cross-platform browser shortcut table** — `app_fastpaths/browser_shortcuts.rs` (new). `shortcut(intent, browser, os)` resolves browser-level intents (address bar, new/close/reopen tab, tab N, next/prev tab, find, reload/hard-reload, back/forward, history, downloads, zoom, private window, fullscreen) per **Chrome/Firefox/Safari/Edge × macOS/Windows/Linux**. Nearly uniform (only ⌘↔Ctrl differs); encodes the real exceptions (Firefox-Linux `Alt+1‑8`, Firefox private window `⌘/Ctrl+Shift+P`, per-browser History/Downloads, Mac `⌘[`/`]` back-forward, `F11` vs `⌃⌘F` fullscreen). Sourced from the official Chrome/Firefox/Edge/Safari docs. `browser.rs` parses commands ("open a new tab", "go back", "reload", "switch to tab 3") and dispatches the resolved chord. (Brave/Arc/Chromium → Chrome family.) + +**(c) App-shortcut fast-path** — `app_fastpaths/app_shortcuts.rs` (new). Same model for **Spotify, Apple Music, Slack**: `shortcut(intent, app, os)` + goal parser. Drives transport/navigation by each app's own global shortcut (faster + more reliable than AX). Media intents (play/pause, next, prev, volume ±, mute, shuffle, repeat, search) for Spotify + Apple Music; Slack nav (quick-switcher, search, new message/compose DM, next/prev unread, next/prev channel, threads, all-unread, mark read). Encodes the quirks: **Spotify uses ↓/↑ for next/prev** (not ←/→); **Apple Music prefixes everything with Ctrl on Windows** (Ctrl+Space vs bare Space on Mac); Slack channel-nav uses **Option/Alt+arrows**. Intents with no simple chord (Spotify volume, Apple Music mute / Windows search access-key) return `None` → fall through. Complementary to `music.rs`, which still owns Apple Music *song search/play*. + +**(d) Keyboard-shortcut plumbing** — + - Extracted shared `pub(crate)` helpers `run_hotkey`/`run_key`/`run_type_text` from `tools/impl/computer/keyboard.rs` (one place for validation + main-thread enigo dispatch, Change 1.15) — no behavior change to the `keyboard` tool. + - New `AutomateBackend` methods (non-breaking defaults): `open_url_in_app`, `key`, `type_text`, `now_playing`. `RealBackend` impls `open -a` per-app, delegates keystrokes to the keyboard helpers, and reads Music state via osascript. + - New **`hotkey` verb** in the general loop + system prompt (`{"action":"hotkey","keys":["Cmd","L"]}`) so the model can use any app's shortcuts — folded into the no-progress signature. + +**(e) Artist-aware Music verification** — `music.rs` + `now_playing`. Apple Music's AX row label is title-only ("Numb - Single"), so a title match lands on the wrong artist. `music.rs` now parses the requested artist (`extract_artist`, "…by Linkin Park") and after Play reads the **now-playing name + artist** via AppleScript (`now_playing` → `osascript … get {name, artist} of current track`). On a match the summary names the verified track + artist; on a **mismatch it says so honestly** ("Now playing 'Numb' by 'Tom Odell'…") instead of claiming the requested track. Closes the wrong-artist false-success. **Follow-up:** *correcting* to the right artist is limited by AX `press`-by-label hitting only the first same-titled row — a library AppleScript `play (track whose name … and artist …)` fallback is the next step. + +**(f) Actionable `automate` responses** — the tool's failures are now agent-actionable, not inert. Four fixes: **(1)** terminal failures (`music.rs` no-match/mismatch, `automate.rs` budget-exhausted/no-progress) carry **next-step guidance + a "don't repeat this" steer**; **(2)** they **list the actual candidates / on-screen labels** (`candidate_labels`, `screen_hint`) instead of a bare negative; **(3)** **verified-only success** — never "Playing 'X'" unless `now_playing` confirms it, and an artist/album press is reported as "navigated, no track started"; **(4)** `pick_row` now **prefers real song rows** (`AXCell`/`AXRow`) over artist/album buttons (the live bug pressed the "LINKIN PARK" artist row) and the step log names the element *type* pressed. + +**Also:** the always-on toggle in `VoicePanel` is now shown in dev/debug builds (`IS_DEV_LIKE`) and hidden in production. + +**Follow-up fix — Full OS access now enables app control.** `ax_interact` (press/set_value) and `automate` gated **only** on `computer_control.ax_interact_mutations`, which is independent of the autonomy level. A user who granted **Full** access in Settings → Agent Access still got *"App control isn't enabled yet…"* because that flag was untouched (confirmed live: `SecurityPolicy autonomy=Full` while the tool refused). Fixed with a shared `app_control_enabled(explicit_opt_in)` gate (`tools/impl/computer/ax_interact.rs`) that returns true when the explicit flag is set **or** the **live** policy (`security::live_policy::current()`) is `AutonomyLevel::Full` — so granting Full access takes effect immediately, no session restart or separate flag flip. Both refusal messages now point at Settings → Agent Access ("Grant Full access (or turn on App UI Control / App Automation)"). Covered by `app_control_enabled_combines_optin_and_full_access` + the two pinned-policy refusal tests. + +**Follow-up fix — Windows browser nav + the re-delegation loop.** Live Windows transcript of *"open my browser, go to youtube.com and play a video"*: the orchestrator delegated to `desktop_control_agent` **7×**, every sub re-`launch_app`-ed Chrome (→ ~10 windows), and the AX path dead-ended — Chromium exposes no page content to UIA, `screenshot`/`vision_click` is unimplemented on Windows, and `ax_interact set_value` on the address bar **can't be submitted** (UIA Invoke on an Edit fails, no Enter). Two fixes: +> - **`open_url_in_app` now works on Windows** (`accessibility/automate.rs`) — was macOS-only (fell back to the *default* browser). Maps the resolved browser display name → the shell `start` App-Paths token (`windows_browser_launch_token`: Chrome→`chrome`, Edge→`msedge`, Brave→`brave`, Firefox→`firefox`; Safari/Arc→`None`→default handler) and runs `cmd /C start "" ""`. Launches/foregrounds the named browser and navigates in **one** step; when the browser is already open this lands in a **new tab of the existing window**, so the fast-path no longer piles up windows. The browser fast-path's deterministic nav is now cross-platform (mac `open -a`, win `start`). Tested by `browser_display_names_map_to_start_tokens` (`#[cfg(target_os="windows")]`). +> - **`desktop_control_agent` prompt steered to `automate` for browsers** (`agent_registry/agents/desktop_control_agent/prompt.md`) — it had `automate` but its only examples were Music/Slack, so it AX-fumbled the browser. Now: web browsers → use `automate{app, goal}` (deterministic URL nav), never type a URL into the address bar via `ax_interact`; **foreground each app at most once per task** (repeated `launch_app` piles up windows); and **stop after two failed attempts at a step** instead of re-launching and looping. (Still a Windows gap: `vision_click` for the final "click first result" — `screenshot` is unimplemented on Windows, so play-from-search-results can't complete deterministically yet; navigation/search now do.) + +**Net for the browser example:** `open -a "Brave Browser" "https://www.youtube.com/results?search_query=music%20video"` + one `vision_click` — instead of 6 delegations and 5 failed AX calls. + +**Tests (all green):** `app_fastpaths` **58**, `accessibility::automate` **21**, `computer::keyboard` **26**. Coverage: browser/app resolution + destination/intent parsers (incl. Spotify ↓/↑, Apple Music Win-Ctrl, Slack Alt-nav, generic-browser→None, percent-encoding); scripted-backend sequences (model-free nav, play→fall-through, media-control hotkey, dispatch order with music-wins-play); artist match→names-track / mismatch→honest; response shapes (candidate listing, wrong-artist steer, non-song honesty, song-row preference, budget/no-progress on-screen hints); a `hotkey`-verb loop test; `#[ignore]` macOS live tests. **Follow-ups:** wire the shortcut tables into the Phase 3 voice `command_router` (voice "pause"/"next" → frontmost-app shortcut); AppleScript library play-by-name+artist correction; expose `vision_click` as a standalone tool. + +--- + ## Windows port — app interaction 🪟 ✅ Implemented Phase 1's app-interaction layer is now ported to Windows. The macOS path uses the @@ -648,6 +691,12 @@ From live agent-in-the-loop testing on 2026-06-03 (grounded in `~/.openhuman/log | 1.5 | M5: richer element model (`enabled`) | ✅ Plumbed; AXEnabled found unreliable → informational only | | 1.5 | Native fast-paths beyond Music (Spotify/Slack) | ➖ Descoped (general loop covers them; Music shipped in M3) | | 1.5 | Vision fallback for Electron apps (Change 1.16) | ✅ Done (`vision_click`: screenshot → vision-locate → guarded click; frontmost guard; F2 coord-transform folded in) | +| 1.5 | Browser fast-path (Change 1.17a) | ✅ Done (`browser.rs`: deterministic `open -a "" url` nav; play→fall-through to vision_click) | +| 1.5 | Cross-platform browser shortcut table (Change 1.17b) | ✅ Done (`browser_shortcuts.rs`: Chrome/Firefox/Safari/Edge × macOS/Win/Linux) | +| 1.5 | App-shortcut fast-path: Spotify/Apple Music/Slack (Change 1.17c) | ✅ Done (`app_shortcuts.rs`: transport + nav via each app's shortcuts) | +| 1.5 | Keyboard-shortcut plumbing: `hotkey` verb + backend methods (Change 1.17d) | ✅ Done (`key`/`type_text`/`open_url_in_app`/`now_playing`; shared keyboard helpers) | +| 1.5 | Artist-aware Music verification (Change 1.17e) | ✅ Done (`now_playing` osascript; honest wrong-artist report, no false success) | +| 1.5 | Actionable `automate` responses (Change 1.17f) | ✅ Done (next-step guidance, candidate/label lists, verified-only success, song-row preference) | | 2 | Always-on microphone loop | ✅ Done (cpal → VAD → STT → agent) | | 2 | `always_on_enabled` config flag + Settings toggle | ✅ Done (RPC + UI + i18n) | | 2 | Privacy hook (screen lock pause) | ✅ Done (macOS; other OSes follow-up) | diff --git a/src/openhuman/accessibility/app_fastpaths/app_shortcuts.rs b/src/openhuman/accessibility/app_fastpaths/app_shortcuts.rs new file mode 100644 index 000000000..a234cc1e5 --- /dev/null +++ b/src/openhuman/accessibility/app_fastpaths/app_shortcuts.rs @@ -0,0 +1,519 @@ +//! Keyboard-shortcut fast-path for Spotify, Apple Music, and Slack (Change 1.17). +//! +//! These desktop apps expose stable global shortcuts for the actions users ask +//! for by voice — "pause", "next song", "turn it up", "jump to a channel". For +//! those, driving the app's own shortcut is far faster and more reliable than +//! walking the AX tree: one `backend.key(chord)` and we're done. (Apple Music +//! *song search/play* still goes through `music.rs`, which types a query; this +//! module is transport/navigation control.) +//! +//! Like `browser_shortcuts.rs`, the table is keyed by `(intent, app, os)` and +//! returns key names the keyboard tool's `parse_key` understands, so the result +//! feeds straight into `AutomateBackend::key`. Sources: the official Spotify, +//! Apple Music (Mac + Windows), and Slack shortcut docs. + +use super::browser_shortcuts::Os; +use super::AutomateBackend; +use super::AutomateOutcome; + +/// Apps this fast-path can drive by shortcut. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AppFamily { + Spotify, + AppleMusic, + Slack, +} + +/// A control action. Media intents apply to Spotify/Apple Music; the rest to +/// Slack. [`shortcut`] returns `None` for an intent an app/OS doesn't support. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AppIntent { + // ── media transport ── + PlayPause, + NextTrack, + PrevTrack, + VolumeUp, + VolumeDown, + Mute, + Shuffle, + Repeat, + MediaSearch, + // ── Slack navigation ── + QuickSwitcher, + SlackSearch, + NewMessage, + ComposeDm, + NextUnread, + PrevUnread, + NextChannel, + PrevChannel, + Threads, + AllUnread, + MarkRead, +} + +/// Resolve the app family from the authoritative `app` arg first (so a stray +/// "music" in the goal text can't mis-route), then from an explicit app name in +/// the goal as a fallback. +fn resolve_family(app: &str, goal: &str) -> Option { + if let Some(f) = family_from_name(&app.to_lowercase()) { + return Some(f); + } + let g = goal.to_lowercase(); + if g.contains("spotify") { + Some(AppFamily::Spotify) + } else if g.contains("slack") { + Some(AppFamily::Slack) + } else if g.contains("apple music") || g.contains("itunes") { + Some(AppFamily::AppleMusic) + } else { + None + } +} + +fn family_from_name(n: &str) -> Option { + if n.contains("spotify") { + Some(AppFamily::Spotify) + } else if n.contains("slack") { + Some(AppFamily::Slack) + } else if n.contains("music") || n.contains("itunes") { + // The macOS app's display name is literally "Music". + Some(AppFamily::AppleMusic) + } else { + None + } +} + +fn keys(parts: &[&str]) -> Vec { + parts.iter().map(|s| s.to_string()).collect() +} + +/// Primary modifier: ⌘ on macOS, Ctrl elsewhere. +fn primary(os: Os) -> &'static str { + match os { + Os::Mac => "Cmd", + _ => "Ctrl", + } +} + +/// Resolve an intent to a key chord for the app + OS, or `None` if that app/OS +/// has no simple chord for it (the loop then falls through). +pub fn shortcut(intent: AppIntent, app: AppFamily, os: Os) -> Option> { + use AppIntent as I; + let p = primary(os); + let v = match (app, intent) { + // ── Spotify (Mac & Windows columns; Linux ≈ Windows) ── + (AppFamily::Spotify, I::PlayPause) => keys(&["space"]), + // Spotify quirk: ↓ = next, ↑ = previous (not ←/→). + (AppFamily::Spotify, I::NextTrack) => keys(&["down"]), + (AppFamily::Spotify, I::PrevTrack) => keys(&["up"]), + (AppFamily::Spotify, I::Mute) => keys(&["m"]), + (AppFamily::Spotify, I::Shuffle) => match os { + Os::Mac => keys(&["alt", "s"]), + _ => keys(&["ctrl", "s"]), + }, + (AppFamily::Spotify, I::Repeat) => match os { + Os::Mac => keys(&["alt", "r"]), + _ => keys(&["ctrl", "r"]), + }, + (AppFamily::Spotify, I::MediaSearch) => keys(&[p, "k"]), + // Spotify has no documented volume shortcut → fall through. + (AppFamily::Spotify, I::VolumeUp | I::VolumeDown) => return None, + + // ── Apple Music ── + (AppFamily::AppleMusic, I::PlayPause) => match os { + Os::Mac => keys(&["space"]), + _ => keys(&["ctrl", "space"]), + }, + (AppFamily::AppleMusic, I::NextTrack) => match os { + Os::Mac => keys(&["right"]), + _ => keys(&["ctrl", "right"]), + }, + (AppFamily::AppleMusic, I::PrevTrack) => match os { + Os::Mac => keys(&["left"]), + _ => keys(&["ctrl", "left"]), + }, + (AppFamily::AppleMusic, I::VolumeUp) => match os { + Os::Mac => keys(&["Cmd", "up"]), + _ => keys(&["ctrl", "up"]), + }, + (AppFamily::AppleMusic, I::VolumeDown) => match os { + Os::Mac => keys(&["Cmd", "down"]), + _ => keys(&["ctrl", "down"]), + }, + // Search field: ⌘F on Mac. Windows uses a sequential access key + // (Alt,N,F) we can't send as one chord → fall through there. + (AppFamily::AppleMusic, I::MediaSearch) => match os { + Os::Mac => keys(&["Cmd", "f"]), + _ => return None, + }, + // No simple mute/shuffle/repeat chord in Apple Music → fall through. + (AppFamily::AppleMusic, I::Mute | I::Shuffle | I::Repeat) => return None, + + // ── Slack ── + (AppFamily::Slack, I::QuickSwitcher) => keys(&[p, "k"]), + (AppFamily::Slack, I::SlackSearch) => keys(&[p, "g"]), + (AppFamily::Slack, I::NewMessage) => keys(&[p, "n"]), + (AppFamily::Slack, I::ComposeDm) => keys(&[p, "shift", "k"]), + // Option (mac) / Alt (win) both map to enigo's Alt key. + (AppFamily::Slack, I::NextUnread) => keys(&["alt", "shift", "down"]), + (AppFamily::Slack, I::PrevUnread) => keys(&["alt", "shift", "up"]), + (AppFamily::Slack, I::NextChannel) => keys(&["alt", "down"]), + (AppFamily::Slack, I::PrevChannel) => keys(&["alt", "up"]), + (AppFamily::Slack, I::Threads) => keys(&[p, "shift", "t"]), + (AppFamily::Slack, I::AllUnread) => keys(&[p, "shift", "a"]), + (AppFamily::Slack, I::MarkRead) => keys(&["esc"]), + + // Any intent not applicable to the app. + _ => return None, + }; + Some(v) +} + +/// Does this (app, goal) name one of these apps with a recognizable control? +pub fn matches(app: &str, goal: &str) -> bool { + match resolve_family(app, goal) { + Some(fam) => extract_intent(goal, fam) + .and_then(|i| shortcut(i, fam, Os::current())) + .is_some(), + None => false, + } +} + +/// Run the shortcut fast-path: resolve the intent, send the chord, done. +pub async fn run(app: &str, goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome { + use super::super::automate::progress; + use crate::openhuman::overlay::OverlayAttentionTone; + + let mut steps: Vec = Vec::new(); + let fam = match resolve_family(app, goal) { + Some(f) => f, + None => return fail("no recognized app", steps), + }; + let intent = match extract_intent(goal, fam) { + Some(i) => i, + None => return fail("no recognized control intent", steps), + }; + let chord = match shortcut(intent, fam, Os::current()) { + Some(c) => c, + None => return fail("no shortcut for this intent on this OS", steps), + }; + let combo = chord.join("+"); + log::info!("[automate::app_shortcuts] ▶ {fam:?} {intent:?} keys={combo}"); + progress(format!("Pressing {combo}…"), OverlayAttentionTone::Accent); + + match backend.key(&chord).await { + Ok(m) => { + steps.push(format!("hotkey {combo}: {m}")); + AutomateOutcome { + success: true, + summary: format!("Sent {combo} to {app}."), + steps, + } + } + Err(e) => { + steps.push(format!("hotkey FAILED: {e}")); + fail("could not send the shortcut", steps) + } + } +} + +/// Parse a control intent from the goal, branching on app domain (media vs +/// Slack) so the same word ("search", "next") maps correctly. +fn extract_intent(goal: &str, fam: AppFamily) -> Option { + let l = goal.to_lowercase(); + match fam { + AppFamily::Slack => parse_slack(&l), + _ => parse_media(&l), + } +} + +fn parse_media(l: &str) -> Option { + use AppIntent as I; + if l.contains("volume up") + || l.contains("louder") + || l.contains("turn it up") + || l.contains("turn up") + || l.contains("increase volume") + || l.contains("raise the volume") + { + return Some(I::VolumeUp); + } + if l.contains("volume down") + || l.contains("quieter") + || l.contains("turn it down") + || l.contains("turn down") + || l.contains("decrease volume") + || l.contains("lower the volume") + || l.contains("lower volume") + { + return Some(I::VolumeDown); + } + if has_word(l, "mute") || has_word(l, "unmute") { + return Some(I::Mute); + } + if has_word(l, "shuffle") { + return Some(I::Shuffle); + } + if has_word(l, "repeat") || has_word(l, "loop") { + return Some(I::Repeat); + } + if has_word(l, "next") || has_word(l, "skip") { + return Some(I::NextTrack); + } + if l.contains("previous") + || l.contains("last song") + || l.contains("last track") + || l.contains("go back a song") + { + return Some(I::PrevTrack); + } + if has_word(l, "pause") || has_word(l, "resume") || has_word(l, "unpause") { + return Some(I::PlayPause); + } + if l.contains("continue playing") || l.contains("keep playing") { + return Some(I::PlayPause); + } + // Bare "play" toggle — but NOT "play " (that's `music.rs`). + if let Some(i) = word_index(l, "play") { + let after = l[i + "play".len()..].trim(); + if after.is_empty() + || matches!( + after, + "music" + | "it" + | "this" + | "the music" + | "the song" + | "song" + | "the track" + | "track" + | "playback" + | "the playback" + ) + { + return Some(I::PlayPause); + } + } + if has_word(l, "search") || has_word(l, "find") { + return Some(I::MediaSearch); + } + None +} + +fn parse_slack(l: &str) -> Option { + use AppIntent as I; + if l.contains("next unread") { + Some(I::NextUnread) + } else if l.contains("previous unread") || l.contains("prev unread") { + Some(I::PrevUnread) + } else if l.contains("next channel") || l.contains("next conversation") || l.contains("next dm") + { + Some(I::NextChannel) + } else if l.contains("previous channel") + || l.contains("prev channel") + || l.contains("previous conversation") + { + Some(I::PrevChannel) + } else if l.contains("all unread") || l.contains("unreads") { + Some(I::AllUnread) + } else if l.contains("thread") { + Some(I::Threads) + } else if l.contains("mark") && l.contains("read") { + Some(I::MarkRead) + } else if l.contains("new message") { + Some(I::NewMessage) + } else if l.contains("compose") + || l.contains("direct message") + || has_word(l, "dm") + || l.contains("message someone") + || l.contains("send a message") + { + Some(I::ComposeDm) + } else if l.contains("jump to") + || l.contains("quick switch") + || l.contains("switch to") + || l.contains("go to channel") + || l.contains("open channel") + || l.contains("open conversation") + || l.contains("find channel") + { + Some(I::QuickSwitcher) + } else if has_word(l, "search") || has_word(l, "find") { + Some(I::SlackSearch) + } else { + None + } +} + +fn fail(msg: &str, steps: Vec) -> AutomateOutcome { + AutomateOutcome { + success: false, + summary: format!("App shortcut fast-path: {msg}"), + steps, + } +} + +/// Whole-word membership test on an already-lowercased string. +fn has_word(haystack: &str, needle: &str) -> bool { + word_index(haystack, needle).is_some() +} + +fn word_index(haystack: &str, needle: &str) -> Option { + let mut from = 0; + while let Some(rel) = haystack[from..].find(needle) { + let idx = from + rel; + let before_ok = idx == 0 + || !haystack[..idx] + .chars() + .next_back() + .map(|c| c.is_alphanumeric()) + .unwrap_or(false); + let after = idx + needle.len(); + let after_ok = haystack[after..] + .chars() + .next() + .map(|c| !c.is_alphanumeric()) + .unwrap_or(true); + if before_ok && after_ok { + return Some(idx); + } + from = idx + needle.len(); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use AppIntent as I; + + #[test] + fn resolves_family_from_app_then_goal() { + assert_eq!(resolve_family("Spotify", "x"), Some(AppFamily::Spotify)); + assert_eq!(resolve_family("Music", "x"), Some(AppFamily::AppleMusic)); + assert_eq!(resolve_family("Slack", "x"), Some(AppFamily::Slack)); + // From goal when app arg is unhelpful. + assert_eq!( + resolve_family("", "pause spotify"), + Some(AppFamily::Spotify) + ); + // A stray "music" in the goal with a browser app must NOT mis-route. + assert_eq!(resolve_family("Brave Browser", "find music videos"), None); + } + + #[test] + fn spotify_transport_quirks() { + let o = Os::Mac; + assert_eq!( + shortcut(I::PlayPause, AppFamily::Spotify, o), + Some(keys(&["space"])) + ); + // ↓ next / ↑ previous (the Spotify quirk). + assert_eq!( + shortcut(I::NextTrack, AppFamily::Spotify, o), + Some(keys(&["down"])) + ); + assert_eq!( + shortcut(I::PrevTrack, AppFamily::Spotify, o), + Some(keys(&["up"])) + ); + assert_eq!( + shortcut(I::Shuffle, AppFamily::Spotify, Os::Windows), + Some(keys(&["ctrl", "s"])) + ); + // No documented volume → None. + assert_eq!(shortcut(I::VolumeUp, AppFamily::Spotify, o), None); + } + + #[test] + fn apple_music_mac_vs_windows() { + assert_eq!( + shortcut(I::PlayPause, AppFamily::AppleMusic, Os::Mac), + Some(keys(&["space"])) + ); + assert_eq!( + shortcut(I::PlayPause, AppFamily::AppleMusic, Os::Windows), + Some(keys(&["ctrl", "space"])) + ); + assert_eq!( + shortcut(I::VolumeUp, AppFamily::AppleMusic, Os::Mac), + Some(keys(&["Cmd", "up"])) + ); + assert_eq!( + shortcut(I::NextTrack, AppFamily::AppleMusic, Os::Windows), + Some(keys(&["ctrl", "right"])) + ); + // Search: Mac ⌘F; Windows access-key sequence → None. + assert_eq!( + shortcut(I::MediaSearch, AppFamily::AppleMusic, Os::Mac), + Some(keys(&["Cmd", "f"])) + ); + assert_eq!( + shortcut(I::MediaSearch, AppFamily::AppleMusic, Os::Windows), + None + ); + } + + #[test] + fn slack_navigation() { + assert_eq!( + shortcut(I::QuickSwitcher, AppFamily::Slack, Os::Mac), + Some(keys(&["Cmd", "k"])) + ); + assert_eq!( + shortcut(I::QuickSwitcher, AppFamily::Slack, Os::Windows), + Some(keys(&["Ctrl", "k"])) + ); + assert_eq!( + shortcut(I::NextUnread, AppFamily::Slack, Os::Mac), + Some(keys(&["alt", "shift", "down"])) + ); + assert_eq!( + shortcut(I::MarkRead, AppFamily::Slack, Os::Mac), + Some(keys(&["esc"])) + ); + // Media intent on Slack → None. + assert_eq!(shortcut(I::Shuffle, AppFamily::Slack, Os::Mac), None); + } + + #[test] + fn parse_media_intents() { + assert_eq!(parse_media("pause"), Some(I::PlayPause)); + assert_eq!(parse_media("resume the music"), Some(I::PlayPause)); + assert_eq!(parse_media("play"), Some(I::PlayPause)); + assert_eq!(parse_media("play the music"), Some(I::PlayPause)); + assert_eq!(parse_media("next song"), Some(I::NextTrack)); + assert_eq!(parse_media("skip this"), Some(I::NextTrack)); + assert_eq!(parse_media("previous track"), Some(I::PrevTrack)); + assert_eq!(parse_media("turn it up"), Some(I::VolumeUp)); + assert_eq!(parse_media("lower the volume"), Some(I::VolumeDown)); + assert_eq!(parse_media("mute"), Some(I::Mute)); + assert_eq!(parse_media("shuffle"), Some(I::Shuffle)); + // "play " is NOT a transport toggle (music.rs owns that). + assert_eq!(parse_media("play despacito"), None); + } + + #[test] + fn parse_slack_intents() { + assert_eq!(parse_slack("jump to a channel"), Some(I::QuickSwitcher)); + assert_eq!(parse_slack("go to channel general"), Some(I::QuickSwitcher)); + assert_eq!(parse_slack("next unread"), Some(I::NextUnread)); + assert_eq!(parse_slack("next channel"), Some(I::NextChannel)); + assert_eq!(parse_slack("compose a message"), Some(I::ComposeDm)); + assert_eq!(parse_slack("open threads"), Some(I::Threads)); + assert_eq!(parse_slack("mark as read"), Some(I::MarkRead)); + assert_eq!(parse_slack("search"), Some(I::SlackSearch)); + assert_eq!(parse_slack("hello"), None); + } + + #[test] + fn matches_gating() { + assert!(matches("Spotify", "next song")); + assert!(matches("Music", "pause")); + assert!(matches("Slack", "jump to a conversation")); + // Unknown app or no intent → no match. + assert!(!matches("Spotify", "hello there")); + assert!(!matches("Notes", "next song")); + } +} diff --git a/src/openhuman/accessibility/app_fastpaths/browser.rs b/src/openhuman/accessibility/app_fastpaths/browser.rs new file mode 100644 index 000000000..f5258ce8c --- /dev/null +++ b/src/openhuman/accessibility/app_fastpaths/browser.rs @@ -0,0 +1,672 @@ +//! Browser fast-path: "open `` and go to `` / search / play". +//! +//! Encodes the deterministic browser flow the live transcript showed the model +//! getting wrong (tracker — Change 1.17): it re-launched the browser 6×, ran +//! `ax_interact` on the wrong app names (Chrome/Safari, never the real +//! `Brave Browser`), and finally clicked a hardcoded coordinate. +//! +//! Instead we navigate in ONE step via `open -a "" ""`: +//! - **Pure navigation** ("open Brave and go to youtube.com", "search +//! youtube for X") completes here — no address-bar typing, no AX, no +//! re-launching. +//! - **Play intents** ("…and play a music video") navigate deterministically +//! to the search-results URL, then return a non-success outcome so the +//! general loop performs the single ambiguous "click the first result" via +//! `vision_click` (Chromium exposes no AX tree). There's no reliable native +//! shortcut that *selects* a search result, so we don't fake success. +//! - **In-page media control** ("pause/next/fullscreen the video") sends the +//! YouTube keyboard shortcut directly — the fastest possible path. +//! +//! Everything goes through the injectable [`AutomateBackend`], so the flow is +//! unit-testable with a scripted backend — no live browser, no model. + +use super::browser_shortcuts::{shortcut, Browser, BrowserShortcut, Os}; +use super::AutomateBackend; +use super::AutomateOutcome; + +/// A resolved navigation target. +struct Destination { + /// Fully-qualified URL to open. + url: String, + /// True when the goal asks to *play* something at a search URL — the + /// fast-path navigates but then defers the final "click first result" to + /// the general loop's `vision_click`. + is_play: bool, +} + +/// Does this (app, goal) look like a browser navigation / control request? +pub fn matches(app: &str, goal: &str) -> bool { + resolve_browser(app, goal).is_some() + && (extract_destination(goal).is_some() + || extract_browser_command(goal).is_some() + || extract_media_control(goal).is_some()) +} + +/// Resolve the browser's macOS display name from the `app` arg first, then the +/// goal text. Returns `None` for a generic "browser" with no named product, so +/// we never guess the wrong app (the original transcript bug). +fn resolve_browser(app: &str, goal: &str) -> Option { + // (alias substring, display name) — longest/most-specific aliases first. + const BROWSERS: &[(&str, &str)] = &[ + ("brave", "Brave Browser"), + ("google chrome", "Google Chrome"), + ("chrome", "Google Chrome"), + ("microsoft edge", "Microsoft Edge"), + ("edge", "Microsoft Edge"), + ("firefox", "Firefox"), + ("safari", "Safari"), + ("arc", "Arc"), + ]; + let app_l = app.to_lowercase(); + for (alias, display) in BROWSERS { + if app_l.contains(alias) { + return Some((*display).to_string()); + } + } + let goal_l = goal.to_lowercase(); + for (alias, display) in BROWSERS { + if goal_l.contains(alias) { + return Some((*display).to_string()); + } + } + None +} + +/// Resolve where to navigate. In priority order: YouTube intent → Google/web +/// search → a bare URL/domain mentioned in the goal. `None` if nothing matches. +fn extract_destination(goal: &str) -> Option { + let lower = goal.to_lowercase(); + let wants_play = has_word(&lower, "play"); + + // 1. YouTube. + if lower.contains("youtube") || lower.contains("you tube") { + if let Some(q) = extract_query(goal) { + return Some(Destination { + url: format!( + "https://www.youtube.com/results?search_query={}", + percent_encode(&q) + ), + is_play: wants_play, + }); + } + // Named YouTube but no query → just open the site. + return Some(Destination { + url: "https://www.youtube.com".to_string(), + is_play: false, + }); + } + + // 2. Google / web search. + if lower.contains("google") || lower.contains("search") { + // "search for X" / "search X for Y" via extract_query, else the text + // right after the word "google" ("google rust async traits"). + let q = extract_query(goal).or_else(|| { + word_index(&lower, "google").and_then(|i| { + let after = clean_query(&goal[i + "google".len()..]); + (!after.is_empty()).then_some(after) + }) + }); + if let Some(q) = q { + return Some(Destination { + url: format!("https://www.google.com/search?q={}", percent_encode(&q)), + is_play: false, + }); + } + } + + // 3. A bare URL / domain anywhere in the goal ("go to example.com"). + if let Some(url) = extract_url(goal) { + return Some(Destination { + url, + is_play: false, + }); + } + + None +} + +/// Pull a search query out of the goal: text after "for" (search phrasing) or +/// after "play", with leading articles and trailing site words stripped. `None` +/// when there's nothing searchable. +fn extract_query(goal: &str) -> Option { + let lower = goal.to_lowercase(); + // "search [youtube|the web|google] for X" / "search for X". + if let Some(p) = lower.find(" for ") { + let after = goal[p + " for ".len()..].trim(); + let q = clean_query(after); + if !q.is_empty() { + return Some(q); + } + } + // "play X", "play X on youtube". + if let Some(idx) = word_index(&lower, "play") { + let after = goal[idx + "play".len()..].trim(); + let q = clean_query(after); + if !q.is_empty() { + return Some(q); + } + } + None +} + +/// Trim filler around an extracted query: leading articles, a leading "me", +/// and a trailing "(on|in|at) " / "video(s)" tail isn't stripped (it's a +/// fine search term), but a trailing "on youtube" etc. is removed. +fn clean_query(raw: &str) -> String { + let mut q = raw.trim().trim_end_matches(['.', '!', '?']).trim(); + // Cut at a clause boundary so "go to youtube and play X then do Y" stops at X. + if let Some(p) = q.to_lowercase().find(" then ") { + q = q[..p].trim(); + } + let mut s = q.to_string(); + for lead in ["me ", "a ", "an ", "the ", "some "] { + if let Some(rest) = strip_prefix_ci(&s, lead) { + s = rest.trim().to_string(); + break; + } + } + // Drop a trailing "(on|in) " navigation tail. + let sl = s.to_lowercase(); + for tail in [" on youtube", " in youtube", " on google", " on the web"] { + if let Some(p) = sl.rfind(tail) { + s.truncate(p); + break; + } + } + s.trim().to_string() +} + +/// Find the first bare URL or `host.tld` token and normalize it to an +/// `https://` URL. Skips obvious non-hosts (must contain a dot, no spaces). +fn extract_url(goal: &str) -> Option { + for tok in goal.split_whitespace() { + let t = + tok.trim_matches(|c: char| !c.is_alphanumeric() && c != '/' && c != ':' && c != '.'); + let tl = t.to_lowercase(); + if tl.starts_with("http://") || tl.starts_with("https://") { + return Some(t.to_string()); + } + // host.tld with a plausible TLD and no scheme. + if t.contains('.') && !t.contains('/') && looks_like_domain(&tl) { + return Some(format!("https://{t}")); + } + } + None +} + +/// Crude domain check: has a dot, and the last label is 2–24 ascii letters. +fn looks_like_domain(s: &str) -> bool { + match s.rsplit_once('.') { + Some((host, tld)) => { + !host.is_empty() + && (2..=24).contains(&tld.len()) + && tld.chars().all(|c| c.is_ascii_alphabetic()) + } + None => false, + } +} + +/// Map a browser-level command ("new tab", "go back", "reload", "switch to tab +/// 3", …) to a [`BrowserShortcut`]. Resolved to actual keys per browser+OS by +/// [`browser_shortcuts::shortcut`]. Navigation (a URL/search) is handled earlier +/// by [`extract_destination`]; this covers tab/window/history/zoom verbs. +fn extract_browser_command(goal: &str) -> Option { + use BrowserShortcut as S; + let l = goal.to_lowercase(); + + // "switch to tab 3" / "go to tab 5" — a digit must follow "tab". + if let Some(n) = tab_number(&l) { + return Some(S::TabN(n)); + } + // Most-specific phrases first. + if l.contains("hard reload") || l.contains("force reload") || l.contains("hard refresh") { + Some(S::HardReload) + } else if l.contains("reload") || l.contains("refresh") { + Some(S::Reload) + } else if l.contains("new tab") { + Some(S::NewTab) + } else if l.contains("close tab") + || l.contains("close this tab") + || l.contains("close the tab") + || l.contains("close current tab") + { + Some(S::CloseTab) + } else if l.contains("reopen") || l.contains("restore tab") || l.contains("undo close") { + Some(S::ReopenTab) + } else if l.contains("incognito") + || l.contains("inprivate") + || l.contains("private window") + || l.contains("private browsing") + { + Some(S::PrivateWindow) + } else if l.contains("new window") { + Some(S::NewWindow) + } else if l.contains("next tab") { + Some(S::NextTab) + } else if l.contains("previous tab") || l.contains("prev tab") { + Some(S::PrevTab) + } else if l.contains("last tab") { + Some(S::LastTab) + } else if l.contains("go back") || l.contains("navigate back") { + Some(S::Back) + } else if l.contains("go forward") || l.contains("navigate forward") { + Some(S::Forward) + } else if l.contains("address bar") || l.contains("url bar") || l.contains("location bar") { + Some(S::FocusAddressBar) + } else if has_word(&l, "history") { + Some(S::History) + } else if has_word(&l, "downloads") || l.contains("download list") { + Some(S::Downloads) + } else if l.contains("bookmark this") + || l.contains("bookmark the page") + || l.contains("bookmark page") + || l.contains("add bookmark") + || l.contains("save bookmark") + { + Some(S::BookmarkPage) + } else if l.contains("zoom in") { + Some(S::ZoomIn) + } else if l.contains("zoom out") { + Some(S::ZoomOut) + } else if l.contains("reset zoom") || l.contains("actual size") || l.contains("default zoom") { + Some(S::ZoomReset) + } else { + None + } +} + +/// Extract a 1‑9 tab number following the word "tab" ("switch to tab 3"). Also +/// accepts "tab number 3". `None` when no digit follows (so "new tab" / "next +/// tab" don't match). +fn tab_number(lower: &str) -> Option { + let idx = lower.find("tab ")?; + let rest = lower[idx + "tab ".len()..].trim_start(); + let mut toks = rest.split_whitespace(); + let first = toks.next()?; + let digit = if first == "number" || first == "no" || first == "#" { + toks.next()? + } else { + first + }; + let n: u8 = digit + .trim_matches(|c: char| !c.is_ascii_digit()) + .parse() + .ok()?; + (1..=9).contains(&n).then_some(n) +} + +/// Map an in-page media-control goal to a YouTube keyboard shortcut, if any. +fn extract_media_control(goal: &str) -> Option> { + let l = goal.to_lowercase(); + // Only treat as a *control* command when there's no navigation verb. + if l.contains("open ") || l.contains("go to") || l.contains("search") { + return None; + } + let key = |k: &str| Some(vec![k.to_string()]); + if has_word(&l, "fullscreen") || l.contains("full screen") { + key("f") + } else if has_word(&l, "mute") || has_word(&l, "unmute") { + key("m") + } else if has_word(&l, "next") || has_word(&l, "skip") { + Some(vec!["shift".to_string(), "n".to_string()]) + } else if has_word(&l, "previous") || has_word(&l, "back") { + Some(vec!["shift".to_string(), "p".to_string()]) + } else if has_word(&l, "pause") || has_word(&l, "resume") || has_word(&l, "play") { + // YouTube `k` toggles play/pause. + key("k") + } else { + None + } +} + +/// Run the browser fast-path. +pub async fn run(app: &str, goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome { + use super::super::automate::progress; + use crate::openhuman::overlay::OverlayAttentionTone; + + let mut steps: Vec = Vec::new(); + let browser = match resolve_browser(app, goal) { + Some(b) => b, + None => return fail("no recognizable browser", steps), + }; + + // Navigation takes priority over in-page control. + if let Some(dest) = extract_destination(goal) { + log::info!( + "[automate::browser] ▶ open browser={browser:?} url={:?} is_play={}", + dest.url, + dest.is_play + ); + progress(format!("Opening {browser}…"), OverlayAttentionTone::Accent); + + match backend.open_url_in_app(&browser, &dest.url).await { + Ok(m) => steps.push(format!("navigate: {m}")), + Err(e) => { + steps.push(format!("navigate FAILED: {e}")); + return fail("could not open the browser/URL", steps); + } + } + backend.settle(&browser).await; + // Give the page network time to render before any follow-up. + backend.wait(1200).await; + + if dest.is_play { + // Deterministic part done; defer the single "click first result" to + // the general loop's vision_click (no reliable shortcut selects a + // search result). Returning non-success makes the loop take over — + // it does NOT re-launch from scratch, since the page is already up. + steps.push("navigated to search results; deferring play-click to vision".to_string()); + return fail( + "navigated; first-result click deferred to general loop", + steps, + ); + } + + return AutomateOutcome { + success: true, + summary: format!("Opened {} in {browser}.", dest.url), + steps, + }; + } + + // Browser-level command ("new tab", "go back", "reload", "tab 3") via the + // cross-platform shortcut table — no AX, no navigation. + if let Some(intent) = extract_browser_command(goal) { + let keys = shortcut(intent, Browser::from_display(&browser), Os::current()); + let combo = keys.join("+"); + log::info!("[automate::browser] ▶ command {intent:?} browser={browser:?} keys={combo}"); + progress(format!("Pressing {combo}…"), OverlayAttentionTone::Accent); + match backend.key(&keys).await { + Ok(m) => { + steps.push(format!("hotkey {combo}: {m}")); + return AutomateOutcome { + success: true, + summary: format!("Sent {combo} to {browser}."), + steps, + }; + } + Err(e) => { + steps.push(format!("hotkey FAILED: {e}")); + return fail("could not send the browser shortcut", steps); + } + } + } + + // In-page media control via a YouTube keyboard shortcut. + if let Some(keys) = extract_media_control(goal) { + let combo = keys.join("+"); + log::info!("[automate::browser] ▶ media control browser={browser:?} keys={combo}"); + progress(format!("Pressing {combo}…"), OverlayAttentionTone::Accent); + match backend.key(&keys).await { + Ok(m) => { + steps.push(format!("hotkey {combo}: {m}")); + return AutomateOutcome { + success: true, + summary: format!("Sent {combo} to {browser}."), + steps, + }; + } + Err(e) => { + steps.push(format!("hotkey FAILED: {e}")); + return fail("could not send the media shortcut", steps); + } + } + } + + fail("no browser destination or control in goal", steps) +} + +fn fail(msg: &str, steps: Vec) -> AutomateOutcome { + AutomateOutcome { + success: false, + summary: format!("Browser fast-path: {msg}"), + steps, + } +} + +// ── small string helpers ──────────────────────────────────────────────────── + +/// True if `needle` appears as a whole word in `haystack` (already lowercased). +fn has_word(haystack: &str, needle: &str) -> bool { + word_index(haystack, needle).is_some() +} + +/// Byte index of `needle` as a whole word in `haystack` (already lowercased). +fn word_index(haystack: &str, needle: &str) -> Option { + let mut from = 0; + while let Some(rel) = haystack[from..].find(needle) { + let idx = from + rel; + let before_ok = idx == 0 + || !haystack[..idx] + .chars() + .next_back() + .map(|c| c.is_alphanumeric()) + .unwrap_or(false); + let after = idx + needle.len(); + let after_ok = haystack[after..] + .chars() + .next() + .map(|c| !c.is_alphanumeric()) + .unwrap_or(true); + if before_ok && after_ok { + return Some(idx); + } + from = idx + needle.len(); + } + None +} + +/// Case-insensitive `strip_prefix`. +fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + if s.len() >= prefix.len() + && s.is_char_boundary(prefix.len()) + && s[..prefix.len()].to_lowercase() == prefix.to_lowercase() + { + Some(&s[prefix.len()..]) + } else { + None + } +} + +/// Percent-encode reserved characters in a query value (enough for a `?q=` +/// search param; not a full RFC-3986 encoder). Mirrors `music::percent_encode`. +fn percent_encode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +#[cfg(test)] +mod unit { + use super::*; + + #[test] + fn resolve_browser_from_app_then_goal_never_guesses() { + // From the app arg. + assert_eq!( + resolve_browser("Brave Browser", "do something").as_deref(), + Some("Brave Browser") + ); + // From the goal text. + assert_eq!( + resolve_browser("", "open chrome and go to x.com").as_deref(), + Some("Google Chrome") + ); + assert_eq!( + resolve_browser("", "use safari to open apple.com").as_deref(), + Some("Safari") + ); + assert_eq!( + resolve_browser("", "open microsoft edge").as_deref(), + Some("Microsoft Edge") + ); + // Generic "browser" with no named product → None (the original bug: we + // must NOT guess Chrome/Safari). + assert_eq!(resolve_browser("", "open my browser"), None); + } + + #[test] + fn matches_requires_browser_and_destination_or_control() { + assert!(matches("Brave Browser", "go to youtube.com")); + assert!(matches("", "open chrome and search youtube for lofi")); + assert!(matches("Brave Browser", "pause the video")); + // Browser but no destination/control. + assert!(!matches("Brave Browser", "hello there")); + // Destination but no resolvable browser. + assert!(!matches("Slack", "go to youtube.com")); + } + + #[test] + fn destination_youtube_search_encodes_query() { + let d = extract_destination("go to youtube and play a music video").unwrap(); + assert_eq!( + d.url, + "https://www.youtube.com/results?search_query=music%20video" + ); + assert!(d.is_play); + } + + #[test] + fn destination_youtube_search_for_phrasing() { + let d = extract_destination("search youtube for lofi beats").unwrap(); + assert_eq!( + d.url, + "https://www.youtube.com/results?search_query=lofi%20beats" + ); + assert!(!d.is_play); // no "play" word → navigation only + } + + #[test] + fn destination_bare_youtube_no_query() { + let d = extract_destination("open youtube").unwrap(); + assert_eq!(d.url, "https://www.youtube.com"); + assert!(!d.is_play); + } + + #[test] + fn destination_bare_domain_normalized_to_https() { + let d = extract_destination("go to example.com").unwrap(); + assert_eq!(d.url, "https://example.com"); + let d2 = extract_destination("open https://news.ycombinator.com").unwrap(); + assert_eq!(d2.url, "https://news.ycombinator.com"); + } + + #[test] + fn destination_google_search() { + let d = extract_destination("google rust async traits").unwrap(); + assert_eq!( + d.url, + "https://www.google.com/search?q=rust%20async%20traits" + ); + } + + #[test] + fn destination_none_when_no_target() { + assert!(extract_destination("just hang out").is_none()); + } + + #[test] + fn media_control_maps_to_youtube_shortcuts() { + assert_eq!( + extract_media_control("pause the video"), + Some(vec!["k".into()]) + ); + assert_eq!( + extract_media_control("resume playback"), + Some(vec!["k".into()]) + ); + assert_eq!(extract_media_control("mute it"), Some(vec!["m".into()])); + assert_eq!( + extract_media_control("next video"), + Some(vec!["shift".into(), "n".into()]) + ); + assert_eq!( + extract_media_control("go fullscreen"), + Some(vec!["f".into()]) + ); + // A navigation goal is NOT a control command. + assert_eq!(extract_media_control("open youtube and play lofi"), None); + } + + #[test] + fn word_index_is_whole_word() { + // "play" must not match inside "display"/"playback". + assert!(has_word("play a song", "play")); + assert!(!has_word("display settings", "play")); + assert!(!has_word("open playback options", "play")); + } + + #[test] + fn browser_command_maps_common_verbs() { + use BrowserShortcut as S; + assert_eq!(extract_browser_command("open a new tab"), Some(S::NewTab)); + assert_eq!(extract_browser_command("close this tab"), Some(S::CloseTab)); + assert_eq!( + extract_browser_command("reopen the closed tab"), + Some(S::ReopenTab) + ); + assert_eq!(extract_browser_command("go back"), Some(S::Back)); + assert_eq!(extract_browser_command("reload the page"), Some(S::Reload)); + assert_eq!(extract_browser_command("hard reload"), Some(S::HardReload)); + assert_eq!(extract_browser_command("next tab"), Some(S::NextTab)); + assert_eq!( + extract_browser_command("open an incognito window"), + Some(S::PrivateWindow) + ); + assert_eq!(extract_browser_command("show my history"), Some(S::History)); + assert_eq!(extract_browser_command("zoom in"), Some(S::ZoomIn)); + // No browser command. + assert_eq!(extract_browser_command("play a music video"), None); + } + + #[test] + fn tab_number_only_with_digit() { + assert_eq!(tab_number("switch to tab 3"), Some(3)); + assert_eq!(tab_number("go to tab number 5"), Some(5)); + // "new tab"/"next tab" have no trailing digit → not a tab-number jump. + assert_eq!(tab_number("open a new tab"), None); + assert_eq!(tab_number("next tab"), None); + assert_eq!( + extract_browser_command("switch to tab 4"), + Some(BrowserShortcut::TabN(4)) + ); + } +} + +/// Live integration test — drives a real browser. Ignored by default (needs a +/// browser installed + Accessibility/Screen-recording permission). Asserts +/// tool-level success only; the visual page state is best-effort. +/// +/// cargo test --lib browser_fastpath_live -- --ignored --nocapture +#[cfg(all(test, target_os = "macos"))] +mod live { + use super::run; + use crate::openhuman::accessibility::automate::RealBackend; + + #[tokio::test] + #[ignore = "requires macOS + a browser + Accessibility permission"] + async fn browser_fastpath_live() { + let backend = RealBackend::new(crate::openhuman::config::Config::default()); + // Use Safari (always present on macOS) for a deterministic nav. + let out = run("Safari", "open safari and go to example.com", &backend).await; + println!( + "[browser_fastpath_live] success={} summary={}", + out.success, out.summary + ); + for s in &out.steps { + println!(" - {s}"); + } + assert!( + out.success, + "nav fast-path reported failure: {}", + out.summary + ); + } +} diff --git a/src/openhuman/accessibility/app_fastpaths/browser_shortcuts.rs b/src/openhuman/accessibility/app_fastpaths/browser_shortcuts.rs new file mode 100644 index 000000000..7ced7f625 --- /dev/null +++ b/src/openhuman/accessibility/app_fastpaths/browser_shortcuts.rs @@ -0,0 +1,351 @@ +//! Cross-platform browser keyboard-shortcut table (Change 1.17). +//! +//! The four major desktop browsers (Chrome, Firefox, Safari, Edge) share almost +//! the same navigation/tab/find shortcuts — they differ mainly by the primary +//! modifier (⌘ on macOS, Ctrl on Windows/Linux), with a handful of real +//! per-browser exceptions (Firefox's `Alt+1‑8` tab selection on Linux, its +//! `Ctrl/⌘+Shift+P` private window, per-browser History/Downloads keys, etc.). +//! +//! [`shortcut`] resolves `(intent, browser, os)` to a key chord expressed as the +//! same key names the keyboard tool's `parse_key` understands (`"Cmd"`, +//! `"Ctrl"`, `"Shift"`, `"Alt"`, single chars, `"left"`, `"tab"`, `"f11"`, …), +//! so the result feeds straight into `AutomateBackend::key`. +//! +//! Sources: Chrome/Firefox/Edge support pages (Win/macOS/Linux columns) and the +//! Safari for Mac guide. In-page media shortcuts (YouTube `k`/`f`/`m`/`Shift+N`) +//! are browser-independent and live in `browser.rs`, not here. + +/// Target operating system — selects the primary modifier and the non-Mac +/// fallbacks (Alt+arrow history, F11 fullscreen, etc.). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Os { + Mac, + Windows, + Linux, +} + +impl Os { + /// The OS this build is running on. + pub fn current() -> Os { + if cfg!(target_os = "macos") { + Os::Mac + } else if cfg!(target_os = "windows") { + Os::Windows + } else { + Os::Linux + } + } +} + +/// Browser family — chosen by [`Browser::from_display`]. Chromium-based browsers +/// (Brave, Arc, Chromium) share Chrome's shortcuts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Browser { + Chrome, + Firefox, + Safari, + Edge, +} + +impl Browser { + /// Map a macOS display name (or alias) to a family. Unknown / generic names + /// default to Chrome, since most other desktop browsers are Chromium-based + /// and share its bindings. + pub fn from_display(name: &str) -> Browser { + let n = name.to_lowercase(); + if n.contains("firefox") { + Browser::Firefox + } else if n.contains("safari") { + Browser::Safari + } else if n.contains("edge") { + Browser::Edge + } else { + // Chrome, Brave, Arc, Chromium, Vivaldi, Opera, … → Chrome bindings. + Browser::Chrome + } + } +} + +/// A browser action we can trigger with a keyboard shortcut. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BrowserShortcut { + FocusAddressBar, + NewTab, + CloseTab, + ReopenTab, + NewWindow, + PrivateWindow, + NextTab, + PrevTab, + /// Jump to tab N (1-based, 1‑8). N≥9 is treated as the last tab. + TabN(u8), + LastTab, + Find, + FindNext, + FindPrev, + Reload, + HardReload, + ZoomIn, + ZoomOut, + ZoomReset, + Print, + BookmarkPage, + Back, + Forward, + History, + Downloads, + Fullscreen, +} + +/// The primary modifier: ⌘ on macOS, Ctrl elsewhere. +fn primary(os: Os) -> &'static str { + match os { + Os::Mac => "Cmd", + _ => "Ctrl", + } +} + +fn keys(parts: &[&str]) -> Vec { + parts.iter().map(|s| s.to_string()).collect() +} + +/// Resolve a shortcut to a key chord for the given browser + OS. +pub fn shortcut(intent: BrowserShortcut, browser: Browser, os: Os) -> Vec { + use BrowserShortcut as S; + let p = primary(os); + match intent { + // ── uniform across all four browsers (only ⌘↔Ctrl differs) ── + S::FocusAddressBar => keys(&[p, "l"]), + S::NewTab => keys(&[p, "t"]), + S::CloseTab => keys(&[p, "w"]), + S::ReopenTab => keys(&[p, "shift", "t"]), + S::NewWindow => keys(&[p, "n"]), + // Ctrl+Tab cycles tabs on every OS/browser (not ⌘ on macOS). + S::NextTab => keys(&["ctrl", "tab"]), + S::PrevTab => keys(&["ctrl", "shift", "tab"]), + S::Find => keys(&[p, "f"]), + S::FindNext => keys(&[p, "g"]), + S::FindPrev => keys(&[p, "shift", "g"]), + S::Reload => keys(&[p, "r"]), + S::HardReload => keys(&[p, "shift", "r"]), + // Use "=" (Ctrl/⌘+=) for zoom-in: it's the unshifted key and every + // browser accepts it as zoom-in, avoiding the Shift needed to type "+". + S::ZoomIn => keys(&[p, "="]), + S::ZoomOut => keys(&[p, "-"]), + S::ZoomReset => keys(&[p, "0"]), + S::Print => keys(&[p, "p"]), + S::BookmarkPage => keys(&[p, "d"]), + + // ── per-browser / per-OS exceptions ── + S::PrivateWindow => match browser { + Browser::Firefox => keys(&[p, "shift", "p"]), + _ => keys(&[p, "shift", "n"]), + }, + S::TabN(n) => { + let d = n.clamp(1, 8).to_string(); + // Firefox on Linux uses Alt+1‑8 (not Ctrl). + if browser == Browser::Firefox && os == Os::Linux { + keys(&["alt", &d]) + } else { + keys(&[p, &d]) + } + } + S::LastTab => { + if browser == Browser::Firefox && os == Os::Linux { + keys(&["alt", "9"]) + } else { + keys(&[p, "9"]) + } + } + S::Back => match os { + Os::Mac => keys(&[p, "["]), + _ => keys(&["alt", "left"]), + }, + S::Forward => match os { + Os::Mac => keys(&[p, "]"]), + _ => keys(&["alt", "right"]), + }, + S::History => match os { + Os::Mac => match browser { + // Chrome/Edge: ⌘Y. Firefox/Safari: ⌘⇧H. + Browser::Chrome | Browser::Edge => keys(&[p, "y"]), + Browser::Firefox | Browser::Safari => keys(&[p, "shift", "h"]), + }, + _ => keys(&["ctrl", "h"]), + }, + S::Downloads => match os { + Os::Mac => match browser { + Browser::Chrome => keys(&[p, "shift", "j"]), + Browser::Firefox => keys(&[p, "j"]), + // Edge / Safari: ⌥⌘L. + Browser::Edge | Browser::Safari => keys(&[p, "option", "l"]), + }, + _ => keys(&["ctrl", "j"]), + }, + S::Fullscreen => match os { + // Avoid the Fn key: ⌃⌘F is the standard macOS window fullscreen + // (Chrome/Edge/Safari); Firefox uses ⌘⇧F. + Os::Mac => match browser { + Browser::Firefox => keys(&[p, "shift", "f"]), + _ => keys(&["ctrl", "cmd", "f"]), + }, + _ => keys(&["f11"]), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use BrowserShortcut as S; + + #[test] + fn primary_modifier_per_os() { + assert_eq!( + shortcut(S::FocusAddressBar, Browser::Chrome, Os::Mac), + vec!["Cmd", "l"] + ); + assert_eq!( + shortcut(S::FocusAddressBar, Browser::Chrome, Os::Windows), + vec!["Ctrl", "l"] + ); + assert_eq!( + shortcut(S::FocusAddressBar, Browser::Safari, Os::Mac), + vec!["Cmd", "l"] + ); + } + + #[test] + fn uniform_core_shortcuts() { + assert_eq!( + shortcut(S::NewTab, Browser::Edge, Os::Linux), + vec!["Ctrl", "t"] + ); + assert_eq!( + shortcut(S::ReopenTab, Browser::Firefox, Os::Mac), + vec!["Cmd", "shift", "t"] + ); + assert_eq!( + shortcut(S::HardReload, Browser::Chrome, Os::Windows), + vec!["Ctrl", "shift", "r"] + ); + // Next/prev tab is Ctrl-based on every OS, even macOS. + assert_eq!( + shortcut(S::NextTab, Browser::Chrome, Os::Mac), + vec!["ctrl", "tab"] + ); + } + + #[test] + fn private_window_firefox_differs() { + assert_eq!( + shortcut(S::PrivateWindow, Browser::Firefox, Os::Windows), + vec!["Ctrl", "shift", "p"] + ); + assert_eq!( + shortcut(S::PrivateWindow, Browser::Chrome, Os::Windows), + vec!["Ctrl", "shift", "n"] + ); + } + + #[test] + fn tab_n_firefox_linux_uses_alt() { + assert_eq!( + shortcut(S::TabN(3), Browser::Firefox, Os::Linux), + vec!["alt", "3"] + ); + assert_eq!( + shortcut(S::TabN(3), Browser::Chrome, Os::Linux), + vec!["Ctrl", "3"] + ); + assert_eq!( + shortcut(S::TabN(3), Browser::Firefox, Os::Mac), + vec!["Cmd", "3"] + ); + // Out-of-range clamps into 1‑8. + assert_eq!( + shortcut(S::TabN(20), Browser::Chrome, Os::Mac), + vec!["Cmd", "8"] + ); + assert_eq!( + shortcut(S::LastTab, Browser::Firefox, Os::Linux), + vec!["alt", "9"] + ); + } + + #[test] + fn back_forward_os_specific() { + assert_eq!( + shortcut(S::Back, Browser::Chrome, Os::Mac), + vec!["Cmd", "["] + ); + assert_eq!( + shortcut(S::Back, Browser::Chrome, Os::Windows), + vec!["alt", "left"] + ); + assert_eq!( + shortcut(S::Forward, Browser::Safari, Os::Mac), + vec!["Cmd", "]"] + ); + } + + #[test] + fn history_and_downloads_per_browser_on_mac() { + assert_eq!( + shortcut(S::History, Browser::Chrome, Os::Mac), + vec!["Cmd", "y"] + ); + assert_eq!( + shortcut(S::History, Browser::Firefox, Os::Mac), + vec!["Cmd", "shift", "h"] + ); + assert_eq!( + shortcut(S::History, Browser::Edge, Os::Windows), + vec!["ctrl", "h"] + ); + assert_eq!( + shortcut(S::Downloads, Browser::Chrome, Os::Mac), + vec!["Cmd", "shift", "j"] + ); + assert_eq!( + shortcut(S::Downloads, Browser::Firefox, Os::Mac), + vec!["Cmd", "j"] + ); + assert_eq!( + shortcut(S::Downloads, Browser::Edge, Os::Mac), + vec!["Cmd", "option", "l"] + ); + assert_eq!( + shortcut(S::Downloads, Browser::Chrome, Os::Linux), + vec!["ctrl", "j"] + ); + } + + #[test] + fn fullscreen_avoids_fn_key() { + assert_eq!( + shortcut(S::Fullscreen, Browser::Chrome, Os::Mac), + vec!["ctrl", "cmd", "f"] + ); + assert_eq!( + shortcut(S::Fullscreen, Browser::Firefox, Os::Mac), + vec!["Cmd", "shift", "f"] + ); + assert_eq!( + shortcut(S::Fullscreen, Browser::Chrome, Os::Windows), + vec!["f11"] + ); + } + + #[test] + fn from_display_maps_families() { + assert_eq!(Browser::from_display("Brave Browser"), Browser::Chrome); + assert_eq!(Browser::from_display("Arc"), Browser::Chrome); + assert_eq!(Browser::from_display("Google Chrome"), Browser::Chrome); + assert_eq!(Browser::from_display("Firefox"), Browser::Firefox); + assert_eq!(Browser::from_display("Microsoft Edge"), Browser::Edge); + assert_eq!(Browser::from_display("Safari"), Browser::Safari); + // Unknown → Chrome default. + assert_eq!(Browser::from_display("Some New Browser"), Browser::Chrome); + } +} diff --git a/src/openhuman/accessibility/app_fastpaths/fastpaths_tests.rs b/src/openhuman/accessibility/app_fastpaths/fastpaths_tests.rs index 29ead72c2..312615744 100644 --- a/src/openhuman/accessibility/app_fastpaths/fastpaths_tests.rs +++ b/src/openhuman/accessibility/app_fastpaths/fastpaths_tests.rs @@ -104,6 +104,8 @@ struct Backend { /// Elements returned by perceive (the search results screen). elements: Vec, press_fail_on: Option, + /// What `now_playing()` reports back (None = backend can't read the track). + now_playing: Option<(String, String)>, } impl Backend { @@ -112,8 +114,13 @@ impl Backend { acts: Mutex::new(Vec::new()), elements, press_fail_on: None, + now_playing: None, } } + fn with_now_playing(mut self, name: &str, artist: &str) -> Self { + self.now_playing = Some((name.to_string(), artist.to_string())); + self + } fn acts(&self) -> Vec { self.acts.lock().unwrap().clone() } @@ -148,6 +155,23 @@ impl AutomateBackend for Backend { self.acts.lock().unwrap().push(format!("open_url:{url}")); Ok("ok".into()) } + async fn open_url_in_app(&self, app: &str, url: &str) -> Result { + self.acts + .lock() + .unwrap() + .push(format!("open_url_in_app:{app}:{url}")); + Ok("ok".into()) + } + async fn key(&self, keys: &[String]) -> Result { + self.acts + .lock() + .unwrap() + .push(format!("key:{}", keys.join("+"))); + Ok("ok".into()) + } + async fn now_playing(&self) -> Option<(String, String)> { + self.now_playing.clone() + } async fn settle(&self, _app: &str) {} async fn wait(&self, _ms: u64) {} } @@ -170,12 +194,88 @@ async fn music_fastpath_full_sequence() { } #[tokio::test] -async fn music_fastpath_no_row_fails_for_fallthrough() { - // Search screen has nothing matching → fast-path fails (loop falls through). - let backend = Backend::new(vec![AXElement::new("AXButton", "Some Unrelated Button")]); - let out = music::run("play Numb", &backend).await; +async fn music_fastpath_reports_verified_track_and_artist() { + // now_playing matches the requested artist → success names the real track. + let backend = Backend::new(vec![song_row("Numb"), AXElement::new("AXButton", "Play")]) + .with_now_playing("Numb", "Linkin Park"); + let out = music::run("play Numb by Linkin Park", &backend).await; + assert!(out.success, "{out:?}"); + assert!( + out.summary.contains("Linkin Park") && out.summary.contains("Numb"), + "summary should name the verified track+artist: {}", + out.summary + ); +} + +#[tokio::test] +async fn music_fastpath_flags_wrong_artist_honestly() { + // The search landed on a same-titled song by a different artist. The + // fast-path must name it AND steer the agent away from re-searching (#1/#3). + let backend = Backend::new(vec![song_row("Numb"), AXElement::new("AXButton", "Play")]) + .with_now_playing("Numb", "Tom Odell"); + let out = music::run("play Numb by Linkin Park", &backend).await; + let s = out.summary.to_lowercase(); + assert!( + s.contains("tom odell"), + "must name what actually played: {}", + out.summary + ); + // Actionable, anti-loop guidance. + assert!( + s.contains("won't surface") && s.contains("library"), + "must steer the agent (no blind re-search): {}", + out.summary + ); +} + +#[tokio::test] +async fn music_fastpath_no_row_lists_candidates_and_warns() { + // Song rows exist but none match the query → response lists what WAS found + // and tells the agent not to repeat the same search (#1/#2). + let backend = Backend::new(vec![ + AXElement::new("AXCell", "Numb - Marshmello & Khalid"), + AXElement::new("AXCell", "Numb - Tom Odell"), + ]); + let out = music::run("play Zelda Theme by Koji Kondo", &backend).await; assert!(!out.success); - assert!(out.summary.contains("no matching song"), "{}", out.summary); + let s = out.summary.to_lowercase(); + assert!( + s.contains("marshmello") && s.contains("tom odell"), + "{}", + out.summary + ); + assert!( + s.contains("won't help") || s.contains("don't repeat"), + "{}", + out.summary + ); +} + +#[tokio::test] +async fn music_fastpath_artist_row_not_claimed_as_playing() { + // Only an artist AXButton matches (no song cell) and the backend can't read + // a track → must NOT claim "Playing"; say it only navigated (#3/#4). + let backend = Backend::new(vec![AXElement::new("AXButton", "LINKIN PARK")]); + let out = music::run("play Linkin Park Numb", &backend).await; + let s = out.summary.to_lowercase(); + assert!( + !s.contains("playing '"), + "must not claim playback: {}", + out.summary + ); + assert!( + s.contains("artist/album") && s.contains("specific song"), + "must explain it only navigated: {}", + out.summary + ); + // It pressed the artist element, flagged as non-song in the step log. + assert!( + out.steps + .iter() + .any(|a| a.contains("artist/album 'LINKIN PARK'")), + "{:?}", + out.steps + ); } #[tokio::test] @@ -203,6 +303,121 @@ async fn try_fastpath_dispatches_music_and_skips_others() { .is_some()); } +// ── Browser fast-path: scripted sequence ──────────────────────────── + +#[tokio::test] +async fn browser_nav_to_domain_succeeds_without_model() { + let backend = Backend::new(vec![]); + let out = super::browser::run( + "Brave Browser", + "open Brave and go to example.com", + &backend, + ) + .await; + assert!(out.success, "pure navigation should succeed: {out:?}"); + let acts = backend.acts(); + // One deterministic open in the named browser — no model `decide` call + // (the scripted backend panics if `decide` is hit). Bare domain is + // normalized to https://. + assert_eq!(acts.len(), 1, "{acts:?}"); + assert_eq!(acts[0], "open_url_in_app:Brave Browser:https://example.com"); +} + +#[tokio::test] +async fn browser_youtube_search_play_navigates_then_falls_through() { + let backend = Backend::new(vec![]); + let out = super::browser::run( + "Brave Browser", + "open my brave browser, go to youtube.com and play a music video", + &backend, + ) + .await; + // Play intent → navigate deterministically, then return non-success so the + // general loop performs the single first-result click via vision_click. + assert!(!out.success, "play must defer the final click: {out:?}"); + let acts = backend.acts(); + assert_eq!(acts.len(), 1, "{acts:?}"); + assert_eq!( + acts[0], + "open_url_in_app:Brave Browser:https://www.youtube.com/results?search_query=music%20video" + ); +} + +#[tokio::test] +async fn browser_media_control_sends_hotkey() { + let backend = Backend::new(vec![]); + let out = super::browser::run("Brave Browser", "pause the video", &backend).await; + assert!(out.success, "media control should succeed: {out:?}"); + assert_eq!(backend.acts(), vec!["key:k".to_string()]); +} + +#[tokio::test] +async fn browser_command_routes_resolved_shortcut() { + // "new tab" → a cross-platform chord resolved per browser+OS. Brave maps to + // the Chrome family; the primary modifier is OS-dependent, so accept either. + let backend = Backend::new(vec![]); + let out = super::browser::run("Brave Browser", "open a new tab", &backend).await; + assert!(out.success, "browser command should succeed: {out:?}"); + let act = &backend.acts()[0]; + assert!( + act == "key:Cmd+t" || act == "key:Ctrl+t", + "expected a new-tab chord, got {act}" + ); +} + +#[tokio::test] +async fn try_fastpath_dispatches_browser() { + let backend = Backend::new(vec![]); + let out = super::try_fastpath( + "Brave Browser", + "open Brave and go to example.com", + &backend, + ) + .await; + assert!( + out.is_some(), + "browser nav should be claimed by a fast-path" + ); + assert!(out.unwrap().success); +} + +// ── App-shortcut fast-path (Spotify / Apple Music / Slack) ────────── + +#[tokio::test] +async fn app_shortcut_spotify_next_sends_down_arrow() { + let backend = Backend::new(vec![]); + let out = super::app_shortcuts::run("Spotify", "next song", &backend).await; + assert!(out.success, "{out:?}"); + // Spotify quirk: next = Down arrow. + assert_eq!(backend.acts(), vec!["key:down".to_string()]); +} + +#[tokio::test] +async fn app_shortcut_slack_quick_switcher() { + let backend = Backend::new(vec![]); + let out = super::app_shortcuts::run("Slack", "jump to a conversation", &backend).await; + assert!(out.success, "{out:?}"); + let act = &backend.acts()[0]; + assert!(act == "key:Cmd+k" || act == "key:Ctrl+k", "got {act}"); +} + +#[tokio::test] +async fn try_fastpath_routes_apps_and_music_still_wins_play() { + let backend = Backend::new(vec![song_row("Numb")]); + // Apple Music "pause" → app-shortcut fast-path (music.rs declines: no query). + assert!(super::try_fastpath("Music", "pause", &backend) + .await + .is_some()); + // Apple Music "play Numb" → still claimed (by music.rs, song search). + assert!(super::try_fastpath("Music", "play Numb", &backend) + .await + .is_some()); + // Spotify "skip" → app-shortcut fast-path. + assert!(super::try_fastpath("Spotify", "skip this track", &backend) + .await + .is_some()); +} + // Outcome type sanity: fast-paths build the same outcome the loop returns. #[test] fn outcome_shape() { diff --git a/src/openhuman/accessibility/app_fastpaths/mod.rs b/src/openhuman/accessibility/app_fastpaths/mod.rs index 534d7299b..70ce780d6 100644 --- a/src/openhuman/accessibility/app_fastpaths/mod.rs +++ b/src/openhuman/accessibility/app_fastpaths/mod.rs @@ -15,6 +15,9 @@ use super::automate::AutomateBackend; use super::automate::AutomateOutcome; +mod app_shortcuts; +mod browser; +mod browser_shortcuts; mod music; /// Try every registered fast-path; return the first that claims the (app, goal). @@ -26,6 +29,12 @@ pub async fn try_fastpath( if music::matches(app, goal) { return Some(music::run(goal, backend).await); } + if browser::matches(app, goal) { + return Some(browser::run(app, goal, backend).await); + } + if app_shortcuts::matches(app, goal) { + return Some(app_shortcuts::run(app, goal, backend).await); + } None } diff --git a/src/openhuman/accessibility/app_fastpaths/music.rs b/src/openhuman/accessibility/app_fastpaths/music.rs index 02f281f96..3e7d414d0 100644 --- a/src/openhuman/accessibility/app_fastpaths/music.rs +++ b/src/openhuman/accessibility/app_fastpaths/music.rs @@ -7,13 +7,32 @@ //! the injectable [`AutomateBackend`], so the whole flow is unit-testable with a //! scripted backend — no live Music, no model. +use super::super::ax_interact::AXElement; use super::AutomateBackend; use super::AutomateOutcome; const APP: &str = "Music"; -/// Element roles that represent a tappable search result / song row. -const ROW_ROLES: &[&str] = &["AXCell", "AXRow", "ListItem", "AXButton", "AXStaticText"]; +/// Roles that represent an actual song/track ROW — the preferred press target, +/// because pressing one navigates into (and can play) the track. +const SONG_ROLES: &[&str] = &["AXCell", "AXRow", "ListItem"]; +/// Secondary roles — artist names, album headers, static text. Pressing these +/// usually only *navigates* (to an artist/album page), so they're a last resort +/// and we flag them so the caller can report honestly. +const OTHER_ROW_ROLES: &[&str] = &["AXButton", "AXStaticText"]; + +fn role_in(role: &str, set: &[&str]) -> bool { + set.iter().any(|r| role.contains(r)) +} + +/// A chosen press target plus whether it's a real song row (vs an artist/album +/// element that only navigates). The caller uses `is_song` to avoid claiming +/// "Playing X" when it merely opened an artist page. +#[derive(Debug, Clone, PartialEq)] +struct PickedRow { + label: String, + is_song: bool, +} /// Does this (app, goal) look like an Apple Music "play X" request? pub fn matches(app: &str, goal: &str) -> bool { @@ -88,6 +107,38 @@ pub fn extract_play_query(goal: &str) -> Option { } } +/// Pull the requested artist out of a "play X by " goal — used to +/// verify we played the *right* track. The Apple Music AX row label is +/// title-only ("Numb - Single"), so a "Numb" search can resolve to the wrong +/// artist; the artist lets us confirm via the now-playing track. `None` when no +/// "by " clause is present. +pub(crate) fn extract_artist(goal: &str) -> Option { + let lower = goal.to_lowercase(); + let p = lower.find(" by ")?; + let after = &goal[p + " by ".len()..]; + // Cut at the first clause boundary. + let mut end = after.len(); + for delim in [",", " and ", " then ", " in ", " on ", " from ", " for "] { + if let Some(q) = after.to_lowercase().find(delim) { + end = end.min(q); + } + } + let artist = after[..end].trim().trim_matches('"').trim().to_string(); + if artist.is_empty() || is_pronoun(&artist) { + None + } else { + Some(artist) + } +} + +/// Loose artist comparison: case-insensitive, matching if either string +/// contains the other (so "Linkin Park" matches "Linkin Park feat. …"). +fn artist_matches(want: &str, got: &str) -> bool { + let w = want.trim().to_lowercase(); + let g = got.trim().to_lowercase(); + !w.is_empty() && !g.is_empty() && (g.contains(&w) || w.contains(&g)) +} + /// Strip a trailing "(in|on) [apple] music" and rewrite " by " → " ". fn clean_query(q: &str) -> String { let mut q = q.trim().to_string(); @@ -216,26 +267,71 @@ fn first_token(query: &str) -> String { .to_string() } -/// Choose the best matching row from a perceive snapshot: an exact label match -/// first, else the first row-role element whose label shares a word with the -/// query. Returns the element label to press. -fn pick_row(elements: &[super::super::ax_interact::AXElement], query: &str) -> Option { +/// Choose the best press target. Preference order: an exact-label **song row**, +/// then a token-matching **song row**, then any exact-label element, and only as +/// a last resort an artist/album element (which merely navigates). Returns the +/// label *and* whether it's a real song row, so the caller never falsely claims +/// playback after pressing an artist/album header. +/// +/// (We deliberately do NOT skip elements whose reported `enabled` is false — +/// Apple Music marks pressable result rows as disabled; see AXElement::enabled.) +fn pick_row(elements: &[AXElement], query: &str) -> Option { let ql = query.to_lowercase(); - // Exact label match wins. (We deliberately do NOT skip elements whose - // reported `enabled` is false — Apple Music marks pressable result rows as - // disabled; see AXElement::enabled docs.) - if let Some(e) = elements.iter().find(|e| e.label.to_lowercase() == ql) { - return Some(e.label.clone()); - } let tokens: Vec<&str> = ql.split_whitespace().filter(|t| t.len() > 2).collect(); + let token_hit = |e: &&AXElement| { + let l = e.label.to_lowercase(); + tokens.iter().any(|t| l.contains(t)) + }; + let song = |e: &&AXElement| role_in(&e.role, SONG_ROLES); + + // 1. Exact match on a song row. + if let Some(e) = elements + .iter() + .find(|e| song(e) && e.label.to_lowercase() == ql) + { + return Some(PickedRow { + label: e.label.clone(), + is_song: true, + }); + } + // 2. Token match on a song row. + if let Some(e) = elements.iter().find(|e| song(e) && token_hit(e)) { + return Some(PickedRow { + label: e.label.clone(), + is_song: true, + }); + } + // 3. Exact match on any element. + if let Some(e) = elements.iter().find(|e| e.label.to_lowercase() == ql) { + return Some(PickedRow { + label: e.label.clone(), + is_song: role_in(&e.role, SONG_ROLES), + }); + } + // 4. Last resort: an artist/album element that token-matches (navigates). elements .iter() - .filter(|e| ROW_ROLES.iter().any(|r| e.role.contains(r))) - .find(|e| { - let l = e.label.to_lowercase(); - tokens.iter().any(|t| l.contains(t)) + .find(|e| role_in(&e.role, OTHER_ROW_ROLES) && token_hit(e)) + .map(|e| PickedRow { + label: e.label.clone(), + is_song: false, }) - .map(|e| e.label.clone()) +} + +/// Up to 5 distinct song-row labels currently visible — surfaced in failure +/// responses so the agent (or user) can see what *was* found and choose. +fn candidate_labels(elements: &[AXElement]) -> Vec { + let mut out: Vec = Vec::new(); + for e in elements.iter().filter(|e| role_in(&e.role, SONG_ROLES)) { + let l = e.label.trim().to_string(); + if !l.is_empty() && !out.contains(&l) { + out.push(l); + if out.len() >= 5 { + break; + } + } + } + out } /// Run the play fast-path. Returns a failed [`AutomateOutcome`] (not a panic) @@ -249,7 +345,10 @@ pub async fn run(goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome { return fail("not a play request", steps); } }; - log::info!("[automate::music] ▶ play query={query:?}"); + // The artist the user asked for (if any), used to confirm we played the + // right track — the AX row label alone can't disambiguate same-titled songs. + let want_artist = extract_artist(goal); + log::info!("[automate::music] ▶ play query={query:?} artist={want_artist:?}"); use super::super::automate::progress; use crate::openhuman::overlay::OverlayAttentionTone; progress( @@ -278,14 +377,17 @@ pub async fn run(goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome { // filter the snapshot by one strong token (a substring filter can't // match a whole multi-word title). let filter = first_token(&query); - let mut row = None; + let mut row: Option = None; + let mut last_els: Vec = Vec::new(); for attempt in 0..6 { backend.settle(APP).await; let els = backend.perceive(APP, &filter).await.unwrap_or_default(); if let Some(r) = pick_row(&els, &query) { + last_els = els; row = Some(r); break; } + last_els = els; // Catalog search results arrive asynchronously (~3-4s); element-count // settle can report "stable" while the network fetch is still pending, // so wait real time between attempts rather than spinning instantly. @@ -293,19 +395,26 @@ pub async fn run(goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome { backend.wait(800).await; } let row = match row { + // #1/#2: when nothing matched, list what WAS found and tell the agent + // not to repeat the same search — so it tries the library / asks the + // user instead of re-delegating the identical query. + None => return fail(&no_match_message(&query, &want_artist, &last_els), steps), Some(r) => r, - None => return fail("no matching song row found", steps), }; // Baseline count of "Play" controls *before* navigating, so we can tell // when the song's detail-page Play has actually rendered (vs. only the // toolbar transport Play that's always present). let plays_before = count_play_buttons(backend).await; - match backend.act_press(APP, &row).await { - Ok(m) => steps.push(format!("open song: {m}")), + // #4: record what KIND of element we pressed — a song row that plays, or an + // artist/album element that only navigates. + let pressed_song = row.is_song; + let kind = if pressed_song { "song" } else { "artist/album" }; + match backend.act_press(APP, &row.label).await { + Ok(m) => steps.push(format!("open {kind} '{}': {m}", row.label)), Err(e) => { - steps.push(format!("open song FAILED: {e}")); - return fail("could not open the song", steps); + steps.push(format!("open {kind} FAILED: {e}")); + return fail("could not open the result", steps); } } @@ -349,25 +458,114 @@ pub async fn run(goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome { } } - match verified { - Some(false) => { - steps.push("verify: player state never reached 'playing'".to_string()); - fail("opened the song but playback didn't start", steps) - } - Some(true) => { - steps.push("verify: playing ✓".to_string()); - progress(format!("Playing {query}"), OverlayAttentionTone::Success); + if matches!(verified, Some(false)) { + steps.push("verify: player state never reached 'playing'".to_string()); + return fail("opened the song but playback didn't start", steps); + } + + // 6. Confirm we played the RIGHT track. The AX row label carries only the + // title, so "Numb" can land on the wrong artist (this is the exact bug we + // hit live). Ask Music for the now-playing name + artist and check it + // against the requested artist — so we never falsely claim success on a + // wrong-artist match, and the summary names what's actually playing. + let now = backend.now_playing().await; + let candidates = candidate_labels(&last_els); + match (&want_artist, &now) { + // #1/#3: played, but the wrong artist — name what's actually playing, + // list the alternatives, and tell the agent NOT to repeat the search + // (success=true so this guidance reaches the user instead of being + // discarded by a fall-through to the model loop). + (Some(want), Some((name, got))) if !artist_matches(want, got) => { + steps.push(format!( + "verify: now playing '{name}' by '{got}' — wanted '{want}' (mismatch)" + )); + progress( + format!("Playing {name} — not {want}"), + OverlayAttentionTone::Neutral, + ); AutomateOutcome { success: true, - summary: format!("Playing '{query}' in Music."), + summary: format!( + "Now playing '{name}' by '{got}', not '{query}' by '{want}'. {}Re-running this search won't surface a different result — try the user's Library, or ask them to confirm the artist.", + candidates_phrase(&candidates), + ), steps, } } - None => AutomateOutcome { - success: true, - summary: format!("Started '{query}' in Music (playback unverified)."), - steps, - }, + // #3: verified the actual track (right artist, or none requested). + (_, Some((name, got))) => { + steps.push(format!("verify: now playing '{name}' by '{got}' ✓")); + progress(format!("Playing {name}"), OverlayAttentionTone::Success); + AutomateOutcome { + success: true, + summary: format!("Playing '{name}' by '{got}' in Music."), + steps, + } + } + // #3/#4: can't read the track. Never claim "Playing X" — and if we only + // pressed an artist/album element (which just navigates), say so. + (_, None) if !pressed_song => { + steps.push("verify: pressed an artist/album element (navigation only)".to_string()); + AutomateOutcome { + success: true, + summary: format!( + "Opened an artist/album page for '{query}' but no specific track started — pressing that only navigates. {}Open a specific song to play it.", + candidates_phrase(&candidates), + ), + steps, + } + } + // Song row pressed, but the backend can't confirm the track (non-macOS). + (_, None) => { + let unverified = matches!(verified, None); + steps.push(if unverified { + "verify: playback unverified".to_string() + } else { + "verify: playing ✓ (track name unknown)".to_string() + }); + if !unverified { + progress(format!("Playing {query}"), OverlayAttentionTone::Success); + } + AutomateOutcome { + success: true, + summary: if unverified { + format!("Started '{query}' in Music (playback unverified).") + } else { + format!("Playing '{query}' in Music.") + }, + steps, + } + } + } +} + +/// "Results seen: a, b, c. " — or empty when nothing was captured. Lets failure +/// responses show the agent/user the actual candidates to choose from. +fn candidates_phrase(cands: &[String]) -> String { + if cands.is_empty() { + String::new() + } else { + format!("Results seen: {}. ", cands.join(", ")) + } +} + +/// Build the "no match" response: list what was found (if anything) and steer +/// the agent away from blindly repeating the identical search. +fn no_match_message(query: &str, want_artist: &Option, els: &[AXElement]) -> String { + let cands = candidate_labels(els); + let by = want_artist + .as_deref() + .map(|a| format!(" by '{a}'")) + .unwrap_or_default(); + if cands.is_empty() { + format!( + "No song results found for '{query}'{by} — the search may not have loaded, or the track isn't in the catalog/library. Don't repeat this exact search; try the user's Library or ask them to confirm the title/artist." + ) + } else { + format!( + "Couldn't find '{query}'{by}. Results seen: {}. Re-running this search won't help — pick the closest match, try the Library, or ask the user.", + cands.join(", ") + ) } } @@ -423,17 +621,73 @@ mod unit { } #[test] - fn pick_row_prefers_exact_then_token() { - use super::super::super::ax_interact::AXElement; + fn extract_artist_pulls_by_clause() { + assert_eq!( + extract_artist("play Numb by Linkin Park").as_deref(), + Some("Linkin Park") + ); + // Trailing clause is cut. + assert_eq!( + extract_artist("play Highway to Hell by AC/DC in Apple Music").as_deref(), + Some("AC/DC") + ); + assert_eq!( + extract_artist("search for \"Numb\" by Linkin Park and play it").as_deref(), + Some("Linkin Park") + ); + // No "by" clause → None. + assert_eq!(extract_artist("play Numb"), None); + } + + #[test] + fn artist_matches_is_loose() { + assert!(artist_matches("Linkin Park", "Linkin Park")); + assert!(artist_matches("Linkin Park", "Linkin Park feat. Jay-Z")); + assert!(artist_matches("ac/dc", "AC/DC")); + assert!(!artist_matches("Linkin Park", "Tom Odell")); + } + + #[test] + fn pick_row_prefers_song_row_over_artist() { + // Token match (query has extra "AC/DC" the row label lacks). let els = vec![ AXElement::new("AXCell", "Highway to Hell"), AXElement::new("AXButton", "Play"), ]; - // Token match (query has extra "AC/DC" the row label lacks). - assert_eq!( - pick_row(&els, "Highway to Hell AC/DC").as_deref(), - Some("Highway to Hell") - ); + let p = pick_row(&els, "Highway to Hell AC/DC").unwrap(); + assert_eq!(p.label, "Highway to Hell"); + assert!(p.is_song); + + // An artist AXButton and a song AXCell both token-match "Linkin Park + // Numb" — the SONG row must win (the live bug pressed the artist). + let els2 = vec![ + AXElement::new("AXButton", "LINKIN PARK"), + AXElement::new("AXCell", "Numb"), + ]; + let p2 = pick_row(&els2, "Linkin Park Numb").unwrap(); + assert_eq!(p2.label, "Numb"); + assert!(p2.is_song); + + // Only an artist element is present → chosen, but flagged non-song. + let els3 = vec![AXElement::new("AXButton", "LINKIN PARK")]; + let p3 = pick_row(&els3, "Linkin Park Numb").unwrap(); + assert_eq!(p3.label, "LINKIN PARK"); + assert!(!p3.is_song); + } + + #[test] + fn no_match_message_lists_candidates_and_warns() { + let els = vec![ + AXElement::new("AXCell", "Numb - Marshmello & Khalid"), + AXElement::new("AXCell", "Numb - Tom Odell"), + AXElement::new("AXButton", "Play"), // not a song row → excluded + ]; + let m = no_match_message("Numb Linkin Park", &Some("Linkin Park".into()), &els); + assert!(m.contains("Marshmello") && m.contains("Tom Odell"), "{m}"); + assert!(m.to_lowercase().contains("won't help"), "{m}"); + // Empty results → still actionable, no fake candidate list. + let m2 = no_match_message("Numb", &None, &[]); + assert!(m2.to_lowercase().contains("don't repeat"), "{m2}"); } } diff --git a/src/openhuman/accessibility/automate.rs b/src/openhuman/accessibility/automate.rs index 3ec3dbf10..b7a16a938 100644 --- a/src/openhuman/accessibility/automate.rs +++ b/src/openhuman/accessibility/automate.rs @@ -71,6 +71,9 @@ pub struct Action { /// Natural-language target for `vision_click` (e.g. "the green Call button"). #[serde(default)] pub description: String, + /// Key chord / single key for `hotkey` (e.g. `["Cmd","L"]`, `["/"]`). + #[serde(default)] + pub keys: Vec, /// Final message for `done` / `fail`. #[serde(default)] pub summary: String, @@ -112,6 +115,25 @@ pub trait AutomateBackend: Send + Sync { /// Open a URL / URI-scheme (e.g. `music://…search?term=…`) via the OS opener. /// Used by deterministic app fast-paths; the general loop does not call it. async fn open_url(&self, url: &str) -> Result; + /// Open a URL in a **specific** app (e.g. a chosen browser) so navigation + /// lands in the app the user named — `open_url` uses the *default* handler, + /// which would send a `https://` link to whatever the default browser is. + /// Default delegates to [`open_url`](Self::open_url) so non-browser backends + /// stay correct. Used by the browser fast-path. + async fn open_url_in_app(&self, _app: &str, url: &str) -> Result { + self.open_url(url).await + } + /// Send a keyboard chord (`["Cmd","L"]`) or a single key (`["/"]`) to the + /// frontmost app. Lets fast-paths and the loop drive app shortcuts (focus + /// the address bar, YouTube `/` search, `k`/space play-pause) instead of + /// hunting AX labels. Default errors so input-less backends can't actuate. + async fn key(&self, _keys: &[String]) -> Result { + Err("keyboard unsupported by this backend".to_string()) + } + /// Type literal text into the frontmost app. Default errors (see [`key`]). + async fn type_text(&self, _text: &str) -> Result { + Err("typing unsupported by this backend".to_string()) + } /// Best-effort: is media currently playing? `None` when the backend can't /// tell (non-macOS, or not applicable). Media fast-paths use this to confirm /// an action *actually started playback* rather than just succeeding at the @@ -119,6 +141,14 @@ pub trait AutomateBackend: Send + Sync { async fn verify_playing(&self) -> Option { None } + /// The currently-playing track as `(name, artist)`, if the backend can read + /// it. Used by the Music fast-path to confirm it played the *right* track + /// (the AX row label carries only the title, so "Numb" can resolve to the + /// wrong artist — see tracker §1.x). `None` when unknown (non-macOS, nothing + /// playing, or not applicable). + async fn now_playing(&self) -> Option<(String, String)> { + None + } /// Capture the target app's window + the geometry needed to map a click /// from image pixels to screen points. Used by the `vision_click` fallback /// for apps with no usable accessibility tree (Electron/Chromium). Default @@ -183,17 +213,21 @@ fn system_prompt() -> String { \n\ Respond with EXACTLY ONE JSON object and nothing else:\n\ {\"thought\":\"...\",\"action\":\"\",\"app\":\"\",\ - \"filter\":\"...\",\"label\":\"...\",\"value\":\"...\",\"summary\":\"...\"}\n\ + \"filter\":\"...\",\"label\":\"...\",\"value\":\"...\",\"keys\":[],\"summary\":\"...\"}\n\ \n\ Verbs:\n\ • launch — open the app (use first if it isn't showing any elements)\n\ • list — re-read elements; set `filter` to a substring to narrow them\n\ • press — activate the element whose label matches `label`\n\ • set_value — type `value` into the field matching `label` (omit label = first field)\n\ + • hotkey — send an app keyboard shortcut; put the chord in `keys` (modifiers \ + first, e.g. [\"Cmd\",\"L\"] to focus a browser address bar, [\"Cmd\",\"T\"] new tab) \ + or a single key (e.g. [\"/\"] to focus YouTube search, [\"k\"] play/pause, [\"f\"] \ + fullscreen). Prefer a known shortcut over hunting labels or clicking.\n\ • vision_click — click an element by sight; put a short `description` of the \ target (e.g. 'the green Call button'). Use this when the element list is \ - EMPTY or missing your target — common for Electron apps (Slack, Discord, \ - VS Code) that expose no accessibility tree.\n\ + EMPTY or missing your target — common for Electron/Chromium apps (browsers, \ + Slack, Discord, VS Code) that expose no accessibility tree.\n\ • done — goal achieved; put a short result in `summary`\n\ • fail — goal cannot be achieved; explain in `summary`\n\ \n\ @@ -203,8 +237,11 @@ fn system_prompt() -> String { (e.g. open a song, THEN press its 'Play'). After such a press, `list` again \ to see the new screen.\n\ - Prefer an exact label match. Keep `filter` specific so the snapshot stays small.\n\ - - If the app shows NO elements, prefer `vision_click` with a clear \ - `description` over guessing labels.\n\ + - For browsers and web apps, prefer `hotkey` for navigation and media control \ + (address bar, search focus, play/pause/next) — it's faster and more reliable \ + than clicking, and works even when the accessibility tree is empty.\n\ + - If the app shows NO elements, prefer `hotkey` (if a known shortcut applies) \ + or `vision_click` with a clear `description` over guessing labels.\n\ - Output JSON only — no prose, no code fences." .to_string() } @@ -300,6 +337,10 @@ pub async fn run( // instead of burning the whole step budget. let mut last_sig = String::new(); let mut repeat_count = 0u32; + // Most recent rendered snapshot — surfaced in terminal failure responses so + // the agent sees what was actually on screen (instead of a bare "budget + // exhausted"), and can pick a real label next time. + let mut last_snapshot = String::new(); for step in 0..opts.step_budget { // ── perceive ── @@ -310,6 +351,7 @@ pub async fn run( format!("(perceive error: {e})") } }; + last_snapshot = snapshot.clone(); // ── decide ── let user = format!( @@ -361,8 +403,12 @@ pub async fn run( // ── no-progress guard ── if !matches!(action.action.as_str(), "done" | "fail") { let sig = format!( - "{}|{}|{}|{}", - action.action, action.label, action.filter, action.description + "{}|{}|{}|{}|{}", + action.action, + action.label, + action.filter, + action.description, + action.keys.join("+") ); if sig == last_sig { repeat_count += 1; @@ -378,7 +424,11 @@ pub async fn run( action.action )); return AutomateOutcome::fail( - "Got stuck repeating the same action with no progress.", + format!( + "Stuck repeating '{}' with no progress — that action isn't advancing the goal.{} Switch tactics: pick a specific label from the screen, take a screenshot + vision_click, or use a keyboard shortcut.", + action.action, + screen_hint(&last_snapshot), + ), steps, ); } @@ -455,6 +505,19 @@ pub async fn run( } backend.settle(target_app).await; } + "hotkey" => { + if action.keys.is_empty() { + steps.push("hotkey skipped: no keys".to_string()); + continue; + } + let combo = action.keys.join("+"); + progress(format!("Pressing {combo}…"), OverlayAttentionTone::Accent); + match backend.key(&action.keys).await { + Ok(msg) => steps.push(format!("hotkey: {msg}")), + Err(e) => steps.push(format!("hotkey FAILED: {e}")), + } + backend.settle(target_app).await; + } "vision_click" => { let description = action.description.trim(); if description.is_empty() { @@ -527,13 +590,55 @@ pub async fn run( log::info!("{LOG_PREFIX} step budget ({}) exhausted", opts.step_budget); AutomateOutcome::fail( format!( - "Step budget ({}) exhausted before the goal was confirmed complete.", - opts.step_budget + "Step budget ({}) exhausted before the goal was confirmed complete.{} Try a different approach (a screenshot/vision_click, a known keyboard shortcut, or a more specific filter) — repeating the same steps won't help.", + opts.step_budget, + screen_hint(&last_snapshot), ), steps, ) } +/// A compact " On screen: [role] a, [role] b, …" hint built from the last +/// rendered snapshot, for failure responses. Empty when there's nothing useful. +fn screen_hint(snapshot: &str) -> String { + let labels: Vec<&str> = snapshot + .lines() + .map(str::trim) + .filter(|l| l.starts_with('[') || l.starts_with("• [")) + .take(10) + .collect(); + if labels.is_empty() { + String::new() + } else { + format!(" On screen: {}.", labels.join("; ")) + } +} + +/// Map a browser **display name** (as resolved by the browser fast-path — +/// `"Google Chrome"`, `"Brave Browser"`, …) to the token the Windows shell +/// `start` verb resolves via the `App Paths` registry. `None` for browsers that +/// don't exist on Windows (Safari/Arc) or any unrecognized name, so the caller +/// falls back to the default URL handler. Matched case-insensitively by +/// substring so aliases ("Chrome", "Microsoft Edge") all resolve. +#[cfg(target_os = "windows")] +pub(crate) fn windows_browser_launch_token(app: &str) -> Option<&'static str> { + let a = app.to_lowercase(); + // Order matters: check the more specific names first ("microsoft edge" + // contains neither "chrome" nor "firefox", but keep edge before a bare + // "chrome" check anyway for clarity). + if a.contains("brave") { + Some("brave") + } else if a.contains("edge") { + Some("msedge") + } else if a.contains("firefox") { + Some("firefox") + } else if a.contains("chrome") || a.contains("chromium") { + Some("chrome") + } else { + None + } +} + /// Production backend: real AX primitives + a fast LLM for decisions. pub struct RealBackend { config: crate::openhuman::config::Config, @@ -664,6 +769,84 @@ impl AutomateBackend for RealBackend { } } + async fn open_url_in_app(&self, app: &str, url: &str) -> Result { + // macOS: `open -a "" ""` both launches/foregrounds the named + // app AND opens the URL in it — exactly the deterministic browser nav we + // want (no address-bar typing, no AX). + #[cfg(target_os = "macos")] + { + match tokio::process::Command::new("open") + .arg("-a") + .arg(app) + .arg(url) + .output() + .await + { + Ok(o) if o.status.success() => Ok(format!("Opened {url} in {app}")), + Ok(o) => Err(format!( + "open -a {app} exited {}: {}", + o.status, + String::from_utf8_lossy(&o.stderr).trim() + )), + Err(e) => Err(format!("failed to launch opener: {e}")), + } + } + // Windows: the shell `start` verb resolves a browser by its registered + // App Paths token (`chrome`, `msedge`, `firefox`, `brave`, …) and opens + // the URL in it. When the browser is already running this lands in a NEW + // TAB of the existing window — so the deterministic fast-path does NOT + // pile up windows (the live bug: each re-delegation `launch_app`-ed Chrome + // again → ~10 windows). Falls back to the default handler when the named + // browser has no known token (e.g. Safari/Arc, which aren't on Windows). + #[cfg(target_os = "windows")] + { + let Some(token) = windows_browser_launch_token(app) else { + log::info!( + "[automate] open_url_in_app: no Windows token for {app:?}; using default handler" + ); + return self.open_url(url).await; + }; + // `cmd /C start "" ""` — the empty "" is `start`'s title + // arg (required so a quoted token isn't mistaken for the title). The + // URL is app-controlled (built by the fast-path), never user free-text. + match tokio::process::Command::new("cmd") + .args(["/C", "start", "", token, url]) + .output() + .await + { + Ok(o) if o.status.success() => Ok(format!("Opened {url} in {app}")), + Ok(o) => { + // `start` failed (token not registered?) — best-effort fall back. + log::warn!( + "[automate] open_url_in_app: start {token} exited {}: {}; falling back", + o.status, + String::from_utf8_lossy(&o.stderr).trim() + ); + self.open_url(url).await + } + Err(e) => Err(format!("failed to launch opener: {e}")), + } + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + let _ = app; + self.open_url(url).await + } + } + + async fn key(&self, keys: &[String]) -> Result { + use crate::openhuman::tools::implementations::computer::keyboard; + match keys.len() { + 0 => Err("no keys provided".to_string()), + 1 => keyboard::run_key(&keys[0]).await, + _ => keyboard::run_hotkey(keys).await, + } + } + + async fn type_text(&self, text: &str) -> Result { + crate::openhuman::tools::implementations::computer::keyboard::run_type_text(text).await + } + async fn verify_playing(&self) -> Option { // macOS: ask Apple Music for ground-truth player state. Other OSes can't // verify this way → None (fast-path treats None as best-effort). @@ -683,6 +866,34 @@ impl AutomateBackend for RealBackend { } } + async fn now_playing(&self) -> Option<(String, String)> { + // macOS: ask Apple Music for the current track's name + artist. We join + // them with a tab (unlikely in titles) so we can split unambiguously. + #[cfg(target_os = "macos")] + { + let script = "tell application \"Music\" to try + set t to current track + return (name of t) & \"\\t\" & (artist of t) + end try"; + let out = tokio::process::Command::new("osascript") + .args(["-e", script]) + .output() + .await + .ok()?; + let line = String::from_utf8_lossy(&out.stdout); + let line = line.trim(); + if line.is_empty() { + return None; + } + let (name, artist) = line.split_once('\t')?; + Some((name.trim().to_string(), artist.trim().to_string())) + } + #[cfg(not(target_os = "macos"))] + { + None + } + } + async fn settle(&self, app: &str) { // M2: poll the element count until the UI stops changing (≤2s), instead // of a blind fixed wait. Removes the timing-race class (tracker §1.11/ @@ -699,3 +910,23 @@ impl AutomateBackend for RealBackend { #[cfg(test)] #[path = "automate_tests.rs"] mod tests; + +#[cfg(all(test, target_os = "windows"))] +mod windows_tests { + use super::windows_browser_launch_token as tok; + + #[test] + fn browser_display_names_map_to_start_tokens() { + assert_eq!(tok("Google Chrome"), Some("chrome")); + assert_eq!(tok("Brave Browser"), Some("brave")); + assert_eq!(tok("Microsoft Edge"), Some("msedge")); + assert_eq!(tok("Firefox"), Some("firefox")); + // Aliases / case-insensitive. + assert_eq!(tok("chrome"), Some("chrome")); + assert_eq!(tok("EDGE"), Some("msedge")); + // Not on Windows / unknown → None → caller uses the default handler. + assert_eq!(tok("Safari"), None); + assert_eq!(tok("Arc"), None); + assert_eq!(tok("Some Random App"), None); + } +} diff --git a/src/openhuman/accessibility/automate_tests.rs b/src/openhuman/accessibility/automate_tests.rs index 1cf15a831..e628000b8 100644 --- a/src/openhuman/accessibility/automate_tests.rs +++ b/src/openhuman/accessibility/automate_tests.rs @@ -128,6 +128,13 @@ impl AutomateBackend for ScriptedBackend { self.acts.lock().unwrap().push(format!("open_url:{url}")); Ok(format!("Opened {url}")) } + async fn key(&self, keys: &[String]) -> Result { + self.acts + .lock() + .unwrap() + .push(format!("key:{}", keys.join("+"))); + Ok(format!("Executed {}", keys.join("+"))) + } async fn settle(&self, _app: &str) {} async fn wait(&self, _ms: u64) {} } @@ -190,6 +197,23 @@ async fn set_value_routes_app_override() { ); } +#[tokio::test] +async fn hotkey_verb_sends_chord_to_backend() { + // The model can drive an app shortcut via the general loop instead of + // hunting AX labels. Use a neutral app/goal so no fast-path intercepts and + // the loop's `hotkey` verb is what runs. + let backend = ScriptedBackend::new(&[ + r#"{"action":"hotkey","keys":["Cmd","L"]}"#, + r#"{"action":"done","summary":"sent shortcut"}"#, + ]); + let out = run("Notes", "do a thing", &backend, opts(5)).await; + assert!(out.success, "expected success, got {out:?}"); + assert_eq!( + backend.acts(), + vec!["launch:Notes", "key:Cmd+L"] // foreground-first, then the chord + ); +} + #[tokio::test] async fn budget_exhaustion_fails() { // Script always lists → never done → budget guard ends the run. @@ -197,6 +221,13 @@ async fn budget_exhaustion_fails() { let out = run("Music", "never finishes", &backend, opts(3)).await; assert!(!out.success); assert!(out.summary.contains("budget"), "got: {}", out.summary); + // #2: surfaces what was on screen + a "try a different approach" steer. + assert!(out.summary.contains("On screen:"), "got: {}", out.summary); + assert!( + out.summary.to_lowercase().contains("different approach"), + "got: {}", + out.summary + ); } #[tokio::test] @@ -211,7 +242,14 @@ async fn no_progress_guard_aborts_repeated_action() { let out = run("Photos", "do something", &backend, opts(10)).await; assert!(!out.success); assert!( - out.summary.contains("stuck repeating"), + out.summary.to_lowercase().contains("stuck repeating"), + "got: {}", + out.summary + ); + // #1/#2: actionable — names the screen and tells it to switch tactics. + assert!(out.summary.contains("On screen:"), "got: {}", out.summary); + assert!( + out.summary.to_lowercase().contains("switch tactics"), "got: {}", out.summary ); diff --git a/src/openhuman/agent_registry/agents/desktop_control_agent/prompt.md b/src/openhuman/agent_registry/agents/desktop_control_agent/prompt.md index 6f4d3e123..033aa3b3f 100644 --- a/src/openhuman/agent_registry/agents/desktop_control_agent/prompt.md +++ b/src/openhuman/agent_registry/agents/desktop_control_agent/prompt.md @@ -5,11 +5,13 @@ You are the desktop-control specialist. Launch apps and operate native desktop U ## Rules - Use `launch_app` for explicit app-launch requests. -- Use `ax_interact` for semantic accessibility interactions. +- **Foreground each app at most ONCE per task.** If the app is already open (a prior step launched it, or the user says it's open), do NOT call `launch_app` again — repeated launches pile up duplicate windows. Re-launch only after a tool result explicitly reports the app isn't running. +- **Web browsers (Chrome, Edge, Brave, Firefox, Arc): use `automate`, not `ax_interact`.** To open a site, search, or play a video, call `automate` with the browser as the app and a plain-English goal — e.g. `automate{app:"Google Chrome", goal:"go to youtube.com and play a lofi music video"}`. It navigates deterministically by URL in one step. Do NOT type a URL into the address bar via `ax_interact`/`set_value`: Chromium exposes no page content to the accessibility tree (only browser chrome), and an address/search field set this way usually cannot be submitted — that path dead-ends and loops. +- Use `ax_interact` for semantic accessibility interactions in **native** (non-Chromium) apps. - Always call `ax_interact` with `action:"list"` before `press` or `set_value`. -- Use `automate` for multi-step app workflows, such as playing a song in Music or sending a message in Slack. -- Before any keyboard or mouse action, foreground the target app with `launch_app`. -- Prefer `automate` or `ax_interact` first. If the accessibility tree is empty, stuck, or only shows menu-bar items, fall back to keyboard-driven control for Electron/Chromium apps. +- Use `automate` for multi-step app workflows: playing a song in Music, sending a message in Slack, or any browser navigation/search/playback (above). +- Before any keyboard or mouse action, foreground the target app with `launch_app` (subject to the once-per-task rule above). +- Prefer `automate` or `ax_interact` first. If the accessibility tree is empty, stuck, or only shows menu-bar items, fall back to keyboard-driven control for Electron/Chromium apps. **Do not retry the same failing approach repeatedly — if two attempts at a step fail, report it and stop rather than re-launching and re-trying in a loop.** - Use `screenshot` plus `mouse` only when semantic or keyboard control cannot target the needed element. - Never invent element labels. Act only on elements returned by `list` or clearly named by the user. - Respect sensitive-app constraints and tool denials. Do not work around password managers, Keychain, System Settings, terminals, or other denied surfaces. diff --git a/src/openhuman/tools/impl/computer/automate.rs b/src/openhuman/tools/impl/computer/automate.rs index 638ab6567..66ae955ba 100644 --- a/src/openhuman/tools/impl/computer/automate.rs +++ b/src/openhuman/tools/impl/computer/automate.rs @@ -11,7 +11,7 @@ //! ApprovalGate, and it refuses the sensitive-app denylist (password managers, //! Keychain, System Settings, terminals) even on auto-approved turns. -use super::ax_interact::is_sensitive_app; +use super::ax_interact::{app_control_enabled, is_sensitive_app}; use crate::openhuman::accessibility::automate::{self, AutomateOptions, RealBackend}; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCallOptions, ToolResult}; use async_trait::async_trait; @@ -121,12 +121,12 @@ impl Tool for AutomateTool { ))); } - if !self.allow_mutations { + if !app_control_enabled(self.allow_mutations) { log::warn!("[automate] refused: mutations disabled"); return Ok(ToolResult::error( - "App control isn't enabled yet. Turn on App Automation in \ - Settings → Agent Access (it grants permission to control apps), \ - then ask again. (Sets computer_control.ax_interact_mutations = true.)", + "App control isn't enabled yet. Grant Full access (or turn on App \ + Automation) in Settings → Agent Access — it grants permission to \ + control apps — then ask again.", )); } @@ -194,13 +194,33 @@ mod tests { #[tokio::test] async fn refuses_when_mutations_disabled() { + // Pin a non-Full live policy so the Full-access bypass can't open the + // gate (other tests in this binary install a Full global policy). + let _env = crate::openhuman::config::TEST_ENV_LOCK + .lock() + .unwrap_or_else(|e| e.into_inner()); + { + use crate::openhuman::security::{live_policy, AutonomyLevel, SecurityPolicy}; + use std::sync::Arc; + let ws = std::env::temp_dir().join("openhuman_automate_gate_test_ws"); + live_policy::install( + Arc::new(SecurityPolicy { + autonomy: AutonomyLevel::Supervised, + workspace_dir: ws.clone(), + ..SecurityPolicy::default() + }), + ws.clone(), + ws, + ); + } let t = AutomateTool::new(false); let r = t .execute(json!({"app": "Music", "goal": "play a song"})) .await .unwrap(); assert!(r.is_error); - assert!(r.output().contains("ax_interact_mutations")); + // Gate closed → the refusal directs the user to Settings → Agent Access. + assert!(r.output().contains("Agent Access")); } #[tokio::test] diff --git a/src/openhuman/tools/impl/computer/ax_interact.rs b/src/openhuman/tools/impl/computer/ax_interact.rs index 8cceb8a3b..5667fb800 100644 --- a/src/openhuman/tools/impl/computer/ax_interact.rs +++ b/src/openhuman/tools/impl/computer/ax_interact.rs @@ -55,6 +55,25 @@ pub(crate) fn is_sensitive_app(app_name: &str) -> bool { SENSITIVE_APPS.iter().any(|s| lower.contains(s)) } +/// Whether the agent may actuate app UI (`press` / `set_value` / `automate`). +/// +/// Enabled when **either** the explicit opt-in +/// (`computer_control.ax_interact_mutations`, captured as `explicit_opt_in` at +/// tool-build time) is set, **or** the agent has been granted **Full** OS access +/// (autonomy level `Full`) in Settings → Agent Access. The autonomy level is +/// read from the **live** policy (`security::live_policy::current`), so granting +/// Full access mid-session takes effect immediately — the previous behaviour +/// gated only on the static config flag, so users who flipped Settings → +/// Agent Access to "Full" still saw "App control isn't enabled yet" because the +/// flag was untouched. `pub(crate)` so `automate` shares the identical gate. +pub(crate) fn app_control_enabled(explicit_opt_in: bool) -> bool { + use crate::openhuman::security::{live_policy, AutonomyLevel}; + explicit_opt_in + || live_policy::current() + .map(|p| matches!(p.autonomy, AutonomyLevel::Full)) + .unwrap_or(false) +} + pub struct AxInteractTool { /// When false, the mutating actions (`press` / `set_value`) are refused /// with guidance to enable `computer_control.ax_interact_mutations`. The @@ -229,14 +248,15 @@ impl Tool for AxInteractTool { ))); } - // Mutating actions are opt-in. Read-only `list` is always allowed. - if mutating && !self.allow_mutations { + // Mutating actions are opt-in (or implied by Full OS access). Read-only + // `list` is always allowed. + if mutating && !app_control_enabled(self.allow_mutations) { log::warn!("[ax_interact] refused: mutations disabled (action={action})"); return Ok(ToolResult::error( "App control isn't enabled yet, so I can't press buttons or type into \ - this app. Turn on App UI Control / App Automation in Settings → Agent \ - Access, then ask again. (Reading the UI still works without it; sets \ - computer_control.ax_interact_mutations = true.)", + this app. Grant Full access (or turn on App UI Control / App \ + Automation) in Settings → Agent Access, then ask again. (Reading the \ + UI still works without it.)", )); } @@ -351,6 +371,43 @@ impl Tool for AxInteractTool { mod tests { use super::*; + /// Force the process-global live policy to a given autonomy level so the + /// `app_control_enabled` Full-access bypass is deterministic. Other tests in + /// this binary (e.g. `security::live_policy`) install a Full global; callers + /// must hold `TEST_ENV_LOCK` while relying on the value they set here. + fn install_live_autonomy(level: crate::openhuman::security::AutonomyLevel) { + use crate::openhuman::security::{live_policy, SecurityPolicy}; + use std::sync::Arc; + let ws = std::env::temp_dir().join("openhuman_ax_interact_gate_test_ws"); + live_policy::install( + Arc::new(SecurityPolicy { + autonomy: level, + workspace_dir: ws.clone(), + ..SecurityPolicy::default() + }), + ws.clone(), + ws, + ); + } + + #[test] + fn app_control_enabled_combines_optin_and_full_access() { + use crate::openhuman::security::AutonomyLevel; + let _env = crate::openhuman::config::TEST_ENV_LOCK + .lock() + .unwrap_or_else(|e| e.into_inner()); + // Explicit opt-in always enables, regardless of the live policy. + assert!(app_control_enabled(true)); + // Supervised + no opt-in → gate closed. + install_live_autonomy(AutonomyLevel::Supervised); + assert!(!app_control_enabled(false)); + // Full OS access opens the gate even without the explicit flag — this is + // the fix: granting "Full" in Settings → Agent Access now enables app + // control without separately flipping `ax_interact_mutations`. + install_live_autonomy(AutonomyLevel::Full); + assert!(app_control_enabled(false)); + } + #[test] fn name_and_permission() { let tool = AxInteractTool::new(true); @@ -416,6 +473,12 @@ mod tests { #[tokio::test] async fn refuses_mutations_when_disabled() { + // Pin a non-Full live policy for the test so the Full-access bypass can't + // open the gate (other tests in this binary install a Full global). + let _env = crate::openhuman::config::TEST_ENV_LOCK + .lock() + .unwrap_or_else(|e| e.into_inner()); + install_live_autonomy(crate::openhuman::security::AutonomyLevel::Supervised); // mutations off → press/set_value blocked, but list still allowed past this guard. let tool = AxInteractTool::new(false); let press = tool @@ -423,7 +486,8 @@ mod tests { .await .unwrap(); assert!(press.is_error); - assert!(press.output().contains("ax_interact_mutations")); + // Gate closed → the refusal points the user at Settings → Agent Access. + assert!(press.output().contains("Agent Access")); } #[tokio::test] diff --git a/src/openhuman/tools/impl/computer/keyboard.rs b/src/openhuman/tools/impl/computer/keyboard.rs index b2a9a581e..0bf593885 100644 --- a/src/openhuman/tools/impl/computer/keyboard.rs +++ b/src/openhuman/tools/impl/computer/keyboard.rs @@ -97,6 +97,135 @@ fn is_modifier(key: &Key) -> bool { matches!(key, Key::Control | Key::Shift | Key::Alt | Key::Meta) } +// ── Shared execution helpers ──────────────────────────────────────────────── +// These hold the validation + main-thread enigo dispatch once, so both the +// `keyboard` tool and the `automate` backend (`RealBackend::{key,type_text}`) +// drive synthetic input through a single code path. They return the raw +// `Result`; the tool wraps it with `into_result`. + +/// Type a literal string via the OS. Runs on the app main thread (macOS TSM +/// requirement, Change 1.15). Validates length like the `type` tool action. +pub(crate) async fn run_type_text(text: &str) -> Result { + if text.is_empty() { + return Err("'text' cannot be empty".to_string()); + } + if text.len() > MAX_TYPE_LENGTH { + return Err(format!( + "Text too long ({} chars). Maximum is {MAX_TYPE_LENGTH}.", + text.len() + )); + } + let text = text.to_string(); + let len = text.len(); + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + enigo + .text(&text) + .map_err(|e| format!("text typing failed: {e}"))?; + Ok(format!("Typed {len} characters")) + }) + .await +} + +/// Tap a single key by name (e.g. "Enter", "/", "k"). Used for app shortcuts +/// like YouTube's `/` (focus search) or `k` (play/pause). +pub(crate) async fn run_key(key_name: &str) -> Result { + let key = parse_key(key_name).ok_or_else(|| { + format!( + "Unknown key '{key_name}'. Use names like Enter, Tab, Escape, F1-F12, a-z, 0-9, Space, etc." + ) + })?; + let key_name = key_name.to_string(); + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + enigo + .key(key, Direction::Click) + .map_err(|e| format!("key press failed: {e}"))?; + Ok(format!("Pressed key '{key_name}'")) + }) + .await +} + +/// Execute a hotkey chord — modifiers first, then exactly one non-modifier +/// final key (e.g. `["Cmd","L"]`). Validates the modifier-first shape, then +/// presses in order and releases in reverse (even on error). +pub(crate) async fn run_hotkey(key_names: &[String]) -> Result { + if key_names.is_empty() { + return Err("'keys' array cannot be empty".to_string()); + } + if key_names.len() > 6 { + return Err("Too many keys in hotkey combination (max 6)".to_string()); + } + if key_names.len() < 2 { + return Err( + "Hotkey requires at least one modifier and one final key (e.g. ['Ctrl', 'C'])" + .to_string(), + ); + } + + let mut keys: Vec = Vec::with_capacity(key_names.len()); + for name in key_names { + let key = + parse_key(name).ok_or_else(|| format!("Unknown key '{name}' in hotkey combination"))?; + keys.push(key); + } + + // Validate modifier-first: all but the last must be modifiers; the last + // must be a non-modifier. + let (modifiers, final_key) = keys.split_at(keys.len() - 1); + for (i, key) in modifiers.iter().enumerate() { + if !is_modifier(key) { + return Err(format!( + "Key '{}' at position {i} must be a modifier (Ctrl/Shift/Alt/Cmd). Non-modifier keys must be last.", + key_names[i] + )); + } + } + if is_modifier(&final_key[0]) { + return Err(format!( + "Last key '{}' cannot be a modifier. Hotkey must end with a non-modifier key (e.g. 'C', 'Enter').", + key_names.last().unwrap() + )); + } + + let combo_desc = key_names.join("+"); + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + + // Press keys in order, tracking which were pressed so we can release + // them on error. + let mut pressed_keys: Vec = Vec::with_capacity(keys.len()); + let press_result: Result<(), String> = (|| { + for key in &keys { + enigo + .key(*key, Direction::Press) + .map_err(|e| format!("key press failed for {key:?}: {e}"))?; + pressed_keys.push(*key); + std::thread::sleep(HOTKEY_INTER_KEY_DELAY); + } + Ok(()) + })(); + + // Always release pressed keys in reverse, even on error. + for key in pressed_keys.iter().rev() { + if let Err(e) = enigo.key(*key, Direction::Release) { + tracing::warn!( + tool = "keyboard", + key = ?key, + error = %e, + "[computer] best-effort key release failed during cleanup" + ); + } + } + press_result?; + Ok(format!("Executed hotkey: {combo_desc}")) + }) + .await +} + #[async_trait] impl Tool for KeyboardTool { fn name(&self) -> &str { @@ -183,57 +312,16 @@ impl Tool for KeyboardTool { let text = args .get("text") .and_then(Value::as_str) - .ok_or_else(|| anyhow::anyhow!("Missing 'text' for type action"))? - .to_string(); - - if text.is_empty() { - return Ok(ToolResult::error("'text' cannot be empty")); - } - if text.len() > MAX_TYPE_LENGTH { - return Ok(ToolResult::error(format!( - "Text too long ({} chars). Maximum is {MAX_TYPE_LENGTH}.", - text.len() - ))); - } - - let len = text.len(); - into_result( - "type", - run_input_on_main(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| format!("Failed to create enigo instance: {e}"))?; - enigo - .text(&text) - .map_err(|e| format!("text typing failed: {e}"))?; - Ok(format!("Typed {len} characters")) - }) - .await, - ) + .ok_or_else(|| anyhow::anyhow!("Missing 'text' for type action"))?; + into_result("type", run_type_text(text).await) } "press" => { let key_name = args .get("key") .and_then(Value::as_str) - .ok_or_else(|| anyhow::anyhow!("Missing 'key' for press action"))? - .to_string(); - - let key = parse_key(&key_name).ok_or_else(|| { - anyhow::anyhow!("Unknown key '{key_name}'. Use names like Enter, Tab, Escape, F1-F12, a-z, 0-9, Space, etc.") - })?; - - into_result( - "press", - run_input_on_main(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| format!("Failed to create enigo instance: {e}"))?; - enigo - .key(key, Direction::Click) - .map_err(|e| format!("key press failed: {e}"))?; - Ok(format!("Pressed key '{key_name}'")) - }) - .await, - ) + .ok_or_else(|| anyhow::anyhow!("Missing 'key' for press action"))?; + into_result("press", run_key(key_name).await) } "hotkey" => { @@ -251,84 +339,7 @@ impl Tool for KeyboardTool { key_names.push(s.to_string()); } - if key_names.is_empty() { - return Ok(ToolResult::error("'keys' array cannot be empty")); - } - if key_names.len() > 6 { - return Ok(ToolResult::error( - "Too many keys in hotkey combination (max 6)", - )); - } - if key_names.len() < 2 { - return Ok(ToolResult::error( - "Hotkey requires at least one modifier and one final key (e.g. ['Ctrl', 'C'])", - )); - } - - // Parse all key names into Key values. - let mut keys: Vec = Vec::with_capacity(key_names.len()); - for name in &key_names { - let key = parse_key(name).ok_or_else(|| { - anyhow::anyhow!("Unknown key '{name}' in hotkey combination") - })?; - keys.push(key); - } - - // Validate modifier-first pattern: all keys except the last - // must be modifiers, and the last must be a non-modifier. - let (modifiers, final_key) = keys.split_at(keys.len() - 1); - for (i, key) in modifiers.iter().enumerate() { - if !is_modifier(key) { - return Ok(ToolResult::error(format!( - "Key '{}' at position {i} must be a modifier (Ctrl/Shift/Alt/Cmd). Non-modifier keys must be last.", - key_names[i] - ))); - } - } - if is_modifier(&final_key[0]) { - return Ok(ToolResult::error(format!( - "Last key '{}' cannot be a modifier. Hotkey must end with a non-modifier key (e.g. 'C', 'Enter').", - key_names.last().unwrap() - ))); - } - - let combo_desc = key_names.join("+"); - into_result( - "hotkey", - run_input_on_main(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| format!("Failed to create enigo instance: {e}"))?; - - // Press keys in order, tracking which were pressed so we - // can release them on error. - let mut pressed_keys: Vec = Vec::with_capacity(keys.len()); - let press_result: Result<(), String> = (|| { - for key in &keys { - enigo - .key(*key, Direction::Press) - .map_err(|e| format!("key press failed for {key:?}: {e}"))?; - pressed_keys.push(*key); - std::thread::sleep(HOTKEY_INTER_KEY_DELAY); - } - Ok(()) - })(); - - // Always release pressed keys in reverse, even on error. - for key in pressed_keys.iter().rev() { - if let Err(e) = enigo.key(*key, Direction::Release) { - tracing::warn!( - tool = "keyboard", - key = ?key, - error = %e, - "[computer] best-effort key release failed during cleanup" - ); - } - } - press_result?; - Ok(format!("Executed hotkey: {combo_desc}")) - }) - .await, - ) + into_result("hotkey", run_hotkey(&key_names).await) } other => Ok(ToolResult::error(format!( diff --git a/src/openhuman/tools/impl/computer/mod.rs b/src/openhuman/tools/impl/computer/mod.rs index 379c8833f..2bb5ec567 100644 --- a/src/openhuman/tools/impl/computer/mod.rs +++ b/src/openhuman/tools/impl/computer/mod.rs @@ -1,7 +1,9 @@ mod automate; mod ax_interact; mod human_path; -mod keyboard; +// `pub(crate)` so the automate backend can reuse the shared input helpers +// (`run_hotkey`/`run_key`/`run_type_text`) without going through the tool. +pub(crate) mod keyboard; mod main_thread; mod mouse; diff --git a/src/openhuman/workflows/ops_tests.rs b/src/openhuman/workflows/ops_tests.rs index fac6d41a1..494ebf038 100644 --- a/src/openhuman/workflows/ops_tests.rs +++ b/src/openhuman/workflows/ops_tests.rs @@ -1637,6 +1637,11 @@ fn uninstall_resolves_agents_skills_legacy_root() { assert!(!dir.exists(), "uninstall should remove the dir"); } +// Unix-only: exercises `std::os::unix::fs::symlink`. Windows symlink creation +// uses a different API and requires elevated privileges / Developer Mode, so +// this case is gated off there (the Windows lib test binary otherwise fails to +// compile on this line). +#[cfg(unix)] #[test] fn symlinked_manifest_file_is_rejected() { // `exists()` follows symlinks; a manifest pointed at an external file would