mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
feat(automate): browser + app keyboard-shortcut fast-paths, artist-aware Music, actionable responses (#3558)
This commit is contained in:
Mega Mind
@@ -18,6 +18,7 @@ import {
|
||||
type VoiceProviderView,
|
||||
type VoiceSettings,
|
||||
} from '../../../services/api/voiceSettingsApi';
|
||||
import { IS_DEV_LIKE } from '../../../utils/config';
|
||||
import {
|
||||
openhumanGetVoiceServerSettings,
|
||||
openhumanUpdateVoiceServerSettings,
|
||||
@@ -87,10 +88,10 @@ interface VoicePanelProps {
|
||||
embedded?: boolean;
|
||||
}
|
||||
|
||||
/** Temporarily hide the always-on listening toggle. Set back to `true` to
|
||||
* restore the control (the backend engine is unchanged). See
|
||||
* docs/voice-system-actions.md. */
|
||||
const SHOW_ALWAYS_ON_TOGGLE = false;
|
||||
/** Always-on listening toggle is hidden in production for now, but shown in
|
||||
* dev/debug builds so the feature can be exercised. Set unconditionally to
|
||||
* `true` to expose it everywhere. See docs/voice-system-actions.md. */
|
||||
const SHOW_ALWAYS_ON_TOGGLE = IS_DEV_LIKE;
|
||||
|
||||
const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => {
|
||||
const { t } = useT();
|
||||
@@ -494,7 +495,7 @@ const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => {
|
||||
|
||||
<div className={embedded ? 'space-y-4' : 'p-4 space-y-4'}>
|
||||
{/* ─── Always-on listening (Phase 2) ──────────────────────────── */}
|
||||
{/* Temporarily hidden — gated on SHOW_ALWAYS_ON_TOGGLE (set to false). */}
|
||||
{/* Gated on SHOW_ALWAYS_ON_TOGGLE — shown in dev/debug builds, hidden in prod. */}
|
||||
{SHOW_ALWAYS_ON_TOGGLE && settings && (
|
||||
<section className="space-y-3">
|
||||
<div className="bg-stone-50 dark:bg-neutral-800/60 rounded-lg border border-stone-200 dark:border-neutral-800 p-4">
|
||||
|
||||
@@ -511,8 +511,7 @@ describe('VoicePanel', () => {
|
||||
|
||||
// ─── Always-on listening toggle ↔ notch indicator ───────────────────────
|
||||
|
||||
// Temporarily hidden in the UI (always-on toggle disabled for now in VoicePanel).
|
||||
it.skip('shows the notch when always-on listening is enabled and hides it when disabled', async () => {
|
||||
it('shows the notch when always-on listening is enabled and hides it when disabled', async () => {
|
||||
renderWithProviders(<VoicePanel />, { initialEntries: ['/settings/voice'] });
|
||||
|
||||
const toggle = await screen.findByTestId('voice-always-on-toggle');
|
||||
@@ -537,8 +536,7 @@ describe('VoicePanel', () => {
|
||||
await waitFor(() => expect(vi.mocked(syncNotchVisibility)).toHaveBeenCalledWith(false));
|
||||
});
|
||||
|
||||
// Temporarily hidden in the UI (always-on toggle disabled for now in VoicePanel).
|
||||
it.skip('does not touch the notch and reverts the toggle when the update RPC fails', async () => {
|
||||
it('does not touch the notch and reverts the toggle when the update RPC fails', async () => {
|
||||
vi.mocked(openhumanUpdateVoiceServerSettings).mockRejectedValueOnce(new Error('rpc down'));
|
||||
|
||||
renderWithProviders(<VoicePanel />, { initialEntries: ['/settings/voice'] });
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
**GitHub Issue:** [#3148](https://github.com/tinyhumansai/openhuman/issues/3148)
|
||||
**Branch:** `feat/voice-always-on-all` (cumulative) — **all merged to `main`.** Phase 1 landed via [#3168](https://github.com/tinyhumansai/openhuman/pull/3168); the full feature was split from the mega-PR [#3307](https://github.com/tinyhumansai/openhuman/pull/3307) (now **closed** in favour of the stack) into an 8-PR stack — **all 8 merged 2026-06-04**: [#3340](https://github.com/tinyhumansai/openhuman/pull/3340) (main-thread input + CEF fix), [#3341](https://github.com/tinyhumansai/openhuman/pull/3341) (AX/UIA perception + automate engine), [#3342](https://github.com/tinyhumansai/openhuman/pull/3342) (wire automate/ax_interact tools), [#3343](https://github.com/tinyhumansai/openhuman/pull/3343) (Phase 2 always-on engine + RPC), [#3344](https://github.com/tinyhumansai/openhuman/pull/3344) (always-on Settings toggle + i18n), [#3345](https://github.com/tinyhumansai/openhuman/pull/3345) (notch status pill — supersedes the closed [#3166](https://github.com/tinyhumansai/openhuman/pull/3166)), [#3346](https://github.com/tinyhumansai/openhuman/pull/3346) (Phase 3 fast command router), [#3362](https://github.com/tinyhumansai/openhuman/pull/3362) (Phase 1.5 vision-click fallback)
|
||||
**Started:** 2026-06-02
|
||||
**Last updated:** 2026-06-09 — verified all 8 stack PRs merged to `main`; code paths confirmed present post-refactor [#3424](https://github.com/tinyhumansai/openhuman/pull/3424). Related: global push-to-talk ([#3090](https://github.com/tinyhumansai/openhuman/issues/3090) via [#3349](https://github.com/tinyhumansai/openhuman/pull/3349)) landed separately as an alternative manual trigger. **Only open item across all phases: the on-device audio wake-word model (Phase 3) — not started; text-based "Hey Tiny" match remains the interim.**
|
||||
**Last updated:** 2026-06-09 — added **Change 1.17** (PR [#3558](https://github.com/tinyhumansai/openhuman/pull/3558)): browser & app (Spotify/Apple Music/Slack) keyboard-shortcut fast-paths, cross-platform shortcut tables, the `hotkey` verb, artist-aware Music verification, and actionable `automate` failure responses — all from live-transcript analysis. Earlier this day: verified all 8 always-on stack PRs merged to `main` (code paths confirmed post-refactor [#3424](https://github.com/tinyhumansai/openhuman/pull/3424)); global push-to-talk ([#3090](https://github.com/tinyhumansai/openhuman/issues/3090) via [#3349](https://github.com/tinyhumansai/openhuman/pull/3349)) landed separately. **Only open item across all phases: the on-device audio wake-word model (Phase 3) — not started; text-based "Hey Tiny" match remains the interim.**
|
||||
|
||||
---
|
||||
|
||||
@@ -330,7 +330,7 @@ test ... ok
|
||||
|
||||
> The bridge between today's `ax_interact` primitives and the always-on voice work. **Prerequisite for Phase 3** — fast voice routing into a slow/fragile action loop still feels slow. This is where "whatever I say happens, live" actually gets delivered.
|
||||
>
|
||||
> **Status:** the Rust inner loop (M1), poll-until-stable settle (M2), Music fast-path (M3, proven live), notch progress streaming (M4), the richer element model (M5), and the **vision fallback for Electron/partial-AX apps** (Change 1.16) are all shipped. (Additional per-app native fast-paths beyond Music — e.g. Spotify/Slack — are **descoped**: the general model-driven loop covers them, so a deterministic accelerator isn't worth the per-app maintenance right now.)
|
||||
> **Status:** the Rust inner loop (M1), poll-until-stable settle (M2), Music fast-path (M3, proven live), notch progress streaming (M4), the richer element model (M5), the **vision fallback for Electron/partial-AX apps** (Change 1.16), and **Change 1.17** (browser & app fast-paths + keyboard-shortcut driving + artist-aware Music + actionable responses) are all shipped. Per-app fast-paths now cover **Music, browsers, Spotify, Apple Music, and Slack** (Spotify/Slack — previously descoped — are now handled by the `app_shortcuts.rs` keyboard-shortcut path).
|
||||
|
||||
**Detailed implementation plan:** [`voice-automate-plan.md`](voice-automate-plan.md) — decided approach: **Rust inner loop + fast model**, first proof target **Music**.
|
||||
|
||||
@@ -392,6 +392,49 @@ enigo's keyboard-layout lookup (`TSMGetInputSourceProperty`) **must run on the a
|
||||
|
||||
---
|
||||
|
||||
### Change 1.17 — browser & app fast-paths, keyboard-shortcut driving, artist-aware Music, actionable responses ✅ Done
|
||||
|
||||
**Status:** ✅ Shipped (PR [#3558](https://github.com/tinyhumansai/openhuman/pull/3558)). One body of work driven by **live-transcript analysis of the `automate` agent** across browsers, Apple Music, Spotify, and Slack. The transcripts surfaced four recurring failures the agent kept hitting:
|
||||
|
||||
1. **Browsers** — for *"open my Brave browser, go to youtube.com and play a music video"* the `desktop_control_agent` was re-delegated **6×** (each sub re-launched Brave + re-navigated), ran `ax_interact` on the **wrong app names** (`Google Chrome`/`Safari`, never the real `Brave Browser`, because Chromium exposes no AX tree), then "played" via a **blind hardcoded mouse click at (400,350)** with no verification.
|
||||
2. **Wrong track** — *"play Numb by Linkin Park"* played a **same-titled song by the wrong artist** ("Numb" by Marshmello & Khalid) and the agent **claimed success**, because the AX row label is title-only and verification only checked *that* something played.
|
||||
3. **Inert responses** — the `automate` tool's replies were *honest but gave no next move*, so the agent **re-ran the same failing search 6×** instead of changing tactics.
|
||||
4. **No keyboard-shortcut lever** — the loop hunted AX labels even where an app's own shortcut would be instant and reliable.
|
||||
|
||||
The six parts below address all four. All fast-paths are consulted by `automate::run` **before** the model loop (`try_fastpath` order: `music` → `browser` → `app_shortcuts`); on any failure they fall through, so they can only *help*.
|
||||
|
||||
**(a) Browser fast-path** — `app_fastpaths/browser.rs` (new). Pure parsers + `run(app, goal, backend)`:
|
||||
- `resolve_browser(app, goal)` — aliases → macOS display names (`brave`→`Brave Browser`, `chrome`→`Google Chrome`, `edge`→`Microsoft Edge`, `safari`/`firefox`/`arc`). Checks the `app` arg first, then the goal; a generic "browser" with no named product → **`None`** so it never guesses the wrong app (the original bug).
|
||||
- `extract_destination(goal)` — YouTube → `youtube.com/results?search_query=…` (percent-encoded); Google/"search for X" → `google.com/search?q=…`; a bare domain/URL → normalized to `https://…`.
|
||||
- **Navigation is one deterministic step:** `backend.open_url_in_app(browser, url)` → `open -a "<browser>" "<url>"` launches/foregrounds + navigates — no address-bar typing, no AX. Pure-navigation goals **complete here**; **play goals** navigate to the results URL then return non-success so the loop does the single "click first result" via `vision_click` (no native shortcut *selects* a result — we don't fake it). In-page media control ("pause/next the video") sends the YouTube shortcut directly.
|
||||
|
||||
**(b) Cross-platform browser shortcut table** — `app_fastpaths/browser_shortcuts.rs` (new). `shortcut(intent, browser, os)` resolves browser-level intents (address bar, new/close/reopen tab, tab N, next/prev tab, find, reload/hard-reload, back/forward, history, downloads, zoom, private window, fullscreen) per **Chrome/Firefox/Safari/Edge × macOS/Windows/Linux**. Nearly uniform (only ⌘↔Ctrl differs); encodes the real exceptions (Firefox-Linux `Alt+1‑8`, Firefox private window `⌘/Ctrl+Shift+P`, per-browser History/Downloads, Mac `⌘[`/`]` back-forward, `F11` vs `⌃⌘F` fullscreen). Sourced from the official Chrome/Firefox/Edge/Safari docs. `browser.rs` parses commands ("open a new tab", "go back", "reload", "switch to tab 3") and dispatches the resolved chord. (Brave/Arc/Chromium → Chrome family.)
|
||||
|
||||
**(c) App-shortcut fast-path** — `app_fastpaths/app_shortcuts.rs` (new). Same model for **Spotify, Apple Music, Slack**: `shortcut(intent, app, os)` + goal parser. Drives transport/navigation by each app's own global shortcut (faster + more reliable than AX). Media intents (play/pause, next, prev, volume ±, mute, shuffle, repeat, search) for Spotify + Apple Music; Slack nav (quick-switcher, search, new message/compose DM, next/prev unread, next/prev channel, threads, all-unread, mark read). Encodes the quirks: **Spotify uses ↓/↑ for next/prev** (not ←/→); **Apple Music prefixes everything with Ctrl on Windows** (Ctrl+Space vs bare Space on Mac); Slack channel-nav uses **Option/Alt+arrows**. Intents with no simple chord (Spotify volume, Apple Music mute / Windows search access-key) return `None` → fall through. Complementary to `music.rs`, which still owns Apple Music *song search/play*.
|
||||
|
||||
**(d) Keyboard-shortcut plumbing** —
|
||||
- Extracted shared `pub(crate)` helpers `run_hotkey`/`run_key`/`run_type_text` from `tools/impl/computer/keyboard.rs` (one place for validation + main-thread enigo dispatch, Change 1.15) — no behavior change to the `keyboard` tool.
|
||||
- New `AutomateBackend` methods (non-breaking defaults): `open_url_in_app`, `key`, `type_text`, `now_playing`. `RealBackend` impls `open -a` per-app, delegates keystrokes to the keyboard helpers, and reads Music state via osascript.
|
||||
- New **`hotkey` verb** in the general loop + system prompt (`{"action":"hotkey","keys":["Cmd","L"]}`) so the model can use any app's shortcuts — folded into the no-progress signature.
|
||||
|
||||
**(e) Artist-aware Music verification** — `music.rs` + `now_playing`. Apple Music's AX row label is title-only ("Numb - Single"), so a title match lands on the wrong artist. `music.rs` now parses the requested artist (`extract_artist`, "…by Linkin Park") and after Play reads the **now-playing name + artist** via AppleScript (`now_playing` → `osascript … get {name, artist} of current track`). On a match the summary names the verified track + artist; on a **mismatch it says so honestly** ("Now playing 'Numb' by 'Tom Odell'…") instead of claiming the requested track. Closes the wrong-artist false-success. **Follow-up:** *correcting* to the right artist is limited by AX `press`-by-label hitting only the first same-titled row — a library AppleScript `play (track whose name … and artist …)` fallback is the next step.
|
||||
|
||||
**(f) Actionable `automate` responses** — the tool's failures are now agent-actionable, not inert. Four fixes: **(1)** terminal failures (`music.rs` no-match/mismatch, `automate.rs` budget-exhausted/no-progress) carry **next-step guidance + a "don't repeat this" steer**; **(2)** they **list the actual candidates / on-screen labels** (`candidate_labels`, `screen_hint`) instead of a bare negative; **(3)** **verified-only success** — never "Playing 'X'" unless `now_playing` confirms it, and an artist/album press is reported as "navigated, no track started"; **(4)** `pick_row` now **prefers real song rows** (`AXCell`/`AXRow`) over artist/album buttons (the live bug pressed the "LINKIN PARK" artist row) and the step log names the element *type* pressed.
|
||||
|
||||
**Also:** the always-on toggle in `VoicePanel` is now shown in dev/debug builds (`IS_DEV_LIKE`) and hidden in production.
|
||||
|
||||
**Follow-up fix — Full OS access now enables app control.** `ax_interact` (press/set_value) and `automate` gated **only** on `computer_control.ax_interact_mutations`, which is independent of the autonomy level. A user who granted **Full** access in Settings → Agent Access still got *"App control isn't enabled yet…"* because that flag was untouched (confirmed live: `SecurityPolicy autonomy=Full` while the tool refused). Fixed with a shared `app_control_enabled(explicit_opt_in)` gate (`tools/impl/computer/ax_interact.rs`) that returns true when the explicit flag is set **or** the **live** policy (`security::live_policy::current()`) is `AutonomyLevel::Full` — so granting Full access takes effect immediately, no session restart or separate flag flip. Both refusal messages now point at Settings → Agent Access ("Grant Full access (or turn on App UI Control / App Automation)"). Covered by `app_control_enabled_combines_optin_and_full_access` + the two pinned-policy refusal tests.
|
||||
|
||||
**Follow-up fix — Windows browser nav + the re-delegation loop.** Live Windows transcript of *"open my browser, go to youtube.com and play a video"*: the orchestrator delegated to `desktop_control_agent` **7×**, every sub re-`launch_app`-ed Chrome (→ ~10 windows), and the AX path dead-ended — Chromium exposes no page content to UIA, `screenshot`/`vision_click` is unimplemented on Windows, and `ax_interact set_value` on the address bar **can't be submitted** (UIA Invoke on an Edit fails, no Enter). Two fixes:
|
||||
> - **`open_url_in_app` now works on Windows** (`accessibility/automate.rs`) — was macOS-only (fell back to the *default* browser). Maps the resolved browser display name → the shell `start` App-Paths token (`windows_browser_launch_token`: Chrome→`chrome`, Edge→`msedge`, Brave→`brave`, Firefox→`firefox`; Safari/Arc→`None`→default handler) and runs `cmd /C start "" <token> "<url>"`. Launches/foregrounds the named browser and navigates in **one** step; when the browser is already open this lands in a **new tab of the existing window**, so the fast-path no longer piles up windows. The browser fast-path's deterministic nav is now cross-platform (mac `open -a`, win `start`). Tested by `browser_display_names_map_to_start_tokens` (`#[cfg(target_os="windows")]`).
|
||||
> - **`desktop_control_agent` prompt steered to `automate` for browsers** (`agent_registry/agents/desktop_control_agent/prompt.md`) — it had `automate` but its only examples were Music/Slack, so it AX-fumbled the browser. Now: web browsers → use `automate{app, goal}` (deterministic URL nav), never type a URL into the address bar via `ax_interact`; **foreground each app at most once per task** (repeated `launch_app` piles up windows); and **stop after two failed attempts at a step** instead of re-launching and looping. (Still a Windows gap: `vision_click` for the final "click first result" — `screenshot` is unimplemented on Windows, so play-from-search-results can't complete deterministically yet; navigation/search now do.)
|
||||
|
||||
**Net for the browser example:** `open -a "Brave Browser" "https://www.youtube.com/results?search_query=music%20video"` + one `vision_click` — instead of 6 delegations and 5 failed AX calls.
|
||||
|
||||
**Tests (all green):** `app_fastpaths` **58**, `accessibility::automate` **21**, `computer::keyboard` **26**. Coverage: browser/app resolution + destination/intent parsers (incl. Spotify ↓/↑, Apple Music Win-Ctrl, Slack Alt-nav, generic-browser→None, percent-encoding); scripted-backend sequences (model-free nav, play→fall-through, media-control hotkey, dispatch order with music-wins-play); artist match→names-track / mismatch→honest; response shapes (candidate listing, wrong-artist steer, non-song honesty, song-row preference, budget/no-progress on-screen hints); a `hotkey`-verb loop test; `#[ignore]` macOS live tests. **Follow-ups:** wire the shortcut tables into the Phase 3 voice `command_router` (voice "pause"/"next" → frontmost-app shortcut); AppleScript library play-by-name+artist correction; expose `vision_click` as a standalone tool.
|
||||
|
||||
---
|
||||
|
||||
## Windows port — app interaction 🪟 ✅ Implemented
|
||||
|
||||
Phase 1's app-interaction layer is now ported to Windows. The macOS path uses the
|
||||
@@ -648,6 +691,12 @@ From live agent-in-the-loop testing on 2026-06-03 (grounded in `~/.openhuman/log
|
||||
| 1.5 | M5: richer element model (`enabled`) | ✅ Plumbed; AXEnabled found unreliable → informational only |
|
||||
| 1.5 | Native fast-paths beyond Music (Spotify/Slack) | ➖ Descoped (general loop covers them; Music shipped in M3) |
|
||||
| 1.5 | Vision fallback for Electron apps (Change 1.16) | ✅ Done (`vision_click`: screenshot → vision-locate → guarded click; frontmost guard; F2 coord-transform folded in) |
|
||||
| 1.5 | Browser fast-path (Change 1.17a) | ✅ Done (`browser.rs`: deterministic `open -a "<browser>" url` nav; play→fall-through to vision_click) |
|
||||
| 1.5 | Cross-platform browser shortcut table (Change 1.17b) | ✅ Done (`browser_shortcuts.rs`: Chrome/Firefox/Safari/Edge × macOS/Win/Linux) |
|
||||
| 1.5 | App-shortcut fast-path: Spotify/Apple Music/Slack (Change 1.17c) | ✅ Done (`app_shortcuts.rs`: transport + nav via each app's shortcuts) |
|
||||
| 1.5 | Keyboard-shortcut plumbing: `hotkey` verb + backend methods (Change 1.17d) | ✅ Done (`key`/`type_text`/`open_url_in_app`/`now_playing`; shared keyboard helpers) |
|
||||
| 1.5 | Artist-aware Music verification (Change 1.17e) | ✅ Done (`now_playing` osascript; honest wrong-artist report, no false success) |
|
||||
| 1.5 | Actionable `automate` responses (Change 1.17f) | ✅ Done (next-step guidance, candidate/label lists, verified-only success, song-row preference) |
|
||||
| 2 | Always-on microphone loop | ✅ Done (cpal → VAD → STT → agent) |
|
||||
| 2 | `always_on_enabled` config flag + Settings toggle | ✅ Done (RPC + UI + i18n) |
|
||||
| 2 | Privacy hook (screen lock pause) | ✅ Done (macOS; other OSes follow-up) |
|
||||
|
||||
@@ -0,0 +1,519 @@
|
||||
//! Keyboard-shortcut fast-path for Spotify, Apple Music, and Slack (Change 1.17).
|
||||
//!
|
||||
//! These desktop apps expose stable global shortcuts for the actions users ask
|
||||
//! for by voice — "pause", "next song", "turn it up", "jump to a channel". For
|
||||
//! those, driving the app's own shortcut is far faster and more reliable than
|
||||
//! walking the AX tree: one `backend.key(chord)` and we're done. (Apple Music
|
||||
//! *song search/play* still goes through `music.rs`, which types a query; this
|
||||
//! module is transport/navigation control.)
|
||||
//!
|
||||
//! Like `browser_shortcuts.rs`, the table is keyed by `(intent, app, os)` and
|
||||
//! returns key names the keyboard tool's `parse_key` understands, so the result
|
||||
//! feeds straight into `AutomateBackend::key`. Sources: the official Spotify,
|
||||
//! Apple Music (Mac + Windows), and Slack shortcut docs.
|
||||
|
||||
use super::browser_shortcuts::Os;
|
||||
use super::AutomateBackend;
|
||||
use super::AutomateOutcome;
|
||||
|
||||
/// Apps this fast-path can drive by shortcut.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AppFamily {
|
||||
Spotify,
|
||||
AppleMusic,
|
||||
Slack,
|
||||
}
|
||||
|
||||
/// A control action. Media intents apply to Spotify/Apple Music; the rest to
|
||||
/// Slack. [`shortcut`] returns `None` for an intent an app/OS doesn't support.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AppIntent {
|
||||
// ── media transport ──
|
||||
PlayPause,
|
||||
NextTrack,
|
||||
PrevTrack,
|
||||
VolumeUp,
|
||||
VolumeDown,
|
||||
Mute,
|
||||
Shuffle,
|
||||
Repeat,
|
||||
MediaSearch,
|
||||
// ── Slack navigation ──
|
||||
QuickSwitcher,
|
||||
SlackSearch,
|
||||
NewMessage,
|
||||
ComposeDm,
|
||||
NextUnread,
|
||||
PrevUnread,
|
||||
NextChannel,
|
||||
PrevChannel,
|
||||
Threads,
|
||||
AllUnread,
|
||||
MarkRead,
|
||||
}
|
||||
|
||||
/// Resolve the app family from the authoritative `app` arg first (so a stray
|
||||
/// "music" in the goal text can't mis-route), then from an explicit app name in
|
||||
/// the goal as a fallback.
|
||||
fn resolve_family(app: &str, goal: &str) -> Option<AppFamily> {
|
||||
if let Some(f) = family_from_name(&app.to_lowercase()) {
|
||||
return Some(f);
|
||||
}
|
||||
let g = goal.to_lowercase();
|
||||
if g.contains("spotify") {
|
||||
Some(AppFamily::Spotify)
|
||||
} else if g.contains("slack") {
|
||||
Some(AppFamily::Slack)
|
||||
} else if g.contains("apple music") || g.contains("itunes") {
|
||||
Some(AppFamily::AppleMusic)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn family_from_name(n: &str) -> Option<AppFamily> {
|
||||
if n.contains("spotify") {
|
||||
Some(AppFamily::Spotify)
|
||||
} else if n.contains("slack") {
|
||||
Some(AppFamily::Slack)
|
||||
} else if n.contains("music") || n.contains("itunes") {
|
||||
// The macOS app's display name is literally "Music".
|
||||
Some(AppFamily::AppleMusic)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn keys(parts: &[&str]) -> Vec<String> {
|
||||
parts.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
/// Primary modifier: ⌘ on macOS, Ctrl elsewhere.
|
||||
fn primary(os: Os) -> &'static str {
|
||||
match os {
|
||||
Os::Mac => "Cmd",
|
||||
_ => "Ctrl",
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve an intent to a key chord for the app + OS, or `None` if that app/OS
|
||||
/// has no simple chord for it (the loop then falls through).
|
||||
pub fn shortcut(intent: AppIntent, app: AppFamily, os: Os) -> Option<Vec<String>> {
|
||||
use AppIntent as I;
|
||||
let p = primary(os);
|
||||
let v = match (app, intent) {
|
||||
// ── Spotify (Mac & Windows columns; Linux ≈ Windows) ──
|
||||
(AppFamily::Spotify, I::PlayPause) => keys(&["space"]),
|
||||
// Spotify quirk: ↓ = next, ↑ = previous (not ←/→).
|
||||
(AppFamily::Spotify, I::NextTrack) => keys(&["down"]),
|
||||
(AppFamily::Spotify, I::PrevTrack) => keys(&["up"]),
|
||||
(AppFamily::Spotify, I::Mute) => keys(&["m"]),
|
||||
(AppFamily::Spotify, I::Shuffle) => match os {
|
||||
Os::Mac => keys(&["alt", "s"]),
|
||||
_ => keys(&["ctrl", "s"]),
|
||||
},
|
||||
(AppFamily::Spotify, I::Repeat) => match os {
|
||||
Os::Mac => keys(&["alt", "r"]),
|
||||
_ => keys(&["ctrl", "r"]),
|
||||
},
|
||||
(AppFamily::Spotify, I::MediaSearch) => keys(&[p, "k"]),
|
||||
// Spotify has no documented volume shortcut → fall through.
|
||||
(AppFamily::Spotify, I::VolumeUp | I::VolumeDown) => return None,
|
||||
|
||||
// ── Apple Music ──
|
||||
(AppFamily::AppleMusic, I::PlayPause) => match os {
|
||||
Os::Mac => keys(&["space"]),
|
||||
_ => keys(&["ctrl", "space"]),
|
||||
},
|
||||
(AppFamily::AppleMusic, I::NextTrack) => match os {
|
||||
Os::Mac => keys(&["right"]),
|
||||
_ => keys(&["ctrl", "right"]),
|
||||
},
|
||||
(AppFamily::AppleMusic, I::PrevTrack) => match os {
|
||||
Os::Mac => keys(&["left"]),
|
||||
_ => keys(&["ctrl", "left"]),
|
||||
},
|
||||
(AppFamily::AppleMusic, I::VolumeUp) => match os {
|
||||
Os::Mac => keys(&["Cmd", "up"]),
|
||||
_ => keys(&["ctrl", "up"]),
|
||||
},
|
||||
(AppFamily::AppleMusic, I::VolumeDown) => match os {
|
||||
Os::Mac => keys(&["Cmd", "down"]),
|
||||
_ => keys(&["ctrl", "down"]),
|
||||
},
|
||||
// Search field: ⌘F on Mac. Windows uses a sequential access key
|
||||
// (Alt,N,F) we can't send as one chord → fall through there.
|
||||
(AppFamily::AppleMusic, I::MediaSearch) => match os {
|
||||
Os::Mac => keys(&["Cmd", "f"]),
|
||||
_ => return None,
|
||||
},
|
||||
// No simple mute/shuffle/repeat chord in Apple Music → fall through.
|
||||
(AppFamily::AppleMusic, I::Mute | I::Shuffle | I::Repeat) => return None,
|
||||
|
||||
// ── Slack ──
|
||||
(AppFamily::Slack, I::QuickSwitcher) => keys(&[p, "k"]),
|
||||
(AppFamily::Slack, I::SlackSearch) => keys(&[p, "g"]),
|
||||
(AppFamily::Slack, I::NewMessage) => keys(&[p, "n"]),
|
||||
(AppFamily::Slack, I::ComposeDm) => keys(&[p, "shift", "k"]),
|
||||
// Option (mac) / Alt (win) both map to enigo's Alt key.
|
||||
(AppFamily::Slack, I::NextUnread) => keys(&["alt", "shift", "down"]),
|
||||
(AppFamily::Slack, I::PrevUnread) => keys(&["alt", "shift", "up"]),
|
||||
(AppFamily::Slack, I::NextChannel) => keys(&["alt", "down"]),
|
||||
(AppFamily::Slack, I::PrevChannel) => keys(&["alt", "up"]),
|
||||
(AppFamily::Slack, I::Threads) => keys(&[p, "shift", "t"]),
|
||||
(AppFamily::Slack, I::AllUnread) => keys(&[p, "shift", "a"]),
|
||||
(AppFamily::Slack, I::MarkRead) => keys(&["esc"]),
|
||||
|
||||
// Any intent not applicable to the app.
|
||||
_ => return None,
|
||||
};
|
||||
Some(v)
|
||||
}
|
||||
|
||||
/// Does this (app, goal) name one of these apps with a recognizable control?
|
||||
pub fn matches(app: &str, goal: &str) -> bool {
|
||||
match resolve_family(app, goal) {
|
||||
Some(fam) => extract_intent(goal, fam)
|
||||
.and_then(|i| shortcut(i, fam, Os::current()))
|
||||
.is_some(),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the shortcut fast-path: resolve the intent, send the chord, done.
|
||||
pub async fn run(app: &str, goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome {
|
||||
use super::super::automate::progress;
|
||||
use crate::openhuman::overlay::OverlayAttentionTone;
|
||||
|
||||
let mut steps: Vec<String> = Vec::new();
|
||||
let fam = match resolve_family(app, goal) {
|
||||
Some(f) => f,
|
||||
None => return fail("no recognized app", steps),
|
||||
};
|
||||
let intent = match extract_intent(goal, fam) {
|
||||
Some(i) => i,
|
||||
None => return fail("no recognized control intent", steps),
|
||||
};
|
||||
let chord = match shortcut(intent, fam, Os::current()) {
|
||||
Some(c) => c,
|
||||
None => return fail("no shortcut for this intent on this OS", steps),
|
||||
};
|
||||
let combo = chord.join("+");
|
||||
log::info!("[automate::app_shortcuts] ▶ {fam:?} {intent:?} keys={combo}");
|
||||
progress(format!("Pressing {combo}…"), OverlayAttentionTone::Accent);
|
||||
|
||||
match backend.key(&chord).await {
|
||||
Ok(m) => {
|
||||
steps.push(format!("hotkey {combo}: {m}"));
|
||||
AutomateOutcome {
|
||||
success: true,
|
||||
summary: format!("Sent {combo} to {app}."),
|
||||
steps,
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
steps.push(format!("hotkey FAILED: {e}"));
|
||||
fail("could not send the shortcut", steps)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a control intent from the goal, branching on app domain (media vs
|
||||
/// Slack) so the same word ("search", "next") maps correctly.
|
||||
fn extract_intent(goal: &str, fam: AppFamily) -> Option<AppIntent> {
|
||||
let l = goal.to_lowercase();
|
||||
match fam {
|
||||
AppFamily::Slack => parse_slack(&l),
|
||||
_ => parse_media(&l),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_media(l: &str) -> Option<AppIntent> {
|
||||
use AppIntent as I;
|
||||
if l.contains("volume up")
|
||||
|| l.contains("louder")
|
||||
|| l.contains("turn it up")
|
||||
|| l.contains("turn up")
|
||||
|| l.contains("increase volume")
|
||||
|| l.contains("raise the volume")
|
||||
{
|
||||
return Some(I::VolumeUp);
|
||||
}
|
||||
if l.contains("volume down")
|
||||
|| l.contains("quieter")
|
||||
|| l.contains("turn it down")
|
||||
|| l.contains("turn down")
|
||||
|| l.contains("decrease volume")
|
||||
|| l.contains("lower the volume")
|
||||
|| l.contains("lower volume")
|
||||
{
|
||||
return Some(I::VolumeDown);
|
||||
}
|
||||
if has_word(l, "mute") || has_word(l, "unmute") {
|
||||
return Some(I::Mute);
|
||||
}
|
||||
if has_word(l, "shuffle") {
|
||||
return Some(I::Shuffle);
|
||||
}
|
||||
if has_word(l, "repeat") || has_word(l, "loop") {
|
||||
return Some(I::Repeat);
|
||||
}
|
||||
if has_word(l, "next") || has_word(l, "skip") {
|
||||
return Some(I::NextTrack);
|
||||
}
|
||||
if l.contains("previous")
|
||||
|| l.contains("last song")
|
||||
|| l.contains("last track")
|
||||
|| l.contains("go back a song")
|
||||
{
|
||||
return Some(I::PrevTrack);
|
||||
}
|
||||
if has_word(l, "pause") || has_word(l, "resume") || has_word(l, "unpause") {
|
||||
return Some(I::PlayPause);
|
||||
}
|
||||
if l.contains("continue playing") || l.contains("keep playing") {
|
||||
return Some(I::PlayPause);
|
||||
}
|
||||
// Bare "play" toggle — but NOT "play <song name>" (that's `music.rs`).
|
||||
if let Some(i) = word_index(l, "play") {
|
||||
let after = l[i + "play".len()..].trim();
|
||||
if after.is_empty()
|
||||
|| matches!(
|
||||
after,
|
||||
"music"
|
||||
| "it"
|
||||
| "this"
|
||||
| "the music"
|
||||
| "the song"
|
||||
| "song"
|
||||
| "the track"
|
||||
| "track"
|
||||
| "playback"
|
||||
| "the playback"
|
||||
)
|
||||
{
|
||||
return Some(I::PlayPause);
|
||||
}
|
||||
}
|
||||
if has_word(l, "search") || has_word(l, "find") {
|
||||
return Some(I::MediaSearch);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn parse_slack(l: &str) -> Option<AppIntent> {
|
||||
use AppIntent as I;
|
||||
if l.contains("next unread") {
|
||||
Some(I::NextUnread)
|
||||
} else if l.contains("previous unread") || l.contains("prev unread") {
|
||||
Some(I::PrevUnread)
|
||||
} else if l.contains("next channel") || l.contains("next conversation") || l.contains("next dm")
|
||||
{
|
||||
Some(I::NextChannel)
|
||||
} else if l.contains("previous channel")
|
||||
|| l.contains("prev channel")
|
||||
|| l.contains("previous conversation")
|
||||
{
|
||||
Some(I::PrevChannel)
|
||||
} else if l.contains("all unread") || l.contains("unreads") {
|
||||
Some(I::AllUnread)
|
||||
} else if l.contains("thread") {
|
||||
Some(I::Threads)
|
||||
} else if l.contains("mark") && l.contains("read") {
|
||||
Some(I::MarkRead)
|
||||
} else if l.contains("new message") {
|
||||
Some(I::NewMessage)
|
||||
} else if l.contains("compose")
|
||||
|| l.contains("direct message")
|
||||
|| has_word(l, "dm")
|
||||
|| l.contains("message someone")
|
||||
|| l.contains("send a message")
|
||||
{
|
||||
Some(I::ComposeDm)
|
||||
} else if l.contains("jump to")
|
||||
|| l.contains("quick switch")
|
||||
|| l.contains("switch to")
|
||||
|| l.contains("go to channel")
|
||||
|| l.contains("open channel")
|
||||
|| l.contains("open conversation")
|
||||
|| l.contains("find channel")
|
||||
{
|
||||
Some(I::QuickSwitcher)
|
||||
} else if has_word(l, "search") || has_word(l, "find") {
|
||||
Some(I::SlackSearch)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn fail(msg: &str, steps: Vec<String>) -> AutomateOutcome {
|
||||
AutomateOutcome {
|
||||
success: false,
|
||||
summary: format!("App shortcut fast-path: {msg}"),
|
||||
steps,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whole-word membership test on an already-lowercased string.
|
||||
fn has_word(haystack: &str, needle: &str) -> bool {
|
||||
word_index(haystack, needle).is_some()
|
||||
}
|
||||
|
||||
fn word_index(haystack: &str, needle: &str) -> Option<usize> {
|
||||
let mut from = 0;
|
||||
while let Some(rel) = haystack[from..].find(needle) {
|
||||
let idx = from + rel;
|
||||
let before_ok = idx == 0
|
||||
|| !haystack[..idx]
|
||||
.chars()
|
||||
.next_back()
|
||||
.map(|c| c.is_alphanumeric())
|
||||
.unwrap_or(false);
|
||||
let after = idx + needle.len();
|
||||
let after_ok = haystack[after..]
|
||||
.chars()
|
||||
.next()
|
||||
.map(|c| !c.is_alphanumeric())
|
||||
.unwrap_or(true);
|
||||
if before_ok && after_ok {
|
||||
return Some(idx);
|
||||
}
|
||||
from = idx + needle.len();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use AppIntent as I;
|
||||
|
||||
#[test]
|
||||
fn resolves_family_from_app_then_goal() {
|
||||
assert_eq!(resolve_family("Spotify", "x"), Some(AppFamily::Spotify));
|
||||
assert_eq!(resolve_family("Music", "x"), Some(AppFamily::AppleMusic));
|
||||
assert_eq!(resolve_family("Slack", "x"), Some(AppFamily::Slack));
|
||||
// From goal when app arg is unhelpful.
|
||||
assert_eq!(
|
||||
resolve_family("", "pause spotify"),
|
||||
Some(AppFamily::Spotify)
|
||||
);
|
||||
// A stray "music" in the goal with a browser app must NOT mis-route.
|
||||
assert_eq!(resolve_family("Brave Browser", "find music videos"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spotify_transport_quirks() {
|
||||
let o = Os::Mac;
|
||||
assert_eq!(
|
||||
shortcut(I::PlayPause, AppFamily::Spotify, o),
|
||||
Some(keys(&["space"]))
|
||||
);
|
||||
// ↓ next / ↑ previous (the Spotify quirk).
|
||||
assert_eq!(
|
||||
shortcut(I::NextTrack, AppFamily::Spotify, o),
|
||||
Some(keys(&["down"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::PrevTrack, AppFamily::Spotify, o),
|
||||
Some(keys(&["up"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::Shuffle, AppFamily::Spotify, Os::Windows),
|
||||
Some(keys(&["ctrl", "s"]))
|
||||
);
|
||||
// No documented volume → None.
|
||||
assert_eq!(shortcut(I::VolumeUp, AppFamily::Spotify, o), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apple_music_mac_vs_windows() {
|
||||
assert_eq!(
|
||||
shortcut(I::PlayPause, AppFamily::AppleMusic, Os::Mac),
|
||||
Some(keys(&["space"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::PlayPause, AppFamily::AppleMusic, Os::Windows),
|
||||
Some(keys(&["ctrl", "space"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::VolumeUp, AppFamily::AppleMusic, Os::Mac),
|
||||
Some(keys(&["Cmd", "up"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::NextTrack, AppFamily::AppleMusic, Os::Windows),
|
||||
Some(keys(&["ctrl", "right"]))
|
||||
);
|
||||
// Search: Mac ⌘F; Windows access-key sequence → None.
|
||||
assert_eq!(
|
||||
shortcut(I::MediaSearch, AppFamily::AppleMusic, Os::Mac),
|
||||
Some(keys(&["Cmd", "f"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::MediaSearch, AppFamily::AppleMusic, Os::Windows),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slack_navigation() {
|
||||
assert_eq!(
|
||||
shortcut(I::QuickSwitcher, AppFamily::Slack, Os::Mac),
|
||||
Some(keys(&["Cmd", "k"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::QuickSwitcher, AppFamily::Slack, Os::Windows),
|
||||
Some(keys(&["Ctrl", "k"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::NextUnread, AppFamily::Slack, Os::Mac),
|
||||
Some(keys(&["alt", "shift", "down"]))
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(I::MarkRead, AppFamily::Slack, Os::Mac),
|
||||
Some(keys(&["esc"]))
|
||||
);
|
||||
// Media intent on Slack → None.
|
||||
assert_eq!(shortcut(I::Shuffle, AppFamily::Slack, Os::Mac), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_media_intents() {
|
||||
assert_eq!(parse_media("pause"), Some(I::PlayPause));
|
||||
assert_eq!(parse_media("resume the music"), Some(I::PlayPause));
|
||||
assert_eq!(parse_media("play"), Some(I::PlayPause));
|
||||
assert_eq!(parse_media("play the music"), Some(I::PlayPause));
|
||||
assert_eq!(parse_media("next song"), Some(I::NextTrack));
|
||||
assert_eq!(parse_media("skip this"), Some(I::NextTrack));
|
||||
assert_eq!(parse_media("previous track"), Some(I::PrevTrack));
|
||||
assert_eq!(parse_media("turn it up"), Some(I::VolumeUp));
|
||||
assert_eq!(parse_media("lower the volume"), Some(I::VolumeDown));
|
||||
assert_eq!(parse_media("mute"), Some(I::Mute));
|
||||
assert_eq!(parse_media("shuffle"), Some(I::Shuffle));
|
||||
// "play <song>" is NOT a transport toggle (music.rs owns that).
|
||||
assert_eq!(parse_media("play despacito"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_slack_intents() {
|
||||
assert_eq!(parse_slack("jump to a channel"), Some(I::QuickSwitcher));
|
||||
assert_eq!(parse_slack("go to channel general"), Some(I::QuickSwitcher));
|
||||
assert_eq!(parse_slack("next unread"), Some(I::NextUnread));
|
||||
assert_eq!(parse_slack("next channel"), Some(I::NextChannel));
|
||||
assert_eq!(parse_slack("compose a message"), Some(I::ComposeDm));
|
||||
assert_eq!(parse_slack("open threads"), Some(I::Threads));
|
||||
assert_eq!(parse_slack("mark as read"), Some(I::MarkRead));
|
||||
assert_eq!(parse_slack("search"), Some(I::SlackSearch));
|
||||
assert_eq!(parse_slack("hello"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_gating() {
|
||||
assert!(matches("Spotify", "next song"));
|
||||
assert!(matches("Music", "pause"));
|
||||
assert!(matches("Slack", "jump to a conversation"));
|
||||
// Unknown app or no intent → no match.
|
||||
assert!(!matches("Spotify", "hello there"));
|
||||
assert!(!matches("Notes", "next song"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,672 @@
|
||||
//! Browser fast-path: "open `<browser>` and go to `<url>` / search / play".
|
||||
//!
|
||||
//! Encodes the deterministic browser flow the live transcript showed the model
|
||||
//! getting wrong (tracker — Change 1.17): it re-launched the browser 6×, ran
|
||||
//! `ax_interact` on the wrong app names (Chrome/Safari, never the real
|
||||
//! `Brave Browser`), and finally clicked a hardcoded coordinate.
|
||||
//!
|
||||
//! Instead we navigate in ONE step via `open -a "<browser>" "<url>"`:
|
||||
//! - **Pure navigation** ("open Brave and go to youtube.com", "search
|
||||
//! youtube for X") completes here — no address-bar typing, no AX, no
|
||||
//! re-launching.
|
||||
//! - **Play intents** ("…and play a music video") navigate deterministically
|
||||
//! to the search-results URL, then return a non-success outcome so the
|
||||
//! general loop performs the single ambiguous "click the first result" via
|
||||
//! `vision_click` (Chromium exposes no AX tree). There's no reliable native
|
||||
//! shortcut that *selects* a search result, so we don't fake success.
|
||||
//! - **In-page media control** ("pause/next/fullscreen the video") sends the
|
||||
//! YouTube keyboard shortcut directly — the fastest possible path.
|
||||
//!
|
||||
//! Everything goes through the injectable [`AutomateBackend`], so the flow is
|
||||
//! unit-testable with a scripted backend — no live browser, no model.
|
||||
|
||||
use super::browser_shortcuts::{shortcut, Browser, BrowserShortcut, Os};
|
||||
use super::AutomateBackend;
|
||||
use super::AutomateOutcome;
|
||||
|
||||
/// A resolved navigation target.
|
||||
struct Destination {
|
||||
/// Fully-qualified URL to open.
|
||||
url: String,
|
||||
/// True when the goal asks to *play* something at a search URL — the
|
||||
/// fast-path navigates but then defers the final "click first result" to
|
||||
/// the general loop's `vision_click`.
|
||||
is_play: bool,
|
||||
}
|
||||
|
||||
/// Does this (app, goal) look like a browser navigation / control request?
|
||||
pub fn matches(app: &str, goal: &str) -> bool {
|
||||
resolve_browser(app, goal).is_some()
|
||||
&& (extract_destination(goal).is_some()
|
||||
|| extract_browser_command(goal).is_some()
|
||||
|| extract_media_control(goal).is_some())
|
||||
}
|
||||
|
||||
/// Resolve the browser's macOS display name from the `app` arg first, then the
|
||||
/// goal text. Returns `None` for a generic "browser" with no named product, so
|
||||
/// we never guess the wrong app (the original transcript bug).
|
||||
fn resolve_browser(app: &str, goal: &str) -> Option<String> {
|
||||
// (alias substring, display name) — longest/most-specific aliases first.
|
||||
const BROWSERS: &[(&str, &str)] = &[
|
||||
("brave", "Brave Browser"),
|
||||
("google chrome", "Google Chrome"),
|
||||
("chrome", "Google Chrome"),
|
||||
("microsoft edge", "Microsoft Edge"),
|
||||
("edge", "Microsoft Edge"),
|
||||
("firefox", "Firefox"),
|
||||
("safari", "Safari"),
|
||||
("arc", "Arc"),
|
||||
];
|
||||
let app_l = app.to_lowercase();
|
||||
for (alias, display) in BROWSERS {
|
||||
if app_l.contains(alias) {
|
||||
return Some((*display).to_string());
|
||||
}
|
||||
}
|
||||
let goal_l = goal.to_lowercase();
|
||||
for (alias, display) in BROWSERS {
|
||||
if goal_l.contains(alias) {
|
||||
return Some((*display).to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Resolve where to navigate. In priority order: YouTube intent → Google/web
|
||||
/// search → a bare URL/domain mentioned in the goal. `None` if nothing matches.
|
||||
fn extract_destination(goal: &str) -> Option<Destination> {
|
||||
let lower = goal.to_lowercase();
|
||||
let wants_play = has_word(&lower, "play");
|
||||
|
||||
// 1. YouTube.
|
||||
if lower.contains("youtube") || lower.contains("you tube") {
|
||||
if let Some(q) = extract_query(goal) {
|
||||
return Some(Destination {
|
||||
url: format!(
|
||||
"https://www.youtube.com/results?search_query={}",
|
||||
percent_encode(&q)
|
||||
),
|
||||
is_play: wants_play,
|
||||
});
|
||||
}
|
||||
// Named YouTube but no query → just open the site.
|
||||
return Some(Destination {
|
||||
url: "https://www.youtube.com".to_string(),
|
||||
is_play: false,
|
||||
});
|
||||
}
|
||||
|
||||
// 2. Google / web search.
|
||||
if lower.contains("google") || lower.contains("search") {
|
||||
// "search for X" / "search X for Y" via extract_query, else the text
|
||||
// right after the word "google" ("google rust async traits").
|
||||
let q = extract_query(goal).or_else(|| {
|
||||
word_index(&lower, "google").and_then(|i| {
|
||||
let after = clean_query(&goal[i + "google".len()..]);
|
||||
(!after.is_empty()).then_some(after)
|
||||
})
|
||||
});
|
||||
if let Some(q) = q {
|
||||
return Some(Destination {
|
||||
url: format!("https://www.google.com/search?q={}", percent_encode(&q)),
|
||||
is_play: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 3. A bare URL / domain anywhere in the goal ("go to example.com").
|
||||
if let Some(url) = extract_url(goal) {
|
||||
return Some(Destination {
|
||||
url,
|
||||
is_play: false,
|
||||
});
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Pull a search query out of the goal: text after "for" (search phrasing) or
|
||||
/// after "play", with leading articles and trailing site words stripped. `None`
|
||||
/// when there's nothing searchable.
|
||||
fn extract_query(goal: &str) -> Option<String> {
|
||||
let lower = goal.to_lowercase();
|
||||
// "search [youtube|the web|google] for X" / "search for X".
|
||||
if let Some(p) = lower.find(" for ") {
|
||||
let after = goal[p + " for ".len()..].trim();
|
||||
let q = clean_query(after);
|
||||
if !q.is_empty() {
|
||||
return Some(q);
|
||||
}
|
||||
}
|
||||
// "play X", "play X on youtube".
|
||||
if let Some(idx) = word_index(&lower, "play") {
|
||||
let after = goal[idx + "play".len()..].trim();
|
||||
let q = clean_query(after);
|
||||
if !q.is_empty() {
|
||||
return Some(q);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Trim filler around an extracted query: leading articles, a leading "me",
|
||||
/// and a trailing "(on|in|at) <site>" / "video(s)" tail isn't stripped (it's a
|
||||
/// fine search term), but a trailing "on youtube" etc. is removed.
|
||||
fn clean_query(raw: &str) -> String {
|
||||
let mut q = raw.trim().trim_end_matches(['.', '!', '?']).trim();
|
||||
// Cut at a clause boundary so "go to youtube and play X then do Y" stops at X.
|
||||
if let Some(p) = q.to_lowercase().find(" then ") {
|
||||
q = q[..p].trim();
|
||||
}
|
||||
let mut s = q.to_string();
|
||||
for lead in ["me ", "a ", "an ", "the ", "some "] {
|
||||
if let Some(rest) = strip_prefix_ci(&s, lead) {
|
||||
s = rest.trim().to_string();
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Drop a trailing "(on|in) <site>" navigation tail.
|
||||
let sl = s.to_lowercase();
|
||||
for tail in [" on youtube", " in youtube", " on google", " on the web"] {
|
||||
if let Some(p) = sl.rfind(tail) {
|
||||
s.truncate(p);
|
||||
break;
|
||||
}
|
||||
}
|
||||
s.trim().to_string()
|
||||
}
|
||||
|
||||
/// Find the first bare URL or `host.tld` token and normalize it to an
|
||||
/// `https://` URL. Skips obvious non-hosts (must contain a dot, no spaces).
|
||||
fn extract_url(goal: &str) -> Option<String> {
|
||||
for tok in goal.split_whitespace() {
|
||||
let t =
|
||||
tok.trim_matches(|c: char| !c.is_alphanumeric() && c != '/' && c != ':' && c != '.');
|
||||
let tl = t.to_lowercase();
|
||||
if tl.starts_with("http://") || tl.starts_with("https://") {
|
||||
return Some(t.to_string());
|
||||
}
|
||||
// host.tld with a plausible TLD and no scheme.
|
||||
if t.contains('.') && !t.contains('/') && looks_like_domain(&tl) {
|
||||
return Some(format!("https://{t}"));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Crude domain check: has a dot, and the last label is 2–24 ascii letters.
|
||||
fn looks_like_domain(s: &str) -> bool {
|
||||
match s.rsplit_once('.') {
|
||||
Some((host, tld)) => {
|
||||
!host.is_empty()
|
||||
&& (2..=24).contains(&tld.len())
|
||||
&& tld.chars().all(|c| c.is_ascii_alphabetic())
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a browser-level command ("new tab", "go back", "reload", "switch to tab
|
||||
/// 3", …) to a [`BrowserShortcut`]. Resolved to actual keys per browser+OS by
|
||||
/// [`browser_shortcuts::shortcut`]. Navigation (a URL/search) is handled earlier
|
||||
/// by [`extract_destination`]; this covers tab/window/history/zoom verbs.
|
||||
fn extract_browser_command(goal: &str) -> Option<BrowserShortcut> {
|
||||
use BrowserShortcut as S;
|
||||
let l = goal.to_lowercase();
|
||||
|
||||
// "switch to tab 3" / "go to tab 5" — a digit must follow "tab".
|
||||
if let Some(n) = tab_number(&l) {
|
||||
return Some(S::TabN(n));
|
||||
}
|
||||
// Most-specific phrases first.
|
||||
if l.contains("hard reload") || l.contains("force reload") || l.contains("hard refresh") {
|
||||
Some(S::HardReload)
|
||||
} else if l.contains("reload") || l.contains("refresh") {
|
||||
Some(S::Reload)
|
||||
} else if l.contains("new tab") {
|
||||
Some(S::NewTab)
|
||||
} else if l.contains("close tab")
|
||||
|| l.contains("close this tab")
|
||||
|| l.contains("close the tab")
|
||||
|| l.contains("close current tab")
|
||||
{
|
||||
Some(S::CloseTab)
|
||||
} else if l.contains("reopen") || l.contains("restore tab") || l.contains("undo close") {
|
||||
Some(S::ReopenTab)
|
||||
} else if l.contains("incognito")
|
||||
|| l.contains("inprivate")
|
||||
|| l.contains("private window")
|
||||
|| l.contains("private browsing")
|
||||
{
|
||||
Some(S::PrivateWindow)
|
||||
} else if l.contains("new window") {
|
||||
Some(S::NewWindow)
|
||||
} else if l.contains("next tab") {
|
||||
Some(S::NextTab)
|
||||
} else if l.contains("previous tab") || l.contains("prev tab") {
|
||||
Some(S::PrevTab)
|
||||
} else if l.contains("last tab") {
|
||||
Some(S::LastTab)
|
||||
} else if l.contains("go back") || l.contains("navigate back") {
|
||||
Some(S::Back)
|
||||
} else if l.contains("go forward") || l.contains("navigate forward") {
|
||||
Some(S::Forward)
|
||||
} else if l.contains("address bar") || l.contains("url bar") || l.contains("location bar") {
|
||||
Some(S::FocusAddressBar)
|
||||
} else if has_word(&l, "history") {
|
||||
Some(S::History)
|
||||
} else if has_word(&l, "downloads") || l.contains("download list") {
|
||||
Some(S::Downloads)
|
||||
} else if l.contains("bookmark this")
|
||||
|| l.contains("bookmark the page")
|
||||
|| l.contains("bookmark page")
|
||||
|| l.contains("add bookmark")
|
||||
|| l.contains("save bookmark")
|
||||
{
|
||||
Some(S::BookmarkPage)
|
||||
} else if l.contains("zoom in") {
|
||||
Some(S::ZoomIn)
|
||||
} else if l.contains("zoom out") {
|
||||
Some(S::ZoomOut)
|
||||
} else if l.contains("reset zoom") || l.contains("actual size") || l.contains("default zoom") {
|
||||
Some(S::ZoomReset)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract a 1‑9 tab number following the word "tab" ("switch to tab 3"). Also
|
||||
/// accepts "tab number 3". `None` when no digit follows (so "new tab" / "next
|
||||
/// tab" don't match).
|
||||
fn tab_number(lower: &str) -> Option<u8> {
|
||||
let idx = lower.find("tab ")?;
|
||||
let rest = lower[idx + "tab ".len()..].trim_start();
|
||||
let mut toks = rest.split_whitespace();
|
||||
let first = toks.next()?;
|
||||
let digit = if first == "number" || first == "no" || first == "#" {
|
||||
toks.next()?
|
||||
} else {
|
||||
first
|
||||
};
|
||||
let n: u8 = digit
|
||||
.trim_matches(|c: char| !c.is_ascii_digit())
|
||||
.parse()
|
||||
.ok()?;
|
||||
(1..=9).contains(&n).then_some(n)
|
||||
}
|
||||
|
||||
/// Map an in-page media-control goal to a YouTube keyboard shortcut, if any.
|
||||
fn extract_media_control(goal: &str) -> Option<Vec<String>> {
|
||||
let l = goal.to_lowercase();
|
||||
// Only treat as a *control* command when there's no navigation verb.
|
||||
if l.contains("open ") || l.contains("go to") || l.contains("search") {
|
||||
return None;
|
||||
}
|
||||
let key = |k: &str| Some(vec![k.to_string()]);
|
||||
if has_word(&l, "fullscreen") || l.contains("full screen") {
|
||||
key("f")
|
||||
} else if has_word(&l, "mute") || has_word(&l, "unmute") {
|
||||
key("m")
|
||||
} else if has_word(&l, "next") || has_word(&l, "skip") {
|
||||
Some(vec!["shift".to_string(), "n".to_string()])
|
||||
} else if has_word(&l, "previous") || has_word(&l, "back") {
|
||||
Some(vec!["shift".to_string(), "p".to_string()])
|
||||
} else if has_word(&l, "pause") || has_word(&l, "resume") || has_word(&l, "play") {
|
||||
// YouTube `k` toggles play/pause.
|
||||
key("k")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the browser fast-path.
|
||||
pub async fn run(app: &str, goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome {
|
||||
use super::super::automate::progress;
|
||||
use crate::openhuman::overlay::OverlayAttentionTone;
|
||||
|
||||
let mut steps: Vec<String> = Vec::new();
|
||||
let browser = match resolve_browser(app, goal) {
|
||||
Some(b) => b,
|
||||
None => return fail("no recognizable browser", steps),
|
||||
};
|
||||
|
||||
// Navigation takes priority over in-page control.
|
||||
if let Some(dest) = extract_destination(goal) {
|
||||
log::info!(
|
||||
"[automate::browser] ▶ open browser={browser:?} url={:?} is_play={}",
|
||||
dest.url,
|
||||
dest.is_play
|
||||
);
|
||||
progress(format!("Opening {browser}…"), OverlayAttentionTone::Accent);
|
||||
|
||||
match backend.open_url_in_app(&browser, &dest.url).await {
|
||||
Ok(m) => steps.push(format!("navigate: {m}")),
|
||||
Err(e) => {
|
||||
steps.push(format!("navigate FAILED: {e}"));
|
||||
return fail("could not open the browser/URL", steps);
|
||||
}
|
||||
}
|
||||
backend.settle(&browser).await;
|
||||
// Give the page network time to render before any follow-up.
|
||||
backend.wait(1200).await;
|
||||
|
||||
if dest.is_play {
|
||||
// Deterministic part done; defer the single "click first result" to
|
||||
// the general loop's vision_click (no reliable shortcut selects a
|
||||
// search result). Returning non-success makes the loop take over —
|
||||
// it does NOT re-launch from scratch, since the page is already up.
|
||||
steps.push("navigated to search results; deferring play-click to vision".to_string());
|
||||
return fail(
|
||||
"navigated; first-result click deferred to general loop",
|
||||
steps,
|
||||
);
|
||||
}
|
||||
|
||||
return AutomateOutcome {
|
||||
success: true,
|
||||
summary: format!("Opened {} in {browser}.", dest.url),
|
||||
steps,
|
||||
};
|
||||
}
|
||||
|
||||
// Browser-level command ("new tab", "go back", "reload", "tab 3") via the
|
||||
// cross-platform shortcut table — no AX, no navigation.
|
||||
if let Some(intent) = extract_browser_command(goal) {
|
||||
let keys = shortcut(intent, Browser::from_display(&browser), Os::current());
|
||||
let combo = keys.join("+");
|
||||
log::info!("[automate::browser] ▶ command {intent:?} browser={browser:?} keys={combo}");
|
||||
progress(format!("Pressing {combo}…"), OverlayAttentionTone::Accent);
|
||||
match backend.key(&keys).await {
|
||||
Ok(m) => {
|
||||
steps.push(format!("hotkey {combo}: {m}"));
|
||||
return AutomateOutcome {
|
||||
success: true,
|
||||
summary: format!("Sent {combo} to {browser}."),
|
||||
steps,
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
steps.push(format!("hotkey FAILED: {e}"));
|
||||
return fail("could not send the browser shortcut", steps);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// In-page media control via a YouTube keyboard shortcut.
|
||||
if let Some(keys) = extract_media_control(goal) {
|
||||
let combo = keys.join("+");
|
||||
log::info!("[automate::browser] ▶ media control browser={browser:?} keys={combo}");
|
||||
progress(format!("Pressing {combo}…"), OverlayAttentionTone::Accent);
|
||||
match backend.key(&keys).await {
|
||||
Ok(m) => {
|
||||
steps.push(format!("hotkey {combo}: {m}"));
|
||||
return AutomateOutcome {
|
||||
success: true,
|
||||
summary: format!("Sent {combo} to {browser}."),
|
||||
steps,
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
steps.push(format!("hotkey FAILED: {e}"));
|
||||
return fail("could not send the media shortcut", steps);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fail("no browser destination or control in goal", steps)
|
||||
}
|
||||
|
||||
fn fail(msg: &str, steps: Vec<String>) -> AutomateOutcome {
|
||||
AutomateOutcome {
|
||||
success: false,
|
||||
summary: format!("Browser fast-path: {msg}"),
|
||||
steps,
|
||||
}
|
||||
}
|
||||
|
||||
// ── small string helpers ────────────────────────────────────────────────────
|
||||
|
||||
/// True if `needle` appears as a whole word in `haystack` (already lowercased).
|
||||
fn has_word(haystack: &str, needle: &str) -> bool {
|
||||
word_index(haystack, needle).is_some()
|
||||
}
|
||||
|
||||
/// Byte index of `needle` as a whole word in `haystack` (already lowercased).
|
||||
fn word_index(haystack: &str, needle: &str) -> Option<usize> {
|
||||
let mut from = 0;
|
||||
while let Some(rel) = haystack[from..].find(needle) {
|
||||
let idx = from + rel;
|
||||
let before_ok = idx == 0
|
||||
|| !haystack[..idx]
|
||||
.chars()
|
||||
.next_back()
|
||||
.map(|c| c.is_alphanumeric())
|
||||
.unwrap_or(false);
|
||||
let after = idx + needle.len();
|
||||
let after_ok = haystack[after..]
|
||||
.chars()
|
||||
.next()
|
||||
.map(|c| !c.is_alphanumeric())
|
||||
.unwrap_or(true);
|
||||
if before_ok && after_ok {
|
||||
return Some(idx);
|
||||
}
|
||||
from = idx + needle.len();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Case-insensitive `strip_prefix`.
|
||||
fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
|
||||
if s.len() >= prefix.len()
|
||||
&& s.is_char_boundary(prefix.len())
|
||||
&& s[..prefix.len()].to_lowercase() == prefix.to_lowercase()
|
||||
{
|
||||
Some(&s[prefix.len()..])
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Percent-encode reserved characters in a query value (enough for a `?q=`
|
||||
/// search param; not a full RFC-3986 encoder). Mirrors `music::percent_encode`.
|
||||
fn percent_encode(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for b in s.bytes() {
|
||||
match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||
out.push(b as char)
|
||||
}
|
||||
_ => out.push_str(&format!("%{b:02X}")),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod unit {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resolve_browser_from_app_then_goal_never_guesses() {
|
||||
// From the app arg.
|
||||
assert_eq!(
|
||||
resolve_browser("Brave Browser", "do something").as_deref(),
|
||||
Some("Brave Browser")
|
||||
);
|
||||
// From the goal text.
|
||||
assert_eq!(
|
||||
resolve_browser("", "open chrome and go to x.com").as_deref(),
|
||||
Some("Google Chrome")
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_browser("", "use safari to open apple.com").as_deref(),
|
||||
Some("Safari")
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_browser("", "open microsoft edge").as_deref(),
|
||||
Some("Microsoft Edge")
|
||||
);
|
||||
// Generic "browser" with no named product → None (the original bug: we
|
||||
// must NOT guess Chrome/Safari).
|
||||
assert_eq!(resolve_browser("", "open my browser"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_requires_browser_and_destination_or_control() {
|
||||
assert!(matches("Brave Browser", "go to youtube.com"));
|
||||
assert!(matches("", "open chrome and search youtube for lofi"));
|
||||
assert!(matches("Brave Browser", "pause the video"));
|
||||
// Browser but no destination/control.
|
||||
assert!(!matches("Brave Browser", "hello there"));
|
||||
// Destination but no resolvable browser.
|
||||
assert!(!matches("Slack", "go to youtube.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destination_youtube_search_encodes_query() {
|
||||
let d = extract_destination("go to youtube and play a music video").unwrap();
|
||||
assert_eq!(
|
||||
d.url,
|
||||
"https://www.youtube.com/results?search_query=music%20video"
|
||||
);
|
||||
assert!(d.is_play);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destination_youtube_search_for_phrasing() {
|
||||
let d = extract_destination("search youtube for lofi beats").unwrap();
|
||||
assert_eq!(
|
||||
d.url,
|
||||
"https://www.youtube.com/results?search_query=lofi%20beats"
|
||||
);
|
||||
assert!(!d.is_play); // no "play" word → navigation only
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destination_bare_youtube_no_query() {
|
||||
let d = extract_destination("open youtube").unwrap();
|
||||
assert_eq!(d.url, "https://www.youtube.com");
|
||||
assert!(!d.is_play);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destination_bare_domain_normalized_to_https() {
|
||||
let d = extract_destination("go to example.com").unwrap();
|
||||
assert_eq!(d.url, "https://example.com");
|
||||
let d2 = extract_destination("open https://news.ycombinator.com").unwrap();
|
||||
assert_eq!(d2.url, "https://news.ycombinator.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destination_google_search() {
|
||||
let d = extract_destination("google rust async traits").unwrap();
|
||||
assert_eq!(
|
||||
d.url,
|
||||
"https://www.google.com/search?q=rust%20async%20traits"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destination_none_when_no_target() {
|
||||
assert!(extract_destination("just hang out").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_control_maps_to_youtube_shortcuts() {
|
||||
assert_eq!(
|
||||
extract_media_control("pause the video"),
|
||||
Some(vec!["k".into()])
|
||||
);
|
||||
assert_eq!(
|
||||
extract_media_control("resume playback"),
|
||||
Some(vec!["k".into()])
|
||||
);
|
||||
assert_eq!(extract_media_control("mute it"), Some(vec!["m".into()]));
|
||||
assert_eq!(
|
||||
extract_media_control("next video"),
|
||||
Some(vec!["shift".into(), "n".into()])
|
||||
);
|
||||
assert_eq!(
|
||||
extract_media_control("go fullscreen"),
|
||||
Some(vec!["f".into()])
|
||||
);
|
||||
// A navigation goal is NOT a control command.
|
||||
assert_eq!(extract_media_control("open youtube and play lofi"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn word_index_is_whole_word() {
|
||||
// "play" must not match inside "display"/"playback".
|
||||
assert!(has_word("play a song", "play"));
|
||||
assert!(!has_word("display settings", "play"));
|
||||
assert!(!has_word("open playback options", "play"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_command_maps_common_verbs() {
|
||||
use BrowserShortcut as S;
|
||||
assert_eq!(extract_browser_command("open a new tab"), Some(S::NewTab));
|
||||
assert_eq!(extract_browser_command("close this tab"), Some(S::CloseTab));
|
||||
assert_eq!(
|
||||
extract_browser_command("reopen the closed tab"),
|
||||
Some(S::ReopenTab)
|
||||
);
|
||||
assert_eq!(extract_browser_command("go back"), Some(S::Back));
|
||||
assert_eq!(extract_browser_command("reload the page"), Some(S::Reload));
|
||||
assert_eq!(extract_browser_command("hard reload"), Some(S::HardReload));
|
||||
assert_eq!(extract_browser_command("next tab"), Some(S::NextTab));
|
||||
assert_eq!(
|
||||
extract_browser_command("open an incognito window"),
|
||||
Some(S::PrivateWindow)
|
||||
);
|
||||
assert_eq!(extract_browser_command("show my history"), Some(S::History));
|
||||
assert_eq!(extract_browser_command("zoom in"), Some(S::ZoomIn));
|
||||
// No browser command.
|
||||
assert_eq!(extract_browser_command("play a music video"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tab_number_only_with_digit() {
|
||||
assert_eq!(tab_number("switch to tab 3"), Some(3));
|
||||
assert_eq!(tab_number("go to tab number 5"), Some(5));
|
||||
// "new tab"/"next tab" have no trailing digit → not a tab-number jump.
|
||||
assert_eq!(tab_number("open a new tab"), None);
|
||||
assert_eq!(tab_number("next tab"), None);
|
||||
assert_eq!(
|
||||
extract_browser_command("switch to tab 4"),
|
||||
Some(BrowserShortcut::TabN(4))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Live integration test — drives a real browser. Ignored by default (needs a
|
||||
/// browser installed + Accessibility/Screen-recording permission). Asserts
|
||||
/// tool-level success only; the visual page state is best-effort.
|
||||
///
|
||||
/// cargo test --lib browser_fastpath_live -- --ignored --nocapture
|
||||
#[cfg(all(test, target_os = "macos"))]
|
||||
mod live {
|
||||
use super::run;
|
||||
use crate::openhuman::accessibility::automate::RealBackend;
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires macOS + a browser + Accessibility permission"]
|
||||
async fn browser_fastpath_live() {
|
||||
let backend = RealBackend::new(crate::openhuman::config::Config::default());
|
||||
// Use Safari (always present on macOS) for a deterministic nav.
|
||||
let out = run("Safari", "open safari and go to example.com", &backend).await;
|
||||
println!(
|
||||
"[browser_fastpath_live] success={} summary={}",
|
||||
out.success, out.summary
|
||||
);
|
||||
for s in &out.steps {
|
||||
println!(" - {s}");
|
||||
}
|
||||
assert!(
|
||||
out.success,
|
||||
"nav fast-path reported failure: {}",
|
||||
out.summary
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
//! Cross-platform browser keyboard-shortcut table (Change 1.17).
|
||||
//!
|
||||
//! The four major desktop browsers (Chrome, Firefox, Safari, Edge) share almost
|
||||
//! the same navigation/tab/find shortcuts — they differ mainly by the primary
|
||||
//! modifier (⌘ on macOS, Ctrl on Windows/Linux), with a handful of real
|
||||
//! per-browser exceptions (Firefox's `Alt+1‑8` tab selection on Linux, its
|
||||
//! `Ctrl/⌘+Shift+P` private window, per-browser History/Downloads keys, etc.).
|
||||
//!
|
||||
//! [`shortcut`] resolves `(intent, browser, os)` to a key chord expressed as the
|
||||
//! same key names the keyboard tool's `parse_key` understands (`"Cmd"`,
|
||||
//! `"Ctrl"`, `"Shift"`, `"Alt"`, single chars, `"left"`, `"tab"`, `"f11"`, …),
|
||||
//! so the result feeds straight into `AutomateBackend::key`.
|
||||
//!
|
||||
//! Sources: Chrome/Firefox/Edge support pages (Win/macOS/Linux columns) and the
|
||||
//! Safari for Mac guide. In-page media shortcuts (YouTube `k`/`f`/`m`/`Shift+N`)
|
||||
//! are browser-independent and live in `browser.rs`, not here.
|
||||
|
||||
/// Target operating system — selects the primary modifier and the non-Mac
|
||||
/// fallbacks (Alt+arrow history, F11 fullscreen, etc.).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Os {
|
||||
Mac,
|
||||
Windows,
|
||||
Linux,
|
||||
}
|
||||
|
||||
impl Os {
|
||||
/// The OS this build is running on.
|
||||
pub fn current() -> Os {
|
||||
if cfg!(target_os = "macos") {
|
||||
Os::Mac
|
||||
} else if cfg!(target_os = "windows") {
|
||||
Os::Windows
|
||||
} else {
|
||||
Os::Linux
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Browser family — chosen by [`Browser::from_display`]. Chromium-based browsers
|
||||
/// (Brave, Arc, Chromium) share Chrome's shortcuts.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Browser {
|
||||
Chrome,
|
||||
Firefox,
|
||||
Safari,
|
||||
Edge,
|
||||
}
|
||||
|
||||
impl Browser {
|
||||
/// Map a macOS display name (or alias) to a family. Unknown / generic names
|
||||
/// default to Chrome, since most other desktop browsers are Chromium-based
|
||||
/// and share its bindings.
|
||||
pub fn from_display(name: &str) -> Browser {
|
||||
let n = name.to_lowercase();
|
||||
if n.contains("firefox") {
|
||||
Browser::Firefox
|
||||
} else if n.contains("safari") {
|
||||
Browser::Safari
|
||||
} else if n.contains("edge") {
|
||||
Browser::Edge
|
||||
} else {
|
||||
// Chrome, Brave, Arc, Chromium, Vivaldi, Opera, … → Chrome bindings.
|
||||
Browser::Chrome
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A browser action we can trigger with a keyboard shortcut.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BrowserShortcut {
|
||||
FocusAddressBar,
|
||||
NewTab,
|
||||
CloseTab,
|
||||
ReopenTab,
|
||||
NewWindow,
|
||||
PrivateWindow,
|
||||
NextTab,
|
||||
PrevTab,
|
||||
/// Jump to tab N (1-based, 1‑8). N≥9 is treated as the last tab.
|
||||
TabN(u8),
|
||||
LastTab,
|
||||
Find,
|
||||
FindNext,
|
||||
FindPrev,
|
||||
Reload,
|
||||
HardReload,
|
||||
ZoomIn,
|
||||
ZoomOut,
|
||||
ZoomReset,
|
||||
Print,
|
||||
BookmarkPage,
|
||||
Back,
|
||||
Forward,
|
||||
History,
|
||||
Downloads,
|
||||
Fullscreen,
|
||||
}
|
||||
|
||||
/// The primary modifier: ⌘ on macOS, Ctrl elsewhere.
|
||||
fn primary(os: Os) -> &'static str {
|
||||
match os {
|
||||
Os::Mac => "Cmd",
|
||||
_ => "Ctrl",
|
||||
}
|
||||
}
|
||||
|
||||
fn keys(parts: &[&str]) -> Vec<String> {
|
||||
parts.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
/// Resolve a shortcut to a key chord for the given browser + OS.
|
||||
pub fn shortcut(intent: BrowserShortcut, browser: Browser, os: Os) -> Vec<String> {
|
||||
use BrowserShortcut as S;
|
||||
let p = primary(os);
|
||||
match intent {
|
||||
// ── uniform across all four browsers (only ⌘↔Ctrl differs) ──
|
||||
S::FocusAddressBar => keys(&[p, "l"]),
|
||||
S::NewTab => keys(&[p, "t"]),
|
||||
S::CloseTab => keys(&[p, "w"]),
|
||||
S::ReopenTab => keys(&[p, "shift", "t"]),
|
||||
S::NewWindow => keys(&[p, "n"]),
|
||||
// Ctrl+Tab cycles tabs on every OS/browser (not ⌘ on macOS).
|
||||
S::NextTab => keys(&["ctrl", "tab"]),
|
||||
S::PrevTab => keys(&["ctrl", "shift", "tab"]),
|
||||
S::Find => keys(&[p, "f"]),
|
||||
S::FindNext => keys(&[p, "g"]),
|
||||
S::FindPrev => keys(&[p, "shift", "g"]),
|
||||
S::Reload => keys(&[p, "r"]),
|
||||
S::HardReload => keys(&[p, "shift", "r"]),
|
||||
// Use "=" (Ctrl/⌘+=) for zoom-in: it's the unshifted key and every
|
||||
// browser accepts it as zoom-in, avoiding the Shift needed to type "+".
|
||||
S::ZoomIn => keys(&[p, "="]),
|
||||
S::ZoomOut => keys(&[p, "-"]),
|
||||
S::ZoomReset => keys(&[p, "0"]),
|
||||
S::Print => keys(&[p, "p"]),
|
||||
S::BookmarkPage => keys(&[p, "d"]),
|
||||
|
||||
// ── per-browser / per-OS exceptions ──
|
||||
S::PrivateWindow => match browser {
|
||||
Browser::Firefox => keys(&[p, "shift", "p"]),
|
||||
_ => keys(&[p, "shift", "n"]),
|
||||
},
|
||||
S::TabN(n) => {
|
||||
let d = n.clamp(1, 8).to_string();
|
||||
// Firefox on Linux uses Alt+1‑8 (not Ctrl).
|
||||
if browser == Browser::Firefox && os == Os::Linux {
|
||||
keys(&["alt", &d])
|
||||
} else {
|
||||
keys(&[p, &d])
|
||||
}
|
||||
}
|
||||
S::LastTab => {
|
||||
if browser == Browser::Firefox && os == Os::Linux {
|
||||
keys(&["alt", "9"])
|
||||
} else {
|
||||
keys(&[p, "9"])
|
||||
}
|
||||
}
|
||||
S::Back => match os {
|
||||
Os::Mac => keys(&[p, "["]),
|
||||
_ => keys(&["alt", "left"]),
|
||||
},
|
||||
S::Forward => match os {
|
||||
Os::Mac => keys(&[p, "]"]),
|
||||
_ => keys(&["alt", "right"]),
|
||||
},
|
||||
S::History => match os {
|
||||
Os::Mac => match browser {
|
||||
// Chrome/Edge: ⌘Y. Firefox/Safari: ⌘⇧H.
|
||||
Browser::Chrome | Browser::Edge => keys(&[p, "y"]),
|
||||
Browser::Firefox | Browser::Safari => keys(&[p, "shift", "h"]),
|
||||
},
|
||||
_ => keys(&["ctrl", "h"]),
|
||||
},
|
||||
S::Downloads => match os {
|
||||
Os::Mac => match browser {
|
||||
Browser::Chrome => keys(&[p, "shift", "j"]),
|
||||
Browser::Firefox => keys(&[p, "j"]),
|
||||
// Edge / Safari: ⌥⌘L.
|
||||
Browser::Edge | Browser::Safari => keys(&[p, "option", "l"]),
|
||||
},
|
||||
_ => keys(&["ctrl", "j"]),
|
||||
},
|
||||
S::Fullscreen => match os {
|
||||
// Avoid the Fn key: ⌃⌘F is the standard macOS window fullscreen
|
||||
// (Chrome/Edge/Safari); Firefox uses ⌘⇧F.
|
||||
Os::Mac => match browser {
|
||||
Browser::Firefox => keys(&[p, "shift", "f"]),
|
||||
_ => keys(&["ctrl", "cmd", "f"]),
|
||||
},
|
||||
_ => keys(&["f11"]),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use BrowserShortcut as S;
|
||||
|
||||
#[test]
|
||||
fn primary_modifier_per_os() {
|
||||
assert_eq!(
|
||||
shortcut(S::FocusAddressBar, Browser::Chrome, Os::Mac),
|
||||
vec!["Cmd", "l"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::FocusAddressBar, Browser::Chrome, Os::Windows),
|
||||
vec!["Ctrl", "l"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::FocusAddressBar, Browser::Safari, Os::Mac),
|
||||
vec!["Cmd", "l"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uniform_core_shortcuts() {
|
||||
assert_eq!(
|
||||
shortcut(S::NewTab, Browser::Edge, Os::Linux),
|
||||
vec!["Ctrl", "t"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::ReopenTab, Browser::Firefox, Os::Mac),
|
||||
vec!["Cmd", "shift", "t"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::HardReload, Browser::Chrome, Os::Windows),
|
||||
vec!["Ctrl", "shift", "r"]
|
||||
);
|
||||
// Next/prev tab is Ctrl-based on every OS, even macOS.
|
||||
assert_eq!(
|
||||
shortcut(S::NextTab, Browser::Chrome, Os::Mac),
|
||||
vec!["ctrl", "tab"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_window_firefox_differs() {
|
||||
assert_eq!(
|
||||
shortcut(S::PrivateWindow, Browser::Firefox, Os::Windows),
|
||||
vec!["Ctrl", "shift", "p"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::PrivateWindow, Browser::Chrome, Os::Windows),
|
||||
vec!["Ctrl", "shift", "n"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tab_n_firefox_linux_uses_alt() {
|
||||
assert_eq!(
|
||||
shortcut(S::TabN(3), Browser::Firefox, Os::Linux),
|
||||
vec!["alt", "3"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::TabN(3), Browser::Chrome, Os::Linux),
|
||||
vec!["Ctrl", "3"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::TabN(3), Browser::Firefox, Os::Mac),
|
||||
vec!["Cmd", "3"]
|
||||
);
|
||||
// Out-of-range clamps into 1‑8.
|
||||
assert_eq!(
|
||||
shortcut(S::TabN(20), Browser::Chrome, Os::Mac),
|
||||
vec!["Cmd", "8"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::LastTab, Browser::Firefox, Os::Linux),
|
||||
vec!["alt", "9"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn back_forward_os_specific() {
|
||||
assert_eq!(
|
||||
shortcut(S::Back, Browser::Chrome, Os::Mac),
|
||||
vec!["Cmd", "["]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::Back, Browser::Chrome, Os::Windows),
|
||||
vec!["alt", "left"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::Forward, Browser::Safari, Os::Mac),
|
||||
vec!["Cmd", "]"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_and_downloads_per_browser_on_mac() {
|
||||
assert_eq!(
|
||||
shortcut(S::History, Browser::Chrome, Os::Mac),
|
||||
vec!["Cmd", "y"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::History, Browser::Firefox, Os::Mac),
|
||||
vec!["Cmd", "shift", "h"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::History, Browser::Edge, Os::Windows),
|
||||
vec!["ctrl", "h"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::Downloads, Browser::Chrome, Os::Mac),
|
||||
vec!["Cmd", "shift", "j"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::Downloads, Browser::Firefox, Os::Mac),
|
||||
vec!["Cmd", "j"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::Downloads, Browser::Edge, Os::Mac),
|
||||
vec!["Cmd", "option", "l"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::Downloads, Browser::Chrome, Os::Linux),
|
||||
vec!["ctrl", "j"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fullscreen_avoids_fn_key() {
|
||||
assert_eq!(
|
||||
shortcut(S::Fullscreen, Browser::Chrome, Os::Mac),
|
||||
vec!["ctrl", "cmd", "f"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::Fullscreen, Browser::Firefox, Os::Mac),
|
||||
vec!["Cmd", "shift", "f"]
|
||||
);
|
||||
assert_eq!(
|
||||
shortcut(S::Fullscreen, Browser::Chrome, Os::Windows),
|
||||
vec!["f11"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_display_maps_families() {
|
||||
assert_eq!(Browser::from_display("Brave Browser"), Browser::Chrome);
|
||||
assert_eq!(Browser::from_display("Arc"), Browser::Chrome);
|
||||
assert_eq!(Browser::from_display("Google Chrome"), Browser::Chrome);
|
||||
assert_eq!(Browser::from_display("Firefox"), Browser::Firefox);
|
||||
assert_eq!(Browser::from_display("Microsoft Edge"), Browser::Edge);
|
||||
assert_eq!(Browser::from_display("Safari"), Browser::Safari);
|
||||
// Unknown → Chrome default.
|
||||
assert_eq!(Browser::from_display("Some New Browser"), Browser::Chrome);
|
||||
}
|
||||
}
|
||||
@@ -104,6 +104,8 @@ struct Backend {
|
||||
/// Elements returned by perceive (the search results screen).
|
||||
elements: Vec<AXElement>,
|
||||
press_fail_on: Option<String>,
|
||||
/// What `now_playing()` reports back (None = backend can't read the track).
|
||||
now_playing: Option<(String, String)>,
|
||||
}
|
||||
|
||||
impl Backend {
|
||||
@@ -112,8 +114,13 @@ impl Backend {
|
||||
acts: Mutex::new(Vec::new()),
|
||||
elements,
|
||||
press_fail_on: None,
|
||||
now_playing: None,
|
||||
}
|
||||
}
|
||||
fn with_now_playing(mut self, name: &str, artist: &str) -> Self {
|
||||
self.now_playing = Some((name.to_string(), artist.to_string()));
|
||||
self
|
||||
}
|
||||
fn acts(&self) -> Vec<String> {
|
||||
self.acts.lock().unwrap().clone()
|
||||
}
|
||||
@@ -148,6 +155,23 @@ impl AutomateBackend for Backend {
|
||||
self.acts.lock().unwrap().push(format!("open_url:{url}"));
|
||||
Ok("ok".into())
|
||||
}
|
||||
async fn open_url_in_app(&self, app: &str, url: &str) -> Result<String, String> {
|
||||
self.acts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(format!("open_url_in_app:{app}:{url}"));
|
||||
Ok("ok".into())
|
||||
}
|
||||
async fn key(&self, keys: &[String]) -> Result<String, String> {
|
||||
self.acts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(format!("key:{}", keys.join("+")));
|
||||
Ok("ok".into())
|
||||
}
|
||||
async fn now_playing(&self) -> Option<(String, String)> {
|
||||
self.now_playing.clone()
|
||||
}
|
||||
async fn settle(&self, _app: &str) {}
|
||||
async fn wait(&self, _ms: u64) {}
|
||||
}
|
||||
@@ -170,12 +194,88 @@ async fn music_fastpath_full_sequence() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn music_fastpath_no_row_fails_for_fallthrough() {
|
||||
// Search screen has nothing matching → fast-path fails (loop falls through).
|
||||
let backend = Backend::new(vec![AXElement::new("AXButton", "Some Unrelated Button")]);
|
||||
let out = music::run("play Numb", &backend).await;
|
||||
async fn music_fastpath_reports_verified_track_and_artist() {
|
||||
// now_playing matches the requested artist → success names the real track.
|
||||
let backend = Backend::new(vec![song_row("Numb"), AXElement::new("AXButton", "Play")])
|
||||
.with_now_playing("Numb", "Linkin Park");
|
||||
let out = music::run("play Numb by Linkin Park", &backend).await;
|
||||
assert!(out.success, "{out:?}");
|
||||
assert!(
|
||||
out.summary.contains("Linkin Park") && out.summary.contains("Numb"),
|
||||
"summary should name the verified track+artist: {}",
|
||||
out.summary
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn music_fastpath_flags_wrong_artist_honestly() {
|
||||
// The search landed on a same-titled song by a different artist. The
|
||||
// fast-path must name it AND steer the agent away from re-searching (#1/#3).
|
||||
let backend = Backend::new(vec![song_row("Numb"), AXElement::new("AXButton", "Play")])
|
||||
.with_now_playing("Numb", "Tom Odell");
|
||||
let out = music::run("play Numb by Linkin Park", &backend).await;
|
||||
let s = out.summary.to_lowercase();
|
||||
assert!(
|
||||
s.contains("tom odell"),
|
||||
"must name what actually played: {}",
|
||||
out.summary
|
||||
);
|
||||
// Actionable, anti-loop guidance.
|
||||
assert!(
|
||||
s.contains("won't surface") && s.contains("library"),
|
||||
"must steer the agent (no blind re-search): {}",
|
||||
out.summary
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn music_fastpath_no_row_lists_candidates_and_warns() {
|
||||
// Song rows exist but none match the query → response lists what WAS found
|
||||
// and tells the agent not to repeat the same search (#1/#2).
|
||||
let backend = Backend::new(vec![
|
||||
AXElement::new("AXCell", "Numb - Marshmello & Khalid"),
|
||||
AXElement::new("AXCell", "Numb - Tom Odell"),
|
||||
]);
|
||||
let out = music::run("play Zelda Theme by Koji Kondo", &backend).await;
|
||||
assert!(!out.success);
|
||||
assert!(out.summary.contains("no matching song"), "{}", out.summary);
|
||||
let s = out.summary.to_lowercase();
|
||||
assert!(
|
||||
s.contains("marshmello") && s.contains("tom odell"),
|
||||
"{}",
|
||||
out.summary
|
||||
);
|
||||
assert!(
|
||||
s.contains("won't help") || s.contains("don't repeat"),
|
||||
"{}",
|
||||
out.summary
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn music_fastpath_artist_row_not_claimed_as_playing() {
|
||||
// Only an artist AXButton matches (no song cell) and the backend can't read
|
||||
// a track → must NOT claim "Playing"; say it only navigated (#3/#4).
|
||||
let backend = Backend::new(vec![AXElement::new("AXButton", "LINKIN PARK")]);
|
||||
let out = music::run("play Linkin Park Numb", &backend).await;
|
||||
let s = out.summary.to_lowercase();
|
||||
assert!(
|
||||
!s.contains("playing '"),
|
||||
"must not claim playback: {}",
|
||||
out.summary
|
||||
);
|
||||
assert!(
|
||||
s.contains("artist/album") && s.contains("specific song"),
|
||||
"must explain it only navigated: {}",
|
||||
out.summary
|
||||
);
|
||||
// It pressed the artist element, flagged as non-song in the step log.
|
||||
assert!(
|
||||
out.steps
|
||||
.iter()
|
||||
.any(|a| a.contains("artist/album 'LINKIN PARK'")),
|
||||
"{:?}",
|
||||
out.steps
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -203,6 +303,121 @@ async fn try_fastpath_dispatches_music_and_skips_others() {
|
||||
.is_some());
|
||||
}
|
||||
|
||||
// ── Browser fast-path: scripted sequence ────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn browser_nav_to_domain_succeeds_without_model() {
|
||||
let backend = Backend::new(vec![]);
|
||||
let out = super::browser::run(
|
||||
"Brave Browser",
|
||||
"open Brave and go to example.com",
|
||||
&backend,
|
||||
)
|
||||
.await;
|
||||
assert!(out.success, "pure navigation should succeed: {out:?}");
|
||||
let acts = backend.acts();
|
||||
// One deterministic open in the named browser — no model `decide` call
|
||||
// (the scripted backend panics if `decide` is hit). Bare domain is
|
||||
// normalized to https://.
|
||||
assert_eq!(acts.len(), 1, "{acts:?}");
|
||||
assert_eq!(acts[0], "open_url_in_app:Brave Browser:https://example.com");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn browser_youtube_search_play_navigates_then_falls_through() {
|
||||
let backend = Backend::new(vec![]);
|
||||
let out = super::browser::run(
|
||||
"Brave Browser",
|
||||
"open my brave browser, go to youtube.com and play a music video",
|
||||
&backend,
|
||||
)
|
||||
.await;
|
||||
// Play intent → navigate deterministically, then return non-success so the
|
||||
// general loop performs the single first-result click via vision_click.
|
||||
assert!(!out.success, "play must defer the final click: {out:?}");
|
||||
let acts = backend.acts();
|
||||
assert_eq!(acts.len(), 1, "{acts:?}");
|
||||
assert_eq!(
|
||||
acts[0],
|
||||
"open_url_in_app:Brave Browser:https://www.youtube.com/results?search_query=music%20video"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn browser_media_control_sends_hotkey() {
|
||||
let backend = Backend::new(vec![]);
|
||||
let out = super::browser::run("Brave Browser", "pause the video", &backend).await;
|
||||
assert!(out.success, "media control should succeed: {out:?}");
|
||||
assert_eq!(backend.acts(), vec!["key:k".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn browser_command_routes_resolved_shortcut() {
|
||||
// "new tab" → a cross-platform chord resolved per browser+OS. Brave maps to
|
||||
// the Chrome family; the primary modifier is OS-dependent, so accept either.
|
||||
let backend = Backend::new(vec![]);
|
||||
let out = super::browser::run("Brave Browser", "open a new tab", &backend).await;
|
||||
assert!(out.success, "browser command should succeed: {out:?}");
|
||||
let act = &backend.acts()[0];
|
||||
assert!(
|
||||
act == "key:Cmd+t" || act == "key:Ctrl+t",
|
||||
"expected a new-tab chord, got {act}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_fastpath_dispatches_browser() {
|
||||
let backend = Backend::new(vec![]);
|
||||
let out = super::try_fastpath(
|
||||
"Brave Browser",
|
||||
"open Brave and go to example.com",
|
||||
&backend,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
out.is_some(),
|
||||
"browser nav should be claimed by a fast-path"
|
||||
);
|
||||
assert!(out.unwrap().success);
|
||||
}
|
||||
|
||||
// ── App-shortcut fast-path (Spotify / Apple Music / Slack) ──────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn app_shortcut_spotify_next_sends_down_arrow() {
|
||||
let backend = Backend::new(vec![]);
|
||||
let out = super::app_shortcuts::run("Spotify", "next song", &backend).await;
|
||||
assert!(out.success, "{out:?}");
|
||||
// Spotify quirk: next = Down arrow.
|
||||
assert_eq!(backend.acts(), vec!["key:down".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn app_shortcut_slack_quick_switcher() {
|
||||
let backend = Backend::new(vec![]);
|
||||
let out = super::app_shortcuts::run("Slack", "jump to a conversation", &backend).await;
|
||||
assert!(out.success, "{out:?}");
|
||||
let act = &backend.acts()[0];
|
||||
assert!(act == "key:Cmd+k" || act == "key:Ctrl+k", "got {act}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_fastpath_routes_apps_and_music_still_wins_play() {
|
||||
let backend = Backend::new(vec![song_row("Numb")]);
|
||||
// Apple Music "pause" → app-shortcut fast-path (music.rs declines: no query).
|
||||
assert!(super::try_fastpath("Music", "pause", &backend)
|
||||
.await
|
||||
.is_some());
|
||||
// Apple Music "play Numb" → still claimed (by music.rs, song search).
|
||||
assert!(super::try_fastpath("Music", "play Numb", &backend)
|
||||
.await
|
||||
.is_some());
|
||||
// Spotify "skip" → app-shortcut fast-path.
|
||||
assert!(super::try_fastpath("Spotify", "skip this track", &backend)
|
||||
.await
|
||||
.is_some());
|
||||
}
|
||||
|
||||
// Outcome type sanity: fast-paths build the same outcome the loop returns.
|
||||
#[test]
|
||||
fn outcome_shape() {
|
||||
|
||||
@@ -15,6 +15,9 @@
|
||||
use super::automate::AutomateBackend;
|
||||
use super::automate::AutomateOutcome;
|
||||
|
||||
mod app_shortcuts;
|
||||
mod browser;
|
||||
mod browser_shortcuts;
|
||||
mod music;
|
||||
|
||||
/// Try every registered fast-path; return the first that claims the (app, goal).
|
||||
@@ -26,6 +29,12 @@ pub async fn try_fastpath(
|
||||
if music::matches(app, goal) {
|
||||
return Some(music::run(goal, backend).await);
|
||||
}
|
||||
if browser::matches(app, goal) {
|
||||
return Some(browser::run(app, goal, backend).await);
|
||||
}
|
||||
if app_shortcuts::matches(app, goal) {
|
||||
return Some(app_shortcuts::run(app, goal, backend).await);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
|
||||
@@ -7,13 +7,32 @@
|
||||
//! the injectable [`AutomateBackend`], so the whole flow is unit-testable with a
|
||||
//! scripted backend — no live Music, no model.
|
||||
|
||||
use super::super::ax_interact::AXElement;
|
||||
use super::AutomateBackend;
|
||||
use super::AutomateOutcome;
|
||||
|
||||
const APP: &str = "Music";
|
||||
|
||||
/// Element roles that represent a tappable search result / song row.
|
||||
const ROW_ROLES: &[&str] = &["AXCell", "AXRow", "ListItem", "AXButton", "AXStaticText"];
|
||||
/// Roles that represent an actual song/track ROW — the preferred press target,
|
||||
/// because pressing one navigates into (and can play) the track.
|
||||
const SONG_ROLES: &[&str] = &["AXCell", "AXRow", "ListItem"];
|
||||
/// Secondary roles — artist names, album headers, static text. Pressing these
|
||||
/// usually only *navigates* (to an artist/album page), so they're a last resort
|
||||
/// and we flag them so the caller can report honestly.
|
||||
const OTHER_ROW_ROLES: &[&str] = &["AXButton", "AXStaticText"];
|
||||
|
||||
fn role_in(role: &str, set: &[&str]) -> bool {
|
||||
set.iter().any(|r| role.contains(r))
|
||||
}
|
||||
|
||||
/// A chosen press target plus whether it's a real song row (vs an artist/album
|
||||
/// element that only navigates). The caller uses `is_song` to avoid claiming
|
||||
/// "Playing X" when it merely opened an artist page.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct PickedRow {
|
||||
label: String,
|
||||
is_song: bool,
|
||||
}
|
||||
|
||||
/// Does this (app, goal) look like an Apple Music "play X" request?
|
||||
pub fn matches(app: &str, goal: &str) -> bool {
|
||||
@@ -88,6 +107,38 @@ pub fn extract_play_query(goal: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Pull the requested artist out of a "play X by <artist>" goal — used to
|
||||
/// verify we played the *right* track. The Apple Music AX row label is
|
||||
/// title-only ("Numb - Single"), so a "Numb" search can resolve to the wrong
|
||||
/// artist; the artist lets us confirm via the now-playing track. `None` when no
|
||||
/// "by <artist>" clause is present.
|
||||
pub(crate) fn extract_artist(goal: &str) -> Option<String> {
|
||||
let lower = goal.to_lowercase();
|
||||
let p = lower.find(" by ")?;
|
||||
let after = &goal[p + " by ".len()..];
|
||||
// Cut at the first clause boundary.
|
||||
let mut end = after.len();
|
||||
for delim in [",", " and ", " then ", " in ", " on ", " from ", " for "] {
|
||||
if let Some(q) = after.to_lowercase().find(delim) {
|
||||
end = end.min(q);
|
||||
}
|
||||
}
|
||||
let artist = after[..end].trim().trim_matches('"').trim().to_string();
|
||||
if artist.is_empty() || is_pronoun(&artist) {
|
||||
None
|
||||
} else {
|
||||
Some(artist)
|
||||
}
|
||||
}
|
||||
|
||||
/// Loose artist comparison: case-insensitive, matching if either string
|
||||
/// contains the other (so "Linkin Park" matches "Linkin Park feat. …").
|
||||
fn artist_matches(want: &str, got: &str) -> bool {
|
||||
let w = want.trim().to_lowercase();
|
||||
let g = got.trim().to_lowercase();
|
||||
!w.is_empty() && !g.is_empty() && (g.contains(&w) || w.contains(&g))
|
||||
}
|
||||
|
||||
/// Strip a trailing "(in|on) [apple] music" and rewrite " by " → " ".
|
||||
fn clean_query(q: &str) -> String {
|
||||
let mut q = q.trim().to_string();
|
||||
@@ -216,26 +267,71 @@ fn first_token(query: &str) -> String {
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Choose the best matching row from a perceive snapshot: an exact label match
|
||||
/// first, else the first row-role element whose label shares a word with the
|
||||
/// query. Returns the element label to press.
|
||||
fn pick_row(elements: &[super::super::ax_interact::AXElement], query: &str) -> Option<String> {
|
||||
/// Choose the best press target. Preference order: an exact-label **song row**,
|
||||
/// then a token-matching **song row**, then any exact-label element, and only as
|
||||
/// a last resort an artist/album element (which merely navigates). Returns the
|
||||
/// label *and* whether it's a real song row, so the caller never falsely claims
|
||||
/// playback after pressing an artist/album header.
|
||||
///
|
||||
/// (We deliberately do NOT skip elements whose reported `enabled` is false —
|
||||
/// Apple Music marks pressable result rows as disabled; see AXElement::enabled.)
|
||||
fn pick_row(elements: &[AXElement], query: &str) -> Option<PickedRow> {
|
||||
let ql = query.to_lowercase();
|
||||
// Exact label match wins. (We deliberately do NOT skip elements whose
|
||||
// reported `enabled` is false — Apple Music marks pressable result rows as
|
||||
// disabled; see AXElement::enabled docs.)
|
||||
if let Some(e) = elements.iter().find(|e| e.label.to_lowercase() == ql) {
|
||||
return Some(e.label.clone());
|
||||
}
|
||||
let tokens: Vec<&str> = ql.split_whitespace().filter(|t| t.len() > 2).collect();
|
||||
let token_hit = |e: &&AXElement| {
|
||||
let l = e.label.to_lowercase();
|
||||
tokens.iter().any(|t| l.contains(t))
|
||||
};
|
||||
let song = |e: &&AXElement| role_in(&e.role, SONG_ROLES);
|
||||
|
||||
// 1. Exact match on a song row.
|
||||
if let Some(e) = elements
|
||||
.iter()
|
||||
.find(|e| song(e) && e.label.to_lowercase() == ql)
|
||||
{
|
||||
return Some(PickedRow {
|
||||
label: e.label.clone(),
|
||||
is_song: true,
|
||||
});
|
||||
}
|
||||
// 2. Token match on a song row.
|
||||
if let Some(e) = elements.iter().find(|e| song(e) && token_hit(e)) {
|
||||
return Some(PickedRow {
|
||||
label: e.label.clone(),
|
||||
is_song: true,
|
||||
});
|
||||
}
|
||||
// 3. Exact match on any element.
|
||||
if let Some(e) = elements.iter().find(|e| e.label.to_lowercase() == ql) {
|
||||
return Some(PickedRow {
|
||||
label: e.label.clone(),
|
||||
is_song: role_in(&e.role, SONG_ROLES),
|
||||
});
|
||||
}
|
||||
// 4. Last resort: an artist/album element that token-matches (navigates).
|
||||
elements
|
||||
.iter()
|
||||
.filter(|e| ROW_ROLES.iter().any(|r| e.role.contains(r)))
|
||||
.find(|e| {
|
||||
let l = e.label.to_lowercase();
|
||||
tokens.iter().any(|t| l.contains(t))
|
||||
.find(|e| role_in(&e.role, OTHER_ROW_ROLES) && token_hit(e))
|
||||
.map(|e| PickedRow {
|
||||
label: e.label.clone(),
|
||||
is_song: false,
|
||||
})
|
||||
.map(|e| e.label.clone())
|
||||
}
|
||||
|
||||
/// Up to 5 distinct song-row labels currently visible — surfaced in failure
|
||||
/// responses so the agent (or user) can see what *was* found and choose.
|
||||
fn candidate_labels(elements: &[AXElement]) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for e in elements.iter().filter(|e| role_in(&e.role, SONG_ROLES)) {
|
||||
let l = e.label.trim().to_string();
|
||||
if !l.is_empty() && !out.contains(&l) {
|
||||
out.push(l);
|
||||
if out.len() >= 5 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Run the play fast-path. Returns a failed [`AutomateOutcome`] (not a panic)
|
||||
@@ -249,7 +345,10 @@ pub async fn run(goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome {
|
||||
return fail("not a play request", steps);
|
||||
}
|
||||
};
|
||||
log::info!("[automate::music] ▶ play query={query:?}");
|
||||
// The artist the user asked for (if any), used to confirm we played the
|
||||
// right track — the AX row label alone can't disambiguate same-titled songs.
|
||||
let want_artist = extract_artist(goal);
|
||||
log::info!("[automate::music] ▶ play query={query:?} artist={want_artist:?}");
|
||||
use super::super::automate::progress;
|
||||
use crate::openhuman::overlay::OverlayAttentionTone;
|
||||
progress(
|
||||
@@ -278,14 +377,17 @@ pub async fn run(goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome {
|
||||
// filter the snapshot by one strong token (a substring filter can't
|
||||
// match a whole multi-word title).
|
||||
let filter = first_token(&query);
|
||||
let mut row = None;
|
||||
let mut row: Option<PickedRow> = None;
|
||||
let mut last_els: Vec<AXElement> = Vec::new();
|
||||
for attempt in 0..6 {
|
||||
backend.settle(APP).await;
|
||||
let els = backend.perceive(APP, &filter).await.unwrap_or_default();
|
||||
if let Some(r) = pick_row(&els, &query) {
|
||||
last_els = els;
|
||||
row = Some(r);
|
||||
break;
|
||||
}
|
||||
last_els = els;
|
||||
// Catalog search results arrive asynchronously (~3-4s); element-count
|
||||
// settle can report "stable" while the network fetch is still pending,
|
||||
// so wait real time between attempts rather than spinning instantly.
|
||||
@@ -293,19 +395,26 @@ pub async fn run(goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome {
|
||||
backend.wait(800).await;
|
||||
}
|
||||
let row = match row {
|
||||
// #1/#2: when nothing matched, list what WAS found and tell the agent
|
||||
// not to repeat the same search — so it tries the library / asks the
|
||||
// user instead of re-delegating the identical query.
|
||||
None => return fail(&no_match_message(&query, &want_artist, &last_els), steps),
|
||||
Some(r) => r,
|
||||
None => return fail("no matching song row found", steps),
|
||||
};
|
||||
// Baseline count of "Play" controls *before* navigating, so we can tell
|
||||
// when the song's detail-page Play has actually rendered (vs. only the
|
||||
// toolbar transport Play that's always present).
|
||||
let plays_before = count_play_buttons(backend).await;
|
||||
|
||||
match backend.act_press(APP, &row).await {
|
||||
Ok(m) => steps.push(format!("open song: {m}")),
|
||||
// #4: record what KIND of element we pressed — a song row that plays, or an
|
||||
// artist/album element that only navigates.
|
||||
let pressed_song = row.is_song;
|
||||
let kind = if pressed_song { "song" } else { "artist/album" };
|
||||
match backend.act_press(APP, &row.label).await {
|
||||
Ok(m) => steps.push(format!("open {kind} '{}': {m}", row.label)),
|
||||
Err(e) => {
|
||||
steps.push(format!("open song FAILED: {e}"));
|
||||
return fail("could not open the song", steps);
|
||||
steps.push(format!("open {kind} FAILED: {e}"));
|
||||
return fail("could not open the result", steps);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -349,25 +458,114 @@ pub async fn run(goal: &str, backend: &dyn AutomateBackend) -> AutomateOutcome {
|
||||
}
|
||||
}
|
||||
|
||||
match verified {
|
||||
Some(false) => {
|
||||
steps.push("verify: player state never reached 'playing'".to_string());
|
||||
fail("opened the song but playback didn't start", steps)
|
||||
}
|
||||
Some(true) => {
|
||||
steps.push("verify: playing ✓".to_string());
|
||||
progress(format!("Playing {query}"), OverlayAttentionTone::Success);
|
||||
if matches!(verified, Some(false)) {
|
||||
steps.push("verify: player state never reached 'playing'".to_string());
|
||||
return fail("opened the song but playback didn't start", steps);
|
||||
}
|
||||
|
||||
// 6. Confirm we played the RIGHT track. The AX row label carries only the
|
||||
// title, so "Numb" can land on the wrong artist (this is the exact bug we
|
||||
// hit live). Ask Music for the now-playing name + artist and check it
|
||||
// against the requested artist — so we never falsely claim success on a
|
||||
// wrong-artist match, and the summary names what's actually playing.
|
||||
let now = backend.now_playing().await;
|
||||
let candidates = candidate_labels(&last_els);
|
||||
match (&want_artist, &now) {
|
||||
// #1/#3: played, but the wrong artist — name what's actually playing,
|
||||
// list the alternatives, and tell the agent NOT to repeat the search
|
||||
// (success=true so this guidance reaches the user instead of being
|
||||
// discarded by a fall-through to the model loop).
|
||||
(Some(want), Some((name, got))) if !artist_matches(want, got) => {
|
||||
steps.push(format!(
|
||||
"verify: now playing '{name}' by '{got}' — wanted '{want}' (mismatch)"
|
||||
));
|
||||
progress(
|
||||
format!("Playing {name} — not {want}"),
|
||||
OverlayAttentionTone::Neutral,
|
||||
);
|
||||
AutomateOutcome {
|
||||
success: true,
|
||||
summary: format!("Playing '{query}' in Music."),
|
||||
summary: format!(
|
||||
"Now playing '{name}' by '{got}', not '{query}' by '{want}'. {}Re-running this search won't surface a different result — try the user's Library, or ask them to confirm the artist.",
|
||||
candidates_phrase(&candidates),
|
||||
),
|
||||
steps,
|
||||
}
|
||||
}
|
||||
None => AutomateOutcome {
|
||||
success: true,
|
||||
summary: format!("Started '{query}' in Music (playback unverified)."),
|
||||
steps,
|
||||
},
|
||||
// #3: verified the actual track (right artist, or none requested).
|
||||
(_, Some((name, got))) => {
|
||||
steps.push(format!("verify: now playing '{name}' by '{got}' ✓"));
|
||||
progress(format!("Playing {name}"), OverlayAttentionTone::Success);
|
||||
AutomateOutcome {
|
||||
success: true,
|
||||
summary: format!("Playing '{name}' by '{got}' in Music."),
|
||||
steps,
|
||||
}
|
||||
}
|
||||
// #3/#4: can't read the track. Never claim "Playing X" — and if we only
|
||||
// pressed an artist/album element (which just navigates), say so.
|
||||
(_, None) if !pressed_song => {
|
||||
steps.push("verify: pressed an artist/album element (navigation only)".to_string());
|
||||
AutomateOutcome {
|
||||
success: true,
|
||||
summary: format!(
|
||||
"Opened an artist/album page for '{query}' but no specific track started — pressing that only navigates. {}Open a specific song to play it.",
|
||||
candidates_phrase(&candidates),
|
||||
),
|
||||
steps,
|
||||
}
|
||||
}
|
||||
// Song row pressed, but the backend can't confirm the track (non-macOS).
|
||||
(_, None) => {
|
||||
let unverified = matches!(verified, None);
|
||||
steps.push(if unverified {
|
||||
"verify: playback unverified".to_string()
|
||||
} else {
|
||||
"verify: playing ✓ (track name unknown)".to_string()
|
||||
});
|
||||
if !unverified {
|
||||
progress(format!("Playing {query}"), OverlayAttentionTone::Success);
|
||||
}
|
||||
AutomateOutcome {
|
||||
success: true,
|
||||
summary: if unverified {
|
||||
format!("Started '{query}' in Music (playback unverified).")
|
||||
} else {
|
||||
format!("Playing '{query}' in Music.")
|
||||
},
|
||||
steps,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// "Results seen: a, b, c. " — or empty when nothing was captured. Lets failure
|
||||
/// responses show the agent/user the actual candidates to choose from.
|
||||
fn candidates_phrase(cands: &[String]) -> String {
|
||||
if cands.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("Results seen: {}. ", cands.join(", "))
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the "no match" response: list what was found (if anything) and steer
|
||||
/// the agent away from blindly repeating the identical search.
|
||||
fn no_match_message(query: &str, want_artist: &Option<String>, els: &[AXElement]) -> String {
|
||||
let cands = candidate_labels(els);
|
||||
let by = want_artist
|
||||
.as_deref()
|
||||
.map(|a| format!(" by '{a}'"))
|
||||
.unwrap_or_default();
|
||||
if cands.is_empty() {
|
||||
format!(
|
||||
"No song results found for '{query}'{by} — the search may not have loaded, or the track isn't in the catalog/library. Don't repeat this exact search; try the user's Library or ask them to confirm the title/artist."
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"Couldn't find '{query}'{by}. Results seen: {}. Re-running this search won't help — pick the closest match, try the Library, or ask the user.",
|
||||
cands.join(", ")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -423,17 +621,73 @@ mod unit {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pick_row_prefers_exact_then_token() {
|
||||
use super::super::super::ax_interact::AXElement;
|
||||
fn extract_artist_pulls_by_clause() {
|
||||
assert_eq!(
|
||||
extract_artist("play Numb by Linkin Park").as_deref(),
|
||||
Some("Linkin Park")
|
||||
);
|
||||
// Trailing clause is cut.
|
||||
assert_eq!(
|
||||
extract_artist("play Highway to Hell by AC/DC in Apple Music").as_deref(),
|
||||
Some("AC/DC")
|
||||
);
|
||||
assert_eq!(
|
||||
extract_artist("search for \"Numb\" by Linkin Park and play it").as_deref(),
|
||||
Some("Linkin Park")
|
||||
);
|
||||
// No "by" clause → None.
|
||||
assert_eq!(extract_artist("play Numb"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artist_matches_is_loose() {
|
||||
assert!(artist_matches("Linkin Park", "Linkin Park"));
|
||||
assert!(artist_matches("Linkin Park", "Linkin Park feat. Jay-Z"));
|
||||
assert!(artist_matches("ac/dc", "AC/DC"));
|
||||
assert!(!artist_matches("Linkin Park", "Tom Odell"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pick_row_prefers_song_row_over_artist() {
|
||||
// Token match (query has extra "AC/DC" the row label lacks).
|
||||
let els = vec![
|
||||
AXElement::new("AXCell", "Highway to Hell"),
|
||||
AXElement::new("AXButton", "Play"),
|
||||
];
|
||||
// Token match (query has extra "AC/DC" the row label lacks).
|
||||
assert_eq!(
|
||||
pick_row(&els, "Highway to Hell AC/DC").as_deref(),
|
||||
Some("Highway to Hell")
|
||||
);
|
||||
let p = pick_row(&els, "Highway to Hell AC/DC").unwrap();
|
||||
assert_eq!(p.label, "Highway to Hell");
|
||||
assert!(p.is_song);
|
||||
|
||||
// An artist AXButton and a song AXCell both token-match "Linkin Park
|
||||
// Numb" — the SONG row must win (the live bug pressed the artist).
|
||||
let els2 = vec![
|
||||
AXElement::new("AXButton", "LINKIN PARK"),
|
||||
AXElement::new("AXCell", "Numb"),
|
||||
];
|
||||
let p2 = pick_row(&els2, "Linkin Park Numb").unwrap();
|
||||
assert_eq!(p2.label, "Numb");
|
||||
assert!(p2.is_song);
|
||||
|
||||
// Only an artist element is present → chosen, but flagged non-song.
|
||||
let els3 = vec![AXElement::new("AXButton", "LINKIN PARK")];
|
||||
let p3 = pick_row(&els3, "Linkin Park Numb").unwrap();
|
||||
assert_eq!(p3.label, "LINKIN PARK");
|
||||
assert!(!p3.is_song);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_match_message_lists_candidates_and_warns() {
|
||||
let els = vec![
|
||||
AXElement::new("AXCell", "Numb - Marshmello & Khalid"),
|
||||
AXElement::new("AXCell", "Numb - Tom Odell"),
|
||||
AXElement::new("AXButton", "Play"), // not a song row → excluded
|
||||
];
|
||||
let m = no_match_message("Numb Linkin Park", &Some("Linkin Park".into()), &els);
|
||||
assert!(m.contains("Marshmello") && m.contains("Tom Odell"), "{m}");
|
||||
assert!(m.to_lowercase().contains("won't help"), "{m}");
|
||||
// Empty results → still actionable, no fake candidate list.
|
||||
let m2 = no_match_message("Numb", &None, &[]);
|
||||
assert!(m2.to_lowercase().contains("don't repeat"), "{m2}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -71,6 +71,9 @@ pub struct Action {
|
||||
/// Natural-language target for `vision_click` (e.g. "the green Call button").
|
||||
#[serde(default)]
|
||||
pub description: String,
|
||||
/// Key chord / single key for `hotkey` (e.g. `["Cmd","L"]`, `["/"]`).
|
||||
#[serde(default)]
|
||||
pub keys: Vec<String>,
|
||||
/// Final message for `done` / `fail`.
|
||||
#[serde(default)]
|
||||
pub summary: String,
|
||||
@@ -112,6 +115,25 @@ pub trait AutomateBackend: Send + Sync {
|
||||
/// Open a URL / URI-scheme (e.g. `music://…search?term=…`) via the OS opener.
|
||||
/// Used by deterministic app fast-paths; the general loop does not call it.
|
||||
async fn open_url(&self, url: &str) -> Result<String, String>;
|
||||
/// Open a URL in a **specific** app (e.g. a chosen browser) so navigation
|
||||
/// lands in the app the user named — `open_url` uses the *default* handler,
|
||||
/// which would send a `https://` link to whatever the default browser is.
|
||||
/// Default delegates to [`open_url`](Self::open_url) so non-browser backends
|
||||
/// stay correct. Used by the browser fast-path.
|
||||
async fn open_url_in_app(&self, _app: &str, url: &str) -> Result<String, String> {
|
||||
self.open_url(url).await
|
||||
}
|
||||
/// Send a keyboard chord (`["Cmd","L"]`) or a single key (`["/"]`) to the
|
||||
/// frontmost app. Lets fast-paths and the loop drive app shortcuts (focus
|
||||
/// the address bar, YouTube `/` search, `k`/space play-pause) instead of
|
||||
/// hunting AX labels. Default errors so input-less backends can't actuate.
|
||||
async fn key(&self, _keys: &[String]) -> Result<String, String> {
|
||||
Err("keyboard unsupported by this backend".to_string())
|
||||
}
|
||||
/// Type literal text into the frontmost app. Default errors (see [`key`]).
|
||||
async fn type_text(&self, _text: &str) -> Result<String, String> {
|
||||
Err("typing unsupported by this backend".to_string())
|
||||
}
|
||||
/// Best-effort: is media currently playing? `None` when the backend can't
|
||||
/// tell (non-macOS, or not applicable). Media fast-paths use this to confirm
|
||||
/// an action *actually started playback* rather than just succeeding at the
|
||||
@@ -119,6 +141,14 @@ pub trait AutomateBackend: Send + Sync {
|
||||
async fn verify_playing(&self) -> Option<bool> {
|
||||
None
|
||||
}
|
||||
/// The currently-playing track as `(name, artist)`, if the backend can read
|
||||
/// it. Used by the Music fast-path to confirm it played the *right* track
|
||||
/// (the AX row label carries only the title, so "Numb" can resolve to the
|
||||
/// wrong artist — see tracker §1.x). `None` when unknown (non-macOS, nothing
|
||||
/// playing, or not applicable).
|
||||
async fn now_playing(&self) -> Option<(String, String)> {
|
||||
None
|
||||
}
|
||||
/// Capture the target app's window + the geometry needed to map a click
|
||||
/// from image pixels to screen points. Used by the `vision_click` fallback
|
||||
/// for apps with no usable accessibility tree (Electron/Chromium). Default
|
||||
@@ -183,17 +213,21 @@ fn system_prompt() -> String {
|
||||
\n\
|
||||
Respond with EXACTLY ONE JSON object and nothing else:\n\
|
||||
{\"thought\":\"...\",\"action\":\"<verb>\",\"app\":\"<optional>\",\
|
||||
\"filter\":\"...\",\"label\":\"...\",\"value\":\"...\",\"summary\":\"...\"}\n\
|
||||
\"filter\":\"...\",\"label\":\"...\",\"value\":\"...\",\"keys\":[],\"summary\":\"...\"}\n\
|
||||
\n\
|
||||
Verbs:\n\
|
||||
• launch — open the app (use first if it isn't showing any elements)\n\
|
||||
• list — re-read elements; set `filter` to a substring to narrow them\n\
|
||||
• press — activate the element whose label matches `label`\n\
|
||||
• set_value — type `value` into the field matching `label` (omit label = first field)\n\
|
||||
• hotkey — send an app keyboard shortcut; put the chord in `keys` (modifiers \
|
||||
first, e.g. [\"Cmd\",\"L\"] to focus a browser address bar, [\"Cmd\",\"T\"] new tab) \
|
||||
or a single key (e.g. [\"/\"] to focus YouTube search, [\"k\"] play/pause, [\"f\"] \
|
||||
fullscreen). Prefer a known shortcut over hunting labels or clicking.\n\
|
||||
• vision_click — click an element by sight; put a short `description` of the \
|
||||
target (e.g. 'the green Call button'). Use this when the element list is \
|
||||
EMPTY or missing your target — common for Electron apps (Slack, Discord, \
|
||||
VS Code) that expose no accessibility tree.\n\
|
||||
EMPTY or missing your target — common for Electron/Chromium apps (browsers, \
|
||||
Slack, Discord, VS Code) that expose no accessibility tree.\n\
|
||||
• done — goal achieved; put a short result in `summary`\n\
|
||||
• fail — goal cannot be achieved; explain in `summary`\n\
|
||||
\n\
|
||||
@@ -203,8 +237,11 @@ fn system_prompt() -> String {
|
||||
(e.g. open a song, THEN press its 'Play'). After such a press, `list` again \
|
||||
to see the new screen.\n\
|
||||
- Prefer an exact label match. Keep `filter` specific so the snapshot stays small.\n\
|
||||
- If the app shows NO elements, prefer `vision_click` with a clear \
|
||||
`description` over guessing labels.\n\
|
||||
- For browsers and web apps, prefer `hotkey` for navigation and media control \
|
||||
(address bar, search focus, play/pause/next) — it's faster and more reliable \
|
||||
than clicking, and works even when the accessibility tree is empty.\n\
|
||||
- If the app shows NO elements, prefer `hotkey` (if a known shortcut applies) \
|
||||
or `vision_click` with a clear `description` over guessing labels.\n\
|
||||
- Output JSON only — no prose, no code fences."
|
||||
.to_string()
|
||||
}
|
||||
@@ -300,6 +337,10 @@ pub async fn run(
|
||||
// instead of burning the whole step budget.
|
||||
let mut last_sig = String::new();
|
||||
let mut repeat_count = 0u32;
|
||||
// Most recent rendered snapshot — surfaced in terminal failure responses so
|
||||
// the agent sees what was actually on screen (instead of a bare "budget
|
||||
// exhausted"), and can pick a real label next time.
|
||||
let mut last_snapshot = String::new();
|
||||
|
||||
for step in 0..opts.step_budget {
|
||||
// ── perceive ──
|
||||
@@ -310,6 +351,7 @@ pub async fn run(
|
||||
format!("(perceive error: {e})")
|
||||
}
|
||||
};
|
||||
last_snapshot = snapshot.clone();
|
||||
|
||||
// ── decide ──
|
||||
let user = format!(
|
||||
@@ -361,8 +403,12 @@ pub async fn run(
|
||||
// ── no-progress guard ──
|
||||
if !matches!(action.action.as_str(), "done" | "fail") {
|
||||
let sig = format!(
|
||||
"{}|{}|{}|{}",
|
||||
action.action, action.label, action.filter, action.description
|
||||
"{}|{}|{}|{}|{}",
|
||||
action.action,
|
||||
action.label,
|
||||
action.filter,
|
||||
action.description,
|
||||
action.keys.join("+")
|
||||
);
|
||||
if sig == last_sig {
|
||||
repeat_count += 1;
|
||||
@@ -378,7 +424,11 @@ pub async fn run(
|
||||
action.action
|
||||
));
|
||||
return AutomateOutcome::fail(
|
||||
"Got stuck repeating the same action with no progress.",
|
||||
format!(
|
||||
"Stuck repeating '{}' with no progress — that action isn't advancing the goal.{} Switch tactics: pick a specific label from the screen, take a screenshot + vision_click, or use a keyboard shortcut.",
|
||||
action.action,
|
||||
screen_hint(&last_snapshot),
|
||||
),
|
||||
steps,
|
||||
);
|
||||
}
|
||||
@@ -455,6 +505,19 @@ pub async fn run(
|
||||
}
|
||||
backend.settle(target_app).await;
|
||||
}
|
||||
"hotkey" => {
|
||||
if action.keys.is_empty() {
|
||||
steps.push("hotkey skipped: no keys".to_string());
|
||||
continue;
|
||||
}
|
||||
let combo = action.keys.join("+");
|
||||
progress(format!("Pressing {combo}…"), OverlayAttentionTone::Accent);
|
||||
match backend.key(&action.keys).await {
|
||||
Ok(msg) => steps.push(format!("hotkey: {msg}")),
|
||||
Err(e) => steps.push(format!("hotkey FAILED: {e}")),
|
||||
}
|
||||
backend.settle(target_app).await;
|
||||
}
|
||||
"vision_click" => {
|
||||
let description = action.description.trim();
|
||||
if description.is_empty() {
|
||||
@@ -527,13 +590,55 @@ pub async fn run(
|
||||
log::info!("{LOG_PREFIX} step budget ({}) exhausted", opts.step_budget);
|
||||
AutomateOutcome::fail(
|
||||
format!(
|
||||
"Step budget ({}) exhausted before the goal was confirmed complete.",
|
||||
opts.step_budget
|
||||
"Step budget ({}) exhausted before the goal was confirmed complete.{} Try a different approach (a screenshot/vision_click, a known keyboard shortcut, or a more specific filter) — repeating the same steps won't help.",
|
||||
opts.step_budget,
|
||||
screen_hint(&last_snapshot),
|
||||
),
|
||||
steps,
|
||||
)
|
||||
}
|
||||
|
||||
/// A compact " On screen: [role] a, [role] b, …" hint built from the last
|
||||
/// rendered snapshot, for failure responses. Empty when there's nothing useful.
|
||||
fn screen_hint(snapshot: &str) -> String {
|
||||
let labels: Vec<&str> = snapshot
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| l.starts_with('[') || l.starts_with("• ["))
|
||||
.take(10)
|
||||
.collect();
|
||||
if labels.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" On screen: {}.", labels.join("; "))
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a browser **display name** (as resolved by the browser fast-path —
|
||||
/// `"Google Chrome"`, `"Brave Browser"`, …) to the token the Windows shell
|
||||
/// `start` verb resolves via the `App Paths` registry. `None` for browsers that
|
||||
/// don't exist on Windows (Safari/Arc) or any unrecognized name, so the caller
|
||||
/// falls back to the default URL handler. Matched case-insensitively by
|
||||
/// substring so aliases ("Chrome", "Microsoft Edge") all resolve.
|
||||
#[cfg(target_os = "windows")]
|
||||
pub(crate) fn windows_browser_launch_token(app: &str) -> Option<&'static str> {
|
||||
let a = app.to_lowercase();
|
||||
// Order matters: check the more specific names first ("microsoft edge"
|
||||
// contains neither "chrome" nor "firefox", but keep edge before a bare
|
||||
// "chrome" check anyway for clarity).
|
||||
if a.contains("brave") {
|
||||
Some("brave")
|
||||
} else if a.contains("edge") {
|
||||
Some("msedge")
|
||||
} else if a.contains("firefox") {
|
||||
Some("firefox")
|
||||
} else if a.contains("chrome") || a.contains("chromium") {
|
||||
Some("chrome")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Production backend: real AX primitives + a fast LLM for decisions.
|
||||
pub struct RealBackend {
|
||||
config: crate::openhuman::config::Config,
|
||||
@@ -664,6 +769,84 @@ impl AutomateBackend for RealBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_url_in_app(&self, app: &str, url: &str) -> Result<String, String> {
|
||||
// macOS: `open -a "<app>" "<url>"` both launches/foregrounds the named
|
||||
// app AND opens the URL in it — exactly the deterministic browser nav we
|
||||
// want (no address-bar typing, no AX).
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
match tokio::process::Command::new("open")
|
||||
.arg("-a")
|
||||
.arg(app)
|
||||
.arg(url)
|
||||
.output()
|
||||
.await
|
||||
{
|
||||
Ok(o) if o.status.success() => Ok(format!("Opened {url} in {app}")),
|
||||
Ok(o) => Err(format!(
|
||||
"open -a {app} exited {}: {}",
|
||||
o.status,
|
||||
String::from_utf8_lossy(&o.stderr).trim()
|
||||
)),
|
||||
Err(e) => Err(format!("failed to launch opener: {e}")),
|
||||
}
|
||||
}
|
||||
// Windows: the shell `start` verb resolves a browser by its registered
|
||||
// App Paths token (`chrome`, `msedge`, `firefox`, `brave`, …) and opens
|
||||
// the URL in it. When the browser is already running this lands in a NEW
|
||||
// TAB of the existing window — so the deterministic fast-path does NOT
|
||||
// pile up windows (the live bug: each re-delegation `launch_app`-ed Chrome
|
||||
// again → ~10 windows). Falls back to the default handler when the named
|
||||
// browser has no known token (e.g. Safari/Arc, which aren't on Windows).
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
let Some(token) = windows_browser_launch_token(app) else {
|
||||
log::info!(
|
||||
"[automate] open_url_in_app: no Windows token for {app:?}; using default handler"
|
||||
);
|
||||
return self.open_url(url).await;
|
||||
};
|
||||
// `cmd /C start "" <token> "<url>"` — the empty "" is `start`'s title
|
||||
// arg (required so a quoted token isn't mistaken for the title). The
|
||||
// URL is app-controlled (built by the fast-path), never user free-text.
|
||||
match tokio::process::Command::new("cmd")
|
||||
.args(["/C", "start", "", token, url])
|
||||
.output()
|
||||
.await
|
||||
{
|
||||
Ok(o) if o.status.success() => Ok(format!("Opened {url} in {app}")),
|
||||
Ok(o) => {
|
||||
// `start` failed (token not registered?) — best-effort fall back.
|
||||
log::warn!(
|
||||
"[automate] open_url_in_app: start {token} exited {}: {}; falling back",
|
||||
o.status,
|
||||
String::from_utf8_lossy(&o.stderr).trim()
|
||||
);
|
||||
self.open_url(url).await
|
||||
}
|
||||
Err(e) => Err(format!("failed to launch opener: {e}")),
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
{
|
||||
let _ = app;
|
||||
self.open_url(url).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn key(&self, keys: &[String]) -> Result<String, String> {
|
||||
use crate::openhuman::tools::implementations::computer::keyboard;
|
||||
match keys.len() {
|
||||
0 => Err("no keys provided".to_string()),
|
||||
1 => keyboard::run_key(&keys[0]).await,
|
||||
_ => keyboard::run_hotkey(keys).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn type_text(&self, text: &str) -> Result<String, String> {
|
||||
crate::openhuman::tools::implementations::computer::keyboard::run_type_text(text).await
|
||||
}
|
||||
|
||||
async fn verify_playing(&self) -> Option<bool> {
|
||||
// macOS: ask Apple Music for ground-truth player state. Other OSes can't
|
||||
// verify this way → None (fast-path treats None as best-effort).
|
||||
@@ -683,6 +866,34 @@ impl AutomateBackend for RealBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn now_playing(&self) -> Option<(String, String)> {
|
||||
// macOS: ask Apple Music for the current track's name + artist. We join
|
||||
// them with a tab (unlikely in titles) so we can split unambiguously.
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let script = "tell application \"Music\" to try
|
||||
set t to current track
|
||||
return (name of t) & \"\\t\" & (artist of t)
|
||||
end try";
|
||||
let out = tokio::process::Command::new("osascript")
|
||||
.args(["-e", script])
|
||||
.output()
|
||||
.await
|
||||
.ok()?;
|
||||
let line = String::from_utf8_lossy(&out.stdout);
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let (name, artist) = line.split_once('\t')?;
|
||||
Some((name.trim().to_string(), artist.trim().to_string()))
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(&self, app: &str) {
|
||||
// M2: poll the element count until the UI stops changing (≤2s), instead
|
||||
// of a blind fixed wait. Removes the timing-race class (tracker §1.11/
|
||||
@@ -699,3 +910,23 @@ impl AutomateBackend for RealBackend {
|
||||
#[cfg(test)]
|
||||
#[path = "automate_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
#[cfg(all(test, target_os = "windows"))]
|
||||
mod windows_tests {
|
||||
use super::windows_browser_launch_token as tok;
|
||||
|
||||
#[test]
|
||||
fn browser_display_names_map_to_start_tokens() {
|
||||
assert_eq!(tok("Google Chrome"), Some("chrome"));
|
||||
assert_eq!(tok("Brave Browser"), Some("brave"));
|
||||
assert_eq!(tok("Microsoft Edge"), Some("msedge"));
|
||||
assert_eq!(tok("Firefox"), Some("firefox"));
|
||||
// Aliases / case-insensitive.
|
||||
assert_eq!(tok("chrome"), Some("chrome"));
|
||||
assert_eq!(tok("EDGE"), Some("msedge"));
|
||||
// Not on Windows / unknown → None → caller uses the default handler.
|
||||
assert_eq!(tok("Safari"), None);
|
||||
assert_eq!(tok("Arc"), None);
|
||||
assert_eq!(tok("Some Random App"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -128,6 +128,13 @@ impl AutomateBackend for ScriptedBackend {
|
||||
self.acts.lock().unwrap().push(format!("open_url:{url}"));
|
||||
Ok(format!("Opened {url}"))
|
||||
}
|
||||
async fn key(&self, keys: &[String]) -> Result<String, String> {
|
||||
self.acts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(format!("key:{}", keys.join("+")));
|
||||
Ok(format!("Executed {}", keys.join("+")))
|
||||
}
|
||||
async fn settle(&self, _app: &str) {}
|
||||
async fn wait(&self, _ms: u64) {}
|
||||
}
|
||||
@@ -190,6 +197,23 @@ async fn set_value_routes_app_override() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hotkey_verb_sends_chord_to_backend() {
|
||||
// The model can drive an app shortcut via the general loop instead of
|
||||
// hunting AX labels. Use a neutral app/goal so no fast-path intercepts and
|
||||
// the loop's `hotkey` verb is what runs.
|
||||
let backend = ScriptedBackend::new(&[
|
||||
r#"{"action":"hotkey","keys":["Cmd","L"]}"#,
|
||||
r#"{"action":"done","summary":"sent shortcut"}"#,
|
||||
]);
|
||||
let out = run("Notes", "do a thing", &backend, opts(5)).await;
|
||||
assert!(out.success, "expected success, got {out:?}");
|
||||
assert_eq!(
|
||||
backend.acts(),
|
||||
vec!["launch:Notes", "key:Cmd+L"] // foreground-first, then the chord
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn budget_exhaustion_fails() {
|
||||
// Script always lists → never done → budget guard ends the run.
|
||||
@@ -197,6 +221,13 @@ async fn budget_exhaustion_fails() {
|
||||
let out = run("Music", "never finishes", &backend, opts(3)).await;
|
||||
assert!(!out.success);
|
||||
assert!(out.summary.contains("budget"), "got: {}", out.summary);
|
||||
// #2: surfaces what was on screen + a "try a different approach" steer.
|
||||
assert!(out.summary.contains("On screen:"), "got: {}", out.summary);
|
||||
assert!(
|
||||
out.summary.to_lowercase().contains("different approach"),
|
||||
"got: {}",
|
||||
out.summary
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -211,7 +242,14 @@ async fn no_progress_guard_aborts_repeated_action() {
|
||||
let out = run("Photos", "do something", &backend, opts(10)).await;
|
||||
assert!(!out.success);
|
||||
assert!(
|
||||
out.summary.contains("stuck repeating"),
|
||||
out.summary.to_lowercase().contains("stuck repeating"),
|
||||
"got: {}",
|
||||
out.summary
|
||||
);
|
||||
// #1/#2: actionable — names the screen and tells it to switch tactics.
|
||||
assert!(out.summary.contains("On screen:"), "got: {}", out.summary);
|
||||
assert!(
|
||||
out.summary.to_lowercase().contains("switch tactics"),
|
||||
"got: {}",
|
||||
out.summary
|
||||
);
|
||||
|
||||
@@ -5,11 +5,13 @@ You are the desktop-control specialist. Launch apps and operate native desktop U
|
||||
## Rules
|
||||
|
||||
- Use `launch_app` for explicit app-launch requests.
|
||||
- Use `ax_interact` for semantic accessibility interactions.
|
||||
- **Foreground each app at most ONCE per task.** If the app is already open (a prior step launched it, or the user says it's open), do NOT call `launch_app` again — repeated launches pile up duplicate windows. Re-launch only after a tool result explicitly reports the app isn't running.
|
||||
- **Web browsers (Chrome, Edge, Brave, Firefox, Arc): use `automate`, not `ax_interact`.** To open a site, search, or play a video, call `automate` with the browser as the app and a plain-English goal — e.g. `automate{app:"Google Chrome", goal:"go to youtube.com and play a lofi music video"}`. It navigates deterministically by URL in one step. Do NOT type a URL into the address bar via `ax_interact`/`set_value`: Chromium exposes no page content to the accessibility tree (only browser chrome), and an address/search field set this way usually cannot be submitted — that path dead-ends and loops.
|
||||
- Use `ax_interact` for semantic accessibility interactions in **native** (non-Chromium) apps.
|
||||
- Always call `ax_interact` with `action:"list"` before `press` or `set_value`.
|
||||
- Use `automate` for multi-step app workflows, such as playing a song in Music or sending a message in Slack.
|
||||
- Before any keyboard or mouse action, foreground the target app with `launch_app`.
|
||||
- Prefer `automate` or `ax_interact` first. If the accessibility tree is empty, stuck, or only shows menu-bar items, fall back to keyboard-driven control for Electron/Chromium apps.
|
||||
- Use `automate` for multi-step app workflows: playing a song in Music, sending a message in Slack, or any browser navigation/search/playback (above).
|
||||
- Before any keyboard or mouse action, foreground the target app with `launch_app` (subject to the once-per-task rule above).
|
||||
- Prefer `automate` or `ax_interact` first. If the accessibility tree is empty, stuck, or only shows menu-bar items, fall back to keyboard-driven control for Electron/Chromium apps. **Do not retry the same failing approach repeatedly — if two attempts at a step fail, report it and stop rather than re-launching and re-trying in a loop.**
|
||||
- Use `screenshot` plus `mouse` only when semantic or keyboard control cannot target the needed element.
|
||||
- Never invent element labels. Act only on elements returned by `list` or clearly named by the user.
|
||||
- Respect sensitive-app constraints and tool denials. Do not work around password managers, Keychain, System Settings, terminals, or other denied surfaces.
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
//! ApprovalGate, and it refuses the sensitive-app denylist (password managers,
|
||||
//! Keychain, System Settings, terminals) even on auto-approved turns.
|
||||
|
||||
use super::ax_interact::is_sensitive_app;
|
||||
use super::ax_interact::{app_control_enabled, is_sensitive_app};
|
||||
use crate::openhuman::accessibility::automate::{self, AutomateOptions, RealBackend};
|
||||
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCallOptions, ToolResult};
|
||||
use async_trait::async_trait;
|
||||
@@ -121,12 +121,12 @@ impl Tool for AutomateTool {
|
||||
)));
|
||||
}
|
||||
|
||||
if !self.allow_mutations {
|
||||
if !app_control_enabled(self.allow_mutations) {
|
||||
log::warn!("[automate] refused: mutations disabled");
|
||||
return Ok(ToolResult::error(
|
||||
"App control isn't enabled yet. Turn on App Automation in \
|
||||
Settings → Agent Access (it grants permission to control apps), \
|
||||
then ask again. (Sets computer_control.ax_interact_mutations = true.)",
|
||||
"App control isn't enabled yet. Grant Full access (or turn on App \
|
||||
Automation) in Settings → Agent Access — it grants permission to \
|
||||
control apps — then ask again.",
|
||||
));
|
||||
}
|
||||
|
||||
@@ -194,13 +194,33 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_when_mutations_disabled() {
|
||||
// Pin a non-Full live policy so the Full-access bypass can't open the
|
||||
// gate (other tests in this binary install a Full global policy).
|
||||
let _env = crate::openhuman::config::TEST_ENV_LOCK
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
{
|
||||
use crate::openhuman::security::{live_policy, AutonomyLevel, SecurityPolicy};
|
||||
use std::sync::Arc;
|
||||
let ws = std::env::temp_dir().join("openhuman_automate_gate_test_ws");
|
||||
live_policy::install(
|
||||
Arc::new(SecurityPolicy {
|
||||
autonomy: AutonomyLevel::Supervised,
|
||||
workspace_dir: ws.clone(),
|
||||
..SecurityPolicy::default()
|
||||
}),
|
||||
ws.clone(),
|
||||
ws,
|
||||
);
|
||||
}
|
||||
let t = AutomateTool::new(false);
|
||||
let r = t
|
||||
.execute(json!({"app": "Music", "goal": "play a song"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(r.is_error);
|
||||
assert!(r.output().contains("ax_interact_mutations"));
|
||||
// Gate closed → the refusal directs the user to Settings → Agent Access.
|
||||
assert!(r.output().contains("Agent Access"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -55,6 +55,25 @@ pub(crate) fn is_sensitive_app(app_name: &str) -> bool {
|
||||
SENSITIVE_APPS.iter().any(|s| lower.contains(s))
|
||||
}
|
||||
|
||||
/// Whether the agent may actuate app UI (`press` / `set_value` / `automate`).
|
||||
///
|
||||
/// Enabled when **either** the explicit opt-in
|
||||
/// (`computer_control.ax_interact_mutations`, captured as `explicit_opt_in` at
|
||||
/// tool-build time) is set, **or** the agent has been granted **Full** OS access
|
||||
/// (autonomy level `Full`) in Settings → Agent Access. The autonomy level is
|
||||
/// read from the **live** policy (`security::live_policy::current`), so granting
|
||||
/// Full access mid-session takes effect immediately — the previous behaviour
|
||||
/// gated only on the static config flag, so users who flipped Settings →
|
||||
/// Agent Access to "Full" still saw "App control isn't enabled yet" because the
|
||||
/// flag was untouched. `pub(crate)` so `automate` shares the identical gate.
|
||||
pub(crate) fn app_control_enabled(explicit_opt_in: bool) -> bool {
|
||||
use crate::openhuman::security::{live_policy, AutonomyLevel};
|
||||
explicit_opt_in
|
||||
|| live_policy::current()
|
||||
.map(|p| matches!(p.autonomy, AutonomyLevel::Full))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub struct AxInteractTool {
|
||||
/// When false, the mutating actions (`press` / `set_value`) are refused
|
||||
/// with guidance to enable `computer_control.ax_interact_mutations`. The
|
||||
@@ -229,14 +248,15 @@ impl Tool for AxInteractTool {
|
||||
)));
|
||||
}
|
||||
|
||||
// Mutating actions are opt-in. Read-only `list` is always allowed.
|
||||
if mutating && !self.allow_mutations {
|
||||
// Mutating actions are opt-in (or implied by Full OS access). Read-only
|
||||
// `list` is always allowed.
|
||||
if mutating && !app_control_enabled(self.allow_mutations) {
|
||||
log::warn!("[ax_interact] refused: mutations disabled (action={action})");
|
||||
return Ok(ToolResult::error(
|
||||
"App control isn't enabled yet, so I can't press buttons or type into \
|
||||
this app. Turn on App UI Control / App Automation in Settings → Agent \
|
||||
Access, then ask again. (Reading the UI still works without it; sets \
|
||||
computer_control.ax_interact_mutations = true.)",
|
||||
this app. Grant Full access (or turn on App UI Control / App \
|
||||
Automation) in Settings → Agent Access, then ask again. (Reading the \
|
||||
UI still works without it.)",
|
||||
));
|
||||
}
|
||||
|
||||
@@ -351,6 +371,43 @@ impl Tool for AxInteractTool {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Force the process-global live policy to a given autonomy level so the
|
||||
/// `app_control_enabled` Full-access bypass is deterministic. Other tests in
|
||||
/// this binary (e.g. `security::live_policy`) install a Full global; callers
|
||||
/// must hold `TEST_ENV_LOCK` while relying on the value they set here.
|
||||
fn install_live_autonomy(level: crate::openhuman::security::AutonomyLevel) {
|
||||
use crate::openhuman::security::{live_policy, SecurityPolicy};
|
||||
use std::sync::Arc;
|
||||
let ws = std::env::temp_dir().join("openhuman_ax_interact_gate_test_ws");
|
||||
live_policy::install(
|
||||
Arc::new(SecurityPolicy {
|
||||
autonomy: level,
|
||||
workspace_dir: ws.clone(),
|
||||
..SecurityPolicy::default()
|
||||
}),
|
||||
ws.clone(),
|
||||
ws,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn app_control_enabled_combines_optin_and_full_access() {
|
||||
use crate::openhuman::security::AutonomyLevel;
|
||||
let _env = crate::openhuman::config::TEST_ENV_LOCK
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
// Explicit opt-in always enables, regardless of the live policy.
|
||||
assert!(app_control_enabled(true));
|
||||
// Supervised + no opt-in → gate closed.
|
||||
install_live_autonomy(AutonomyLevel::Supervised);
|
||||
assert!(!app_control_enabled(false));
|
||||
// Full OS access opens the gate even without the explicit flag — this is
|
||||
// the fix: granting "Full" in Settings → Agent Access now enables app
|
||||
// control without separately flipping `ax_interact_mutations`.
|
||||
install_live_autonomy(AutonomyLevel::Full);
|
||||
assert!(app_control_enabled(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_and_permission() {
|
||||
let tool = AxInteractTool::new(true);
|
||||
@@ -416,6 +473,12 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_mutations_when_disabled() {
|
||||
// Pin a non-Full live policy for the test so the Full-access bypass can't
|
||||
// open the gate (other tests in this binary install a Full global).
|
||||
let _env = crate::openhuman::config::TEST_ENV_LOCK
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
install_live_autonomy(crate::openhuman::security::AutonomyLevel::Supervised);
|
||||
// mutations off → press/set_value blocked, but list still allowed past this guard.
|
||||
let tool = AxInteractTool::new(false);
|
||||
let press = tool
|
||||
@@ -423,7 +486,8 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(press.is_error);
|
||||
assert!(press.output().contains("ax_interact_mutations"));
|
||||
// Gate closed → the refusal points the user at Settings → Agent Access.
|
||||
assert!(press.output().contains("Agent Access"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -97,6 +97,135 @@ fn is_modifier(key: &Key) -> bool {
|
||||
matches!(key, Key::Control | Key::Shift | Key::Alt | Key::Meta)
|
||||
}
|
||||
|
||||
// ── Shared execution helpers ────────────────────────────────────────────────
|
||||
// These hold the validation + main-thread enigo dispatch once, so both the
|
||||
// `keyboard` tool and the `automate` backend (`RealBackend::{key,type_text}`)
|
||||
// drive synthetic input through a single code path. They return the raw
|
||||
// `Result<String, String>`; the tool wraps it with `into_result`.
|
||||
|
||||
/// Type a literal string via the OS. Runs on the app main thread (macOS TSM
|
||||
/// requirement, Change 1.15). Validates length like the `type` tool action.
|
||||
pub(crate) async fn run_type_text(text: &str) -> Result<String, String> {
|
||||
if text.is_empty() {
|
||||
return Err("'text' cannot be empty".to_string());
|
||||
}
|
||||
if text.len() > MAX_TYPE_LENGTH {
|
||||
return Err(format!(
|
||||
"Text too long ({} chars). Maximum is {MAX_TYPE_LENGTH}.",
|
||||
text.len()
|
||||
));
|
||||
}
|
||||
let text = text.to_string();
|
||||
let len = text.len();
|
||||
run_input_on_main(move || {
|
||||
let mut enigo = Enigo::new(&Settings::default())
|
||||
.map_err(|e| format!("Failed to create enigo instance: {e}"))?;
|
||||
enigo
|
||||
.text(&text)
|
||||
.map_err(|e| format!("text typing failed: {e}"))?;
|
||||
Ok(format!("Typed {len} characters"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Tap a single key by name (e.g. "Enter", "/", "k"). Used for app shortcuts
|
||||
/// like YouTube's `/` (focus search) or `k` (play/pause).
|
||||
pub(crate) async fn run_key(key_name: &str) -> Result<String, String> {
|
||||
let key = parse_key(key_name).ok_or_else(|| {
|
||||
format!(
|
||||
"Unknown key '{key_name}'. Use names like Enter, Tab, Escape, F1-F12, a-z, 0-9, Space, etc."
|
||||
)
|
||||
})?;
|
||||
let key_name = key_name.to_string();
|
||||
run_input_on_main(move || {
|
||||
let mut enigo = Enigo::new(&Settings::default())
|
||||
.map_err(|e| format!("Failed to create enigo instance: {e}"))?;
|
||||
enigo
|
||||
.key(key, Direction::Click)
|
||||
.map_err(|e| format!("key press failed: {e}"))?;
|
||||
Ok(format!("Pressed key '{key_name}'"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Execute a hotkey chord — modifiers first, then exactly one non-modifier
|
||||
/// final key (e.g. `["Cmd","L"]`). Validates the modifier-first shape, then
|
||||
/// presses in order and releases in reverse (even on error).
|
||||
pub(crate) async fn run_hotkey(key_names: &[String]) -> Result<String, String> {
|
||||
if key_names.is_empty() {
|
||||
return Err("'keys' array cannot be empty".to_string());
|
||||
}
|
||||
if key_names.len() > 6 {
|
||||
return Err("Too many keys in hotkey combination (max 6)".to_string());
|
||||
}
|
||||
if key_names.len() < 2 {
|
||||
return Err(
|
||||
"Hotkey requires at least one modifier and one final key (e.g. ['Ctrl', 'C'])"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let mut keys: Vec<Key> = Vec::with_capacity(key_names.len());
|
||||
for name in key_names {
|
||||
let key =
|
||||
parse_key(name).ok_or_else(|| format!("Unknown key '{name}' in hotkey combination"))?;
|
||||
keys.push(key);
|
||||
}
|
||||
|
||||
// Validate modifier-first: all but the last must be modifiers; the last
|
||||
// must be a non-modifier.
|
||||
let (modifiers, final_key) = keys.split_at(keys.len() - 1);
|
||||
for (i, key) in modifiers.iter().enumerate() {
|
||||
if !is_modifier(key) {
|
||||
return Err(format!(
|
||||
"Key '{}' at position {i} must be a modifier (Ctrl/Shift/Alt/Cmd). Non-modifier keys must be last.",
|
||||
key_names[i]
|
||||
));
|
||||
}
|
||||
}
|
||||
if is_modifier(&final_key[0]) {
|
||||
return Err(format!(
|
||||
"Last key '{}' cannot be a modifier. Hotkey must end with a non-modifier key (e.g. 'C', 'Enter').",
|
||||
key_names.last().unwrap()
|
||||
));
|
||||
}
|
||||
|
||||
let combo_desc = key_names.join("+");
|
||||
run_input_on_main(move || {
|
||||
let mut enigo = Enigo::new(&Settings::default())
|
||||
.map_err(|e| format!("Failed to create enigo instance: {e}"))?;
|
||||
|
||||
// Press keys in order, tracking which were pressed so we can release
|
||||
// them on error.
|
||||
let mut pressed_keys: Vec<Key> = Vec::with_capacity(keys.len());
|
||||
let press_result: Result<(), String> = (|| {
|
||||
for key in &keys {
|
||||
enigo
|
||||
.key(*key, Direction::Press)
|
||||
.map_err(|e| format!("key press failed for {key:?}: {e}"))?;
|
||||
pressed_keys.push(*key);
|
||||
std::thread::sleep(HOTKEY_INTER_KEY_DELAY);
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
|
||||
// Always release pressed keys in reverse, even on error.
|
||||
for key in pressed_keys.iter().rev() {
|
||||
if let Err(e) = enigo.key(*key, Direction::Release) {
|
||||
tracing::warn!(
|
||||
tool = "keyboard",
|
||||
key = ?key,
|
||||
error = %e,
|
||||
"[computer] best-effort key release failed during cleanup"
|
||||
);
|
||||
}
|
||||
}
|
||||
press_result?;
|
||||
Ok(format!("Executed hotkey: {combo_desc}"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for KeyboardTool {
|
||||
fn name(&self) -> &str {
|
||||
@@ -183,57 +312,16 @@ impl Tool for KeyboardTool {
|
||||
let text = args
|
||||
.get("text")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing 'text' for type action"))?
|
||||
.to_string();
|
||||
|
||||
if text.is_empty() {
|
||||
return Ok(ToolResult::error("'text' cannot be empty"));
|
||||
}
|
||||
if text.len() > MAX_TYPE_LENGTH {
|
||||
return Ok(ToolResult::error(format!(
|
||||
"Text too long ({} chars). Maximum is {MAX_TYPE_LENGTH}.",
|
||||
text.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let len = text.len();
|
||||
into_result(
|
||||
"type",
|
||||
run_input_on_main(move || {
|
||||
let mut enigo = Enigo::new(&Settings::default())
|
||||
.map_err(|e| format!("Failed to create enigo instance: {e}"))?;
|
||||
enigo
|
||||
.text(&text)
|
||||
.map_err(|e| format!("text typing failed: {e}"))?;
|
||||
Ok(format!("Typed {len} characters"))
|
||||
})
|
||||
.await,
|
||||
)
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing 'text' for type action"))?;
|
||||
into_result("type", run_type_text(text).await)
|
||||
}
|
||||
|
||||
"press" => {
|
||||
let key_name = args
|
||||
.get("key")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing 'key' for press action"))?
|
||||
.to_string();
|
||||
|
||||
let key = parse_key(&key_name).ok_or_else(|| {
|
||||
anyhow::anyhow!("Unknown key '{key_name}'. Use names like Enter, Tab, Escape, F1-F12, a-z, 0-9, Space, etc.")
|
||||
})?;
|
||||
|
||||
into_result(
|
||||
"press",
|
||||
run_input_on_main(move || {
|
||||
let mut enigo = Enigo::new(&Settings::default())
|
||||
.map_err(|e| format!("Failed to create enigo instance: {e}"))?;
|
||||
enigo
|
||||
.key(key, Direction::Click)
|
||||
.map_err(|e| format!("key press failed: {e}"))?;
|
||||
Ok(format!("Pressed key '{key_name}'"))
|
||||
})
|
||||
.await,
|
||||
)
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing 'key' for press action"))?;
|
||||
into_result("press", run_key(key_name).await)
|
||||
}
|
||||
|
||||
"hotkey" => {
|
||||
@@ -251,84 +339,7 @@ impl Tool for KeyboardTool {
|
||||
key_names.push(s.to_string());
|
||||
}
|
||||
|
||||
if key_names.is_empty() {
|
||||
return Ok(ToolResult::error("'keys' array cannot be empty"));
|
||||
}
|
||||
if key_names.len() > 6 {
|
||||
return Ok(ToolResult::error(
|
||||
"Too many keys in hotkey combination (max 6)",
|
||||
));
|
||||
}
|
||||
if key_names.len() < 2 {
|
||||
return Ok(ToolResult::error(
|
||||
"Hotkey requires at least one modifier and one final key (e.g. ['Ctrl', 'C'])",
|
||||
));
|
||||
}
|
||||
|
||||
// Parse all key names into Key values.
|
||||
let mut keys: Vec<Key> = Vec::with_capacity(key_names.len());
|
||||
for name in &key_names {
|
||||
let key = parse_key(name).ok_or_else(|| {
|
||||
anyhow::anyhow!("Unknown key '{name}' in hotkey combination")
|
||||
})?;
|
||||
keys.push(key);
|
||||
}
|
||||
|
||||
// Validate modifier-first pattern: all keys except the last
|
||||
// must be modifiers, and the last must be a non-modifier.
|
||||
let (modifiers, final_key) = keys.split_at(keys.len() - 1);
|
||||
for (i, key) in modifiers.iter().enumerate() {
|
||||
if !is_modifier(key) {
|
||||
return Ok(ToolResult::error(format!(
|
||||
"Key '{}' at position {i} must be a modifier (Ctrl/Shift/Alt/Cmd). Non-modifier keys must be last.",
|
||||
key_names[i]
|
||||
)));
|
||||
}
|
||||
}
|
||||
if is_modifier(&final_key[0]) {
|
||||
return Ok(ToolResult::error(format!(
|
||||
"Last key '{}' cannot be a modifier. Hotkey must end with a non-modifier key (e.g. 'C', 'Enter').",
|
||||
key_names.last().unwrap()
|
||||
)));
|
||||
}
|
||||
|
||||
let combo_desc = key_names.join("+");
|
||||
into_result(
|
||||
"hotkey",
|
||||
run_input_on_main(move || {
|
||||
let mut enigo = Enigo::new(&Settings::default())
|
||||
.map_err(|e| format!("Failed to create enigo instance: {e}"))?;
|
||||
|
||||
// Press keys in order, tracking which were pressed so we
|
||||
// can release them on error.
|
||||
let mut pressed_keys: Vec<Key> = Vec::with_capacity(keys.len());
|
||||
let press_result: Result<(), String> = (|| {
|
||||
for key in &keys {
|
||||
enigo
|
||||
.key(*key, Direction::Press)
|
||||
.map_err(|e| format!("key press failed for {key:?}: {e}"))?;
|
||||
pressed_keys.push(*key);
|
||||
std::thread::sleep(HOTKEY_INTER_KEY_DELAY);
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
|
||||
// Always release pressed keys in reverse, even on error.
|
||||
for key in pressed_keys.iter().rev() {
|
||||
if let Err(e) = enigo.key(*key, Direction::Release) {
|
||||
tracing::warn!(
|
||||
tool = "keyboard",
|
||||
key = ?key,
|
||||
error = %e,
|
||||
"[computer] best-effort key release failed during cleanup"
|
||||
);
|
||||
}
|
||||
}
|
||||
press_result?;
|
||||
Ok(format!("Executed hotkey: {combo_desc}"))
|
||||
})
|
||||
.await,
|
||||
)
|
||||
into_result("hotkey", run_hotkey(&key_names).await)
|
||||
}
|
||||
|
||||
other => Ok(ToolResult::error(format!(
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
mod automate;
|
||||
mod ax_interact;
|
||||
mod human_path;
|
||||
mod keyboard;
|
||||
// `pub(crate)` so the automate backend can reuse the shared input helpers
|
||||
// (`run_hotkey`/`run_key`/`run_type_text`) without going through the tool.
|
||||
pub(crate) mod keyboard;
|
||||
mod main_thread;
|
||||
mod mouse;
|
||||
|
||||
|
||||
@@ -1637,6 +1637,11 @@ fn uninstall_resolves_agents_skills_legacy_root() {
|
||||
assert!(!dir.exists(), "uninstall should remove the dir");
|
||||
}
|
||||
|
||||
// Unix-only: exercises `std::os::unix::fs::symlink`. Windows symlink creation
|
||||
// uses a different API and requires elevated privileges / Developer Mode, so
|
||||
// this case is gated off there (the Windows lib test binary otherwise fails to
|
||||
// compile on this line).
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn symlinked_manifest_file_is_rejected() {
|
||||
// `exists()` follows symlinks; a manifest pointed at an external file would
|
||||
|
||||
Reference in New Issue
Block a user