mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
This commit is contained in:
Mega Mind
Generated
+75
@@ -5234,6 +5234,7 @@ dependencies = [
|
||||
"tracing-appender",
|
||||
"tracing-log",
|
||||
"tracing-subscriber",
|
||||
"uiautomation",
|
||||
"unicode-normalization",
|
||||
"unicode-segmentation",
|
||||
"unicode-width",
|
||||
@@ -8388,6 +8389,29 @@ version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e36a83ea2b3c704935a01b4642946aadd445cea40b10935e3f8bd8052b8193d6"
|
||||
|
||||
[[package]]
|
||||
name = "uiautomation"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c68495a701b9f2f21f29353ac446f0d27dd0d7ce97aa9ccf9061bca0446cd744"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"uiautomation_derive",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uiautomation_derive"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffcc4d404aa1c03a848f95cf5feadc3e63946d7f095bf388770b85550093d388"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uint"
|
||||
version = "0.9.5"
|
||||
@@ -9287,6 +9311,27 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
|
||||
dependencies = [
|
||||
"windows-collections",
|
||||
"windows-core 0.62.2",
|
||||
"windows-future",
|
||||
"windows-numerics",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-collections"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.54.0"
|
||||
@@ -9335,6 +9380,17 @@ dependencies = [
|
||||
"windows-strings 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-future"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
"windows-link",
|
||||
"windows-threading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.57.0"
|
||||
@@ -9407,6 +9463,16 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-numerics"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-registry"
|
||||
version = "0.6.1"
|
||||
@@ -9572,6 +9638,15 @@ dependencies = [
|
||||
"windows_x86_64_msvc 0.53.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.42.2"
|
||||
|
||||
+8
-1
@@ -207,16 +207,23 @@ tokio-tungstenite = { version = "0.24", default-features = false, features = ["c
|
||||
# AppContainer / process-jail backend in `openhuman::cwd_jail`.
|
||||
# Feature list mirrors the Win32 surface used by cwd_jail/windows.rs:
|
||||
# AppContainer profile APIs, ACL editing, STARTUPINFOEXW process spawn,
|
||||
# and the GENERIC_* file access masks.
|
||||
# and the GENERIC_* file access masks. `Win32_System_Com` is used by the
|
||||
# UIA accessibility backend (`accessibility::uia_interact`) to initialise COM
|
||||
# on the worker thread before creating the UI Automation client.
|
||||
windows-sys = { version = "0.61", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_Security",
|
||||
"Win32_Security_Authorization",
|
||||
"Win32_Security_Isolation",
|
||||
"Win32_Storage_FileSystem",
|
||||
"Win32_System_Com",
|
||||
"Win32_System_Memory",
|
||||
"Win32_System_Threading",
|
||||
] }
|
||||
# Microsoft UI Automation (UIA) bindings — the Windows backend for the
|
||||
# `ax_interact` tool (`accessibility::uia_interact`). Safe Rust wrappers over
|
||||
# the UIA COM API; the Windows analogue of the macOS AXUIElement Swift helper.
|
||||
uiautomation = "0.25"
|
||||
|
||||
[target.'cfg(not(windows))'.dependencies]
|
||||
# macOS / Linux: keep rustls + Mozilla webpki-roots — the historical
|
||||
|
||||
Generated
+79
-4
@@ -5443,6 +5443,7 @@ dependencies = [
|
||||
"tracing-appender",
|
||||
"tracing-log",
|
||||
"tracing-subscriber",
|
||||
"uiautomation",
|
||||
"unicode-normalization",
|
||||
"unicode-segmentation",
|
||||
"unicode-width",
|
||||
@@ -9286,6 +9287,29 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uiautomation"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c68495a701b9f2f21f29353ac446f0d27dd0d7ce97aa9ccf9061bca0446cd744"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"uiautomation_derive",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uiautomation_derive"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffcc4d404aa1c03a848f95cf5feadc3e63946d7f095bf388770b85550093d388"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uint"
|
||||
version = "0.9.5"
|
||||
@@ -10087,11 +10111,23 @@ version = "0.61.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
|
||||
dependencies = [
|
||||
"windows-collections",
|
||||
"windows-collections 0.2.0",
|
||||
"windows-core 0.61.2",
|
||||
"windows-future",
|
||||
"windows-future 0.2.1",
|
||||
"windows-link 0.1.3",
|
||||
"windows-numerics",
|
||||
"windows-numerics 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
|
||||
dependencies = [
|
||||
"windows-collections 0.3.2",
|
||||
"windows-core 0.62.2",
|
||||
"windows-future 0.3.2",
|
||||
"windows-numerics 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10103,6 +10139,15 @@ dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-collections"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.54.0"
|
||||
@@ -10172,7 +10217,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
|
||||
dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
"windows-link 0.1.3",
|
||||
"windows-threading",
|
||||
"windows-threading 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-future"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
"windows-link 0.2.1",
|
||||
"windows-threading 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10263,6 +10319,16 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-numerics"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-registry"
|
||||
version = "0.5.3"
|
||||
@@ -10464,6 +10530,15 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-version"
|
||||
version = "0.1.7"
|
||||
|
||||
@@ -79,4 +79,43 @@ describe('useDictationHotkey', () => {
|
||||
});
|
||||
expect(hoisted.mockSocket.on).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('dispatches an autoSend insert-text event on transcription', async () => {
|
||||
renderHook(() => useDictationHotkey());
|
||||
await waitFor(() => {
|
||||
expect(hoisted.handlers['dictation:transcription']).toBeDefined();
|
||||
});
|
||||
|
||||
const received: CustomEvent<{ text?: string; autoSend?: boolean }>[] = [];
|
||||
const listener = (e: Event) =>
|
||||
received.push(e as CustomEvent<{ text?: string; autoSend?: boolean }>);
|
||||
window.addEventListener('dictation://insert-text', listener);
|
||||
try {
|
||||
hoisted.handlers['dictation:transcription']({ text: ' hello world ' });
|
||||
} finally {
|
||||
window.removeEventListener('dictation://insert-text', listener);
|
||||
}
|
||||
|
||||
expect(received).toHaveLength(1);
|
||||
// Trimmed text, and autoSend flag set so Conversations sends it straight to the agent.
|
||||
expect(received[0].detail).toEqual({ text: 'hello world', autoSend: true });
|
||||
});
|
||||
|
||||
it('ignores blank transcription text (no event dispatched)', async () => {
|
||||
renderHook(() => useDictationHotkey());
|
||||
await waitFor(() => {
|
||||
expect(hoisted.handlers['dictation:transcription']).toBeDefined();
|
||||
});
|
||||
|
||||
const received: Event[] = [];
|
||||
const listener = (e: Event) => received.push(e);
|
||||
window.addEventListener('dictation://insert-text', listener);
|
||||
try {
|
||||
hoisted.handlers['dictation:transcription']({ text: ' ' });
|
||||
} finally {
|
||||
window.removeEventListener('dictation://insert-text', listener);
|
||||
}
|
||||
|
||||
expect(received).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -150,7 +150,9 @@ export function useDictationHotkey(): DictationHotkeyState {
|
||||
if (!text) return;
|
||||
console.debug(`[dictation] transcription received: ${text.length} chars — "${text}"`);
|
||||
|
||||
window.dispatchEvent(new CustomEvent('dictation://insert-text', { detail: { text } }));
|
||||
window.dispatchEvent(
|
||||
new CustomEvent('dictation://insert-text', { detail: { text, autoSend: true } })
|
||||
);
|
||||
});
|
||||
|
||||
socket.connect();
|
||||
|
||||
@@ -302,6 +302,8 @@ const Conversations = ({
|
||||
// from `selectedThreadId` so switching threads mid-turn doesn't move the
|
||||
// timer's reference point.
|
||||
const sendingThreadIdRef = useRef<string | null>(null);
|
||||
// Ref so the mount-time dictation event handler can call the latest send fn.
|
||||
const handleSendMessageRef = useRef<((text?: string) => Promise<void>) | null>(null);
|
||||
// Previous inference status for the sending thread; lets the rearm effect
|
||||
// distinguish "status was just cleared (chat_done / chat_error)" from
|
||||
// "status was never set yet (in-flight turn pre-status)".
|
||||
@@ -462,11 +464,19 @@ const Conversations = ({
|
||||
|
||||
useEffect(() => {
|
||||
const onDictationInsert = (event: Event) => {
|
||||
const customEvent = event as CustomEvent<{ text?: string }>;
|
||||
const customEvent = event as CustomEvent<{ text?: string; autoSend?: boolean }>;
|
||||
const text = customEvent.detail?.text?.trim();
|
||||
if (!text) return;
|
||||
|
||||
customEvent.preventDefault();
|
||||
|
||||
// When autoSend is set (hotkey dictation), dispatch the transcript directly
|
||||
// to the agent without going through the text composer.
|
||||
if (customEvent.detail?.autoSend) {
|
||||
void handleSendMessageRef.current?.(text);
|
||||
return;
|
||||
}
|
||||
|
||||
setInputMode('text');
|
||||
setInputValue(prev => {
|
||||
const base = prev.trim();
|
||||
@@ -844,6 +854,8 @@ const Conversations = ({
|
||||
}
|
||||
};
|
||||
|
||||
handleSendMessageRef.current = handleSendMessage;
|
||||
|
||||
const transcribeAndSendAudio = async (mimeType: string) => {
|
||||
setIsRecording(false);
|
||||
mediaRecorderRef.current = null;
|
||||
|
||||
@@ -675,6 +675,44 @@ describe('Conversations — smoke render (#1123 welcome-lock removal)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('auto-sends a dictation transcript (autoSend) straight to chat without the composer', async () => {
|
||||
const { thread } = await renderSelectedConversation();
|
||||
|
||||
// Hotkey dictation dispatches this event with autoSend:true (see
|
||||
// useDictationHotkey). Conversations must route it directly to chatSend,
|
||||
// bypassing the text composer.
|
||||
await act(async () => {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent('dictation://insert-text', {
|
||||
detail: { text: ' play highway to hell ', autoSend: true },
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(chatSend).toHaveBeenCalledWith({
|
||||
threadId: thread.id,
|
||||
message: 'play highway to hell',
|
||||
model: 'reasoning-v1',
|
||||
profileId: 'default',
|
||||
locale: 'en',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('ignores a blank autoSend dictation event (no send)', async () => {
|
||||
await renderSelectedConversation();
|
||||
vi.mocked(chatSend).mockClear();
|
||||
|
||||
await act(async () => {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent('dictation://insert-text', { detail: { text: ' ', autoSend: true } })
|
||||
);
|
||||
});
|
||||
|
||||
expect(chatSend).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('blocks duplicate sends while the first send is still pending', async () => {
|
||||
let resolveSend: (() => void) | undefined;
|
||||
vi.mocked(chatSend).mockImplementationOnce(
|
||||
|
||||
@@ -28,6 +28,23 @@ export const TOOL_CATALOG: ToolDefinition[] = [
|
||||
defaultEnabled: true,
|
||||
rustToolNames: ['shell'],
|
||||
},
|
||||
{
|
||||
id: 'launch_app',
|
||||
displayName: 'Launch Applications',
|
||||
description: 'Open apps on your desktop by name (e.g. Music, Spotify, Safari).',
|
||||
category: 'System',
|
||||
defaultEnabled: true,
|
||||
rustToolNames: ['launch_app'],
|
||||
},
|
||||
{
|
||||
id: 'ax_interact',
|
||||
displayName: 'App UI Control',
|
||||
description:
|
||||
'Interact with desktop app UI by element label via the platform accessibility API — click buttons, type in fields, without needing screen coordinates.',
|
||||
category: 'System',
|
||||
defaultEnabled: true,
|
||||
rustToolNames: ['ax_interact'],
|
||||
},
|
||||
{
|
||||
id: 'git_operations',
|
||||
displayName: 'Git Operations',
|
||||
|
||||
@@ -0,0 +1,481 @@
|
||||
# Voice → System Action Feature Tracker
|
||||
|
||||
**GitHub Issue:** [#3148](https://github.com/tinyhumansai/openhuman/issues/3148)
|
||||
**Branch:** `feat/voice-always-on`
|
||||
**PR:** [#3168](https://github.com/tinyhumansai/openhuman/pull/3168)
|
||||
**Started:** 2026-06-02
|
||||
|
||||
---
|
||||
|
||||
## Goal
|
||||
|
||||
Enable the app to continuously listen to the user, understand spoken commands, and perform system actions on the laptop — e.g., saying *"open my Music player"* causes the app to open it, without any hotkey press or manual send.
|
||||
|
||||
---
|
||||
|
||||
## Companion Feature (Separate PR)
|
||||
|
||||
**Notch Live Activity Indicator** — [PR #3166](https://github.com/tinyhumansai/openhuman/pull/3166)
|
||||
A transparent NSPanel pill at the top of the primary screen (the macOS notch area) that shows live voice/agent status. Built alongside this feature; will light up automatically once always-on listening is implemented.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Quick Wins ✅ Complete
|
||||
|
||||
> Low-effort changes that make the existing hotkey-triggered dictation flow work end-to-end without manual sends or approval prompts.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.1 — Auto-send after transcription
|
||||
|
||||
**Status:** ✅ Done
|
||||
**Commit:** `7269f4373`
|
||||
|
||||
**Problem:** After speaking via the dictation hotkey, the transcript appeared in the chat composer but the user had to press Enter manually to send it.
|
||||
|
||||
**Fix:**
|
||||
- `app/src/hooks/useDictationHotkey.ts` — added `autoSend: true` to the `dictation://insert-text` event detail
|
||||
- `app/src/pages/Conversations.tsx` — `onDictationInsert` now checks the flag; when set, calls `handleSendMessage(text)` directly instead of inserting into the textarea. Added `handleSendMessageRef` (updated every render) so the mount-time effect can access the latest send function
|
||||
|
||||
**Result:** Press hotkey → speak → message auto-sends to agent. No Enter key needed.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.2 — Shell allowlist for app-launching — ⚠️ REVERTED / SUPERSEDED
|
||||
|
||||
**Status:** ❌ Reverted — superseded by Change 1.4 (`launch_app`) and the security review.
|
||||
|
||||
**What was tried (commit `7269f4373`):** added `"open"` / `"xdg-open"` to `READ_ONLY_BASES` so `open -a Music` would run without an approval prompt.
|
||||
|
||||
**Why reverted:** base-command classification can't see args, and `open`/`xdg-open`/`start` can open arbitrary `https://` URLs and custom URI handlers — too broad for the `Read` (no-approval) path (maintainer security review). They were **removed** from `READ_ONLY_BASES`; the current code (`policy_command.rs:514-520`) deliberately keeps them out, with a comment. App launching now goes through the dedicated, gated `launch_app` tool (Change 1.4), which is scoped to named applications only.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.3 — Shell tool description fix
|
||||
|
||||
**Status:** ✅ Done
|
||||
**Commit:** `ec8f5be2e`
|
||||
|
||||
**Problem:** Shell tool description said *"Execute a shell command in the workspace directory"* — the LLM was reasoning that it could only run workspace commands, not launch apps.
|
||||
|
||||
**Fix:**
|
||||
- `src/openhuman/tools/impl/system/shell.rs` — updated description to explicitly mention system actions and app launching examples
|
||||
|
||||
**Result:** Agent now understands the shell tool can perform system actions, not just workspace file operations.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.4 — Dedicated `launch_app` tool
|
||||
|
||||
**Status:** ✅ Done
|
||||
**Commit:** `802fbca76`
|
||||
|
||||
**Problem:** Using the `shell` tool for app launching requires loosening `workspace_only` and expanding `allowed_commands` — a security regression. The `shell` tool also couldn't be used because the orchestrator's strict `named` tool list excluded it.
|
||||
|
||||
**Fix (production approach):**
|
||||
- `src/openhuman/tools/impl/system/launch_app.rs` — **new tool** with `PermissionLevel::ReadOnly` (never triggers approval gate)
|
||||
- macOS: `open -a "<app_name>"` via `tokio::process::Command`
|
||||
- Linux: `gtk-launch`, fallback `xdg-open`
|
||||
- Windows: `Start-Process` via PowerShell
|
||||
- Input validation: rejects paths, metacharacters, empty names
|
||||
- Unit tests: name, permission, schema, validation, error cases
|
||||
- `src/openhuman/tools/impl/system/mod.rs` — registered module + pub use
|
||||
- `src/openhuman/tools/ops.rs` — added `LaunchAppTool` to `all_tools_with_runtime`
|
||||
- `src/openhuman/tools/user_filter.rs` — added `"launch_app"` family, `default_enabled = true`
|
||||
- `app/src/utils/toolDefinitions.ts` — added to frontend tool catalog (Settings → Agent Access toggle)
|
||||
|
||||
**Result:** Agent has a purpose-built, always-allow tool for launching apps. No shell exposure, no path security concerns.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.5 — Orchestrator agent tool scope
|
||||
|
||||
**Status:** ✅ Done
|
||||
**Commit:** `7d04fc4bc`
|
||||
|
||||
**Problem:** Even though `launch_app` was registered, it was invisible to the agent. The orchestrator (`src/openhuman/agent_registry/agents/orchestrator/agent.toml`) has a strict `named = [...]` allowlist. `launch_app` was not in it, so it was filtered out. Confirmed via logs: `visible=24, names=[...no launch_app...]`.
|
||||
|
||||
**Fix:**
|
||||
- `src/openhuman/agent_registry/agents/orchestrator/agent.toml` — added `"launch_app"` to the `[tools] named` list, alongside `"current_time"` (same pattern: direct answer without delegation)
|
||||
|
||||
**Confirmed working via logs:**
|
||||
```text
|
||||
visible=25, names=[..., launch_app, ...]
|
||||
[launch_app] ▶ execute called app_name="Music"
|
||||
[launch_app] macOS: running `open -a "Music"`
|
||||
[launch_app] macOS: `open -a` exit=exit status: 0 stderr=
|
||||
[launch_app] ✓ launch succeeded msg="Opened 'Music'."
|
||||
```
|
||||
|
||||
**Result:** Saying "open my Music app" now opens Music directly. No approval prompt, no delegation, no refusal.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.6 — SOUL.md capability hint
|
||||
|
||||
**Status:** ✅ Done
|
||||
**Commit:** `cdd3bb4a4`
|
||||
|
||||
**Problem:** Even with the tool available, the agent was refusing ("I can't open apps on your device") because its training overrides the function-calling schema.
|
||||
|
||||
**Fix:**
|
||||
- `src/openhuman/agent/prompts/SOUL.md` — added explicit *"What you can do on the user's machine"* section listing `launch_app`, `shell`, `file_read`/`file_write` with the instruction: *"Never say 'I can't open apps' when you have a tool to do it. Use the tool."*
|
||||
|
||||
**Result:** Agent now knows it has these capabilities and is instructed to use them.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.7 — Diagnostic logging
|
||||
|
||||
**Status:** ✅ Done
|
||||
**Commit:** `cdd3bb4a4`
|
||||
|
||||
**Added logging to:**
|
||||
- `src/openhuman/tools/impl/system/launch_app.rs` — logs every step: `▶ execute`, validation pass/fail, platform dispatch, `open -a` exit code + stderr, fallback result
|
||||
- `src/openhuman/agent/harness/session/builder.rs` — logs the **full list** of visible tool names at session build time (previously only logged count)
|
||||
|
||||
**Result:** Can now confirm at a glance whether `launch_app` is in the tool list and trace every step of its execution.
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
### Change 1.8 — Computer control (mouse + keyboard) — ⚠️ REVERTED
|
||||
|
||||
**Status:** ❌ Reverted (commits `50ca434b7` add, `bi0rd96sa` revert)
|
||||
|
||||
**Problem:** Agent could open apps but couldn't interact with their UI.
|
||||
|
||||
**What was tried:** Enabled the existing `mouse` + `keyboard` tools (enigo / `CGEventPost`), wired into the orchestrator, user filter, and frontend catalog.
|
||||
|
||||
**Why reverted:** `CGEventPost` injects synthetic events to the **currently focused window**. When the focused window was OpenHuman's own CEF renderer (the chat UI), a Space keypress crashed the app — `EXC_BREAKPOINT / SIGTRAP` in `CFRelease → NSKeyValueWillChangeWithPerThreadPendingNotifications → -[NSApplication stop:]`. CEF can't handle arbitrary key injection. Confirmed via crash report `OpenHuman-2026-06-02-035139.ips`.
|
||||
|
||||
**Replaced by:** Change 1.9 (`ax_interact`) — AXUIElement targets elements directly by label with no synthetic events and no CEF crash risk.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.9 — AXUIElement app UI interaction (`ax_interact`)
|
||||
|
||||
**Status:** ✅ Done
|
||||
**Commits:** `4f9ca1cad` (feature), `2c32b59c9` (exact-match fix), `betuerj11`/test commits
|
||||
|
||||
**Problem:** Need to interact with desktop app UIs reliably, without the CEF crash from synthetic events.
|
||||
|
||||
**Fix — uses the macOS Accessibility API (AXUIElement) instead of CGEventPost:**
|
||||
- `src/openhuman/accessibility/helper.rs` — extended the unified Swift helper with three commands:
|
||||
- `ax_list` → walk the AX tree, return interactive elements (buttons, fields, cells)
|
||||
- `ax_press` → `AXUIElementPerformAction(kAXPressAction)` by label, **exact match preferred over contains** (so "Play" beats "Playlist")
|
||||
- `ax_set_value` → `AXUIElementSetAttributeValue(kAXValueAttribute)` by label
|
||||
- `src/openhuman/accessibility/ax_interact.rs` (new) — Rust wrappers `ax_list_elements`, `ax_press_element`, `ax_set_field_value`
|
||||
- `src/openhuman/tools/impl/computer/ax_interact.rs` (new) — `AxInteractTool` with actions `list` / `press` / `set_value`, `PermissionLevel::ReadOnly`
|
||||
- `src/openhuman/accessibility/ax_interact_tests.rs` (new) — integration tests (open Music → search AC/DC → find row → press)
|
||||
- Wired into `tools/ops.rs`, `tools/user_filter.rs`, `toolDefinitions.ts` (App UI Control), `orchestrator/agent.toml`, `SOUL.md`
|
||||
|
||||
**Why it's better than mouse/keyboard:**
|
||||
|
||||
| | mouse/keyboard (reverted) | ax_interact |
|
||||
|---|---|---|
|
||||
| Mechanism | `CGEventPost` synthetic events | `AXUIElementPerformAction` direct API |
|
||||
| CEF crash risk | Yes | None |
|
||||
| Coordinates | Required (needs screenshot) | None — finds by label |
|
||||
| Works when app unfocused | No | Yes |
|
||||
|
||||
**Verified working:** Direct AX test against Music listed 256 elements including `Bollywood Hits`, `Play`, etc.; pressing `Bollywood Hits` then `Play` both returned `exact=true` and acted correctly.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.10 — Multi-step UI workflow guidance
|
||||
|
||||
**Status:** ✅ Done
|
||||
|
||||
**Problem:** When asked to "play Highway to Hell by AC/DC", the agent ran: launch → list → press Library → press Songs → press "Show Filter Field" → set_value "Highway to Hell" → **press "Play"**. The final press hit the **global playback bar Play button** (plays last queue item), not the specific song row. Result: app navigated correctly but the wrong/no track played.
|
||||
|
||||
**Fix:**
|
||||
- `src/openhuman/agent/prompts/SOUL.md` — added explicit multi-step workflow:
|
||||
1. `list` → discover elements
|
||||
2. `set_value` → type in filter/search
|
||||
3. `list` **again** → see filtered results
|
||||
4. `press` the **specific item** (song row), not the generic Play button
|
||||
- Added Apple Music guidance: use `shell` to open `music://music.apple.com/search?term=...`, then `ax_interact list` to see song rows as AXCells, then press the specific row. More reliable than the Library filter field.
|
||||
|
||||
**Result:** Agent is directed to select the specific item before pressing playback, instead of pressing the global Play button after filtering.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.11 — Apple Music two-step play (navigate then play)
|
||||
|
||||
**Status:** ✅ Done
|
||||
|
||||
**Problem:** When asked to "play Highway to Hell by AC/DC", the agent navigated to the right screen but **nothing played**. Pressing a search-result row in Apple Music only *selects/navigates* — it does not start playback. The agent then pressed the global transport Play button, but nothing was queued.
|
||||
|
||||
**Investigation (empirical AX probing against live Music):**
|
||||
- Every "Highway to Hell" element (AXCell, AXGroup, AXButton) exposes only the `AXPress` action — which selects/navigates, never plays.
|
||||
- Double `AXPress`, a real CGEvent double-click on the Top-Results card, and AX-select + Return key **all left player state `stopped`**.
|
||||
- **Working sequence found:** AXPress the search-result card to **navigate into the song's detail page**, then AXPress the **Play button on that detail page** → `player state: playing` ✅
|
||||
|
||||
**Fix:**
|
||||
- `src/openhuman/agent/prompts/SOUL.md` — replaced the Apple Music guidance with the exact 5-step sequence: URL-scheme search → list → press song row (navigates in) → list detail page → press detail-page Play. Explicitly warns that pressing a search result only navigates, and the second Play press is mandatory.
|
||||
- `src/openhuman/accessibility/ax_interact_tests.rs` — `test_full_flow_search_and_play_acdc` exercises the full navigate-then-play flow and **logs** the `osascript ... get player state` outcome. Playback is **best-effort, not hard-asserted** (Apple Music's UI is nondeterministic — see change 1.13); the test hard-asserts only the tool-level press/list successes.
|
||||
|
||||
**Verified:**
|
||||
```text
|
||||
[step 4] navigate into song: Ok("Pressed 'Highway to Hell' in 'Music'.")
|
||||
[step 5] press detail Play: Ok("Pressed 'Play' in 'Music'.")
|
||||
[step 6] player state: playing
|
||||
test ... ok
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Change 1.12 — One-shot `play_music` tool (root-cause fix)
|
||||
|
||||
**Status:** ✅ Done
|
||||
|
||||
**Problem:** Even after change 1.11, the agent still used the broken filter-field approach and didn't play. Transcript analysis (`~/.openhuman/users/<id>/workspace/session_raw/*.jsonl`) revealed two real root causes:
|
||||
|
||||
1. **The orchestrator has no `shell` tool.** Change 1.11 put the play guidance in `SOUL.md` — but the orchestrator runs with `omit_identity = true` and **never sees SOUL.md**. Change 1.11b moved it to the `ax_interact` description, which told the agent to "use the shell tool to open `music://...`" — but the orchestrator can't run shell (it delegates). The agent wrapped the command in a `prompt` arg to a delegation tool; it never executed, and it fell back to the filter approach.
|
||||
2. **Cross-chat memory contamination.** The user message was prefixed with `[Cross-chat context — historical]` containing prior filter-approach "Progress Checkpoint" steps, biasing the agent back to the wrong method.
|
||||
|
||||
**Fix — stop relying on the LLM to orchestrate a fragile multi-step flow with a tool it lacks. Encapsulate the whole proven sequence in native Rust:**
|
||||
- `src/openhuman/accessibility/ax_interact.rs` — `play_apple_music(query)`: open search URL → AX-find + press song cell (navigate) → press detail-page Play → verify `player state == playing`
|
||||
- `src/openhuman/tools/impl/computer/play_music.rs` (new) — `PlayMusicTool`, single call `play_music{query}`, `PermissionLevel::ReadOnly`, runs the blocking flow via `spawn_blocking`
|
||||
- Registered in `ops.rs`, `user_filter.rs`, `orchestrator/agent.toml`, `toolDefinitions.ts`
|
||||
|
||||
**Result:** Agent calls `play_music{query:'Highway to Hell AC/DC'}` **once**; Rust does search→navigate→play. No shell dependency, no multi-step LLM orchestration, no filter-field fallback. Unit tests pass; the underlying flow is exercised by `test_full_flow_search_and_play_acdc` (tool-level success hard-asserted, playback best-effort). **Note:** `play_music` was later removed in change 1.13 in favour of the generic `ax_interact` tool — this entry documents the investigation that led there.
|
||||
|
||||
**Key learning:** The orchestrator (chat agent) only reads **tool descriptions + agent.toml** — NOT SOUL.md (omit_identity=true). Behavior guidance for the chat agent must live in tool descriptions or be encapsulated in the tool itself.
|
||||
|
||||
---
|
||||
|
||||
### Change 1.13 — Generic any-app tool + filtered list (remove play_music)
|
||||
|
||||
**Status:** ✅ Done
|
||||
|
||||
**Problem:** "Play Numb by Linkin Park" still failed, and the agent **hallucinated**. Transcript (`session_raw/*.jsonl`) showed:
|
||||
1. `play_music` hit a 4s timing race — results hadn't rendered, so it returned "No matching song found. Top result cells: [empty]".
|
||||
2. The agent fell back to `ax_interact list`, which dumped **273 elements**. The tool result was **truncated mid-list**, so the model reasoned over a partial view and hallucinated a wrong result ("Numb - Single by Marshmello").
|
||||
|
||||
**Feedback:** A music-specific tool is the wrong abstraction. Build a generic tool that interacts with **any** app.
|
||||
|
||||
**Fix:**
|
||||
- **Removed** `play_music` tool + `play_apple_music` helper and all registrations.
|
||||
- **`ax_interact` is now a robust generic any-app tool:**
|
||||
- `ax_list_elements_filtered(app, filter)` — Rust-side label filter so `list` returns only relevant elements (fixes the truncation→hallucination root cause).
|
||||
- `list` action takes a new `filter` param; output capped at 60 elements with a "narrow your filter" hint; empty-match returns a "UI may still be loading" hint instead of failing hard.
|
||||
- Description rewritten to be app-agnostic and document the general **navigate-then-activate** pattern (pressing a list row/search result selects/opens it; press the action button afterward) — no hardcoded Apple Music steps.
|
||||
|
||||
**Key learning:** Dumping a full AX tree (hundreds of elements) overflows the tool-result budget; the truncated view makes the model hallucinate. Always filter list results to keep them small and accurate.
|
||||
|
||||
---
|
||||
|
||||
## Windows port — app interaction 🪟 ✅ Implemented
|
||||
|
||||
Phase 1's app-interaction layer is now ported to Windows. The macOS path uses the
|
||||
Accessibility API via a Swift helper; the Windows path uses **Microsoft UI
|
||||
Automation (UIA)** called directly from Rust (no helper process). The
|
||||
agent-facing tool is a single `ax_interact` tool on both platforms — only the
|
||||
backend differs, via cfg-dispatch. The sections below were the design plan; see
|
||||
**"Windows port — implementation status"** at the end of this part for what
|
||||
shipped and the test evidence.
|
||||
|
||||
### What already works cross-platform
|
||||
|
||||
| Capability | macOS (done) | Windows status |
|
||||
|---|---|---|
|
||||
| Auto-send dictation transcript | TS (`useDictationHotkey`/`Conversations`) | ✅ Already cross-platform (frontend) |
|
||||
| App launching | `launch_app` / `policy_command.rs` | ✅ Launchers (`open`/`xdg-open`/`start`) stay OUT of `READ_ONLY_BASES` (can open arbitrary URLs/handlers); app launching goes through the gated `launch_app` tool. |
|
||||
| `launch_app` | `open -a` | ⚠️ Already has a Windows branch (`Start-Process`) — verify it resolves app display names |
|
||||
| `ax_interact` (list/press/set_value) | AXUIElement Swift helper | ❌ Needs a UI Automation (UIA) backend |
|
||||
|
||||
### 1. Launching apps on Windows (`launch_app`)
|
||||
|
||||
`launch_app.rs` already has a `#[cfg(target_os = "windows")]` branch using PowerShell `Start-Process "<app_name>"`. Caveats to verify on the Windows machine:
|
||||
- `Start-Process "Spotify"` works for apps on PATH or registered App Paths, but **Store/UWP apps** (e.g. the Windows "Media Player", "Spotify" from the Store) need their AUMID: `Start-Process "shell:AppsFolder\<AUMID>"`. Consider enumerating Store apps via `Get-StartApps` (returns Name + AppID) and matching by display name.
|
||||
- For URIs (e.g. `spotify:`, `mailto:`), `Start-Process "<uri>"` works the same as macOS `open`.
|
||||
|
||||
### 2. App UI interaction (`ax_interact` → UI Automation)
|
||||
|
||||
**The Windows analog of macOS AXUIElement is Microsoft UI Automation (UIA)** — the OS-level accessibility tree. It exposes the same concepts:
|
||||
|
||||
| macOS AX concept | Windows UIA equivalent |
|
||||
|---|---|
|
||||
| `AXUIElement` | `IUIAutomationElement` |
|
||||
| `kAXRoleAttribute` (AXButton, AXCell…) | `ControlType` (Button, ListItem, Edit, Text…) |
|
||||
| `kAXTitleAttribute` / `kAXDescriptionAttribute` | `Name` property (+ `AutomationId`, `HelpText`) |
|
||||
| `AXUIElementPerformAction(kAXPressAction)` | `InvokePattern.Invoke()` (buttons) / `SelectionItemPattern.Select()` (list rows) |
|
||||
| `AXUIElementSetAttributeValue(kAXValueAttribute)` | `ValuePattern.SetValue()` (text fields) |
|
||||
| `AXUIElementCopyAttributeValue(kAXChildrenAttribute)` | `TreeWalker` / `FindAll(TreeScope.Descendants, …)` |
|
||||
| Walk tree from app PID | `IUIAutomation.ElementFromHandle(hwnd)` or root + `ProcessId` condition |
|
||||
|
||||
**Recommended implementation path (Rust-native, no helper process needed):**
|
||||
- Use the [`uiautomation`](https://crates.io/crates/uiautomation) crate (safe Rust bindings over the UIA COM API). This is cleaner than macOS, where we had to shell out to a Swift helper — on Windows the COM API is callable directly from Rust.
|
||||
- Mirror the existing `accessibility::ax_interact` surface so the **tool stays identical**:
|
||||
- `list(app, filter)` → `CreateTreeWalker` over the app's window, collect elements whose `Name` matches `filter`, return `[{control_type, name}]`.
|
||||
- `press(app, label)` → find element by `Name` (exact-first), then call `InvokePattern` if supported, else `SelectionItemPattern.Select()`, else `LegacyIAccessiblePattern.DoDefaultAction()`.
|
||||
- `set_value(app, label, value)` → find `Edit`/`ComboBox`, call `ValuePattern.SetValue()`.
|
||||
- **Key win over macOS:** UIA Invoke is generally a real "activate" (it triggers the control's default action), so the navigate-then-activate two-step that plagued Apple Music is less likely. A list-item Invoke on most Windows media apps plays directly. Still expect per-app quirks.
|
||||
|
||||
**Suggested module layout (parallel to macOS):**
|
||||
```text
|
||||
src/openhuman/accessibility/
|
||||
ax_interact.rs # macOS (existing)
|
||||
uia_interact.rs # NEW — Windows UIA backend, same fn signatures
|
||||
mod.rs # cfg-dispatch: pub use the right backend per-OS
|
||||
```
|
||||
Then `tools/impl/computer/ax_interact.rs` calls a thin cfg-gated facade so the **agent-facing tool is one tool on both platforms** (same name `ax_interact`, same actions). Update its description to be OS-neutral ("uses the platform accessibility API").
|
||||
|
||||
### 3. Permissions
|
||||
|
||||
- macOS needs the Accessibility permission. **Windows UIA needs no special permission** for same-session, same-integrity-level apps — a big simplification. Caveat: a non-elevated process can't drive an **elevated** app's UI (UIPI). If the agent must control an elevated app, OpenHuman would need to run elevated too (avoid unless necessary).
|
||||
|
||||
### 4. Diagnostics
|
||||
|
||||
Keep the same `[ax_interact]`/`[uia_interact]` log prefixes and the verbose step logging (`▶ action`, found-count, press result) — they were essential for diagnosing the macOS flow and will be just as useful on Windows.
|
||||
|
||||
### 5. Testing
|
||||
|
||||
Port the integration tests using a built-in Windows app that's always present and UIA-friendly:
|
||||
- **Calculator** (`calc.exe`) — press digit/operator buttons by Name, read the result `Text` element. Deterministic, no network, ideal smoke test.
|
||||
- **Notepad** — `set_value` into the `Edit` control, verify via `ValuePattern.Value`.
|
||||
- Media: **Windows Media Player** or the Store **Media Player** for a play test, but expect the same nondeterminism caveat as Apple Music — assert tool-level success, log playback as best-effort.
|
||||
|
||||
### 6. Known-different behaviors to watch for
|
||||
|
||||
- **Element naming:** Windows apps often populate `AutomationId` (stable) where macOS only had a visible title. Prefer matching `Name`, fall back to `AutomationId`.
|
||||
- **Chromium/Electron apps** (Slack, Discord, VS Code, Spotify desktop): on Windows these expose a partial UIA tree by default; some require the app to have accessibility enabled. Same class of limitation as the macOS `chromiumAppPatterns` special-casing already in `helper.rs`.
|
||||
- **Focus/foreground:** UIA generally doesn't require foregrounding to read/invoke, like macOS AX. No CGEventPost-style CEF crash risk because UIA Invoke is not synthetic input injection.
|
||||
|
||||
### Quick start for the Windows machine
|
||||
|
||||
1. `launch_app` should already work — test `"open notepad"` / `"open calculator"` first.
|
||||
2. Do NOT add launchers to `READ_ONLY_BASES` — `launch_app` (gated, URI-rejecting) is the Windows app-launch path. `Start-Process` lives inside that tool, not the shell allowlist.
|
||||
3. Build `uia_interact.rs` against the `uiautomation` crate, mirroring the three `ax_interact` fns.
|
||||
4. cfg-dispatch in `accessibility/mod.rs` so `ax_interact` the tool resolves to UIA on Windows.
|
||||
5. Smoke-test with Calculator (deterministic), then a media app (best-effort).
|
||||
|
||||
### Cross-platform compatibility audit (current state)
|
||||
|
||||
Every Phase 1 change was written to **compile on all platforms** — nothing here breaks the Windows build. macOS-specific native code is `#[cfg(target_os = "macos")]`-gated and the non-macOS branches return a clean `"…macOS-only"` error at runtime rather than failing to build.
|
||||
|
||||
| Change | Cross-platform status | Notes for Windows |
|
||||
|---|---|---|
|
||||
| Auto-send dictation transcript (TS) | ✅ Fully portable | Pure frontend; no OS code. Works as-is. |
|
||||
| `launch_app` | ✅ macOS / Linux / Windows branches | Windows branch now uses `Start-Process` with a **Store/UWP (`Get-StartApps` AUMID) fallback** and injection-safe env passing (§1). |
|
||||
| `ax_interact` **tool** (`tools/impl/computer/ax_interact.rs`) | ✅ Functional on Windows | Delegates to `accessibility::ax_interact` fns, which now cfg-dispatch to the UIA backend on Windows. Description made OS-neutral. |
|
||||
| `accessibility::ax_interact` helpers | ✅ cfg-dispatched | macOS → Swift helper; Windows → `uia_interact.rs`; other → clean runtime error. |
|
||||
| `accessibility::uia_interact` (NEW) | ✅ Windows backend | UIA `list`/`press`/`set_value` via the `uiautomation` crate; same fn signatures as the macOS path. |
|
||||
| Swift unified helper (`accessibility/helper.rs`) | ⚠️ macOS-only by design | Windows needs no helper process — UIA COM API is called directly from Rust. |
|
||||
| App launching | ✅ Done | Launchers stay out of `READ_ONLY_BASES`; `launch_app` (gated) handles Windows `Start-Process`. |
|
||||
| Notch indicator (separate PR #3166) | ⚠️ macOS NSPanel | A Windows equivalent would be a borderless always-on-top WebView2 window or a tray flyout — out of scope for this branch. |
|
||||
|
||||
**Before merging the Windows port, confirm the whole branch still builds and runs on macOS too** (`cargo check` on both `Cargo.toml` and `app/src-tauri/Cargo.toml`) so the cfg-dispatch doesn't regress the working macOS path.
|
||||
|
||||
### ⚠️ Mandatory: extensive E2E testing on Windows
|
||||
|
||||
The macOS path was hardened only through repeated real-app runs (each bug — CEF crash, select-vs-play, list truncation/hallucination — surfaced only by actually driving live apps, not by unit tests). **Do the same on Windows before considering it done.** Treat the following as the required E2E matrix:
|
||||
|
||||
1. **App launch** — `launch_app` for: a Win32 app (Notepad), a Store/UWP app (Media Player / Spotify from Store), and a URI (`spotify:`). Confirm each actually opens.
|
||||
2. **Deterministic UI control** — Calculator: `list filter='5'` → `press '5'`, `press '+'`, `press '='`, then read the result element. Assert the computed value. This is the Windows analog of the AC/DC test and should be a **hard-asserted** automated test (Calculator is deterministic).
|
||||
3. **Text entry** — Notepad: `set_value` into the Edit control, verify via `ValuePattern.Value`.
|
||||
4. **Filtered list correctness** — confirm `list` with a `filter` returns a small, accurate set (the truncation→hallucination bug must not recur; verify the 60-element cap + filter behaves on a busy app like Settings or a browser).
|
||||
5. **Real-world app** — drive a media app end-to-end (open → search → play). Assert tool-level success; treat playback state as **best-effort** (same nondeterminism caveat as Apple Music).
|
||||
6. **Chromium/Electron app** — Slack/Discord/VS Code: confirm whether their UIA tree is exposed; document any app that needs accessibility explicitly enabled.
|
||||
7. **Permissions/elevation** — verify behavior against a normal app vs an elevated one (UIPI boundary); document what fails and why.
|
||||
8. **Agent-in-the-loop run** — the real test: ask the running agent (chat) to perform each action and confirm it picks `launch_app` / `ax_interact` and the action lands. Watch `[ax_interact]`/`[launch_app]` logs exactly as we did on macOS.
|
||||
9. **Regression** — re-run the macOS E2E suite after the Windows changes land to prove cfg-dispatch didn't break the Mac path.
|
||||
|
||||
Add the deterministic ones (Calculator, Notepad, launch) as `#[cfg(target_os = "windows")]` `#[ignore]` integration tests mirroring `ax_interact_tests.rs`, runnable with `cargo test ... -- --include-ignored` on the Windows machine.
|
||||
|
||||
### Windows port — implementation status ✅
|
||||
|
||||
Shipped on the Windows machine (2026-06-02):
|
||||
|
||||
**Code**
|
||||
- `Cargo.toml` — `uiautomation = "0.25"` under `[target.'cfg(windows)'.dependencies]`; `Win32_System_Com` feature added to `windows-sys` for COM init.
|
||||
- `src/openhuman/accessibility/uia_interact.rs` (**new**) — UIA backend. `list` / `press` / `set_value` over the UIA COM tree via the `uiautomation` crate, mirroring the macOS `ax_interact` fn signatures. `press` activates via UIA patterns in order — `Invoke` → `SelectionItem.Select` → `LegacyIAccessible.DoDefaultAction` — never injecting synthetic input. `set_value` finds an editable field, preferring `Edit`, then `ComboBox`, then `Document` (so the Win11 RichEdit Notepad, whose editor is a `Document`, works). Exact-label match preferred over substring. Per-thread COM init via `CoInitializeEx(MTA)`.
|
||||
- `src/openhuman/accessibility/ax_interact.rs` — the three public helpers now cfg-dispatch: macOS → Swift helper, Windows → `uia_interact`, else → clean runtime error. Module + tool docs made OS-neutral.
|
||||
- `src/openhuman/accessibility/mod.rs` — declares `uia_interact` (cfg-gated to Windows).
|
||||
- `src/openhuman/tools/impl/computer/ax_interact.rs` — description rewritten to be platform-neutral ("platform accessibility API (macOS AXUIElement / Windows UI Automation)").
|
||||
- `src/openhuman/tools/impl/system/launch_app.rs` — Windows launcher hardened: app name passed via env var (no string interpolation → no injection), `Start-Process` first, then Store/UWP fallback by display name via `Get-StartApps` → AUMID (`shell:AppsFolder\<AppID>`); stderr surfaced on failure.
|
||||
- `src/openhuman/security/policy_command.rs` — launchers (`open`/`xdg-open`/`start`) deliberately kept OUT of `READ_ONLY_BASES`; `launch_app` is the gated launch path.
|
||||
- `src/openhuman/accessibility/uia_interact_tests.rs` (**new**) — `#[cfg(all(test, target_os = "windows"))]` integration tests, wired into `ax_interact.rs`.
|
||||
|
||||
**Test evidence (real apps on Windows 11)**
|
||||
- `test_uia_calculator_five_plus_five` ✅ — drove the live Calculator entirely by element label: `list` → 41 interactive elements; pressed `Five`/`Plus`/`Five`/`Equals`; **hard-asserted** the readout `[Text] "Display is 10"` (5 + 5 = 10). Deterministic — the Windows analogue of the macOS AC/DC test.
|
||||
- `test_uia_notepad_set_value` ✅ — `set_value` wrote into the live Win11 Notepad's `Document` "Text editor" (`Ok("Set 'Text editor' in 'Notepad' to the provided value.")`). The `Document` fallback is what makes the redesigned Notepad work.
|
||||
- `test_uia_list_nonexistent_app` ✅ (non-ignored) — exercises COM init + window walk + error path deterministically.
|
||||
- `launch_app` (×8) and `ax_interact` tool (×4) unit tests ✅.
|
||||
- Full `cargo test --lib --no-run` compiles clean on Windows (warnings only, all pre-existing).
|
||||
|
||||
**Environment gotcha (this machine):** Norton real-time protection blocks `link.exe` from writing the freshly-linked ~150 MB test `.exe` (LNK1104, "Access denied" creating the file). Fix: exclude the repo's `target` dir under Norton's **Auto-Protect / SONAR / Download Intelligence** exclusion list (not the separate "Scans" list), and restore the file from Quarantine if already flagged.
|
||||
|
||||
**Follow-ups / not done here**
|
||||
- **macOS regression check** — the cfg-dispatch is additive (the `#[cfg(target_os="macos")]` arms are untouched; only the non-macOS catch-all message changed), but per the branch note, re-run `cargo check` + the macOS E2E suite on a Mac before merge to prove the Mac path didn't regress (can't be done from the Windows machine).
|
||||
- **Agent-in-the-loop run** (E2E item 8) — the full Tauri desktop app was built and run on Windows (`pnpm dev:app:win`) and the in-process core booted fine (verbose `[launch_app]`/`[ax_interact]`/`[uia_interact]` logging wired via `RUST_LOG`). The first chat attempt couldn't complete because the configured **local AI model was still downloading** (`kind="empty_provider_response"` — the agent returned an empty response, so it never reached a tool call). **Still pending:** a working model (finish the local download or select a configured cloud model), then ask the agent "open Calculator" / "press 5 in Calculator" and confirm it picks `launch_app`/`ax_interact` and the action lands.
|
||||
- Chromium/Electron UIA coverage, elevation/UIPI behavior, and a busy-app filtered-list check (E2E items 4/6/7) remain to be spot-checked manually.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Always-On Listening ⏳ Not Started
|
||||
|
||||
> Continuous microphone listening without requiring a hotkey press.
|
||||
|
||||
**Planned files:**
|
||||
- `src/openhuman/voice/always_on.rs` (new) — dedicated tokio task holding the mic open, running VAD, emitting utterances to the STT pipeline
|
||||
- `src/openhuman/config/schema/voice_server.rs` — add `always_on_enabled: bool` config flag
|
||||
- Privacy hook: pause always-on when screen is locked
|
||||
|
||||
**Acceptance criteria:**
|
||||
- [ ] User can speak without pressing any hotkey
|
||||
- [ ] VAD detects end of utterance and sends to agent
|
||||
- [ ] Toggle in Settings → Voice
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — Wake-Word + Fast Routing ⏳ Not Started
|
||||
|
||||
> Activate only on a trigger phrase; route simple commands locally without a full LLM turn.
|
||||
|
||||
**Planned files:**
|
||||
- `src/openhuman/inference/voice/wake_word.rs` (new) — lightweight always-on model (Porcupine or custom ONNX)
|
||||
- `src/openhuman/voice/command_router.rs` (new) — intent→tool mapping for high-confidence commands, LLM fallback for ambiguous input
|
||||
|
||||
**Acceptance criteria:**
|
||||
- [ ] Wake-word detection runs fully on-device
|
||||
- [ ] Latency from end-of-utterance to action start ≤ 500ms for local-routed commands
|
||||
|
||||
---
|
||||
|
||||
## Phase 4 — Polish ⏳ Not Started
|
||||
|
||||
> Voice confirmation loop, UI indicator, computer control onboarding.
|
||||
|
||||
**Planned:**
|
||||
- TTS confirmation before executing sensitive actions ("Opening Music — confirm?")
|
||||
- Always-on status indicator (notch pill from PR #3166 will handle this automatically)
|
||||
- Computer control (`mouse`/`keyboard` tools) toggle in Settings onboarding
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
| Phase | Item | Status |
|
||||
|---|---|---|
|
||||
| 1 | Auto-send after transcription | ✅ Done |
|
||||
| 1 | Shell allowlist for `open`/`xdg-open` | ✅ Done |
|
||||
| 1 | Shell tool description clarification | ✅ Done |
|
||||
| 1 | Dedicated `launch_app` tool | ✅ Done |
|
||||
| 1 | Orchestrator tool scope | ✅ Done |
|
||||
| 1 | SOUL.md capability hint | ✅ Done |
|
||||
| 1 | Diagnostic logging | ✅ Done |
|
||||
| 1 | Computer control (mouse/keyboard) | ❌ Reverted (CEF crash) |
|
||||
| 1 | AXUIElement app UI interaction (`ax_interact`) | ✅ Done |
|
||||
| 1 | Multi-step UI workflow guidance | ✅ Done |
|
||||
| 1 | Apple Music two-step play (navigate→play) | ✅ Done (playback best-effort) |
|
||||
| 2 | Always-on microphone loop | ⏳ Not started |
|
||||
| 2 | `always_on_enabled` config flag | ⏳ Not started |
|
||||
| 2 | Privacy hook (screen lock pause) | ⏳ Not started |
|
||||
| 3 | Wake-word detection | ⏳ Not started |
|
||||
| 3 | Local command router | ⏳ Not started |
|
||||
| 4 | Voice confirmation loop | ⏳ Not started |
|
||||
| 4 | Always-on UI indicator | ✅ Done (notch PR #3166) |
|
||||
@@ -0,0 +1,152 @@
|
||||
//! Accessibility interaction helpers — list, press, and set-value for named apps.
|
||||
//!
|
||||
//! Cross-platform facade over the OS accessibility tree. Each public fn
|
||||
//! cfg-dispatches to the right backend:
|
||||
//! - macOS: the unified Swift helper (`helper.rs`), which walks the AX tree
|
||||
//! without injecting synthetic events (unlike enigo/CGEventPost).
|
||||
//! Works even when OpenHuman is not focused, and never crashes CEF.
|
||||
//! - Windows: the UI Automation backend (`uia_interact.rs`), which drives the
|
||||
//! UIA COM tree directly — same "no synthetic input" guarantee.
|
||||
//!
|
||||
//! Other platforms return a clean runtime error. The agent-facing `ax_interact`
|
||||
//! tool is a single tool on every platform; only the backend differs.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "ax_interact_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
#[cfg(all(test, target_os = "windows"))]
|
||||
#[path = "uia_interact_tests.rs"]
|
||||
mod uia_tests;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct AXElement {
|
||||
pub role: String,
|
||||
pub label: String,
|
||||
}
|
||||
|
||||
/// List interactive UI elements (buttons, text fields, checkboxes, …) in `app_name`.
|
||||
pub fn ax_list_elements(app_name: &str) -> Result<Vec<AXElement>, String> {
|
||||
ax_list_elements_filtered(app_name, "")
|
||||
}
|
||||
|
||||
/// List interactive UI elements in `app_name`, optionally keeping only those
|
||||
/// whose label contains `filter` (case-insensitive). An empty `filter` returns
|
||||
/// everything. Filtering happens on the Rust side so the tool result stays
|
||||
/// small — dumping every element (apps expose hundreds) overflows the result
|
||||
/// budget and causes the model to hallucinate from a truncated view.
|
||||
pub fn ax_list_elements_filtered(app_name: &str, filter: &str) -> Result<Vec<AXElement>, String> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let req = serde_json::json!({ "type": "ax_list", "app_name": app_name });
|
||||
let resp = super::helper::helper_send_receive(&req)?;
|
||||
if resp.get("ok").and_then(|v| v.as_bool()) == Some(false) {
|
||||
let err = resp
|
||||
.get("error")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown error");
|
||||
return Err(err.to_string());
|
||||
}
|
||||
let mut elements: Vec<AXElement> = resp
|
||||
.get("elements")
|
||||
.and_then(|v| serde_json::from_value(v.clone()).ok())
|
||||
.unwrap_or_default();
|
||||
let needle = filter.trim().to_lowercase();
|
||||
if !needle.is_empty() {
|
||||
elements.retain(|e| e.label.to_lowercase().contains(&needle));
|
||||
}
|
||||
return Ok(elements);
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
return super::uia_interact::list(app_name, filter);
|
||||
}
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
{
|
||||
let _ = (app_name, filter);
|
||||
Err("ax_interact is supported on macOS and Windows only".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Press the first UI element in `app_name` whose label contains `label` (case-insensitive).
|
||||
///
|
||||
/// Rejects a blank `label`: with an empty needle the helper's `contains`
|
||||
/// match degenerates to match-all and would press the first named control it
|
||||
/// finds. Guard here rather than trusting every caller to pre-validate.
|
||||
pub fn ax_press_element(app_name: &str, label: &str) -> Result<String, String> {
|
||||
if label.trim().is_empty() {
|
||||
return Err("label must not be empty for press".into());
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let req = serde_json::json!({
|
||||
"type": "ax_press",
|
||||
"app_name": app_name,
|
||||
"label": label,
|
||||
});
|
||||
let resp = super::helper::helper_send_receive(&req)?;
|
||||
if resp.get("ok").and_then(|v| v.as_bool()) == Some(false) {
|
||||
let err = resp
|
||||
.get("error")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown error");
|
||||
return Err(err.to_string());
|
||||
}
|
||||
let pressed = resp
|
||||
.get("pressed")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or(label)
|
||||
.to_string();
|
||||
return Ok(format!("Pressed '{pressed}' in '{app_name}'."));
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
return super::uia_interact::press(app_name, label);
|
||||
}
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
{
|
||||
let _ = (app_name, label);
|
||||
Err("ax_interact is supported on macOS and Windows only".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the value of the first text field in `app_name` whose label contains `label`.
|
||||
/// Pass an empty `label` to target the first available text field.
|
||||
pub fn ax_set_field_value(app_name: &str, label: &str, value: &str) -> Result<String, String> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let req = serde_json::json!({
|
||||
"type": "ax_set_value",
|
||||
"app_name": app_name,
|
||||
"label": label,
|
||||
"value": value,
|
||||
});
|
||||
let resp = super::helper::helper_send_receive(&req)?;
|
||||
if resp.get("ok").and_then(|v| v.as_bool()) == Some(false) {
|
||||
let err = resp
|
||||
.get("error")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown error");
|
||||
return Err(err.to_string());
|
||||
}
|
||||
let field = resp
|
||||
.get("field")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or(label)
|
||||
.to_string();
|
||||
return Ok(format!(
|
||||
"Set '{field}' in '{app_name}' to the provided value."
|
||||
));
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
return super::uia_interact::set_value(app_name, label, value);
|
||||
}
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
{
|
||||
let _ = (app_name, label, value);
|
||||
Err("ax_interact is supported on macOS and Windows only".into())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
//! Integration tests for AXUIElement-based app interaction.
|
||||
//!
|
||||
//! These tests require:
|
||||
//! 1. macOS with Accessibility permission granted to the test runner
|
||||
//! 2. Apple Music to be running (or openable)
|
||||
//!
|
||||
//! Run with: cargo test ax_interact -- --nocapture --include-ignored
|
||||
|
||||
#![cfg(all(test, target_os = "macos"))]
|
||||
|
||||
use super::{ax_list_elements, ax_press_element, ax_set_field_value};
|
||||
use std::process::Command;
|
||||
use std::thread::sleep;
|
||||
use std::time::Duration;
|
||||
|
||||
fn ensure_music_open() -> bool {
|
||||
let ok = Command::new("open")
|
||||
.arg("-a")
|
||||
.arg("Music")
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false);
|
||||
if ok {
|
||||
sleep(Duration::from_secs(2));
|
||||
}
|
||||
ok
|
||||
}
|
||||
|
||||
fn open_acdc_search() {
|
||||
Command::new("open")
|
||||
.arg("music://music.apple.com/search?term=Highway+to+Hell+ACDC")
|
||||
.status()
|
||||
.ok();
|
||||
sleep(Duration::from_secs(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires macOS Accessibility permission and Apple Music"]
|
||||
fn test_ax_list_returns_elements() {
|
||||
assert!(ensure_music_open(), "Could not open Music");
|
||||
let elements = ax_list_elements("Music").expect("ax_list_elements failed");
|
||||
assert!(
|
||||
!elements.is_empty(),
|
||||
"Expected interactive elements in Music"
|
||||
);
|
||||
println!("Found {} elements:", elements.len());
|
||||
for el in &elements {
|
||||
println!(" [{}] {}", el.role, el.label);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires macOS Accessibility permission and Apple Music"]
|
||||
fn test_ax_press_play_button() {
|
||||
assert!(ensure_music_open(), "Could not open Music");
|
||||
let result = ax_press_element("Music", "Play");
|
||||
println!("press Play: {:?}", result);
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"Expected Play button to be pressable: {:?}",
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires macOS Accessibility permission and Apple Music"]
|
||||
fn test_full_flow_search_and_play_acdc() {
|
||||
assert!(ensure_music_open(), "Could not open Music");
|
||||
|
||||
let elements = ax_list_elements("Music").expect("ax_list failed");
|
||||
assert!(
|
||||
!elements.is_empty(),
|
||||
"Music AX tree empty — check Accessibility permission"
|
||||
);
|
||||
println!("[step 1] AX tree: {} elements", elements.len());
|
||||
|
||||
open_acdc_search();
|
||||
println!("[step 2] search URL opened");
|
||||
|
||||
let after_search = ax_list_elements("Music").expect("ax_list post-search failed");
|
||||
let highway = after_search
|
||||
.iter()
|
||||
.find(|e| e.label.contains("Highway to Hell"));
|
||||
println!(
|
||||
"[step 3] 'Highway to Hell' element: {:?}",
|
||||
highway.map(|e| &e.label)
|
||||
);
|
||||
assert!(
|
||||
highway.is_some(),
|
||||
"Expected 'Highway to Hell' in results. Found:\n{}",
|
||||
after_search
|
||||
.iter()
|
||||
.map(|e| format!(" [{}] {}", e.role, e.label))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
|
||||
// Pressing the search result NAVIGATES into the song detail page
|
||||
// (it does not start playback — Apple Music only selects/opens on press).
|
||||
let nav_result = ax_press_element("Music", "Highway to Hell");
|
||||
println!("[step 4] navigate into song: {:?}", nav_result);
|
||||
assert!(
|
||||
nav_result.is_ok(),
|
||||
"Could not navigate into song: {:?}",
|
||||
nav_result
|
||||
);
|
||||
|
||||
sleep(Duration::from_secs(2));
|
||||
|
||||
// On the detail page, press the prominent Play button to actually play.
|
||||
let play_result = ax_press_element("Music", "Play");
|
||||
println!("[step 5] press detail Play: {:?}", play_result);
|
||||
assert!(
|
||||
play_result.is_ok(),
|
||||
"Could not press detail Play: {:?}",
|
||||
play_result
|
||||
);
|
||||
|
||||
sleep(Duration::from_secs(2));
|
||||
|
||||
// Playback outcome is best-effort and NOT asserted: Apple Music's UI is
|
||||
// nondeterministic here (detail-page render timing varies, and there are
|
||||
// multiple "Play" elements — detail-page vs transport-bar — that AX can't
|
||||
// reliably disambiguate). What this test verifies is that the generic
|
||||
// ax_interact primitives (list / press) work against a real app; the
|
||||
// player state is logged for diagnosis only.
|
||||
let state = Command::new("osascript")
|
||||
.args(["-e", "tell application \"Music\" to get player state"])
|
||||
.output()
|
||||
.expect("osascript player state failed");
|
||||
let state_str = String::from_utf8_lossy(&state.stdout);
|
||||
println!(
|
||||
"[step 6] player state (best-effort, not asserted): {}",
|
||||
state_str.trim()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires macOS Accessibility permission and Apple Music"]
|
||||
fn test_ax_set_search_field() {
|
||||
assert!(ensure_music_open(), "Could not open Music");
|
||||
Command::new("open")
|
||||
.arg("music://music.apple.com/search")
|
||||
.status()
|
||||
.ok();
|
||||
sleep(Duration::from_secs(2));
|
||||
let result = ax_set_field_value("Music", "Search", "Bollywood");
|
||||
println!("set_value Search=Bollywood: {:?}", result);
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"Expected the Search field to accept text: {:?}",
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ax_list_nonexistent_app() {
|
||||
let result = ax_list_elements("NonExistentApp12345");
|
||||
assert!(result.is_err(), "Expected error for non-existent app");
|
||||
println!("Error (expected): {:?}", result.unwrap_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ax_press_nonexistent_app() {
|
||||
let result = ax_press_element("NonExistentApp12345", "Play");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
@@ -80,7 +80,7 @@ const HELPER_RECV_TIMEOUT: Duration = Duration::from_secs(8);
|
||||
/// `UNIFIED_HELPER` before blocking on the channel recv, so fire-and-forget
|
||||
/// callers (`show`/`hide`) are never blocked by an in-flight focus query.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(super) fn helper_send_receive(
|
||||
pub(crate) fn helper_send_receive(
|
||||
request: &serde_json::Value,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
// Serialise request/response pairs — prevents interleaved reads.
|
||||
@@ -645,6 +645,138 @@ func pasteText(id: String?, text: String) -> [String: Any] {
|
||||
return result
|
||||
}
|
||||
|
||||
// MARK: - AXUIElement App Interaction
|
||||
|
||||
/// Find a running application by display name (exact, prefix, or contains match).
|
||||
func findRunningApp(named appName: String) -> NSRunningApplication? {
|
||||
let apps = NSWorkspace.shared.runningApplications
|
||||
let lower = appName.lowercased()
|
||||
if let app = apps.first(where: { $0.localizedName?.lowercased() == lower }) { return app }
|
||||
if let app = apps.first(where: { $0.localizedName?.lowercased().hasPrefix(lower) ?? false }) { return app }
|
||||
return apps.first(where: { $0.localizedName?.lowercased().contains(lower) ?? false })
|
||||
}
|
||||
|
||||
/// Walk the AX element tree depth-first. Visitor returns true to stop early.
|
||||
func axWalk(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = 12,
|
||||
visitor: (AXUIElement, String, String) -> Bool) -> Bool {
|
||||
if depth > maxDepth { return false }
|
||||
var roleRef: AnyObject?
|
||||
AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &roleRef)
|
||||
let role = (roleRef as? String) ?? ""
|
||||
var labelRef: AnyObject?
|
||||
AXUIElementCopyAttributeValue(element, kAXTitleAttribute as CFString, &labelRef)
|
||||
var label = (labelRef as? String) ?? ""
|
||||
if label.isEmpty {
|
||||
var descRef: AnyObject?
|
||||
AXUIElementCopyAttributeValue(element, kAXDescriptionAttribute as CFString, &descRef)
|
||||
label = (descRef as? String) ?? ""
|
||||
}
|
||||
if visitor(element, role, label) { return true }
|
||||
var childrenRef: AnyObject?
|
||||
guard AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &childrenRef) == .success,
|
||||
let children = childrenRef as? [AXUIElement] else { return false }
|
||||
for child in children {
|
||||
if axWalk(child, depth: depth + 1, maxDepth: maxDepth, visitor: visitor) { return true }
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/// List interactive UI elements in the named app.
|
||||
func axListElements(appName: String, id: String?) -> [String: Any] {
|
||||
guard let app = findRunningApp(named: appName) else {
|
||||
return ["type": "ax_list", "id": id ?? "", "ok": false,
|
||||
"error": "App '\(appName)' not found or not running", "elements": [] as [Any]]
|
||||
}
|
||||
let axApp = AXUIElementCreateApplication(app.processIdentifier)
|
||||
let interactiveRoles: Set<String> = [
|
||||
"AXButton", "AXMenuItem", "AXMenuBarItem", "AXTextField", "AXTextArea",
|
||||
"AXCheckBox", "AXRadioButton", "AXSlider", "AXPopUpButton",
|
||||
"AXComboBox", "AXLink", "AXTab"
|
||||
]
|
||||
var elements: [[String: String]] = []
|
||||
axWalk(axApp, maxDepth: 10) { _, role, label in
|
||||
if interactiveRoles.contains(role) && !label.isEmpty {
|
||||
elements.append(["role": role, "label": label])
|
||||
}
|
||||
return false
|
||||
}
|
||||
return ["type": "ax_list", "id": id ?? "", "ok": true, "error": NSNull(), "elements": elements]
|
||||
}
|
||||
|
||||
/// Collect all AX elements whose label contains `label` (case-insensitive).
|
||||
/// Returns matches sorted exact-first so "Play" beats "Playlist".
|
||||
struct AXCandidate {
|
||||
var element: AXUIElement
|
||||
var label: String
|
||||
var exact: Bool
|
||||
}
|
||||
|
||||
func axCollectMatches(_ root: AXUIElement, label: String, depth: Int = 0, maxDepth: Int = 12) -> [AXCandidate] {
|
||||
if depth > maxDepth { return [] }
|
||||
var roleRef: AnyObject?; AXUIElementCopyAttributeValue(root, kAXRoleAttribute as CFString, &roleRef)
|
||||
var titleRef: AnyObject?; AXUIElementCopyAttributeValue(root, kAXTitleAttribute as CFString, &titleRef)
|
||||
var elemLabel = (titleRef as? String) ?? ""
|
||||
if elemLabel.isEmpty { var d: AnyObject?; AXUIElementCopyAttributeValue(root, kAXDescriptionAttribute as CFString, &d); elemLabel = (d as? String) ?? "" }
|
||||
let lower = label.lowercased(); let elLower = elemLabel.lowercased()
|
||||
var results: [AXCandidate] = []
|
||||
if !elemLabel.isEmpty, elLower.contains(lower) {
|
||||
results.append(AXCandidate(element: root, label: elemLabel, exact: elLower == lower))
|
||||
}
|
||||
var childrenRef: AnyObject?
|
||||
guard AXUIElementCopyAttributeValue(root, kAXChildrenAttribute as CFString, &childrenRef) == .success,
|
||||
let children = childrenRef as? [AXUIElement] else { return results }
|
||||
for child in children { results += axCollectMatches(child, label: label, depth: depth+1, maxDepth: maxDepth) }
|
||||
return results
|
||||
}
|
||||
|
||||
/// Press a UI element by label — exact match preferred over contains match.
|
||||
func axPress(appName: String, label: String, id: String?) -> [String: Any] {
|
||||
guard let app = findRunningApp(named: appName) else {
|
||||
return ["type": "ax_press", "id": id ?? "", "ok": false,
|
||||
"error": "App '\(appName)' not found or not running"]
|
||||
}
|
||||
let axApp = AXUIElementCreateApplication(app.processIdentifier)
|
||||
var matches = axCollectMatches(axApp, label: label)
|
||||
// Exact matches first so "Play" beats "Playlist"
|
||||
matches.sort { $0.exact && !$1.exact }
|
||||
for match in matches {
|
||||
if AXUIElementPerformAction(match.element, kAXPressAction as CFString) == .success {
|
||||
return ["type": "ax_press", "id": id ?? "", "ok": true, "error": NSNull(),
|
||||
"pressed": match.label]
|
||||
}
|
||||
}
|
||||
return ["type": "ax_press", "id": id ?? "", "ok": false,
|
||||
"error": "No pressable element matching '\(label)' found in '\(appName)' (\(matches.count) label match(es) not actionable)"]
|
||||
}
|
||||
|
||||
/// Set the value of a text field by partial label match (or first text field if label is empty).
|
||||
func axSetValue(appName: String, label: String, value: String, id: String?) -> [String: Any] {
|
||||
guard let app = findRunningApp(named: appName) else {
|
||||
return ["type": "ax_set_value", "id": id ?? "", "ok": false,
|
||||
"error": "App '\(appName)' not found or not running"]
|
||||
}
|
||||
let axApp = AXUIElementCreateApplication(app.processIdentifier)
|
||||
let textRoles: Set<String> = ["AXTextField", "AXTextArea", "AXSearchField", "AXComboBox"]
|
||||
let lower = label.lowercased()
|
||||
var setLabel = ""
|
||||
let found = axWalk(axApp) { element, role, elemLabel in
|
||||
guard textRoles.contains(role) else { return false }
|
||||
let matchLabel = lower.isEmpty || elemLabel.lowercased().contains(lower)
|
||||
guard matchLabel else { return false }
|
||||
if AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFTypeRef) == .success {
|
||||
setLabel = elemLabel
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
if found {
|
||||
return ["type": "ax_set_value", "id": id ?? "", "ok": true, "error": NSNull(),
|
||||
"field": setLabel]
|
||||
}
|
||||
return ["type": "ax_set_value", "id": id ?? "", "ok": false,
|
||||
"error": "No text field matching '\(label)' found in '\(appName)'"]
|
||||
}
|
||||
|
||||
// MARK: - Overlay Controller
|
||||
|
||||
final class OverlayController {
|
||||
@@ -807,6 +939,21 @@ DispatchQueue.global(qos: .userInitiated).async {
|
||||
let response = pasteText(id: id, text: text)
|
||||
writeResponse(response)
|
||||
|
||||
case "ax_list":
|
||||
let appName = (payload["app_name"] as? String) ?? ""
|
||||
writeResponse(axListElements(appName: appName, id: id))
|
||||
|
||||
case "ax_press":
|
||||
let appName = (payload["app_name"] as? String) ?? ""
|
||||
let label = (payload["label"] as? String) ?? ""
|
||||
writeResponse(axPress(appName: appName, label: label, id: id))
|
||||
|
||||
case "ax_set_value":
|
||||
let appName = (payload["app_name"] as? String) ?? ""
|
||||
let label = (payload["label"] as? String) ?? ""
|
||||
let value = (payload["value"] as? String) ?? ""
|
||||
writeResponse(axSetValue(appName: appName, label: label, value: value, id: id))
|
||||
|
||||
case "show":
|
||||
let x = CGFloat((payload["x"] as? NSNumber)?.doubleValue ?? 0)
|
||||
let y = CGFloat((payload["y"] as? NSNumber)?.doubleValue ?? 0)
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! instead of owning platform-specific code directly.
|
||||
|
||||
mod automation_state;
|
||||
pub mod ax_interact;
|
||||
mod capture;
|
||||
mod focus;
|
||||
mod globe;
|
||||
@@ -17,6 +18,10 @@ mod permissions;
|
||||
mod terminal;
|
||||
mod text_util;
|
||||
mod types;
|
||||
// Windows accessibility backend for `ax_interact` (UI Automation). Sibling of
|
||||
// the macOS Swift-helper path; selected via cfg-dispatch in `ax_interact.rs`.
|
||||
#[cfg(target_os = "windows")]
|
||||
mod uia_interact;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use automation_state::test_lock as automation_state_test_lock;
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
//! Windows UI Automation (UIA) backend for the `ax_interact` tool.
|
||||
//!
|
||||
//! The Windows analogue of the macOS AXUIElement helper (`ax_interact.rs`):
|
||||
//! it provides the same three primitives — `list` / `press` / `set_value` —
|
||||
//! addressing UI elements by their semantic label, using Microsoft UI
|
||||
//! Automation (the OS-level accessibility tree) via the `uiautomation` crate
|
||||
//! (safe Rust wrappers over the UIA COM API).
|
||||
//!
|
||||
//! Why this is cleaner than the macOS path:
|
||||
//! - No helper process. macOS shells out to a Swift helper; on Windows the
|
||||
//! UIA COM API is callable directly from Rust.
|
||||
//! - No synthetic input. `press` activates controls through UIA patterns
|
||||
//! (`Invoke` / `SelectionItem.Select` / `LegacyIAccessible.DoDefaultAction`),
|
||||
//! never injecting mouse/keyboard events — so there is no CEF-crash risk
|
||||
//! (the bug that forced the macOS `mouse`/`keyboard` revert) and it works
|
||||
//! regardless of which window is focused.
|
||||
//! - No special permission for same-integrity-level apps. (UIPI still blocks
|
||||
//! a non-elevated process from driving an *elevated* app's UI.)
|
||||
//!
|
||||
//! Windows only. Reached through cfg-dispatch in `ax_interact.rs`; the
|
||||
//! agent-facing tool stays a single `ax_interact` tool on every platform.
|
||||
|
||||
use super::ax_interact::AXElement;
|
||||
use uiautomation::controls::ControlType;
|
||||
use uiautomation::core::{UIAutomation, UIElement};
|
||||
use uiautomation::patterns::{
|
||||
UIInvokePattern, UILegacyIAccessiblePattern, UISelectionItemPattern, UIValuePattern,
|
||||
};
|
||||
|
||||
/// Matcher retry-wait budget. UIA windows/elements can lag behind a launch or a
|
||||
/// navigation; the matcher polls until this deadline before giving up.
|
||||
const FIND_TIMEOUT_MS: u64 = 2000;
|
||||
|
||||
/// How deep to walk an app's UI subtree. Deep enough for nested panes/lists,
|
||||
/// shallow enough to stay fast; the tool layer caps and filters the output.
|
||||
const TREE_DEPTH: u32 = 40;
|
||||
|
||||
/// Initialise COM on the calling (tokio blocking-pool) thread before creating a
|
||||
/// UIA client. Idempotent — `S_OK` the first time, `S_FALSE` if COM is already
|
||||
/// up on this thread, `RPC_E_CHANGED_MODE` if a different apartment was already
|
||||
/// chosen (all acceptable). We never `CoUninitialize`: the worker thread keeps
|
||||
/// COM live for the process lifetime, which is how `uiautomation` expects to be
|
||||
/// driven.
|
||||
fn ensure_com() {
|
||||
use windows_sys::Win32::System::Com::{CoInitializeEx, COINIT_MULTITHREADED};
|
||||
let _ = unsafe { CoInitializeEx(std::ptr::null(), COINIT_MULTITHREADED as u32) };
|
||||
}
|
||||
|
||||
/// Control types worth surfacing as "interactive" in a `list`. Mirrors the
|
||||
/// macOS helper's button/field/cell focus. `Text` is included because read-only
|
||||
/// readouts (e.g. a Calculator result, a status label) are often the thing a
|
||||
/// caller wants to inspect; everything is still narrowed by the caller's filter
|
||||
/// and capped by the tool layer.
|
||||
fn is_interactive(ct: ControlType) -> bool {
|
||||
matches!(
|
||||
ct,
|
||||
ControlType::Button
|
||||
| ControlType::Edit
|
||||
| ControlType::ListItem
|
||||
| ControlType::MenuItem
|
||||
| ControlType::CheckBox
|
||||
| ControlType::RadioButton
|
||||
| ControlType::ComboBox
|
||||
| ControlType::Hyperlink
|
||||
| ControlType::TabItem
|
||||
| ControlType::TreeItem
|
||||
| ControlType::SplitButton
|
||||
| ControlType::Text
|
||||
)
|
||||
}
|
||||
|
||||
/// Locate the top-level window for `app_name`. Matches a `Window` element whose
|
||||
/// `Name` equals (preferred) or contains `app_name`, case-insensitively. UWP
|
||||
/// apps nest under `ApplicationFrameWindow`, so we allow a few levels of depth.
|
||||
fn find_window(automation: &UIAutomation, app_name: &str) -> Result<UIElement, String> {
|
||||
let root = automation
|
||||
.get_root_element()
|
||||
.map_err(|e| format!("UIA root element unavailable: {e}"))?;
|
||||
let matcher = automation
|
||||
.create_matcher()
|
||||
.from(root)
|
||||
.control_type(ControlType::Window)
|
||||
.depth(6)
|
||||
.timeout(FIND_TIMEOUT_MS);
|
||||
let windows = matcher.find_all().unwrap_or_default();
|
||||
|
||||
let needle = app_name.trim().to_lowercase();
|
||||
let mut contains: Option<UIElement> = None;
|
||||
for w in windows {
|
||||
let Ok(name) = w.get_name() else { continue };
|
||||
let nl = name.trim().to_lowercase();
|
||||
if nl.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if nl == needle {
|
||||
return Ok(w); // exact title match wins
|
||||
}
|
||||
if contains.is_none() && !needle.is_empty() && nl.contains(&needle) {
|
||||
contains = Some(w);
|
||||
}
|
||||
}
|
||||
contains.ok_or_else(|| {
|
||||
format!(
|
||||
"No open window matches app '{app_name}'. Make sure it is running \
|
||||
(try launch_app first), then retry."
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Find an element under `window` by label. Exact (case-insensitive) match is
|
||||
/// preferred over a substring match — so "Play" beats "Playlist", mirroring the
|
||||
/// macOS exact-match-preferred behaviour. Returns the element plus its resolved
|
||||
/// `Name` for messaging.
|
||||
fn find_by_label(
|
||||
automation: &UIAutomation,
|
||||
window: &UIElement,
|
||||
label: &str,
|
||||
) -> Result<(UIElement, String), String> {
|
||||
let matcher = automation
|
||||
.create_matcher()
|
||||
.from(window.clone())
|
||||
.depth(TREE_DEPTH)
|
||||
.timeout(FIND_TIMEOUT_MS);
|
||||
let elements = matcher.find_all().unwrap_or_default();
|
||||
|
||||
let needle = label.trim().to_lowercase();
|
||||
let mut contains: Option<(UIElement, String)> = None;
|
||||
for el in elements {
|
||||
let Ok(name) = el.get_name() else { continue };
|
||||
let nl = name.trim().to_lowercase();
|
||||
if nl.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if nl == needle {
|
||||
return Ok((el, name)); // exact preferred
|
||||
}
|
||||
if contains.is_none() && nl.contains(&needle) {
|
||||
contains = Some((el, name));
|
||||
}
|
||||
}
|
||||
contains.ok_or_else(|| {
|
||||
format!(
|
||||
"No element labelled '{label}' found. Try action='list' with a \
|
||||
filter to see available labels."
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Find the first editable text field under `window`. Prefers a plain `Edit`
|
||||
/// control (classic text boxes, WordPad, classic Notepad), then falls back to a
|
||||
/// `ComboBox` (editable dropdowns) and finally a `Document` control (rich-text
|
||||
/// editors such as the redesigned Windows 11 Notepad, which exposes its editor
|
||||
/// as a `Document` rather than an `Edit`). The caller still requires the chosen
|
||||
/// element to expose the UIA `Value` pattern before writing to it.
|
||||
fn find_first_edit(automation: &UIAutomation, window: &UIElement) -> Result<UIElement, String> {
|
||||
let matcher = automation
|
||||
.create_matcher()
|
||||
.from(window.clone())
|
||||
.depth(TREE_DEPTH)
|
||||
.timeout(FIND_TIMEOUT_MS);
|
||||
let elements = matcher.find_all().unwrap_or_default();
|
||||
|
||||
let mut combo: Option<UIElement> = None;
|
||||
let mut document: Option<UIElement> = None;
|
||||
for el in elements {
|
||||
match el.get_control_type() {
|
||||
Ok(ControlType::Edit) => return Ok(el), // best match — return immediately
|
||||
Ok(ControlType::ComboBox) if combo.is_none() => combo = Some(el),
|
||||
Ok(ControlType::Document) if document.is_none() => document = Some(el),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
combo.or(document).ok_or_else(|| {
|
||||
"no editable text field (Edit / ComboBox / Document) found in the app".to_string()
|
||||
})
|
||||
}
|
||||
|
||||
/// List interactive UI elements in `app_name`, keeping only those whose label
|
||||
/// contains `filter` (case-insensitive; empty = all). Filtering happens here so
|
||||
/// the tool result stays small — dumping a full UIA tree (apps expose hundreds
|
||||
/// of elements) overflows the result budget and makes the model hallucinate
|
||||
/// from a truncated view.
|
||||
pub fn list(app_name: &str, filter: &str) -> Result<Vec<AXElement>, String> {
|
||||
ensure_com();
|
||||
let automation = UIAutomation::new().map_err(|e| format!("UIA init failed: {e}"))?;
|
||||
let window = find_window(&automation, app_name)?;
|
||||
|
||||
let matcher = automation
|
||||
.create_matcher()
|
||||
.from(window)
|
||||
.depth(TREE_DEPTH)
|
||||
.timeout(FIND_TIMEOUT_MS);
|
||||
let elements = match matcher.find_all() {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
log::debug!("[uia_interact] list: tree walk returned empty for '{app_name}': {e}");
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
|
||||
let needle = filter.trim().to_lowercase();
|
||||
let mut out = Vec::new();
|
||||
for el in elements {
|
||||
let Ok(ct) = el.get_control_type() else {
|
||||
continue;
|
||||
};
|
||||
if !is_interactive(ct) {
|
||||
continue;
|
||||
}
|
||||
let label = el.get_name().unwrap_or_default().trim().to_string();
|
||||
if label.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !needle.is_empty() && !label.to_lowercase().contains(&needle) {
|
||||
continue;
|
||||
}
|
||||
out.push(AXElement {
|
||||
role: format!("{ct:?}"),
|
||||
label,
|
||||
});
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"[uia_interact] list app={app_name:?} filter={filter:?} -> {} elements",
|
||||
out.len()
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Activate the element in `app_name` whose label matches `label`. Uses UIA
|
||||
/// patterns in order of preference — `Invoke` (buttons/links/menu items), then
|
||||
/// `SelectionItem.Select` (list rows/tabs), then the `LegacyIAccessible`
|
||||
/// default action — and never injects synthetic mouse/keyboard input.
|
||||
pub fn press(app_name: &str, label: &str) -> Result<String, String> {
|
||||
ensure_com();
|
||||
let automation = UIAutomation::new().map_err(|e| format!("UIA init failed: {e}"))?;
|
||||
let window = find_window(&automation, app_name)?;
|
||||
let (element, matched) = find_by_label(&automation, &window, label)?;
|
||||
|
||||
log::info!("[uia_interact] press app={app_name:?} label={label:?} matched={matched:?}");
|
||||
|
||||
if let Ok(p) = element.get_pattern::<UIInvokePattern>() {
|
||||
p.invoke()
|
||||
.map_err(|e| format!("invoke '{matched}' failed: {e}"))?;
|
||||
return Ok(format!("Pressed '{matched}' in '{app_name}'."));
|
||||
}
|
||||
if let Ok(p) = element.get_pattern::<UISelectionItemPattern>() {
|
||||
p.select()
|
||||
.map_err(|e| format!("select '{matched}' failed: {e}"))?;
|
||||
return Ok(format!("Selected '{matched}' in '{app_name}'."));
|
||||
}
|
||||
if let Ok(p) = element.get_pattern::<UILegacyIAccessiblePattern>() {
|
||||
p.do_default_action()
|
||||
.map_err(|e| format!("default action on '{matched}' failed: {e}"))?;
|
||||
return Ok(format!("Activated '{matched}' in '{app_name}'."));
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"Element '{matched}' in '{app_name}' exposes no Invoke / Select / default \
|
||||
action — it may not be activatable. Try action='list' to find the real control."
|
||||
))
|
||||
}
|
||||
|
||||
/// Set the value of a text field in `app_name`. With an empty `label`, targets
|
||||
/// the first editable field; otherwise finds the field by label. Requires the
|
||||
/// element to expose the UIA `Value` pattern.
|
||||
pub fn set_value(app_name: &str, label: &str, value: &str) -> Result<String, String> {
|
||||
ensure_com();
|
||||
let automation = UIAutomation::new().map_err(|e| format!("UIA init failed: {e}"))?;
|
||||
let window = find_window(&automation, app_name)?;
|
||||
|
||||
let (element, matched) = if label.trim().is_empty() {
|
||||
let el = find_first_edit(&automation, &window)?;
|
||||
let name = el.get_name().unwrap_or_default();
|
||||
let name = if name.trim().is_empty() {
|
||||
"text field".to_string()
|
||||
} else {
|
||||
name
|
||||
};
|
||||
(el, name)
|
||||
} else {
|
||||
find_by_label(&automation, &window, label)?
|
||||
};
|
||||
|
||||
log::info!("[uia_interact] set_value app={app_name:?} field={matched:?}");
|
||||
|
||||
let vp = element
|
||||
.get_pattern::<UIValuePattern>()
|
||||
.map_err(|e| format!("'{matched}' is not a settable text field (no Value pattern): {e}"))?;
|
||||
vp.set_value(value)
|
||||
.map_err(|e| format!("set_value on '{matched}' failed: {e}"))?;
|
||||
Ok(format!(
|
||||
"Set '{matched}' in '{app_name}' to the provided value."
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
//! Integration tests for the Windows UI Automation backend of `ax_interact`.
|
||||
//!
|
||||
//! These exercise the SAME public helpers the tool uses (`ax_list_elements`,
|
||||
//! `ax_press_element`, `ax_set_field_value`) — which cfg-dispatch to
|
||||
//! `uia_interact` on Windows — so they validate the real agent-facing path.
|
||||
//!
|
||||
//! Most need a live desktop session and a real app, so they are `#[ignore]` by
|
||||
//! default. Run them on a Windows machine with:
|
||||
//!
|
||||
//! cargo test --lib uia_interact -- --nocapture --include-ignored
|
||||
//!
|
||||
//! `test_uia_list_nonexistent_app` is deterministic (asserts an error) and runs
|
||||
//! in the normal suite.
|
||||
|
||||
#![cfg(all(test, target_os = "windows"))]
|
||||
|
||||
use super::{ax_list_elements, ax_list_elements_filtered, ax_press_element, ax_set_field_value};
|
||||
use std::process::Command;
|
||||
use std::thread::sleep;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Spawn a launcher exe (e.g. `calc.exe`, `notepad.exe`). Returns whether the
|
||||
/// spawn itself succeeded; the launched app may appear a moment later.
|
||||
fn launch(exe: &str) -> bool {
|
||||
Command::new(exe).spawn().is_ok()
|
||||
}
|
||||
|
||||
/// Deterministic UI control test (the Windows analogue of the macOS AC/DC test):
|
||||
/// drive Calculator through `5 + 5 =` purely by element label, then hard-assert
|
||||
/// the readout shows 10. Calculator is deterministic and always present, so this
|
||||
/// is a real assertion, not best-effort.
|
||||
#[test]
|
||||
#[ignore = "requires a desktop session; launches the Calculator app"]
|
||||
fn test_uia_calculator_five_plus_five() {
|
||||
assert!(launch("calc.exe"), "could not spawn calc.exe");
|
||||
sleep(Duration::from_secs(3));
|
||||
|
||||
// 1. List — Calculator should expose its buttons via UIA.
|
||||
let elements = ax_list_elements("Calculator").expect("ax_list_elements(Calculator) failed");
|
||||
assert!(
|
||||
!elements.is_empty(),
|
||||
"Calculator exposed no interactive elements — UIA tree empty?"
|
||||
);
|
||||
println!("[calc] {} interactive elements", elements.len());
|
||||
|
||||
// 2. Press 5, +, 5, = by their (English) UIA Names.
|
||||
for label in ["Five", "Plus", "Five", "Equals"] {
|
||||
let r = ax_press_element("Calculator", label);
|
||||
println!("[calc] press {label}: {r:?}");
|
||||
assert!(r.is_ok(), "press '{label}' failed: {r:?}");
|
||||
sleep(Duration::from_millis(300));
|
||||
}
|
||||
|
||||
// 3. Assert the result readout computed 10.
|
||||
sleep(Duration::from_millis(500));
|
||||
let readout = ax_list_elements_filtered("Calculator", "Display").unwrap_or_default();
|
||||
println!("[calc] readout (filter='Display'): {readout:?}");
|
||||
let shows_ten = readout.iter().any(|e| e.label.contains("10"))
|
||||
|| ax_list_elements("Calculator")
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.any(|e| e.label.contains("10"));
|
||||
assert!(
|
||||
shows_ten,
|
||||
"expected a result element showing 10 after 5 + 5 =; readout={readout:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Text entry via `set_value` into Notepad's edit control.
|
||||
///
|
||||
/// Win11's redesigned Notepad uses a RichEdit control that may not expose the
|
||||
/// UIA `Value` pattern, whereas classic Notepad does. A `Value`-pattern absence
|
||||
/// is treated as a documented limitation (best-effort); any other failure is a
|
||||
/// real test failure.
|
||||
#[test]
|
||||
#[ignore = "requires a desktop session; launches Notepad"]
|
||||
fn test_uia_notepad_set_value() {
|
||||
assert!(launch("notepad.exe"), "could not spawn notepad.exe");
|
||||
sleep(Duration::from_secs(2));
|
||||
|
||||
let r = ax_set_field_value("Notepad", "", "OpenHuman UIA test");
|
||||
println!("[notepad] set_value: {r:?}");
|
||||
if let Err(e) = &r {
|
||||
// The redesigned Win11 Notepad exposes its editor as a Document/RichEdit
|
||||
// that may not support the UIA Value pattern (or any settable text
|
||||
// field). That is a documented platform limitation, not a code bug, so
|
||||
// treat it as a best-effort skip; classic Notepad / WordPad / ordinary
|
||||
// Edit controls still exercise the real write path.
|
||||
if e.contains("Value pattern")
|
||||
|| e.contains("settable")
|
||||
|| e.contains("no editable text field")
|
||||
{
|
||||
println!(
|
||||
"[notepad] set_value unsupported on this Notepad build \
|
||||
(expected on Win11 RichEdit Notepad): {e}"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
assert!(r.is_ok(), "set_value failed unexpectedly: {r:?}");
|
||||
}
|
||||
|
||||
/// Deterministic, no-app-needed: a non-existent app must surface an error
|
||||
/// (either "no window matches" or, in a session-less environment, a UIA-init
|
||||
/// error — both are `Err` from our wrapper).
|
||||
#[test]
|
||||
fn test_uia_list_nonexistent_app() {
|
||||
let r = ax_list_elements("OpenHuman_NoSuchApp_ZZZ123");
|
||||
assert!(r.is_err(), "expected error for non-existent app, got {r:?}");
|
||||
println!(
|
||||
"[uia] nonexistent app error (expected): {:?}",
|
||||
r.unwrap_err()
|
||||
);
|
||||
}
|
||||
@@ -454,12 +454,15 @@ impl AgentBuilder {
|
||||
let visible_tool_specs: Vec<ToolSpec> =
|
||||
dedup_visible_tool_specs(visible_tool_specs_unfiltered);
|
||||
|
||||
let visible_names_list: Vec<&str> =
|
||||
visible_tool_specs.iter().map(|s| s.name.as_str()).collect();
|
||||
log::info!(
|
||||
"[agent] tool spec filter: total={} visible={} (filter_active={} policy_restricted={})",
|
||||
"[agent] tool spec filter: total={} visible={} (filter_active={} policy_restricted={}) names=[{}]",
|
||||
tool_specs.len(),
|
||||
visible_tool_specs.len(),
|
||||
!visible_names.is_empty(),
|
||||
tool_policy_session.has_restrictions()
|
||||
tool_policy_session.has_restrictions(),
|
||||
visible_names_list.join(", ")
|
||||
);
|
||||
|
||||
// Pull the provider out of the builder once. We store it on
|
||||
|
||||
@@ -17,6 +17,35 @@ You are OpenHuman — the user's AI teammate for productivity, research, and tea
|
||||
- Present alternatives and trade-offs when the call isn't obvious — then let the user pick.
|
||||
- Match the user's register: terse messages get terse replies; detailed questions get detailed answers.
|
||||
|
||||
## What you can do on the user's machine
|
||||
|
||||
You run on the user's own desktop. You have tools that let you act on their behalf:
|
||||
|
||||
- **`launch_app`** — open any application by name (e.g. Music, Spotify, Safari, Calculator, VS Code). When the user asks you to open an app, **always use this tool** — do not tell them to open it themselves.
|
||||
- **`ax_interact`** — interact with a running app's UI via the platform accessibility API (macOS Accessibility / Windows UI Automation). Finds buttons, text fields, and controls by their label — no screen coordinates needed. Always call `action='list'` first to discover available elements, then `action='press'` to click or `action='set_value'` to type.
|
||||
- **`shell`** — run shell commands in the workspace (git, npm, cargo, file operations, etc.).
|
||||
- **`file_read` / `file_write`** — read and edit files in the workspace.
|
||||
|
||||
Never say "I can't open apps" or "that's outside what I can do" when you have a tool to do it. Use the tool.
|
||||
|
||||
**Workflow for interacting with an app's UI:**
|
||||
1. `action='list'` — discover what buttons/fields/rows exist
|
||||
2. `action='set_value'` to type in a filter or search field
|
||||
3. `action='list'` again — see the updated/filtered results that appeared
|
||||
4. `action='press'` — press the specific item (song row, playlist, etc.), NOT the generic Play button
|
||||
5. Only press the playback-bar "Play" button after the right item is selected/playing
|
||||
|
||||
**For playing a specific song in Apple Music (macOS) — use this EXACT sequence:**
|
||||
1. `shell`: `open "music://music.apple.com/search?term=Song+Name+Artist"` (URL-encode the query)
|
||||
2. Wait ~3s for results to load, then `ax_interact action='list' app_name='Music'`
|
||||
3. `ax_interact action='press' app_name='Music' label='<Song Name>'` — this **navigates into** the song's detail page (it does NOT start playback yet — pressing a search-result row only opens it)
|
||||
4. Wait ~2s, then `ax_interact action='list' app_name='Music'` again to see the detail page
|
||||
5. `ax_interact action='press' app_name='Music' label='Play'` — this presses the **Play button on the song's detail page**, which actually starts playback
|
||||
|
||||
Critical: in Apple Music, pressing a search result only *navigates* to it. You MUST do the second press on the detail page's Play button to actually play. Do not stop after step 3. Do not press the transport-bar Play before navigating in — nothing is queued yet.
|
||||
|
||||
The example above is macOS-specific (the `open`/`music://` scheme and Apple Music). On Windows the same **list → press** pattern applies via UI Automation, but `ax_interact action='press'` usually *activates* a control directly (a list-row Invoke often plays/opens in one step), so the second navigate-then-play press is frequently unnecessary. Use `launch_app` to open the player, then `list` with a `filter` and `press` the specific item; re-`list` if a press only navigated.
|
||||
|
||||
## When things go wrong
|
||||
|
||||
- **Tool failure:** try a different approach before escalating. If you're stuck, name what failed and what you'd need to proceed.
|
||||
|
||||
@@ -117,6 +117,19 @@ named = [
|
||||
"spawn_worker_thread",
|
||||
"spawn_parallel_agents",
|
||||
"composio_list_connections",
|
||||
# App launching — lets the orchestrator open desktop applications
|
||||
# directly when asked ("open Music", "launch Spotify", etc.) without
|
||||
# delegating or telling the user it can't. Routes through the ApprovalGate
|
||||
# (external effect), so launching prompts for confirmation per autonomy tier.
|
||||
"launch_app",
|
||||
# AXUIElement / UIA interaction — find UI by semantic label. No CGEventPost,
|
||||
# no coordinate dependency, no CEF crash risk. Always call action='list'
|
||||
# first to discover elements. Only the read-only `list` action is
|
||||
# ReadOnly/unprompted; the mutating `press` / `set_value` actions are
|
||||
# `Dangerous`, gate interactively through the ApprovalGate, are opt-in via
|
||||
# `computer_control.ax_interact_mutations`, and refuse a sensitive-app
|
||||
# denylist (password managers, Keychain, System Settings, terminals).
|
||||
"ax_interact",
|
||||
# Time + scheduling — lets the orchestrator answer "what time is it",
|
||||
# "remind me in 10 minutes", "every morning at 8" directly rather than
|
||||
# delegating or telling the user it can't. `current_time` grounds
|
||||
|
||||
@@ -972,6 +972,12 @@ pub struct ComputerControlConfig {
|
||||
/// the user must explicitly opt in.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Opt-in for the mutating `ax_interact` actions (`press` / `set_value`).
|
||||
/// Disabled by default: the read-only `list` action is always available,
|
||||
/// but actuating arbitrary app controls / typing into arbitrary fields
|
||||
/// requires explicit user opt-in (mirrors `enabled` for mouse/keyboard).
|
||||
#[serde(default)]
|
||||
pub ax_interact_mutations: bool,
|
||||
}
|
||||
|
||||
// ── Agent integration tools (backend-proxied) ───────────────────────
|
||||
|
||||
@@ -511,6 +511,13 @@ const READ_ONLY_BASES: &[&str] = &[
|
||||
"lsblk",
|
||||
"lscpu",
|
||||
"cut",
|
||||
// NOTE: OS-native launchers (`open`, `xdg-open`, `start`) are deliberately
|
||||
// NOT in the read-only set. `classify_command` only sees the base command,
|
||||
// not its args, and these launchers can open arbitrary `https://` URLs and
|
||||
// custom URI handlers — i.e. trigger outbound network / system actions — so
|
||||
// treating them as `Read` (no approval) is too broad. App launching now
|
||||
// goes through the dedicated `launch_app` tool, which is scoped to named
|
||||
// applications only and carries no shell-arg ambiguity.
|
||||
// Windows cmd / PowerShell read verbs + common aliases
|
||||
"dir",
|
||||
"type",
|
||||
|
||||
@@ -0,0 +1,442 @@
|
||||
//! Tool: ax_interact — interact with desktop app UI via the OS accessibility API.
|
||||
//!
|
||||
//! Cross-platform: macOS uses AXUIElement (Swift helper), Windows uses UI
|
||||
//! Automation (UIA COM API). Both back-ends:
|
||||
//! - Never crash CEF (no synthetic key/mouse events injected system-wide)
|
||||
//! - Work regardless of which app is focused
|
||||
//! - Find elements by semantic label, not pixel coordinates
|
||||
//!
|
||||
//! Three actions:
|
||||
//! list — enumerate interactive elements in a running app
|
||||
//! press — activate a button/control by label
|
||||
//! set_value — type text into a field by label
|
||||
//!
|
||||
//! Requires: macOS Accessibility permission granted to OpenHuman. On Windows no
|
||||
//! special permission is needed for same-integrity-level apps (UIPI blocks
|
||||
//! driving an elevated app from a non-elevated process).
|
||||
|
||||
use crate::openhuman::accessibility::ax_interact as ax;
|
||||
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCallOptions, ToolResult};
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
|
||||
/// Apps whose UI must never be actuated by the agent. `press` / `set_value`
|
||||
/// are refused when `app_name` matches any of these (case-insensitive
|
||||
/// substring) — defense-in-depth that holds even on background/auto-approved
|
||||
/// turns where the ApprovalGate may not prompt. `list` is also refused so the
|
||||
/// agent can't enumerate, e.g., a password manager's fields. Matched by display
|
||||
/// name; broad substrings ("keychain", "1password") cover localized variants.
|
||||
const SENSITIVE_APPS: &[&str] = &[
|
||||
"keychain",
|
||||
"1password",
|
||||
"bitwarden",
|
||||
"lastpass",
|
||||
"dashlane",
|
||||
"system settings",
|
||||
"system preferences",
|
||||
"console", // macOS Console (logs)
|
||||
// Terminal emulators — mirror the set helper.rs treats as terminals
|
||||
// (helper.rs ~:557) so "terminals are denied" actually holds.
|
||||
"terminal",
|
||||
"iterm",
|
||||
"wezterm",
|
||||
"warp",
|
||||
"alacritty",
|
||||
"kitty",
|
||||
"ghostty",
|
||||
"hyper",
|
||||
"rio",
|
||||
];
|
||||
|
||||
fn is_sensitive_app(app_name: &str) -> bool {
|
||||
let lower = app_name.to_lowercase();
|
||||
SENSITIVE_APPS.iter().any(|s| lower.contains(s))
|
||||
}
|
||||
|
||||
pub struct AxInteractTool {
|
||||
/// When false, the mutating actions (`press` / `set_value`) are refused
|
||||
/// with guidance to enable `computer_control.ax_interact_mutations`. The
|
||||
/// read-only `list` action is always available. Mirrors the opt-in posture
|
||||
/// of the mouse/keyboard tools (`computer_control.enabled`).
|
||||
allow_mutations: bool,
|
||||
}
|
||||
|
||||
impl AxInteractTool {
|
||||
pub fn new(allow_mutations: bool) -> Self {
|
||||
Self { allow_mutations }
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AxInteractTool {
|
||||
fn default() -> Self {
|
||||
// Default to read-only (mutations opt-in) — safe baseline.
|
||||
Self::new(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for AxInteractTool {
|
||||
fn name(&self) -> &str {
|
||||
"ax_interact"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Interact with ANY desktop application's UI using the platform accessibility API \
|
||||
(macOS AXUIElement / Windows UI Automation). Finds buttons, text fields, list rows, \
|
||||
and controls by their label — no screen coordinates, no synthetic key/mouse events. \
|
||||
Works for any app: a music player, browser, mail, notes, Slack, system settings, etc.\n\
|
||||
\n\
|
||||
Actions:\n\
|
||||
• 'list' → show interactive elements. ALWAYS pass a `filter` substring to narrow \
|
||||
results (apps expose hundreds of elements; an unfiltered list is huge and unreliable). \
|
||||
e.g. filter='Play', filter='Send', filter='Highway'.\n\
|
||||
• 'press' → activate a button/control/row by label (exact match preferred). \
|
||||
e.g. label='Play', label='Send', label='OK'.\n\
|
||||
• 'set_value' → type text into a field by label (omit label for the first text field).\n\
|
||||
\n\
|
||||
General pattern: (1) `list` with a `filter` to find the element, (2) `press` it. \
|
||||
Note that in many apps, pressing a LIST ROW or SEARCH RESULT only selects/opens it — \
|
||||
to trigger an action you then press the relevant action button (e.g. after opening a \
|
||||
song's page, press its 'Play' button). If a press doesn't have the intended effect, \
|
||||
`list` again to see the new screen and press the actual action control.\n\
|
||||
\n\
|
||||
On macOS this requires Accessibility permission for OpenHuman; on Windows no special \
|
||||
permission is needed for normal (non-elevated) apps."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> serde_json::Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["list", "press", "set_value"],
|
||||
"description": "'list' = show interactive elements (use with filter); 'press' = activate a control by label; 'set_value' = type into a text field."
|
||||
},
|
||||
"app_name": {
|
||||
"type": "string",
|
||||
"description": "Display name of the running application (e.g. 'Music', 'Safari', 'Telegram')."
|
||||
},
|
||||
"filter": {
|
||||
"type": "string",
|
||||
"description": "For 'list': only return elements whose label contains this substring (case-insensitive). Strongly recommended — keeps results small and accurate."
|
||||
},
|
||||
"label": {
|
||||
"type": "string",
|
||||
"description": "For 'press'/'set_value': label of the element to target (case-insensitive, exact match preferred). For 'set_value', omit to target the first text field."
|
||||
},
|
||||
"value": {
|
||||
"type": "string",
|
||||
"description": "Text to enter (required for 'set_value')."
|
||||
}
|
||||
},
|
||||
"required": ["action", "app_name"]
|
||||
})
|
||||
}
|
||||
|
||||
fn permission_level(&self) -> PermissionLevel {
|
||||
// Minimum across actions: `list` is read-only. The per-call level
|
||||
// (Dangerous for press/set_value) is enforced by
|
||||
// `permission_level_with_args`. Returning the minimum here keeps the
|
||||
// tool available on channels that can run the read-only `list`.
|
||||
PermissionLevel::ReadOnly
|
||||
}
|
||||
|
||||
fn permission_level_with_args(&self, args: &serde_json::Value) -> PermissionLevel {
|
||||
match args.get("action").and_then(|v| v.as_str()) {
|
||||
// `list` only reads the AX tree — no state change.
|
||||
Some("list") | None => PermissionLevel::ReadOnly,
|
||||
// `press` / `set_value` actuate real controls (click buttons,
|
||||
// type into fields) and change application state, so they must
|
||||
// not ride on the read-only path.
|
||||
_ => PermissionLevel::Dangerous,
|
||||
}
|
||||
}
|
||||
|
||||
fn external_effect_with_args(&self, args: &serde_json::Value) -> bool {
|
||||
// Route mutating actions through the ApprovalGate before execute();
|
||||
// `list` is a pure read and flows through unprompted.
|
||||
!matches!(
|
||||
args.get("action").and_then(|v| v.as_str()),
|
||||
Some("list") | None
|
||||
)
|
||||
}
|
||||
|
||||
async fn execute(&self, args: serde_json::Value) -> anyhow::Result<ToolResult> {
|
||||
self.execute_with_options(args, ToolCallOptions::default())
|
||||
.await
|
||||
}
|
||||
|
||||
async fn execute_with_options(
|
||||
&self,
|
||||
args: serde_json::Value,
|
||||
_options: ToolCallOptions,
|
||||
) -> anyhow::Result<ToolResult> {
|
||||
let action = args
|
||||
.get("action")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
let app_name = args
|
||||
.get("app_name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
let label = args
|
||||
.get("label")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
let value = args
|
||||
.get("value")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let filter = args
|
||||
.get("filter")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
log::info!(
|
||||
"[ax_interact] ▶ action={action:?} app={app_name:?} label={label:?} filter={filter:?}"
|
||||
);
|
||||
|
||||
if app_name.is_empty() {
|
||||
return Ok(ToolResult::error("app_name is required"));
|
||||
}
|
||||
|
||||
let mutating = matches!(action.as_str(), "press" | "set_value");
|
||||
|
||||
// Denylist: never actuate or enumerate sensitive apps (password
|
||||
// managers, Keychain, System Settings, terminals). Defense-in-depth
|
||||
// that holds even when the ApprovalGate doesn't prompt (background /
|
||||
// auto-approved turns).
|
||||
if is_sensitive_app(&app_name) {
|
||||
log::warn!("[ax_interact] refused: sensitive app '{app_name}' (action={action})");
|
||||
return Ok(ToolResult::error(format!(
|
||||
"Refusing to interact with '{app_name}': it is on the sensitive-app denylist \
|
||||
(password managers, Keychain, System Settings, terminals). This is a hard \
|
||||
safety boundary."
|
||||
)));
|
||||
}
|
||||
|
||||
// Mutating actions are opt-in. Read-only `list` is always allowed.
|
||||
if mutating && !self.allow_mutations {
|
||||
log::warn!("[ax_interact] refused: mutations disabled (action={action})");
|
||||
return Ok(ToolResult::error(
|
||||
"ax_interact mutations (press/set_value) are disabled. They actuate arbitrary \
|
||||
app controls and type into arbitrary fields, so they require explicit opt-in: \
|
||||
set `computer_control.ax_interact_mutations = true`. The read-only 'list' \
|
||||
action remains available.",
|
||||
));
|
||||
}
|
||||
|
||||
// Cap how many elements we render so a broad/empty filter can't overflow
|
||||
// the tool-result budget and cause the model to reason over a truncated view.
|
||||
const MAX_LISTED: usize = 60;
|
||||
|
||||
let result = match action.as_str() {
|
||||
"list" => match ax::ax_list_elements_filtered(&app_name, &filter) {
|
||||
Ok(elements) if elements.is_empty() => {
|
||||
log::info!(
|
||||
"[ax_interact] list: no elements in '{app_name}' (filter={filter:?})"
|
||||
);
|
||||
let hint = if filter.is_empty() {
|
||||
format!(
|
||||
"No interactive elements found in '{app_name}'. \
|
||||
The app may not expose its UI tree via Accessibility API, \
|
||||
or OpenHuman may need Accessibility permission."
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"No elements in '{app_name}' match filter '{filter}'. \
|
||||
The UI may still be loading — wait and try again, or call \
|
||||
'list' with a shorter/different filter."
|
||||
)
|
||||
};
|
||||
ToolResult::success(hint)
|
||||
}
|
||||
Ok(elements) => {
|
||||
let total = elements.len();
|
||||
log::info!(
|
||||
"[ax_interact] list: {total} elements in '{app_name}' (filter={filter:?})"
|
||||
);
|
||||
let shown = total.min(MAX_LISTED);
|
||||
let lines: Vec<String> = elements
|
||||
.iter()
|
||||
.take(MAX_LISTED)
|
||||
.map(|e| format!(" [{role}] {label}", role = e.role, label = e.label))
|
||||
.collect();
|
||||
let mut out = if filter.is_empty() {
|
||||
format!("Elements in '{app_name}' (showing {shown} of {total}):\n")
|
||||
} else {
|
||||
format!(
|
||||
"Elements in '{app_name}' matching '{filter}' (showing {shown} of {total}):\n"
|
||||
)
|
||||
};
|
||||
out.push_str(&lines.join("\n"));
|
||||
if total > MAX_LISTED {
|
||||
out.push_str(&format!(
|
||||
"\n… {} more — narrow with a more specific `filter`.",
|
||||
total - MAX_LISTED
|
||||
));
|
||||
}
|
||||
ToolResult::success(out)
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("[ax_interact] list failed: {e}");
|
||||
ToolResult::error(e)
|
||||
}
|
||||
},
|
||||
|
||||
"press" => {
|
||||
if label.is_empty() {
|
||||
return Ok(ToolResult::error(
|
||||
"'label' is required for action='press'. Use action='list' first to discover element labels.",
|
||||
));
|
||||
}
|
||||
match ax::ax_press_element(&app_name, &label) {
|
||||
Ok(msg) => {
|
||||
log::info!("[ax_interact] press succeeded: {msg}");
|
||||
ToolResult::success(msg)
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("[ax_interact] press failed: {e}");
|
||||
ToolResult::error(format!(
|
||||
"{e}. Try action='list' to see available element labels."
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
"set_value" => {
|
||||
if value.is_empty() {
|
||||
return Ok(ToolResult::error(
|
||||
"'value' is required for action='set_value'",
|
||||
));
|
||||
}
|
||||
match ax::ax_set_field_value(&app_name, &label, &value) {
|
||||
Ok(msg) => {
|
||||
log::info!("[ax_interact] set_value succeeded: {msg}");
|
||||
ToolResult::success(msg)
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("[ax_interact] set_value failed: {e}");
|
||||
ToolResult::error(format!(
|
||||
"{e}. Try action='list' to see available text field labels."
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
other => ToolResult::error(format!(
|
||||
"Unknown action '{other}'. Valid actions: 'list', 'press', 'set_value'."
|
||||
)),
|
||||
};
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn name_and_permission() {
|
||||
let tool = AxInteractTool::new(true);
|
||||
assert_eq!(tool.name(), "ax_interact");
|
||||
assert_eq!(tool.permission_level(), PermissionLevel::ReadOnly);
|
||||
// Mutating actions gate per-call.
|
||||
assert_eq!(
|
||||
tool.permission_level_with_args(&json!({"action": "press"})),
|
||||
PermissionLevel::Dangerous
|
||||
);
|
||||
assert!(tool.external_effect_with_args(&json!({"action": "press"})));
|
||||
assert!(!tool.external_effect_with_args(&json!({"action": "list"})));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_requires_action_and_app_name() {
|
||||
let schema = AxInteractTool::new(true).parameters_schema();
|
||||
let required = schema["required"].as_array().unwrap();
|
||||
assert!(required.iter().any(|v| v == "action"));
|
||||
assert!(required.iter().any(|v| v == "app_name"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sensitive_apps_detected() {
|
||||
assert!(is_sensitive_app("Keychain Access"));
|
||||
assert!(is_sensitive_app("1Password 7"));
|
||||
assert!(is_sensitive_app("System Settings"));
|
||||
assert!(is_sensitive_app("Terminal"));
|
||||
// All terminal emulators helper.rs recognizes must also be denied.
|
||||
for t in [
|
||||
"iTerm",
|
||||
"WezTerm",
|
||||
"Warp",
|
||||
"Alacritty",
|
||||
"kitty",
|
||||
"Ghostty",
|
||||
"Hyper",
|
||||
"Rio",
|
||||
] {
|
||||
assert!(is_sensitive_app(t), "expected '{t}' to be denied");
|
||||
}
|
||||
assert!(!is_sensitive_app("Music"));
|
||||
assert!(!is_sensitive_app("Safari"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_missing_app_name() {
|
||||
let result = AxInteractTool::new(true)
|
||||
.execute(json!({"action": "list", "app_name": ""}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_error);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_press_without_label() {
|
||||
let result = AxInteractTool::new(true)
|
||||
.execute(json!({"action": "press", "app_name": "Music"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_error);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_mutations_when_disabled() {
|
||||
// mutations off → press/set_value blocked, but list still allowed past this guard.
|
||||
let tool = AxInteractTool::new(false);
|
||||
let press = tool
|
||||
.execute(json!({"action": "press", "app_name": "Music", "label": "Play"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(press.is_error);
|
||||
assert!(press.output().contains("ax_interact_mutations"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_sensitive_app_even_with_mutations() {
|
||||
let tool = AxInteractTool::new(true);
|
||||
for app in [
|
||||
"Keychain Access",
|
||||
"1Password",
|
||||
"Terminal",
|
||||
"System Settings",
|
||||
] {
|
||||
let r = tool
|
||||
.execute(json!({"action": "press", "app_name": app, "label": "OK"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(r.is_error, "expected refusal for {app}");
|
||||
assert!(r.output().to_lowercase().contains("denylist"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
mod ax_interact;
|
||||
mod human_path;
|
||||
mod keyboard;
|
||||
mod mouse;
|
||||
|
||||
pub use ax_interact::AxInteractTool;
|
||||
pub use keyboard::KeyboardTool;
|
||||
pub use mouse::MouseTool;
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
//! Tool: launch_app — open a named application on the user's desktop.
|
||||
//!
|
||||
//! A dedicated, narrow-scope alternative to using the `shell` tool with
|
||||
//! `open -a <App>` / `xdg-open` / `Start-Process`. It carries no shell
|
||||
//! injection risk and accepts **named applications only** (URI schemes like
|
||||
//! `spotify:` / `mailto:` are rejected — see `validate_app_name`).
|
||||
//!
|
||||
//! Being injection-safe does NOT make it side-effect-free: opening an app
|
||||
//! window (and, on Linux/Windows, potentially firing a registered URI handler)
|
||||
//! is an externally-observable action on the user's machine. So the tool is an
|
||||
//! external-effect tool (`external_effect() == true`) and routes through the
|
||||
//! `ApprovalGate` before executing, like `shell` — it is NOT always-allow.
|
||||
//!
|
||||
//! Platform dispatch:
|
||||
//! macOS — `open -a "<app_name>"` (falls back to `open "<app_name>"`)
|
||||
//! Linux — `gtk-launch "<app_name>"`, fallback `xdg-open "<app_name>"`
|
||||
//! Windows — `Start-Process "<app_name>"`
|
||||
|
||||
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCallOptions, ToolResult};
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
use std::process::Stdio;
|
||||
|
||||
pub struct LaunchAppTool;
|
||||
|
||||
impl LaunchAppTool {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LaunchAppTool {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Reject names that look like path traversal or contain shell metacharacters.
|
||||
fn validate_app_name(name: &str) -> Result<(), String> {
|
||||
let trimmed = name.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err("app_name must not be empty".into());
|
||||
}
|
||||
if trimmed.len() > 128 {
|
||||
return Err("app_name too long (max 128 chars)".into());
|
||||
}
|
||||
// No path separators or traversal sequences.
|
||||
if trimmed.contains('/') || trimmed.contains('\\') || trimmed.contains("..") {
|
||||
return Err(format!(
|
||||
"app_name '{trimmed}' looks like a path; supply a bare application name instead \
|
||||
(e.g. 'Music', 'Spotify')"
|
||||
));
|
||||
}
|
||||
// Reject shell metacharacters — not needed here since we bypass the shell,
|
||||
// but guard against accidental misuse of the API.
|
||||
for ch in ['$', '`', '|', '&', ';', '>', '<', '!', '(', ')', '\n', '\r'] {
|
||||
if trimmed.contains(ch) {
|
||||
return Err(format!("app_name contains disallowed character '{ch}'"));
|
||||
}
|
||||
}
|
||||
// Reject URI schemes (e.g. `spotify:`, `mailto:`, `slack:`, `https:`). On
|
||||
// Linux/Windows the launcher fallbacks (`xdg-open`/`Start-Process`) would
|
||||
// fire an arbitrary registered URI handler — exactly the ungated
|
||||
// network/system reach that `open`/`xdg-open`/`start` were kept out of
|
||||
// READ_ONLY_BASES to avoid. This tool is "named applications only".
|
||||
if is_uri_scheme(trimmed) {
|
||||
return Err(format!(
|
||||
"app_name '{trimmed}' looks like a URI scheme; this tool launches named \
|
||||
applications only, not URIs/handlers"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// True if `s` begins with a URI scheme per RFC 3986: `ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) ":"`.
|
||||
fn is_uri_scheme(s: &str) -> bool {
|
||||
let Some(colon) = s.find(':') else {
|
||||
return false;
|
||||
};
|
||||
if colon == 0 {
|
||||
return false;
|
||||
}
|
||||
let scheme = &s[..colon];
|
||||
let mut chars = scheme.chars();
|
||||
match chars.next() {
|
||||
Some(c) if c.is_ascii_alphabetic() => {}
|
||||
_ => return false,
|
||||
}
|
||||
scheme
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for LaunchAppTool {
|
||||
fn name(&self) -> &str {
|
||||
"launch_app"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Open a named application on the user's desktop. Supply the app's display name \
|
||||
(e.g. 'Music', 'Spotify', 'Safari', 'Calculator', 'VS Code'). \
|
||||
Works on macOS, Linux, and Windows. \
|
||||
Use this instead of the shell tool whenever the goal is simply to open an app."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> serde_json::Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"app_name": {
|
||||
"type": "string",
|
||||
"description": "Display name of the application to open \
|
||||
(e.g. 'Music', 'Spotify', 'Google Chrome'). \
|
||||
Do not supply a file path — use the bare name."
|
||||
}
|
||||
},
|
||||
"required": ["app_name"]
|
||||
})
|
||||
}
|
||||
|
||||
fn permission_level(&self) -> PermissionLevel {
|
||||
// Launching an app executes a process / opens a window on the user's
|
||||
// machine — an execution-class action, not a read.
|
||||
PermissionLevel::Execute
|
||||
}
|
||||
|
||||
fn external_effect(&self) -> bool {
|
||||
// Opening an application is an externally-observable side effect, so the
|
||||
// harness routes this through the ApprovalGate before execute() — same
|
||||
// contract as `shell`. Not always-allow.
|
||||
true
|
||||
}
|
||||
|
||||
async fn execute(&self, args: serde_json::Value) -> anyhow::Result<ToolResult> {
|
||||
self.execute_with_options(args, ToolCallOptions::default())
|
||||
.await
|
||||
}
|
||||
|
||||
async fn execute_with_options(
|
||||
&self,
|
||||
args: serde_json::Value,
|
||||
_options: ToolCallOptions,
|
||||
) -> anyhow::Result<ToolResult> {
|
||||
let app_name = args
|
||||
.get("app_name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
log::info!("[launch_app] ▶ execute called app_name={app_name:?} raw_args={args}");
|
||||
|
||||
if let Err(reason) = validate_app_name(&app_name) {
|
||||
log::warn!("[launch_app] ✗ validation failed app_name={app_name:?} reason={reason}");
|
||||
return Ok(ToolResult::error(reason));
|
||||
}
|
||||
|
||||
log::info!("[launch_app] ✓ validation passed — dispatching to platform launcher");
|
||||
|
||||
let result = launch_platform(&app_name).await;
|
||||
|
||||
match result {
|
||||
Ok(msg) => {
|
||||
log::info!("[launch_app] ✓ launch succeeded app_name={app_name:?} msg={msg:?}");
|
||||
Ok(ToolResult::success(msg))
|
||||
}
|
||||
Err(err) => {
|
||||
log::warn!("[launch_app] ✗ launch failed app_name={app_name:?} error={err}");
|
||||
Ok(ToolResult::error(format!(
|
||||
"Could not open '{app_name}': {err}"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Platform-specific launch dispatch. Returns a human-readable success message.
|
||||
async fn launch_platform(app_name: &str) -> Result<String, String> {
|
||||
log::info!(
|
||||
"[launch_app] platform={} dispatching launch for app_name={app_name:?}",
|
||||
std::env::consts::OS
|
||||
);
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
return launch_macos(app_name).await;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
return launch_linux(app_name).await;
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
return launch_windows(app_name).await;
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
|
||||
Err("launch_app is not supported on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
async fn launch_macos(app_name: &str) -> Result<String, String> {
|
||||
log::info!("[launch_app] macOS: running `open -a {app_name:?}`");
|
||||
|
||||
// `open -a "App Name"` resolves by display name via LaunchServices.
|
||||
let output = tokio::process::Command::new("open")
|
||||
.arg("-a")
|
||||
.arg(app_name)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("failed to invoke `open`: {e}"))?;
|
||||
|
||||
log::info!(
|
||||
"[launch_app] macOS: `open -a` exit={} stderr={}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
);
|
||||
|
||||
if output.status.success() {
|
||||
return Ok(format!("Opened '{app_name}'."));
|
||||
}
|
||||
|
||||
log::info!("[launch_app] macOS: primary failed — trying fallback `open {app_name:?}`");
|
||||
|
||||
// Fallback: `open "<App Name>"` — works for bundle names and some URIs.
|
||||
let fallback = tokio::process::Command::new("open")
|
||||
.arg(app_name)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.await
|
||||
.map_err(|e| format!("failed to invoke `open` (fallback): {e}"))?;
|
||||
|
||||
log::info!("[launch_app] macOS: fallback exit={fallback}");
|
||||
|
||||
if fallback.success() {
|
||||
Ok(format!("Opened '{app_name}'."))
|
||||
} else {
|
||||
Err(format!(
|
||||
"`open -a \"{app_name}\"` failed (exit {}) — check the app name matches its title in /Applications",
|
||||
output.status
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn launch_linux(app_name: &str) -> Result<String, String> {
|
||||
// `gtk-launch` takes a .desktop file ID (e.g. "google-chrome"), NOT a
|
||||
// human-readable display name ("Google Chrome"). Try the name as given
|
||||
// first, then a best-effort desktop-id derived from the display name
|
||||
// (lowercased, spaces → hyphens). `xdg-open` does NOT launch apps by
|
||||
// name — it only opens URIs/paths in the default handler — so it's a
|
||||
// last resort for app_name values that happen to be a URI.
|
||||
let desktop_id = app_name.to_lowercase().replace(' ', "-");
|
||||
let mut candidates = vec![app_name.to_string()];
|
||||
if desktop_id != app_name {
|
||||
candidates.push(desktop_id);
|
||||
}
|
||||
|
||||
for candidate in &candidates {
|
||||
let gtk = tokio::process::Command::new("gtk-launch")
|
||||
.arg(candidate)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.await;
|
||||
if let Ok(s) = gtk {
|
||||
if s.success() {
|
||||
return Ok(format!("Opened '{app_name}'."));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback for URI-shaped inputs only (xdg-open won't resolve app names).
|
||||
let xdg = tokio::process::Command::new("xdg-open")
|
||||
.arg(app_name)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.await
|
||||
.map_err(|e| format!("failed to invoke `xdg-open`: {e}"))?;
|
||||
|
||||
if xdg.success() {
|
||||
Ok(format!("Opened '{app_name}'."))
|
||||
} else {
|
||||
Err(format!(
|
||||
"Could not launch '{app_name}' on Linux. gtk-launch needs a .desktop \
|
||||
ID (e.g. 'google-chrome'); xdg-open only opens URIs/paths, not app names. \
|
||||
Try the .desktop ID, or supply a URI."
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
async fn launch_windows(app_name: &str) -> Result<String, String> {
|
||||
// The app name is passed through an env var (`OH_LAUNCH_APP`) and never
|
||||
// string-interpolated into the script, so a name containing a quote cannot
|
||||
// break out of the command. `validate_app_name` already blocks shell
|
||||
// metacharacters; the static script + env passing is belt-and-suspenders.
|
||||
//
|
||||
// Resolution order:
|
||||
// 1. `Start-Process -FilePath <name>` — resolves PATH executables,
|
||||
// registered App Paths (e.g. "Spotify" desktop), and URIs ("spotify:").
|
||||
// 2. Fallback for Store/UWP apps that have no plain exe: match by display
|
||||
// name via `Get-StartApps` and launch by AUMID
|
||||
// (`shell:AppsFolder\<AppID>`), e.g. the Store "Media Player".
|
||||
const PS_LAUNCH: &str = "\
|
||||
$ErrorActionPreference='Stop'; \
|
||||
$n=$env:OH_LAUNCH_APP; \
|
||||
try { Start-Process -FilePath $n } \
|
||||
catch { \
|
||||
$app = Get-StartApps | Where-Object { $_.Name -like \"*$n*\" } | Select-Object -First 1; \
|
||||
if ($app) { Start-Process -FilePath ('shell:AppsFolder\\' + $app.AppID) } \
|
||||
else { throw } \
|
||||
}";
|
||||
|
||||
log::info!(
|
||||
"[launch_app] windows: launching app_name={app_name:?} (Start-Process, Store fallback)"
|
||||
);
|
||||
|
||||
let output = tokio::process::Command::new("powershell")
|
||||
.args(["-NoProfile", "-NonInteractive", "-Command", PS_LAUNCH])
|
||||
.env("OH_LAUNCH_APP", app_name)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("failed to invoke PowerShell: {e}"))?;
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
log::info!(
|
||||
"[launch_app] windows: exit={} stderr={}",
|
||||
output.status,
|
||||
stderr.trim()
|
||||
);
|
||||
|
||||
if output.status.success() {
|
||||
Ok(format!("Opened '{app_name}'."))
|
||||
} else if stderr.trim().is_empty() {
|
||||
Err(format!(
|
||||
"could not open '{app_name}' (Start-Process and Store-app lookup both failed)"
|
||||
))
|
||||
} else {
|
||||
Err(format!("could not open '{app_name}': {}", stderr.trim()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn name_and_permission() {
|
||||
let tool = LaunchAppTool::new();
|
||||
assert_eq!(tool.name(), "launch_app");
|
||||
// Execution-class + external effect so it routes through the ApprovalGate.
|
||||
assert_eq!(tool.permission_level(), PermissionLevel::Execute);
|
||||
assert!(tool.external_effect());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_uri_schemes() {
|
||||
// URI schemes would fire arbitrary registered handlers via the
|
||||
// Linux/Windows launcher fallbacks — reject them (named apps only).
|
||||
assert!(validate_app_name("spotify:track/123").is_err());
|
||||
assert!(validate_app_name("mailto:a@b.com").is_err());
|
||||
assert!(validate_app_name("slack:").is_err());
|
||||
assert!(validate_app_name("https://evil.example").is_err());
|
||||
assert!(validate_app_name("x-custom-scheme:payload").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_requires_app_name() {
|
||||
let schema = LaunchAppTool::new().parameters_schema();
|
||||
assert_eq!(schema["type"], "object");
|
||||
assert!(schema["required"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|v| v == "app_name"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_empty() {
|
||||
assert!(validate_app_name("").is_err());
|
||||
assert!(validate_app_name(" ").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_paths() {
|
||||
assert!(validate_app_name("/Applications/Music.app").is_err());
|
||||
assert!(validate_app_name("../etc/passwd").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_metacharacters() {
|
||||
assert!(validate_app_name("Music; rm -rf /").is_err());
|
||||
assert!(validate_app_name("$(evil)").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_normal_names() {
|
||||
assert!(validate_app_name("Music").is_ok());
|
||||
assert!(validate_app_name("Google Chrome").is_ok());
|
||||
assert!(validate_app_name("VS Code").is_ok());
|
||||
assert!(validate_app_name("Spotify").is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn returns_error_for_empty_app_name() {
|
||||
let result = LaunchAppTool::new()
|
||||
.execute(json!({"app_name": ""}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_error);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn returns_error_for_path_traversal() {
|
||||
let result = LaunchAppTool::new()
|
||||
.execute(json!({"app_name": "/Applications/Music.app"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_error);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ mod current_time;
|
||||
mod detect_tools;
|
||||
mod insert_sql_record;
|
||||
mod install_tool;
|
||||
mod launch_app;
|
||||
mod lsp;
|
||||
mod node_exec;
|
||||
mod npm_exec;
|
||||
@@ -18,6 +19,7 @@ pub use current_time::CurrentTimeTool;
|
||||
pub use detect_tools::DetectToolsTool;
|
||||
pub use insert_sql_record::InsertSqlRecordTool;
|
||||
pub use install_tool::InstallToolTool;
|
||||
pub use launch_app::LaunchAppTool;
|
||||
pub use lsp::{lsp_capability_enabled, LspTool, LSP_ENABLED_ENV};
|
||||
pub use node_exec::NodeExecTool;
|
||||
pub use npm_exec::NpmExecTool;
|
||||
|
||||
@@ -120,7 +120,9 @@ impl Tool for ShellTool {
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Execute a shell command in the workspace directory"
|
||||
"Execute a shell command. Use this to run code, manipulate files in the workspace, \
|
||||
or perform system actions on the user's machine — including launching applications \
|
||||
(e.g. `open -a Music` on macOS, `xdg-open music://` on Linux)."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> serde_json::Value {
|
||||
|
||||
@@ -168,6 +168,10 @@ pub fn all_tools_with_runtime(
|
||||
// and tool callers share one spawn path.
|
||||
Box::new(RunSkillTool::new()),
|
||||
Box::new(CurrentTimeTool::new()),
|
||||
Box::new(LaunchAppTool::new()),
|
||||
Box::new(AxInteractTool::new(
|
||||
root_config.computer_control.ax_interact_mutations,
|
||||
)),
|
||||
Box::new(CodegraphIndexTool::new(
|
||||
config.clone(),
|
||||
action_dir.to_path_buf(),
|
||||
|
||||
@@ -28,6 +28,27 @@ const TOOL_FAMILIES: &[ToolFamily] = &[
|
||||
rust_names: &["shell"],
|
||||
default_enabled: true,
|
||||
},
|
||||
// Dedicated app-launcher: always-allow, no shell exposure, no workspace_only concern.
|
||||
ToolFamily {
|
||||
id: "launch_app",
|
||||
rust_names: &["launch_app"],
|
||||
default_enabled: true,
|
||||
},
|
||||
// AXUIElement interaction: semantic UI control via macOS Accessibility API.
|
||||
// No CGEventPost, no coordinate dependency, no CEF crash risk.
|
||||
ToolFamily {
|
||||
id: "ax_interact",
|
||||
rust_names: &["ax_interact"],
|
||||
default_enabled: true,
|
||||
},
|
||||
// Computer control — mouse and keyboard. Gated by computer_control.enabled
|
||||
// in config (tools only register when that flag is true). PermissionLevel::Dangerous
|
||||
// so the approval gate fires per-action; user opts in explicitly.
|
||||
ToolFamily {
|
||||
id: "computer_control",
|
||||
rust_names: &["mouse", "keyboard"],
|
||||
default_enabled: true,
|
||||
},
|
||||
// detect_tools / install_tool are filterable but not surfaced in the
|
||||
// default-ON catalog, so they stay opt-in (default-OFF).
|
||||
ToolFamily {
|
||||
|
||||
Reference in New Issue
Block a user