From d61d72c092f88e3f5bc4ac8a06ac2c6c7b1511e9 Mon Sep 17 00:00:00 2001 From: Mega Mind <146339422+M3gA-Mind@users.noreply.github.com> Date: Sat, 13 Jun 2026 19:20:32 +0530 Subject: [PATCH] feat(settings): Desktop Agent setup panel (permissions, seamless mode, always-on, wake-word) (#3634) Co-authored-by: Steven Enamakel --- .../settings/hooks/useSettingsNavigation.ts | 1 + .../settings/layout/settingsNavIcons.tsx | 5 + .../settings/panels/DesktopAgentPanel.tsx | 421 ++++++++++++++++++ .../settings/panels/VoiceDebugPanel.tsx | 24 +- .../components/settings/panels/VoicePanel.tsx | 39 +- .../__tests__/DesktopAgentPanel.test.tsx | 278 ++++++++++++ .../panels/__tests__/VoiceDebugPanel.test.tsx | 56 +-- .../panels/__tests__/VoicePanel.test.tsx | 43 +- .../settings/settingsRouteRegistry.ts | 22 +- app/src/lib/i18n/ar.ts | 16 + app/src/lib/i18n/bn.ts | 19 + app/src/lib/i18n/de.ts | 20 + app/src/lib/i18n/en.ts | 17 + app/src/lib/i18n/es.ts | 19 + app/src/lib/i18n/fr.ts | 20 + app/src/lib/i18n/hi.ts | 18 + app/src/lib/i18n/id.ts | 19 + app/src/lib/i18n/it.ts | 19 + app/src/lib/i18n/ko.ts | 18 + app/src/lib/i18n/pl.ts | 19 + app/src/lib/i18n/pt.ts | 20 + app/src/lib/i18n/ru.ts | 20 + app/src/lib/i18n/zh-CN.ts | 16 + app/src/pages/Settings.tsx | 2 + app/src/utils/tauriCommands/accessibility.ts | 12 +- app/test/e2e/specs/desktop-agent.spec.ts | 74 +++ docs/voice-system-actions.md | 80 ++++ 27 files changed, 1169 insertions(+), 148 deletions(-) create mode 100644 app/src/components/settings/panels/DesktopAgentPanel.tsx create mode 100644 app/src/components/settings/panels/__tests__/DesktopAgentPanel.test.tsx create mode 100644 app/test/e2e/specs/desktop-agent.spec.ts diff --git a/app/src/components/settings/hooks/useSettingsNavigation.ts b/app/src/components/settings/hooks/useSettingsNavigation.ts index f9dce3500..8e8933e35 100644 --- a/app/src/components/settings/hooks/useSettingsNavigation.ts +++ b/app/src/components/settings/hooks/useSettingsNavigation.ts @@ -20,6 +20,7 @@ export type SettingsRoute = | 'account' | 'cron-jobs' | 'screen-intelligence' + | 'desktop-agent' | 'autocomplete' | 'privacy' | 'billing' diff --git a/app/src/components/settings/layout/settingsNavIcons.tsx b/app/src/components/settings/layout/settingsNavIcons.tsx index 8b3fca180..7fddc1521 100644 --- a/app/src/components/settings/layout/settingsNavIcons.tsx +++ b/app/src/components/settings/layout/settingsNavIcons.tsx @@ -61,6 +61,11 @@ export const SETTINGS_NAV_ICONS: Record = { ), integrations: icon(stroke('M13 10V3L4 14h7v7l9-11h-7z')), 'screen-intelligence': icon(stroke('M3 5h18v12H3zM8 21h8m-4-4v4')), + 'desktop-agent': icon( + stroke( + 'M9 17v2m6-2v2M5 5h14a1 1 0 011 1v8a1 1 0 01-1 1H5a1 1 0 01-1-1V6a1 1 0 011-1zm4 4l-2 2 2 2m6-4l2 2-2 2' + ) + ), tools: icon( stroke( 'M10.325 4.317c.426-1.756 2.924-1.756 3.35 0a1.724 1.724 0 002.573 1.066c1.543-.94 3.31.826 2.37 2.37a1.724 1.724 0 001.066 2.573c1.756.426 1.756 2.924 0 3.35a1.724 1.724 0 00-1.066 2.573c.94 1.543-.826 3.31-2.37 2.37a1.724 1.724 0 00-2.573 1.066c-.426 1.756-2.924 1.756-3.35 0a1.724 1.724 0 00-2.573-1.066c-1.543.94-3.31-.826-2.37-2.37a1.724 1.724 0 00-1.066-2.573c-1.756-.426-1.756-2.924 0-3.35a1.724 1.724 0 001.066-2.573c-.94-1.543.826-3.31 2.37-2.37.996.608 2.296.07 2.572-1.065zM15 12a3 3 0 11-6 0 3 3 0 016 0z' diff --git a/app/src/components/settings/panels/DesktopAgentPanel.tsx b/app/src/components/settings/panels/DesktopAgentPanel.tsx new file mode 100644 index 000000000..49c56c7cd --- /dev/null +++ b/app/src/components/settings/panels/DesktopAgentPanel.tsx @@ -0,0 +1,421 @@ +import { useCallback, useEffect, useState } from 'react'; + +import { + fetchScreenIntelligenceStatus, + refreshScreenIntelligencePermissionsWithRestart, + requestScreenIntelligencePermission, +} from '../../../features/screen-intelligence/api'; +import { useT } from '../../../lib/i18n/I18nContext'; +import { + type AccessibilityPermissionKind, + type AccessibilityPermissionState, + type AccessibilityStatus, + openhumanGetAutonomySettings, + openhumanGetVoiceServerSettings, + openhumanUpdateAutonomySettings, + openhumanUpdateVoiceServerSettings, + syncNotchVisibility, +} from '../../../utils/tauriCommands'; +import SettingsHeader from '../components/SettingsHeader'; +import { + SettingsBadge, + type SettingsBadgeVariant, + SettingsRow, + SettingsSection, + SettingsSwitch, +} from '../controls'; +import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; + +/** + * Desktop Agent setup panel — Settings → Features → Desktop Agent. + * + * Two things in one place, no new backend: + * 1. Check + grant the four OS permissions the desktop agent needs (Microphone, + * Accessibility, Screen Recording, Input Monitoring) — reuses the existing + * screen-intelligence permission RPCs. + * 2. "Let the agent act without asking" — a seamless-mode toggle that grants Full + * access and auto-approves the desktop-control tools so the agent runs them + * without an in-app approval card (reuses the autonomy RPCs). + */ + +/** + * The four permissions the desktop agent depends on, in setup order. + * + * Accessibility / Screen Recording / Input Monitoring intentionally reuse the + * existing `settings.screenIntel.permissions.*` labels (same OS permissions as the + * Screen Intelligence panel) to avoid duplicate translations; only Microphone needs + * a desktop-agent-specific key. + */ +const PERMISSIONS: ReadonlyArray<{ kind: AccessibilityPermissionKind; labelKey: string }> = [ + { kind: 'microphone', labelKey: 'settings.desktopAgent.microphone' }, + { kind: 'accessibility', labelKey: 'settings.screenIntel.permissions.accessibility' }, + { kind: 'screen_recording', labelKey: 'settings.screenIntel.permissions.screenRecording' }, + { kind: 'input_monitoring', labelKey: 'settings.screenIntel.permissions.inputMonitoring' }, +]; + +/** + * Desktop-control tools auto-approved by seamless mode so the agent actuates apps + * without an approval card. Deliberately excludes shell / file / network / install + * tools — those keep prompting. Matched by exact tool name in `autonomy.auto_approve`. + */ +const SEAMLESS_TOOLS = ['automate', 'ax_interact', 'launch_app', 'keyboard', 'mouse'] as const; + +const STATE_VARIANT: Record = { + granted: 'success', + denied: 'danger', + unknown: 'warning', + unsupported: 'neutral', +}; + +const errorMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +const stateFor = ( + status: AccessibilityStatus | null, + kind: AccessibilityPermissionKind +): AccessibilityPermissionState => status?.permissions?.[kind] ?? 'unknown'; + +const DesktopAgentPanel = () => { + const { t } = useT(); + const { navigateBack, breadcrumbs } = useSettingsNavigation(); + + const [status, setStatus] = useState(null); + const [isLoading, setIsLoading] = useState(false); + const [requestingKind, setRequestingKind] = useState(null); + const [isRestarting, setIsRestarting] = useState(false); + const [error, setError] = useState(null); + const [restartSummary, setRestartSummary] = useState(null); + + // Seamless mode (auto-approve allowlist). Null until the autonomy settings load. + const [autoApprove, setAutoApprove] = useState(null); + const [isUpdatingSeamless, setIsUpdatingSeamless] = useState(false); + + // Always-on listening (relocated from the Voice panel). Null until loaded. + const [alwaysOn, setAlwaysOn] = useState(null); + const [isUpdatingAlwaysOn, setIsUpdatingAlwaysOn] = useState(false); + + const refresh = useCallback(async () => { + setIsLoading(true); + setError(null); + try { + setStatus(await fetchScreenIntelligenceStatus()); + } catch (err) { + setError(errorMessage(err)); + } finally { + setIsLoading(false); + } + }, []); + + const loadAutonomy = useCallback(async () => { + setError(null); + try { + const resp = await openhumanGetAutonomySettings(); + setAutoApprove(resp.result.auto_approve ?? []); + } catch (err) { + setError(errorMessage(err)); + } + }, []); + + const loadVoice = useCallback(async () => { + setError(null); + try { + const resp = await openhumanGetVoiceServerSettings(); + setAlwaysOn(resp.result.always_on_enabled); + } catch (err) { + setError(errorMessage(err)); + } + }, []); + + useEffect(() => { + // eslint-disable-next-line react-hooks/set-state-in-effect + void refresh(); + void loadAutonomy(); + void loadVoice(); + }, [refresh, loadAutonomy, loadVoice]); + + const toggleAlwaysOn = useCallback( + async (next: boolean) => { + // Guard against rapid re-toggles racing and committing out of order. + if (isUpdatingAlwaysOn) return; + const previous = alwaysOn; + setIsUpdatingAlwaysOn(true); + setAlwaysOn(next); // optimistic flip + setError(null); + try { + await openhumanUpdateVoiceServerSettings({ always_on_enabled: next }); + } catch (err) { + // Persist failed — revert the optimistic flip and surface the error. + setAlwaysOn(previous); + setError(errorMessage(err)); + setIsUpdatingAlwaysOn(false); + return; + } + try { + // The notch pill is the always-on listening HUD: show on, hide off. + // Persistence already succeeded, so keep the UI state even if this fails. + await syncNotchVisibility(next); + } catch (err) { + setError(errorMessage(err)); + } finally { + setIsUpdatingAlwaysOn(false); + } + }, + [alwaysOn, isUpdatingAlwaysOn] + ); + + const seamlessOn = + autoApprove !== null && SEAMLESS_TOOLS.every(tool => autoApprove.includes(tool)); + + const toggleSeamless = useCallback( + async (next: boolean) => { + const current = autoApprove ?? []; + setIsUpdatingSeamless(true); + setError(null); + try { + if (next) { + // Grant Full access (satisfies app_control_enabled) + auto-approve the + // desktop tools + drop the task-plan approval prompt. + const merged = Array.from(new Set([...current, ...SEAMLESS_TOOLS])); + await openhumanUpdateAutonomySettings({ + level: 'full', + require_task_plan_approval: false, + auto_approve: merged, + }); + } else { + // Remove only the desktop tools + restore the plan-approval prompt. Leave + // the autonomy tier as-is (managed in Settings → Agent Access). + const seamlessSet = new Set(SEAMLESS_TOOLS); + const pruned = current.filter(tool => !seamlessSet.has(tool)); + await openhumanUpdateAutonomySettings({ + require_task_plan_approval: true, + auto_approve: pruned, + }); + } + await loadAutonomy(); + } catch (err) { + setError(errorMessage(err)); + } finally { + setIsUpdatingSeamless(false); + } + }, + [autoApprove, loadAutonomy] + ); + + const grant = useCallback(async (kind: AccessibilityPermissionKind) => { + setRequestingKind(kind); + setError(null); + setRestartSummary(null); + try { + setStatus(await requestScreenIntelligencePermission(kind)); + } catch (err) { + setError(errorMessage(err)); + } finally { + setRequestingKind(null); + } + }, []); + + const restartAndRecheck = useCallback(async () => { + setIsRestarting(true); + setError(null); + setRestartSummary(null); + try { + const result = await refreshScreenIntelligencePermissionsWithRestart(status); + setStatus(result.status); + setRestartSummary(result.restartSummary); + } catch (err) { + setError(errorMessage(err)); + } finally { + setIsRestarting(false); + } + }, [status]); + + const busy = isLoading || isRestarting || requestingKind !== null; + const actionable = PERMISSIONS.map(p => stateFor(status, p.kind)).filter( + s => s === 'denied' || s === 'unknown' + ); + const allGranted = + status !== null && + PERMISSIONS.every(p => { + const s = stateFor(status, p.kind); + return s === 'granted' || s === 'unsupported'; + }); + + return ( +
+ + +
+
+ {t('settings.desktopAgent.beta')} +
+

+ {t('settings.desktopAgent.description')} +

+ + + {PERMISSIONS.map(({ kind, labelKey }) => { + const state = stateFor(status, kind); + const unsupported = state === 'unsupported'; + const canGrant = state === 'denied' || state === 'unknown'; + return ( + + {state} + {unsupported ? ( + + {t('settings.desktopAgent.notRequiredOnOs')} + + ) : canGrant ? ( + + ) : null} +
+ } + /> + ); + })} + + + + void toggleSeamless(next)} + aria-label={t('settings.desktopAgent.seamless.label')} + /> + } + /> + +

+ {t('settings.desktopAgent.seamless.note')} +

+ + + void toggleAlwaysOn(next)} + aria-label={t('voice.debug.alwaysOn')} + /> + } + /> + + +
+ +

+ {t('settings.desktopAgent.wakeWordHint')} +

+
+ + {allGranted ? ( +
+ {t('settings.desktopAgent.allGranted')} +
+ ) : ( +
+ {t('settings.screenIntel.permissions.grantHint')} + {status?.permission_check_process_path ? ( +

+ {t('settings.screenIntel.permissions.macosAppliesPrivacy')}{' '} + + {status.permission_check_process_path} + +

+ ) : null} +
+ )} + + {restartSummary ? ( +
+ {restartSummary} +
+ ) : null} + + {error ? ( +
+ {error} +
+ ) : null} + +
+ + {actionable.length > 0 ? ( + + ) : null} +
+
+ + ); +}; + +export default DesktopAgentPanel; diff --git a/app/src/components/settings/panels/VoiceDebugPanel.tsx b/app/src/components/settings/panels/VoiceDebugPanel.tsx index 1aa28d3ef..75eb98ff9 100644 --- a/app/src/components/settings/panels/VoiceDebugPanel.tsx +++ b/app/src/components/settings/panels/VoiceDebugPanel.tsx @@ -13,13 +13,7 @@ import { import PanelPage from '../../layout/PanelPage'; import Button from '../../ui/Button'; import SettingsBackButton from '../components/SettingsBackButton'; -import { - SettingsNumberField, - SettingsRow, - SettingsSection, - SettingsStatusLine, - SettingsSwitch, -} from '../controls'; +import { SettingsNumberField, SettingsRow, SettingsSection, SettingsStatusLine } from '../controls'; import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; const VoiceDebugPanel = () => { @@ -111,7 +105,6 @@ const VoiceDebugPanel = () => { min_duration_secs: settings.min_duration_secs, silence_threshold: settings.silence_threshold, custom_dictionary: settings.custom_dictionary, - always_on_enabled: settings.always_on_enabled, }); setNotice(t('voice.debug.settingsSaved')); await loadData(true); @@ -201,20 +194,7 @@ const VoiceDebugPanel = () => { description={t('voice.debug.advancedSettingsDesc')}> {settings && ( <> - updateSetting('always_on_enabled', next)} - aria-label={t('voice.debug.alwaysOn')} - data-testid="voice-always-on-toggle" - /> - } - /> + {/* Always-on listening moved to Settings → Features → Desktop Agent. */} { description={embedded ? undefined : t('pages.settings.ai.voiceDesc')} leading={embedded ? undefined : }>
- {/* ─── Always-on listening (Phase 2) ──────────────────────────── */} - {settings && ( - - { - setSettings(current => - current ? { ...current, always_on_enabled: next } : current - ); - try { - await openhumanUpdateVoiceServerSettings({ always_on_enabled: next }); - // The notch pill is the always-on listening HUD: show it - // when listening is enabled, drop it when disabled. - await syncNotchVisibility(next); - } catch (err) { - // Revert on failure so the UI reflects the persisted value. - setSettings(current => - current ? { ...current, always_on_enabled: !next } : current - ); - console.error('[VoicePanel] failed to toggle always-on', err); - } - }} - aria-label={t('voice.debug.alwaysOn')} - data-testid="voice-always-on-toggle" - /> - } - /> - - )} + {/* Always-on listening moved to Settings → Features → Desktop Agent. */} {/* ─── Section 1: Voice Provider Chips ─────────────────────────── */} {/* Provider chips are intentional bespoke UI — kept as-is. */} diff --git a/app/src/components/settings/panels/__tests__/DesktopAgentPanel.test.tsx b/app/src/components/settings/panels/__tests__/DesktopAgentPanel.test.tsx new file mode 100644 index 000000000..c047d2c46 --- /dev/null +++ b/app/src/components/settings/panels/__tests__/DesktopAgentPanel.test.tsx @@ -0,0 +1,278 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + fetchScreenIntelligenceStatus, + refreshScreenIntelligencePermissionsWithRestart, + requestScreenIntelligencePermission, +} from '../../../../features/screen-intelligence/api'; +import { + type AccessibilityPermissionState, + type AccessibilityStatus, + openhumanGetAutonomySettings, + openhumanGetVoiceServerSettings, + openhumanUpdateAutonomySettings, + openhumanUpdateVoiceServerSettings, + syncNotchVisibility, +} from '../../../../utils/tauriCommands'; +import DesktopAgentPanel from '../DesktopAgentPanel'; + +vi.mock('../../../../features/screen-intelligence/api', () => ({ + fetchScreenIntelligenceStatus: vi.fn(), + requestScreenIntelligencePermission: vi.fn(), + refreshScreenIntelligencePermissionsWithRestart: vi.fn(), +})); + +vi.mock('../../../../utils/tauriCommands', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + openhumanGetAutonomySettings: vi.fn(), + openhumanUpdateAutonomySettings: vi.fn(), + openhumanGetVoiceServerSettings: vi.fn(), + openhumanUpdateVoiceServerSettings: vi.fn(), + syncNotchVisibility: vi.fn(), + }; +}); + +function mockVoice(alwaysOn: boolean) { + vi.mocked(openhumanGetVoiceServerSettings).mockResolvedValue({ + result: { always_on_enabled: alwaysOn }, + logs: [], + } as unknown as Awaited>); +} + +const SEAMLESS_TOOLS = ['automate', 'ax_interact', 'launch_app', 'keyboard', 'mouse']; + +function mockAutonomy(autoApprove: string[]) { + vi.mocked(openhumanGetAutonomySettings).mockResolvedValue({ + result: { auto_approve: autoApprove }, + logs: [], + } as unknown as Awaited>); +} + +type Perms = { + microphone: AccessibilityPermissionState; + accessibility: AccessibilityPermissionState; + screen_recording: AccessibilityPermissionState; + input_monitoring: AccessibilityPermissionState; +}; + +function makeStatus(permissions: Perms): AccessibilityStatus { + return { + platform_supported: true, + permissions, + permission_check_process_path: '/tmp/openhuman-core', + } as unknown as AccessibilityStatus; +} + +function renderPanel() { + render( + + + + ); +} + +describe('DesktopAgentPanel', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockAutonomy([]); + mockVoice(false); + vi.mocked(openhumanUpdateAutonomySettings).mockResolvedValue( + {} as unknown as Awaited> + ); + vi.mocked(openhumanUpdateVoiceServerSettings).mockResolvedValue( + {} as unknown as Awaited> + ); + vi.mocked(syncNotchVisibility).mockResolvedValue(undefined); + }); + + it('renders a row per permission with the right grant affordance', async () => { + vi.mocked(fetchScreenIntelligenceStatus).mockResolvedValue( + makeStatus({ + microphone: 'granted', + accessibility: 'denied', + screen_recording: 'unknown', + input_monitoring: 'unsupported', + }) + ); + + renderPanel(); + + // Granted permission → badge, no grant button. + const micRow = await screen.findByTestId('desktop-agent-perm-microphone'); + expect(within(micRow).getByText('granted')).toBeInTheDocument(); + expect(screen.queryByTestId('desktop-agent-grant-microphone')).not.toBeInTheDocument(); + + // Denied + unknown → grant buttons. + expect(screen.getByTestId('desktop-agent-grant-accessibility')).toBeInTheDocument(); + expect(screen.getByTestId('desktop-agent-grant-screen_recording')).toBeInTheDocument(); + + // Unsupported → no grant button (muted "not required" instead). + expect(screen.queryByTestId('desktop-agent-grant-input_monitoring')).not.toBeInTheDocument(); + const inputRow = screen.getByTestId('desktop-agent-perm-input_monitoring'); + expect(within(inputRow).getByText('Not required on this OS')).toBeInTheDocument(); + }); + + it('requests the permission and reflects the new status when Grant is clicked', async () => { + vi.mocked(fetchScreenIntelligenceStatus).mockResolvedValue( + makeStatus({ + microphone: 'granted', + accessibility: 'denied', + screen_recording: 'granted', + input_monitoring: 'unsupported', + }) + ); + vi.mocked(requestScreenIntelligencePermission).mockResolvedValue( + makeStatus({ + microphone: 'granted', + accessibility: 'granted', + screen_recording: 'granted', + input_monitoring: 'unsupported', + }) + ); + + renderPanel(); + + const grantBtn = await screen.findByTestId('desktop-agent-grant-accessibility'); + fireEvent.click(grantBtn); + + await waitFor(() => + expect(requestScreenIntelligencePermission).toHaveBeenCalledWith('accessibility') + ); + // Once granted, the grant button disappears and the all-granted banner shows. + await waitFor(() => + expect(screen.queryByTestId('desktop-agent-grant-accessibility')).not.toBeInTheDocument() + ); + expect(screen.getByTestId('desktop-agent-all-granted')).toBeInTheDocument(); + }); + + it('surfaces an error when a permission request fails', async () => { + vi.mocked(fetchScreenIntelligenceStatus).mockResolvedValue( + makeStatus({ + microphone: 'denied', + accessibility: 'granted', + screen_recording: 'granted', + input_monitoring: 'unsupported', + }) + ); + vi.mocked(requestScreenIntelligencePermission).mockRejectedValue( + new Error('permission request blew up') + ); + + renderPanel(); + + fireEvent.click(await screen.findByTestId('desktop-agent-grant-microphone')); + + const errorBox = await screen.findByTestId('desktop-agent-error'); + expect(errorBox).toHaveTextContent('permission request blew up'); + // refresh-with-restart is unrelated to a failed grant. + expect(refreshScreenIntelligencePermissionsWithRestart).not.toHaveBeenCalled(); + }); + + describe('seamless mode (act without asking)', () => { + beforeEach(() => { + vi.mocked(fetchScreenIntelligenceStatus).mockResolvedValue( + makeStatus({ + microphone: 'granted', + accessibility: 'granted', + screen_recording: 'granted', + input_monitoring: 'unsupported', + }) + ); + }); + + it('reflects the auto-approve allowlist: off when the desktop tools are absent', async () => { + mockAutonomy([]); + renderPanel(); + const toggle = await screen.findByTestId('desktop-agent-seamless-toggle'); + await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'false')); + }); + + it('reflects the auto-approve allowlist: on when all desktop tools are present', async () => { + mockAutonomy([...SEAMLESS_TOOLS]); + renderPanel(); + const toggle = await screen.findByTestId('desktop-agent-seamless-toggle'); + await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'true')); + }); + + it('enabling grants Full access, drops plan approval, and auto-approves the desktop tools', async () => { + mockAutonomy([]); + renderPanel(); + const toggle = await screen.findByTestId('desktop-agent-seamless-toggle'); + await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'false')); + + fireEvent.click(toggle); + + await waitFor(() => expect(openhumanUpdateAutonomySettings).toHaveBeenCalledTimes(1)); + const payload = vi.mocked(openhumanUpdateAutonomySettings).mock.calls[0][0]; + expect(payload).toMatchObject({ level: 'full', require_task_plan_approval: false }); + expect(payload.auto_approve).toEqual(expect.arrayContaining(SEAMLESS_TOOLS)); + }); + + it('disabling removes the desktop tools and restores plan approval, keeping other allows', async () => { + mockAutonomy([...SEAMLESS_TOOLS, 'shell']); + renderPanel(); + const toggle = await screen.findByTestId('desktop-agent-seamless-toggle'); + await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'true')); + + fireEvent.click(toggle); + + await waitFor(() => expect(openhumanUpdateAutonomySettings).toHaveBeenCalledTimes(1)); + const payload = vi.mocked(openhumanUpdateAutonomySettings).mock.calls[0][0]; + expect(payload).toMatchObject({ require_task_plan_approval: true }); + expect(payload.auto_approve).toEqual(['shell']); + // Disabling must not downgrade the tier. + expect(payload.level).toBeUndefined(); + }); + }); + + describe('always-on listening (relocated from Voice)', () => { + beforeEach(() => { + vi.mocked(fetchScreenIntelligenceStatus).mockResolvedValue( + makeStatus({ + microphone: 'granted', + accessibility: 'granted', + screen_recording: 'granted', + input_monitoring: 'unsupported', + }) + ); + }); + + it('reflects the persisted always-on flag', async () => { + mockVoice(true); + renderPanel(); + const toggle = await screen.findByTestId('voice-always-on-toggle'); + await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'true')); + }); + + it('persists the flag and syncs the notch when toggled on', async () => { + mockVoice(false); + renderPanel(); + const toggle = await screen.findByTestId('voice-always-on-toggle'); + await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'false')); + + fireEvent.click(toggle); + + await waitFor(() => + expect(openhumanUpdateVoiceServerSettings).toHaveBeenCalledWith({ always_on_enabled: true }) + ); + await waitFor(() => expect(syncNotchVisibility).toHaveBeenCalledWith(true)); + }); + + it('reverts and never touches the notch when the update fails', async () => { + mockVoice(false); + vi.mocked(openhumanUpdateVoiceServerSettings).mockRejectedValueOnce(new Error('rpc down')); + renderPanel(); + const toggle = await screen.findByTestId('voice-always-on-toggle'); + await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'false')); + + fireEvent.click(toggle); + + await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'false')); + expect(syncNotchVisibility).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/app/src/components/settings/panels/__tests__/VoiceDebugPanel.test.tsx b/app/src/components/settings/panels/__tests__/VoiceDebugPanel.test.tsx index 8c1d522b5..01d87d78e 100644 --- a/app/src/components/settings/panels/__tests__/VoiceDebugPanel.test.tsx +++ b/app/src/components/settings/panels/__tests__/VoiceDebugPanel.test.tsx @@ -76,40 +76,9 @@ const VOICE_STATUS: VoiceStatus = { tts_provider: 'cloud', }; -describe('VoiceDebugPanel — always-on toggle', () => { - beforeEach(() => { - vi.clearAllMocks(); - vi.mocked(openhumanGetVoiceServerSettings).mockResolvedValue({ - result: { ...SETTINGS }, - logs: [], - }); - vi.mocked(openhumanUpdateVoiceServerSettings).mockResolvedValue({ - result: {} as unknown as ConfigSnapshot, - logs: [], - }); - vi.mocked(openhumanVoiceServerStatus).mockResolvedValue(SERVER_STATUS); - vi.mocked(openhumanVoiceStatus).mockResolvedValue(VOICE_STATUS); - }); - - it('toggles always-on and persists it via the update RPC on save', async () => { - render(); - - const toggle = await screen.findByTestId('voice-always-on-toggle'); - expect(toggle).toHaveAttribute('aria-checked', 'false'); - - // Local optimistic flip (creates an unsaved change → enables Save). - fireEvent.click(toggle); - expect(toggle).toHaveAttribute('aria-checked', 'true'); - - fireEvent.click(screen.getByText('common.save')); - - await waitFor(() => - expect(vi.mocked(openhumanUpdateVoiceServerSettings)).toHaveBeenCalledWith( - expect.objectContaining({ always_on_enabled: true }) - ) - ); - }); -}); +// Always-on listening was relocated to the Desktop Agent panel; the Voice debug +// panel no longer hosts that toggle. Save-flow coverage below drives the +// minimum-recording-seconds field instead. describe('VoiceDebugPanel — runtime status section (uncovered lines)', () => { beforeEach(() => { @@ -216,8 +185,9 @@ describe('VoiceDebugPanel — runtime status section (uncovered lines)', () => { it('renders advanced settings section when settings are loaded (line 204)', async () => { render(); - await waitFor(() => expect(screen.getByText('voice.debug.alwaysOn')).toBeInTheDocument()); - expect(screen.getByText('voice.debug.minimumRecordingSeconds')).toBeInTheDocument(); + await waitFor(() => + expect(screen.getByText('voice.debug.minimumRecordingSeconds')).toBeInTheDocument() + ); expect(screen.getByText('voice.debug.silenceThreshold')).toBeInTheDocument(); }); @@ -231,11 +201,11 @@ describe('VoiceDebugPanel — runtime status section (uncovered lines)', () => { expect(saveBtn.disabled).toBe(true); }); - it('enables Save after toggling always-on (line 228)', async () => { + it('enables Save after editing a setting (line 228)', async () => { render(); - const toggle = await screen.findByTestId('voice-always-on-toggle'); - fireEvent.click(toggle); + const input = await screen.findByLabelText('voice.debug.minimumRecordingSeconds'); + fireEvent.change(input, { target: { value: '5' } }); const saveBtn = screen.getByText('common.save').closest('button') as HTMLButtonElement; expect(saveBtn.disabled).toBe(false); @@ -246,8 +216,8 @@ describe('VoiceDebugPanel — runtime status section (uncovered lines)', () => { it('shows settingsSaved notice after successful save (line 242-243)', async () => { render(); - const toggle = await screen.findByTestId('voice-always-on-toggle'); - fireEvent.click(toggle); + const input = await screen.findByLabelText('voice.debug.minimumRecordingSeconds'); + fireEvent.change(input, { target: { value: '5' } }); fireEvent.click(screen.getByText('common.save')); @@ -260,8 +230,8 @@ describe('VoiceDebugPanel — runtime status section (uncovered lines)', () => { ); render(); - const toggle = await screen.findByTestId('voice-always-on-toggle'); - fireEvent.click(toggle); + const input = await screen.findByLabelText('voice.debug.minimumRecordingSeconds'); + fireEvent.change(input, { target: { value: '5' } }); fireEvent.click(screen.getByText('common.save')); diff --git a/app/src/components/settings/panels/__tests__/VoicePanel.test.tsx b/app/src/components/settings/panels/__tests__/VoicePanel.test.tsx index 8b7ae9960..6d90782b9 100644 --- a/app/src/components/settings/panels/__tests__/VoicePanel.test.tsx +++ b/app/src/components/settings/panels/__tests__/VoicePanel.test.tsx @@ -509,48 +509,7 @@ describe('VoicePanel', () => { await waitFor(() => expect(screen.getByText('core offline')).toBeInTheDocument()); }); - // ─── Always-on listening toggle ↔ notch indicator ─────────────────────── - - it('shows the notch when always-on listening is enabled and hides it when disabled', async () => { - renderWithProviders(, { initialEntries: ['/settings/voice'] }); - - const toggle = await screen.findByTestId('voice-always-on-toggle'); - await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'false')); - - // Turn always-on ON → persists the flag and shows the notch HUD. - fireEvent.click(toggle); - await waitFor(() => - expect(vi.mocked(openhumanUpdateVoiceServerSettings)).toHaveBeenCalledWith({ - always_on_enabled: true, - }) - ); - await waitFor(() => expect(vi.mocked(syncNotchVisibility)).toHaveBeenCalledWith(true)); - - // Turn always-on OFF → hides the notch HUD. - fireEvent.click(toggle); - await waitFor(() => - expect(vi.mocked(openhumanUpdateVoiceServerSettings)).toHaveBeenCalledWith({ - always_on_enabled: false, - }) - ); - await waitFor(() => expect(vi.mocked(syncNotchVisibility)).toHaveBeenCalledWith(false)); - }); - - it('does not touch the notch and reverts the toggle when the update RPC fails', async () => { - vi.mocked(openhumanUpdateVoiceServerSettings).mockRejectedValueOnce(new Error('rpc down')); - - renderWithProviders(, { initialEntries: ['/settings/voice'] }); - - const toggle = await screen.findByTestId('voice-always-on-toggle'); - await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'false')); - - fireEvent.click(toggle); - - // The optimistic flip is reverted back to off after the RPC rejects, and - // the notch is never shown because the persist failed. - await waitFor(() => expect(toggle).toHaveAttribute('aria-checked', 'false')); - expect(vi.mocked(syncNotchVisibility)).not.toHaveBeenCalled(); - }); + // Always-on listening toggle moved to DesktopAgentPanel (see its test). // ─── STT / TTS Test buttons ──────────────────────────────────────────────── diff --git a/app/src/components/settings/settingsRouteRegistry.ts b/app/src/components/settings/settingsRouteRegistry.ts index afff5a009..06942b24d 100644 --- a/app/src/components/settings/settingsRouteRegistry.ts +++ b/app/src/components/settings/settingsRouteRegistry.ts @@ -437,6 +437,24 @@ export const SETTINGS_ROUTE_REGISTRY: SettingsRegistryEntry[] = [ navGroup: 'connections', navOrder: 1, }, + { + id: 'desktop-agent', + titleKey: 'settings.desktopAgent.title', + descriptionKey: 'settings.desktopAgent.description', + section: 'features', + searchKeywords: [ + 'desktop', + 'agent', + 'automation', + 'permissions', + 'microphone', + 'accessibility', + 'screen recording', + 'input monitoring', + ], + navGroup: 'connections', + navOrder: 2, + }, { id: 'tools', titleKey: 'pages.settings.features.tools', @@ -444,7 +462,7 @@ export const SETTINGS_ROUTE_REGISTRY: SettingsRegistryEntry[] = [ section: 'features', searchKeywords: ['tools', 'capabilities', 'functions'], navGroup: 'connections', - navOrder: 2, + navOrder: 3, }, { id: 'companion', @@ -453,7 +471,7 @@ export const SETTINGS_ROUTE_REGISTRY: SettingsRegistryEntry[] = [ section: 'features', searchKeywords: ['desktop', 'overlay', 'companion'], navGroup: 'connections', - navOrder: 3, + navOrder: 4, }, // ========================================================================= diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 3dc2c1305..fdcbfcba4 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -4229,6 +4229,22 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'استبدال التطبيق', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'وكيل سطح المكتب', + 'settings.desktopAgent.description': 'تحقّق من الأذونات التي يحتاجها وكيل سطح المكتب وامنحها.', + 'settings.desktopAgent.microphone': 'الميكروفون', + 'settings.desktopAgent.grant': 'منح', + 'settings.desktopAgent.notRequiredOnOs': 'غير مطلوب على هذا النظام', + 'settings.desktopAgent.allGranted': 'كل شيء جاهز — يمتلك وكيل سطح المكتب جميع الأذونات اللازمة.', + 'settings.desktopAgent.recheck': 'إعادة الفحص', + 'settings.desktopAgent.restartAndRecheck': 'إعادة تشغيل النواة وإعادة الفحص', + 'settings.desktopAgent.seamless.title': 'إجراءات الوكيل', + 'settings.desktopAgent.seamless.label': 'دع الوكيل يتصرّف دون طلب الإذن', + 'settings.desktopAgent.seamless.description': + 'امنح الوصول الكامل واعتمد إجراءات سطح المكتب تلقائيًا (فتح التطبيقات والتحكّم في واجهاتها) حتى ينفّذها الوكيل دون مطالبة بالموافقة.', + 'settings.desktopAgent.seamless.note': + 'إجراءات سطح المكتب وحدها هي التي تتخطّى المطالبة — أما تنفيذ أوامر الصدفة وتحرير الملفات وطلبات الشبكة وتثبيت البرامج فما زالت تطلب الإذن. إيقاف هذا الخيار يوقف الاعتماد التلقائي لإجراءات سطح المكتب لكنه يترك مستوى الوصول دون تغيير.', + 'settings.desktopAgent.beta': 'تجريبي — وكيل سطح المكتب تجريبي وقد لا يعمل دائمًا كما هو متوقّع.', + 'settings.desktopAgent.wakeWordHint': 'قل ”Hey Tiny“ قبل أمرك لتنبيه الوكيل.', 'settings.screenIntel.permissions.accessibility': 'إمكانية الوصول', 'settings.screenIntel.permissions.grantHint': 'تلميح المنح', 'settings.screenIntel.permissions.inputMonitoring': 'مراقبة الإدخال', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index 6c4c2cd9c..ae6400738 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -4315,6 +4315,25 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': '& বদলাও', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'ডেস্কটপ এজেন্ট', + 'settings.desktopAgent.description': + 'ডেস্কটপ এজেন্টের প্রয়োজনীয় অনুমতিগুলি যাচাই করুন ও মঞ্জুর করুন।', + 'settings.desktopAgent.microphone': 'মাইক্রোফোন', + 'settings.desktopAgent.grant': 'মঞ্জুর করুন', + 'settings.desktopAgent.notRequiredOnOs': 'এই অপারেটিং সিস্টেমে প্রয়োজন নেই', + 'settings.desktopAgent.allGranted': + 'সব প্রস্তুত — ডেস্কটপ এজেন্টের কাছে প্রয়োজনীয় সব অনুমতি রয়েছে।', + 'settings.desktopAgent.recheck': 'পুনরায় যাচাই', + 'settings.desktopAgent.restartAndRecheck': 'কোর পুনরায় চালু করে পুনরায় যাচাই করুন', + 'settings.desktopAgent.seamless.title': 'এজেন্টের কাজ', + 'settings.desktopAgent.seamless.label': 'এজেন্টকে অনুমতি না চেয়ে কাজ করতে দিন', + 'settings.desktopAgent.seamless.description': + 'সম্পূর্ণ অ্যাক্সেস দিন এবং ডেস্কটপের কাজ (অ্যাপ খোলা, অ্যাপের ইন্টারফেস নিয়ন্ত্রণ) স্বয়ংক্রিয়ভাবে অনুমোদন করুন, যাতে এজেন্ট অনুমোদনের অনুরোধ ছাড়াই সেগুলি চালায়।', + 'settings.desktopAgent.seamless.note': + 'শুধু ডেস্কটপের কাজই অনুরোধ এড়িয়ে যায় — শেল কমান্ড চালানো, ফাইল সম্পাদনা, নেটওয়ার্ক অনুরোধ এবং সফটওয়্যার ইনস্টল করা এখনও অনুমতি চায়। এটি বন্ধ করলে ডেস্কটপের কাজের স্বয়ংক্রিয় অনুমোদন থামে, তবে অ্যাক্সেসের স্তর অপরিবর্তিত থাকে।', + 'settings.desktopAgent.beta': + 'বিটা — ডেস্কটপ এজেন্ট পরীক্ষামূলক এবং সবসময় প্রত্যাশিতভাবে কাজ নাও করতে পারে।', + 'settings.desktopAgent.wakeWordHint': 'এজেন্টকে জাগাতে আপনার আদেশের আগে ”Hey Tiny” বলুন।', 'settings.screenIntel.permissions.accessibility': 'অ্যাক্সেসিবিলিটি', 'settings.screenIntel.permissions.grantHint': 'গ্রান্ট হিন্ট', 'settings.screenIntel.permissions.inputMonitoring': 'ইনপুট মনিটরিং', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index 80ae3772a..28ff7fe6f 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -4426,6 +4426,26 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'Ersatz anwenden', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'Desktop-Agent', + 'settings.desktopAgent.description': + 'Überprüfen und erteilen Sie die Berechtigungen, die der Desktop-Agent benötigt.', + 'settings.desktopAgent.microphone': 'Mikrofon', + 'settings.desktopAgent.grant': 'Erlauben', + 'settings.desktopAgent.notRequiredOnOs': 'Auf diesem Betriebssystem nicht erforderlich', + 'settings.desktopAgent.allGranted': + 'Alles bereit – der Desktop-Agent hat alle erforderlichen Berechtigungen.', + 'settings.desktopAgent.recheck': 'Erneut prüfen', + 'settings.desktopAgent.restartAndRecheck': 'Kern neu starten und erneut prüfen', + 'settings.desktopAgent.seamless.title': 'Agentenaktionen', + 'settings.desktopAgent.seamless.label': 'Den Agenten ohne Nachfrage handeln lassen', + 'settings.desktopAgent.seamless.description': + 'Vollzugriff erteilen und Desktop-Aktionen (Apps öffnen, App-Oberflächen steuern) automatisch genehmigen, damit der Agent sie ohne Bestätigungsabfrage ausführt.', + 'settings.desktopAgent.seamless.note': + 'Nur Desktop-Aktionen überspringen die Abfrage — das Ausführen von Shell-Befehlen, das Bearbeiten von Dateien, Netzwerkanfragen und das Installieren von Software fragen weiterhin nach. Wird dies deaktiviert, endet die automatische Genehmigung von Desktop-Aktionen, die Zugriffsstufe bleibt jedoch unverändert.', + 'settings.desktopAgent.beta': + 'Beta – der Desktop-Agent ist experimentell und funktioniert möglicherweise nicht immer wie erwartet.', + 'settings.desktopAgent.wakeWordHint': + 'Sagen Sie „Hey Tiny“ vor Ihrem Befehl, um den Agenten zu wecken.', 'settings.screenIntel.permissions.accessibility': 'Barrierefreiheit', 'settings.screenIntel.permissions.grantHint': 'Grant-Hinweis', 'settings.screenIntel.permissions.inputMonitoring': 'Eingabeüberwachung', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 48c9383be..edbd050a1 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -4857,6 +4857,23 @@ const en: TranslationMap = { 'settings.modelHealth.modal.apply': 'Apply Replacement', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'Desktop Agent', + 'settings.desktopAgent.beta': + 'Beta — the desktop agent is experimental and may not always work as expected.', + 'settings.desktopAgent.wakeWordHint': 'Say “Hey Tiny” before your command to wake the agent.', + 'settings.desktopAgent.description': 'Check and grant the permissions the desktop agent needs.', + 'settings.desktopAgent.microphone': 'Microphone', + 'settings.desktopAgent.grant': 'Grant', + 'settings.desktopAgent.notRequiredOnOs': 'Not required on this OS', + 'settings.desktopAgent.allGranted': 'All set — the desktop agent has every permission it needs.', + 'settings.desktopAgent.recheck': 'Recheck', + 'settings.desktopAgent.restartAndRecheck': 'Restart core & recheck', + 'settings.desktopAgent.seamless.title': 'Agent actions', + 'settings.desktopAgent.seamless.label': 'Let the agent act without asking', + 'settings.desktopAgent.seamless.description': + 'Grant Full access and auto-approve desktop actions (opening apps, controlling app interfaces) so the agent runs them without an approval prompt.', + 'settings.desktopAgent.seamless.note': + 'Only desktop actions skip the prompt — running shell commands, editing files, network requests, and installing software still ask. Turning this off stops auto-approving desktop actions but leaves the access level unchanged.', 'settings.screenIntel.permissions.accessibility': 'Accessibility', 'settings.screenIntel.permissions.grantHint': 'Grant these permissions in System Settings, then restart the core.', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 1bb7dd68d..a1df648f3 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -4399,6 +4399,25 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'Aplicar reemplazo', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'Agente de escritorio', + 'settings.desktopAgent.description': + 'Comprueba y concede los permisos que necesita el agente de escritorio.', + 'settings.desktopAgent.microphone': 'Micrófono', + 'settings.desktopAgent.grant': 'Conceder', + 'settings.desktopAgent.notRequiredOnOs': 'No es necesario en este sistema operativo', + 'settings.desktopAgent.allGranted': + 'Todo listo: el agente de escritorio tiene todos los permisos que necesita.', + 'settings.desktopAgent.recheck': 'Volver a comprobar', + 'settings.desktopAgent.restartAndRecheck': 'Reiniciar el núcleo y volver a comprobar', + 'settings.desktopAgent.seamless.title': 'Acciones del agente', + 'settings.desktopAgent.seamless.label': 'Deja que el agente actúe sin preguntar', + 'settings.desktopAgent.seamless.description': + 'Concede acceso completo y aprueba automáticamente las acciones de escritorio (abrir aplicaciones, controlar sus interfaces) para que el agente las ejecute sin pedir confirmación.', + 'settings.desktopAgent.seamless.note': + 'Solo las acciones de escritorio omiten la confirmación: ejecutar comandos de shell, editar archivos, peticiones de red e instalar software siguen preguntando. Al desactivarlo se deja de aprobar automáticamente las acciones de escritorio, pero el nivel de acceso no cambia.', + 'settings.desktopAgent.beta': + 'Beta: el agente de escritorio es experimental y puede no funcionar siempre como se espera.', + 'settings.desktopAgent.wakeWordHint': 'Di “Hey Tiny” antes de tu orden para activar el agente.', 'settings.screenIntel.permissions.accessibility': 'Accesibilidad', 'settings.screenIntel.permissions.grantHint': 'Sugerencia de permiso', 'settings.screenIntel.permissions.inputMonitoring': 'Monitoreo de entrada', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index 8028bfb6d..e50b44edc 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -4418,6 +4418,26 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'Appliquer le remplacement', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'Agent de bureau', + 'settings.desktopAgent.description': + 'Vérifiez et accordez les autorisations dont l’agent de bureau a besoin.', + 'settings.desktopAgent.microphone': 'Microphone', + 'settings.desktopAgent.grant': 'Accorder', + 'settings.desktopAgent.notRequiredOnOs': 'Non requis sur ce système d’exploitation', + 'settings.desktopAgent.allGranted': + 'Tout est prêt : l’agent de bureau dispose de toutes les autorisations nécessaires.', + 'settings.desktopAgent.recheck': 'Revérifier', + 'settings.desktopAgent.restartAndRecheck': 'Redémarrer le cœur et revérifier', + 'settings.desktopAgent.seamless.title': 'Actions de l’agent', + 'settings.desktopAgent.seamless.label': 'Laisser l’agent agir sans demander', + 'settings.desktopAgent.seamless.description': + 'Accordez l’accès complet et approuvez automatiquement les actions de bureau (ouvrir des applications, contrôler leurs interfaces) afin que l’agent les exécute sans demande de confirmation.', + 'settings.desktopAgent.seamless.note': + 'Seules les actions de bureau évitent la demande — exécuter des commandes shell, modifier des fichiers, les requêtes réseau et l’installation de logiciels demandent toujours. Désactiver ceci arrête l’approbation automatique des actions de bureau mais laisse le niveau d’accès inchangé.', + 'settings.desktopAgent.beta': + 'Bêta — l’agent de bureau est expérimental et peut ne pas toujours fonctionner comme prévu.', + 'settings.desktopAgent.wakeWordHint': + 'Dites « Hey Tiny » avant votre commande pour réveiller l’agent.', 'settings.screenIntel.permissions.accessibility': 'Accessibilité', 'settings.screenIntel.permissions.grantHint': "Indice d'autorisation", 'settings.screenIntel.permissions.inputMonitoring': 'Surveillance des entrées', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index c093bbfe3..5ea589269 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -4318,6 +4318,24 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'प्रतिस्थापन लागू करें', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'डेस्कटॉप एजेंट', + 'settings.desktopAgent.description': + 'डेस्कटॉप एजेंट को जिन अनुमतियों की आवश्यकता है उन्हें जाँचें और प्रदान करें।', + 'settings.desktopAgent.microphone': 'माइक्रोफ़ोन', + 'settings.desktopAgent.grant': 'प्रदान करें', + 'settings.desktopAgent.notRequiredOnOs': 'इस ऑपरेटिंग सिस्टम पर आवश्यक नहीं', + 'settings.desktopAgent.allGranted': 'सब तैयार — डेस्कटॉप एजेंट के पास सभी आवश्यक अनुमतियाँ हैं।', + 'settings.desktopAgent.recheck': 'पुनः जाँचें', + 'settings.desktopAgent.restartAndRecheck': 'कोर पुनः आरंभ करें और पुनः जाँचें', + 'settings.desktopAgent.seamless.title': 'एजेंट की कार्रवाइयाँ', + 'settings.desktopAgent.seamless.label': 'एजेंट को बिना पूछे कार्य करने दें', + 'settings.desktopAgent.seamless.description': + 'पूर्ण पहुँच दें और डेस्कटॉप की कार्रवाइयों (ऐप खोलना, ऐप के इंटरफ़ेस को नियंत्रित करना) को स्वतः स्वीकृत करें, ताकि एजेंट उन्हें अनुमोदन के अनुरोध के बिना चलाए।', + 'settings.desktopAgent.seamless.note': + 'केवल डेस्कटॉप की कार्रवाइयाँ ही अनुरोध को छोड़ती हैं — शेल कमांड चलाना, फ़ाइलें संपादित करना, नेटवर्क अनुरोध और सॉफ़्टवेयर इंस्टॉल करना अब भी पूछते हैं। इसे बंद करने पर डेस्कटॉप कार्रवाइयों की स्वतः स्वीकृति रुक जाती है, पर पहुँच का स्तर अपरिवर्तित रहता है।', + 'settings.desktopAgent.beta': + 'बीटा — डेस्कटॉप एजेंट प्रायोगिक है और हमेशा अपेक्षित रूप से काम नहीं कर सकता।', + 'settings.desktopAgent.wakeWordHint': 'एजेंट को जगाने के लिए अपने आदेश से पहले ”Hey Tiny” कहें।', 'settings.screenIntel.permissions.accessibility': 'एक्सेसिबिलिटी', 'settings.screenIntel.permissions.grantHint': 'स्वीकृति संकेत', 'settings.screenIntel.permissions.inputMonitoring': 'इनपुट मॉनिटरिंग', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index ea0cb52d7..c300f51a0 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -4332,6 +4332,25 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'Terapkan Penggantian', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'Agen Desktop', + 'settings.desktopAgent.description': 'Periksa dan berikan izin yang dibutuhkan agen desktop.', + 'settings.desktopAgent.microphone': 'Mikrofon', + 'settings.desktopAgent.grant': 'Berikan', + 'settings.desktopAgent.notRequiredOnOs': 'Tidak diperlukan di sistem operasi ini', + 'settings.desktopAgent.allGranted': + 'Semua siap — agen desktop memiliki semua izin yang dibutuhkan.', + 'settings.desktopAgent.recheck': 'Periksa ulang', + 'settings.desktopAgent.restartAndRecheck': 'Mulai ulang inti dan periksa ulang', + 'settings.desktopAgent.seamless.title': 'Tindakan agen', + 'settings.desktopAgent.seamless.label': 'Biarkan agen bertindak tanpa bertanya', + 'settings.desktopAgent.seamless.description': + 'Berikan akses penuh dan setujui otomatis tindakan desktop (membuka aplikasi, mengendalikan antarmuka aplikasi) agar agen menjalankannya tanpa permintaan persetujuan.', + 'settings.desktopAgent.seamless.note': + 'Hanya tindakan desktop yang melewati permintaan — menjalankan perintah shell, menyunting berkas, permintaan jaringan, dan memasang perangkat lunak tetap meminta izin. Mematikan ini menghentikan persetujuan otomatis tindakan desktop, tetapi tingkat akses tidak berubah.', + 'settings.desktopAgent.beta': + 'Beta — agen desktop masih eksperimental dan mungkin tidak selalu bekerja seperti yang diharapkan.', + 'settings.desktopAgent.wakeWordHint': + 'Ucapkan “Hey Tiny” sebelum perintah Anda untuk membangunkan agen.', 'settings.screenIntel.permissions.accessibility': 'Aksesibilitas', 'settings.screenIntel.permissions.grantHint': 'Petunjuk izin', 'settings.screenIntel.permissions.inputMonitoring': 'Pemantauan Input', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index 8e3693b00..67ccf8c6c 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -4393,6 +4393,25 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'Applica sostituzione', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'Agente desktop', + 'settings.desktopAgent.description': + 'Verifica e concedi le autorizzazioni necessarie all’agente desktop.', + 'settings.desktopAgent.microphone': 'Microfono', + 'settings.desktopAgent.grant': 'Concedi', + 'settings.desktopAgent.notRequiredOnOs': 'Non necessario su questo sistema operativo', + 'settings.desktopAgent.allGranted': + 'Tutto pronto: l’agente desktop dispone di tutte le autorizzazioni necessarie.', + 'settings.desktopAgent.recheck': 'Verifica di nuovo', + 'settings.desktopAgent.restartAndRecheck': 'Riavvia il core e verifica di nuovo', + 'settings.desktopAgent.seamless.title': 'Azioni dell’agente', + 'settings.desktopAgent.seamless.label': 'Lascia che l’agente agisca senza chiedere', + 'settings.desktopAgent.seamless.description': + 'Concedi l’accesso completo e approva automaticamente le azioni desktop (aprire app, controllarne le interfacce) così l’agente le esegue senza una richiesta di conferma.', + 'settings.desktopAgent.seamless.note': + 'Solo le azioni desktop saltano la richiesta — eseguire comandi shell, modificare file, le richieste di rete e l’installazione di software continuano a chiedere. Disattivando questa opzione si interrompe l’approvazione automatica delle azioni desktop, ma il livello di accesso resta invariato.', + 'settings.desktopAgent.beta': + 'Beta — l’agente desktop è sperimentale e potrebbe non funzionare sempre come previsto.', + 'settings.desktopAgent.wakeWordHint': 'Di’ “Hey Tiny” prima del comando per attivare l’agente.', 'settings.screenIntel.permissions.accessibility': 'Accessibilità', 'settings.screenIntel.permissions.grantHint': 'Suggerimento concessione', 'settings.screenIntel.permissions.inputMonitoring': 'Monitoraggio input', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 27060c460..7c87a73c8 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -4273,6 +4273,24 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': '교체 적용', 'settings.modelHealth.tag.cheaper': '더 저렴함', 'settings.modelHealth.tag.better': '더 좋음', + 'settings.desktopAgent.title': '데스크톱 에이전트', + 'settings.desktopAgent.description': '데스크톱 에이전트에 필요한 권한을 확인하고 허용하세요.', + 'settings.desktopAgent.microphone': '마이크', + 'settings.desktopAgent.grant': '허용', + 'settings.desktopAgent.notRequiredOnOs': '이 운영 체제에서는 필요하지 않음', + 'settings.desktopAgent.allGranted': + '모두 완료 — 데스크톱 에이전트에 필요한 모든 권한이 있습니다.', + 'settings.desktopAgent.recheck': '다시 확인', + 'settings.desktopAgent.restartAndRecheck': '코어 재시작 후 다시 확인', + 'settings.desktopAgent.seamless.title': '에이전트 작업', + 'settings.desktopAgent.seamless.label': '에이전트가 묻지 않고 실행하도록 허용', + 'settings.desktopAgent.seamless.description': + '전체 접근을 허용하고 데스크톱 작업(앱 열기, 앱 인터페이스 제어)을 자동 승인하여 에이전트가 승인 요청 없이 실행하도록 합니다.', + 'settings.desktopAgent.seamless.note': + '데스크톱 작업만 확인을 건너뜁니다 — 셸 명령 실행, 파일 편집, 네트워크 요청, 소프트웨어 설치는 여전히 확인을 요청합니다. 이 기능을 끄면 데스크톱 작업의 자동 승인이 중단되지만 접근 수준은 그대로 유지됩니다.', + 'settings.desktopAgent.beta': + '베타 — 데스크톱 에이전트는 실험적 기능이며 항상 기대대로 작동하지 않을 수 있습니다.', + 'settings.desktopAgent.wakeWordHint': '에이전트를 깨우려면 명령 앞에 “Hey Tiny”라고 말하세요.', 'settings.screenIntel.permissions.accessibility': '접근성', 'settings.screenIntel.permissions.grantHint': '권한 허용 안내', 'settings.screenIntel.permissions.inputMonitoring': '입력 모니터링', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 72fb41d4a..6f2b8c0f1 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -4385,6 +4385,25 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'Zastosuj zamianę', 'settings.modelHealth.tag.cheaper': 'TAŃSZY', 'settings.modelHealth.tag.better': 'LEPSZY', + 'settings.desktopAgent.title': 'Agent pulpitu', + 'settings.desktopAgent.description': + 'Sprawdź i przyznaj uprawnienia wymagane przez agenta pulpitu.', + 'settings.desktopAgent.microphone': 'Mikrofon', + 'settings.desktopAgent.grant': 'Przyznaj', + 'settings.desktopAgent.notRequiredOnOs': 'Niewymagane w tym systemie operacyjnym', + 'settings.desktopAgent.allGranted': + 'Wszystko gotowe — agent pulpitu ma wszystkie wymagane uprawnienia.', + 'settings.desktopAgent.recheck': 'Sprawdź ponownie', + 'settings.desktopAgent.restartAndRecheck': 'Uruchom ponownie rdzeń i sprawdź ponownie', + 'settings.desktopAgent.seamless.title': 'Działania agenta', + 'settings.desktopAgent.seamless.label': 'Pozwól agentowi działać bez pytania', + 'settings.desktopAgent.seamless.description': + 'Przyznaj pełny dostęp i automatycznie zatwierdzaj działania na pulpicie (otwieranie aplikacji, sterowanie ich interfejsami), aby agent wykonywał je bez prośby o zatwierdzenie.', + 'settings.desktopAgent.seamless.note': + 'Tylko działania na pulpicie pomijają prośbę — uruchamianie poleceń powłoki, edycja plików, żądania sieciowe i instalowanie oprogramowania nadal pytają. Wyłączenie tego zatrzymuje automatyczne zatwierdzanie działań na pulpicie, ale poziom dostępu pozostaje bez zmian.', + 'settings.desktopAgent.beta': + 'Beta — agent pulpitu jest eksperymentalny i może nie zawsze działać zgodnie z oczekiwaniami.', + 'settings.desktopAgent.wakeWordHint': 'Powiedz „Hey Tiny” przed poleceniem, aby obudzić agenta.', 'settings.screenIntel.permissions.accessibility': 'Dostępność', 'settings.screenIntel.permissions.grantHint': 'System otworzy okno żądania uprawnień.', 'settings.screenIntel.permissions.inputMonitoring': 'Monitorowanie wejścia', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index fb9e062c9..2546a6571 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -4393,6 +4393,26 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'Aplicar Substituição', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'Agente de desktop', + 'settings.desktopAgent.description': + 'Verifique e conceda as permissões necessárias para o agente de desktop.', + 'settings.desktopAgent.microphone': 'Microfone', + 'settings.desktopAgent.grant': 'Conceder', + 'settings.desktopAgent.notRequiredOnOs': 'Não necessário neste sistema operacional', + 'settings.desktopAgent.allGranted': + 'Tudo pronto: o agente de desktop tem todas as permissões necessárias.', + 'settings.desktopAgent.recheck': 'Verificar novamente', + 'settings.desktopAgent.restartAndRecheck': 'Reiniciar o núcleo e verificar novamente', + 'settings.desktopAgent.seamless.title': 'Ações do agente', + 'settings.desktopAgent.seamless.label': 'Deixe o agente agir sem perguntar', + 'settings.desktopAgent.seamless.description': + 'Conceda acesso total e aprove automaticamente as ações de área de trabalho (abrir aplicativos, controlar suas interfaces) para que o agente as execute sem pedir confirmação.', + 'settings.desktopAgent.seamless.note': + 'Apenas as ações de área de trabalho ignoram a confirmação — executar comandos de shell, editar arquivos, solicitações de rede e instalar software continuam perguntando. Desativar isto interrompe a aprovação automática das ações de área de trabalho, mas mantém o nível de acesso inalterado.', + 'settings.desktopAgent.beta': + 'Beta — o agente de desktop é experimental e pode nem sempre funcionar como esperado.', + 'settings.desktopAgent.wakeWordHint': + 'Diga “Hey Tiny” antes do seu comando para ativar o agente.', 'settings.screenIntel.permissions.accessibility': 'Acessibilidade', 'settings.screenIntel.permissions.grantHint': 'Dica de concessão', 'settings.screenIntel.permissions.inputMonitoring': 'Monitoramento de Entrada', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index df2f3768e..cae20b548 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -4358,6 +4358,26 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': 'Применить замену', 'settings.modelHealth.tag.cheaper': 'CHEAPER', 'settings.modelHealth.tag.better': 'BETTER', + 'settings.desktopAgent.title': 'Агент рабочего стола', + 'settings.desktopAgent.description': + 'Проверьте и предоставьте разрешения, необходимые агенту рабочего стола.', + 'settings.desktopAgent.microphone': 'Микрофон', + 'settings.desktopAgent.grant': 'Предоставить', + 'settings.desktopAgent.notRequiredOnOs': 'Не требуется в этой операционной системе', + 'settings.desktopAgent.allGranted': + 'Всё готово — у агента рабочего стола есть все необходимые разрешения.', + 'settings.desktopAgent.recheck': 'Проверить снова', + 'settings.desktopAgent.restartAndRecheck': 'Перезапустить ядро и проверить снова', + 'settings.desktopAgent.seamless.title': 'Действия агента', + 'settings.desktopAgent.seamless.label': 'Разрешить агенту действовать без запроса', + 'settings.desktopAgent.seamless.description': + 'Предоставьте полный доступ и автоматически одобряйте действия на рабочем столе (открытие приложений, управление их интерфейсами), чтобы агент выполнял их без запроса на подтверждение.', + 'settings.desktopAgent.seamless.note': + 'Запрос пропускают только действия на рабочем столе — выполнение команд оболочки, редактирование файлов, сетевые запросы и установка программ по-прежнему запрашивают подтверждение. Отключение этой опции прекращает автоматическое одобрение действий на рабочем столе, но уровень доступа остаётся прежним.', + 'settings.desktopAgent.beta': + 'Бета — агент рабочего стола является экспериментальным и может не всегда работать как ожидается.', + 'settings.desktopAgent.wakeWordHint': + 'Скажите «Hey Tiny» перед командой, чтобы разбудить агента.', 'settings.screenIntel.permissions.accessibility': 'Доступность', 'settings.screenIntel.permissions.grantHint': 'Подсказка по предоставлению', 'settings.screenIntel.permissions.inputMonitoring': 'Мониторинг ввода', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 21c0b1322..c42b1e7c7 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -4106,6 +4106,22 @@ const messages: TranslationMap = { 'settings.modelHealth.modal.apply': '应用替换', 'settings.modelHealth.tag.cheaper': '更便宜', 'settings.modelHealth.tag.better': '更好', + 'settings.desktopAgent.title': '桌面智能体', + 'settings.desktopAgent.description': '检查并授予桌面智能体所需的权限。', + 'settings.desktopAgent.microphone': '麦克风', + 'settings.desktopAgent.grant': '授予', + 'settings.desktopAgent.notRequiredOnOs': '此操作系统无需此权限', + 'settings.desktopAgent.allGranted': '全部就绪——桌面智能体已拥有所需的全部权限。', + 'settings.desktopAgent.recheck': '重新检查', + 'settings.desktopAgent.restartAndRecheck': '重启内核并重新检查', + 'settings.desktopAgent.seamless.title': '智能体操作', + 'settings.desktopAgent.seamless.label': '让智能体无需询问即可执行', + 'settings.desktopAgent.seamless.description': + '授予完全访问权限并自动批准桌面操作（打开应用、控制应用界面），使智能体无需确认提示即可执行。', + 'settings.desktopAgent.seamless.note': + '只有桌面操作会跳过提示——运行 shell 命令、编辑文件、网络请求和安装软件仍会询问。关闭此选项会停止自动批准桌面操作，但访问级别保持不变。', + 'settings.desktopAgent.beta': '测试版——桌面智能体为实验性功能，可能并不总是按预期工作。', + 'settings.desktopAgent.wakeWordHint': '在指令前说“Hey Tiny”来唤醒智能体。', 'settings.screenIntel.permissions.accessibility': '辅助功能', 'settings.screenIntel.permissions.grantHint': '在系统设置中授权后，点击下方刷新', 'settings.screenIntel.permissions.inputMonitoring': '输入监控', diff --git a/app/src/pages/Settings.tsx b/app/src/pages/Settings.tsx index 9e5099613..9a90b4d0c 100644 --- a/app/src/pages/Settings.tsx +++ b/app/src/pages/Settings.tsx @@ -19,6 +19,7 @@ import BillingPanel from '../components/settings/panels/BillingPanel'; import CompanionPanel from '../components/settings/panels/CompanionPanel'; import ComposioTriagePanel from '../components/settings/panels/ComposioTriagePanel'; import CronJobsPanel from '../components/settings/panels/CronJobsPanel'; +import DesktopAgentPanel from '../components/settings/panels/DesktopAgentPanel'; import DeveloperOptionsPanel from '../components/settings/panels/DeveloperOptionsPanel'; import DevicesPanel from '../components/settings/panels/DevicesPanel'; import DevWorkflowPanel from '../components/settings/panels/DevWorkflowPanel'; @@ -139,6 +140,7 @@ const Settings = () => { path="screen-intelligence" element={wrapSettingsPage()} /> + )} /> )} /> )} /> )} /> diff --git a/app/src/utils/tauriCommands/accessibility.ts b/app/src/utils/tauriCommands/accessibility.ts index 0f8b6b63a..a0b2f0ca2 100644 --- a/app/src/utils/tauriCommands/accessibility.ts +++ b/app/src/utils/tauriCommands/accessibility.ts @@ -5,12 +5,22 @@ import { callCoreRpc } from '../../services/coreRpcClient'; import { CommandResponse, isTauri } from './common'; export type AccessibilityPermissionState = 'granted' | 'denied' | 'unknown' | 'unsupported'; -export type AccessibilityPermissionKind = 'screen_recording' | 'accessibility' | 'input_monitoring'; +export type AccessibilityPermissionKind = + | 'screen_recording' + | 'accessibility' + | 'input_monitoring' + | 'microphone'; export interface AccessibilityPermissionStatus { screen_recording: AccessibilityPermissionState; accessibility: AccessibilityPermissionState; input_monitoring: AccessibilityPermissionState; + /** + * Microphone permission. The Rust `PermissionStatus` always serializes this, + * but it's optional here so existing object literals (e.g. screen-intelligence + * status fixtures) that predate the desktop-agent panel still typecheck. + */ + microphone?: AccessibilityPermissionState; } export interface AccessibilityFeatures { diff --git a/app/test/e2e/specs/desktop-agent.spec.ts b/app/test/e2e/specs/desktop-agent.spec.ts new file mode 100644 index 000000000..3ddb7ed45 --- /dev/null +++ b/app/test/e2e/specs/desktop-agent.spec.ts @@ -0,0 +1,74 @@ +import { browser, expect } from '@wdio/globals'; + +import { waitForApp } from '../helpers/app-helpers'; +import { textExists, waitForText } from '../helpers/element-helpers'; +import { isTauriDriver } from '../helpers/platform'; +import { resetApp } from '../helpers/reset-app'; +import { startMockServer, stopMockServer } from '../mock-server'; + +function stepLog(message: string, context?: unknown): void { + const stamp = new Date().toISOString(); + if (context === undefined) { + console.log(`[DesktopAgentE2E][${stamp}] ${message}`); + return; + } + console.log(`[DesktopAgentE2E][${stamp}] ${message}`, JSON.stringify(context, null, 2)); +} + +describe('Desktop Agent settings panel', () => { + before(async function () { + stepLog('Starting Desktop Agent E2E'); + await startMockServer(); + await waitForApp(); + await resetApp('e2e-desktop-agent-user'); + }); + + after(async () => { + await stopMockServer(); + }); + + it('renders the Desktop Agent panel with permissions, toggles, and wake-word hint', async function () { + if (!isTauriDriver()) { + this.skip(); + return; + } + + // Load the settings shell first so nested routes are available. + await browser.execute(() => { + window.location.hash = '/settings'; + }); + await browser.pause(2_000); + + // Navigate to the nested desktop-agent route. Retry on hash bounce + // (lazy component load may briefly redirect), mirroring the + // Screen Intelligence spec. + for (let attempt = 0; attempt < 3; attempt++) { + await browser.execute(() => { + window.location.hash = '/settings/desktop-agent'; + }); + await browser.pause(3_000); + const h = String(await browser.execute(() => window.location.hash)); + if (h.includes('/settings/desktop-agent')) break; + stepLog(`hash bounce attempt ${attempt}`, { hash: h }); + } + + const currentHash = await browser.execute(() => window.location.hash); + stepLog('Navigated to desktop agent route', { currentHash }); + + // Title + beta notice render on every platform. + await waitForText('Desktop Agent', 15_000); + expect(await textExists('Beta')).toBe(true); + + // Permission checklist (section title is reused from Screen Intelligence) and + // the Microphone row — the desktop-agent panel surfaces Microphone, which the + // Screen Intelligence panel does not. Rows render on every platform (non-macOS + // permissions just show as unsupported). + expect(await textExists('Permissions')).toBe(true); + expect(await textExists('Microphone')).toBe(true); + + // Seamless "act without asking" toggle + the relocated always-on listening + // section with its "Hey Tiny" wake-word hint. + expect(await textExists('Let the agent act without asking')).toBe(true); + expect(await textExists('Hey Tiny')).toBe(true); + }); +}); diff --git a/docs/voice-system-actions.md b/docs/voice-system-actions.md index 736da098c..dcb40cc80 100644 --- a/docs/voice-system-actions.md +++ b/docs/voice-system-actions.md @@ -665,6 +665,86 @@ From live agent-in-the-loop testing on 2026-06-03 (grounded in `~/.openhuman/log --- +## Permissions matrix — what the desktop agent needs to run + +> Grounded in `src/openhuman/accessibility/permissions.rs` (detection), +> `src/openhuman/accessibility/types.rs` (`PermissionKind`), +> `app/src-tauri/Info.plist` (usage strings), and +> `app/src-tauri/entitlements.sidecar.plist` (Hardened-Runtime entitlements). +> Use this as the pre-test checklist when building for macOS (Apple Silicon + +> Intel) and Windows. + +### The four OS permissions the app tracks + +`detect_permissions()` returns a `PermissionStatus` over these four +`PermissionKind`s: + +| Permission | Detected via (macOS) | Agent capability that needs it | +|---|---|---| +| **Microphone** | CPAL input-device probe (`detect_microphone_permission`) | All voice capture — dictation hotkey **and** always-on listening (`voice/always_on.rs` mic stream). **Required for "voice on" to capture anything.** | +| **Accessibility** | `AXIsProcessTrusted()` | The whole app-control surface: `ax_interact` (AX tree read/press), `automate`, autocomplete focus query, **and synthetic keyboard/mouse injection** (enigo / CGEvent). | +| **Screen Recording** | `CGPreflightScreenCaptureAccess()` | `screenshot` → `vision_click` (vision fallback for Electron/partial-AX apps) and screen-intelligence capture. | +| **Input Monitoring** | `IOHIDCheckAccess(LISTEN_EVENT)` | Global hotkey **listening** (dictation hotkey / rdev / Globe-key listener — `accessibility/keys.rs`, `globe.rs`). Not needed for always-on (no hotkey). | + +### Supporting macOS entitlements + usage strings (Hardened Runtime) + +These are baked into the **signed build**, not runtime toggles. If missing, the +OS blocks the call *before* any consent dialog renders — so they must be correct +in the signed `.app`/DMG, and **cannot be validated with `tauri dev` alone**. + +- **`Info.plist` usage strings:** `NSMicrophoneUsageDescription`, + `NSCameraUsageDescription`, `NSAppleEventsUsageDescription` (+ Bluetooth / + Location / Contacts / Calendar / folder strings — those are mostly for the + **embedded provider webviews**, not the agent itself). +- **`entitlements.sidecar.plist`:** `device.audio-input`, `device.camera`, + `automation.apple-events` (drives `osascript` / System Events — Music + transport, volume, foreground detection), `network.client` / `network.server`. + +### Capability → permission map (agent surfaces) + +| Capability | Microphone | Accessibility | Screen Recording | Apple Events / Automation | Input Monitoring | +|---|:---:|:---:|:---:|:---:|:---:| +| Always-on capture (mic → VAD → STT) | ✅ | — | — | — | — | +| Dictation hotkey | ✅ | — | — | — | ✅ (listen) | +| `launch_app` / `OpenApp` intent | — | — | — | — | — | +| `Pause`/`Next`/volume intents (osascript) | — | — | — | ✅ | — | +| `Play` (Music fast-path) | — | ✅ | — | ✅ (verify `player state`) | — | +| `ax_interact` (list/press/set_value) | — | ✅ | — | — | — | +| `automate` general loop | — | ✅ | ✅ (only for `vision_click`) | ✅ (where it uses osascript) | — | +| `vision_click` (Electron fallback) | — | ✅ | ✅ | — | — | +| `keyboard` / `mouse` (synthetic input) | — | ✅ | — | — | — | + +**Minimal "voice on" path:** Microphone to capture; then per intent — +`OpenApp` needs nothing extra, transport/volume need Automation, `Play` needs +Accessibility + Automation. + +### Per-OS differences (the three test machines) + +| | macOS (M-chip & Intel — identical) | Windows | Linux | +|---|---|---|---| +| **Microphone** | Real TCC prompt; **does** prompt | Real privacy gate — Settings → Privacy → Microphone | Ungated on standard desktops; Flatpak needs an XDG portal | +| **Accessibility** | Manual grant in System Settings → Privacy & Security; **does not reliably auto-prompt**; usually needs an app **restart** after granting | No analog — UIA needs **no permission** for same-integrity apps; **cannot drive elevated apps** (UIPI) | App-interaction effectively **unsupported** (backend is macOS Swift helper / Windows UIA only; Linux returns a clean runtime error) | +| **Screen Recording** | Manual grant; no reliable auto-prompt | `screenshot`/`vision_click` is **currently unimplemented on Windows** — the vision fallback can't run there yet | Unsupported | +| **Input Monitoring** | Manual grant for hotkey listening | Not required | Not required | +| **Entitlements** | Must be present in the signed build | n/a | n/a | + +### Gotchas to flag before testing + +1. **Detection ≠ entitlement.** On macOS the runtime checks report + `Denied`/`Unknown` if the *entitlement* is missing from the signed build, + independent of what the user clicked — so test on a **properly signed build**, + not `tauri dev`. +2. **Microphone detection is a proxy.** `detect_microphone_permission` infers + from CPAL device enumeration, so "no device connected" and "permission denied" + both surface as `Unknown` on macOS/Windows (`permissions.rs`). The always-on + capture logging (`[voice::always_on] microphone permission: …` + device name + + first-chunk confirmation) disambiguates this live. +3. **Restart after granting** Accessibility / Screen Recording / Input + Monitoring on macOS — TCC changes are not always picked up by a running + process. + +--- + ## Summary | Phase | Item | Status |