From ee1fa76081bc03a9e18d7684e9150ecd0f186f69 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:18:42 +0300 Subject: [PATCH] refactor: remove screen awareness and capture surfaces (#5186) --- .github/tauri-cef-expected-sha | 2 +- AGENTS.md | 2 +- app/schema.json | 225 ---- app/scripts/e2e-run-all-flows.sh | 3 +- app/src-tauri/entitlements.sidecar.plist | 15 +- .../permissions/allow-core-process.toml | 9 +- .../permissions/allow-webview-recipe.toml | 5 +- app/src-tauri/src/cdp/session.rs | 12 - app/src-tauri/src/companion/mod.rs | 51 +- app/src-tauri/src/companion/pipeline.rs | 118 +- app/src-tauri/src/companion/pipeline_tests.rs | 51 +- app/src-tauri/src/companion/pointing.rs | 128 -- app/src-tauri/src/companion/pointing_tests.rs | 170 --- app/src-tauri/src/companion/session.rs | 12 +- app/src-tauri/src/companion/session_tests.rs | 25 - app/src-tauri/src/companion/types.rs | 18 +- app/src-tauri/src/lib.rs | 5 - .../src/meet_audio/captions_bridge.js | 4 +- app/src-tauri/src/screen_capture/mod.rs | 1068 ----------------- app/src-tauri/src/webview_accounts/runtime.js | 706 +---------- app/src-tauri/vendor/tauri-cef | 2 +- .../ScreenIntelligenceDebugPanel.tsx | 266 ---- .../components/intelligence/WorkflowsTab.tsx | 3 +- .../ScreenIntelligenceDebugPanel.test.tsx | 161 --- .../__tests__/oauthAuthReadiness.test.ts | 4 +- .../settings/hooks/useSettingsNavigation.ts | 2 - .../settings/layout/settingsNavIcons.tsx | 1 - .../settings/panels/CompanionPanel.tsx | 18 +- .../settings/panels/DeveloperOptionsPanel.tsx | 20 +- .../panels/ScreenAwarenessDebugPanel.tsx | 292 ----- .../panels/ScreenIntelligencePanel.tsx | 281 ----- .../panels/__tests__/CompanionPanel.test.tsx | 21 +- .../__tests__/DeveloperOptionsPanel.test.tsx | 4 +- .../panels/__tests__/EmbeddingsPanel.test.tsx | 2 +- .../ScreenAwarenessDebugPanel.test.tsx | 310 ----- .../ScreenIntelligencePanel.test.tsx | 339 ------ .../PermissionsSection.tsx | 144 --- .../__tests__/PermissionsSection.test.tsx | 130 -- .../settings/settingsRouteElements.test.tsx | 29 + .../settings/settingsRouteElements.tsx | 15 - .../settings/settingsRouteRegistry.ts | 25 +- .../skills/ScreenIntelligenceSetupModal.tsx | 332 ----- .../ScreenIntelligenceSetupModal.test.tsx | 147 --- app/src/components/skills/skillIcons.tsx | 2 - app/src/features/human/Mascot/Ghosty.tsx | 12 - app/src/features/human/Mascot/riveMaps.ts | 6 +- app/src/features/human/useHumanMascot.test.ts | 11 + app/src/features/human/useHumanMascot.ts | 6 +- app/src/features/screen-intelligence/api.ts | 156 --- .../useScreenIntelligenceSkillStatus.ts | 66 - .../useScreenIntelligenceState.ts | 255 ---- app/src/lib/coreState/__tests__/store.test.ts | 2 +- app/src/lib/coreState/store.ts | 4 +- app/src/lib/i18n/ar.ts | 136 --- app/src/lib/i18n/bn.ts | 137 --- app/src/lib/i18n/de.ts | 143 --- app/src/lib/i18n/en.ts | 148 +-- app/src/lib/i18n/es.ts | 140 --- app/src/lib/i18n/fr.ts | 140 --- app/src/lib/i18n/hi.ts | 139 +-- app/src/lib/i18n/id.ts | 138 --- app/src/lib/i18n/it.ts | 143 +-- app/src/lib/i18n/ko.ts | 137 --- app/src/lib/i18n/pl.ts | 140 --- app/src/lib/i18n/pt.ts | 139 --- app/src/lib/i18n/ru.ts | 140 --- app/src/lib/i18n/zh-CN.ts | 132 -- app/src/overlay/CompanionPointer.tsx | 45 - app/src/overlay/OverlayApp.tsx | 2 - .../__tests__/CompanionPointer.test.tsx | 80 -- .../__tests__/companionStateLabel.test.ts | 1 - app/src/pages/Skills.tsx | 73 +- .../__tests__/OnboardingLayout.test.tsx | 2 +- app/src/providers/CoreStateProvider.tsx | 6 +- .../CoreStateProvider.identityFlip.test.tsx | 4 +- .../__tests__/CoreStateProvider.test.tsx | 4 +- .../__tests__/companionPayload.test.ts | 2 +- .../services/__tests__/coreRpcClient.test.ts | 99 -- app/src/services/__tests__/rpcMethods.test.ts | 49 +- app/src/services/companionEvents.ts | 1 - app/src/services/coreStateApi.test.ts | 2 +- app/src/services/coreStateApi.ts | 7 +- app/src/services/rpcMethods.ts | 8 - .../store/__tests__/socketSelectors.test.ts | 2 +- app/src/store/companionSlice.test.ts | 24 +- app/src/store/companionSlice.ts | 4 +- app/src/test/mockDefaultSkillStatusHooks.ts | 10 +- .../__tests__/toolTimelineFormatting.test.ts | 7 - app/src/utils/tauriCommands/aboutApp.ts | 1 - app/src/utils/tauriCommands/accessibility.ts | 331 ----- app/src/utils/tauriCommands/config.ts | 25 - app/src/utils/tauriCommands/index.ts | 1 - app/src/utils/toolDefinitions.ts | 10 +- app/src/utils/toolTimelineFormatting.ts | 6 - app/test/e2e/specs/mega-flow.spec.ts | 8 - .../e2e/specs/screen-intelligence.spec.ts | 136 --- .../settings-feature-preferences.spec.ts | 27 +- app/test/e2e/specs/tool-browser-flow.spec.ts | 62 +- .../playwright/specs/mcp-tab-flow.spec.ts | 1 - .../specs/screen-intelligence.spec.ts | 33 - .../settings-feature-preferences.spec.ts | 9 +- docs/RELEASE-MANUAL-SMOKE.md | 3 - docs/TEST-COVERAGE-MATRIX.md | 29 +- docs/library-minimal-recipe.md | 4 +- ...ve-screen-awareness-implementation-plan.md | 364 ++++++ ...26-07-24-remove-screen-awareness-design.md | 139 +++ docs/tinyagents-inference-migration-plan.md | 4 +- gitbooks/SUMMARY.md | 1 - .../developing/architecture/tauri-shell.md | 13 +- gitbooks/developing/testing-strategy.md | 16 +- gitbooks/features/native-tools/README.md | 2 +- .../native-tools/browser-and-computer.md | 9 +- gitbooks/features/native-tools/image-tools.md | 1 - gitbooks/features/screen-intelligence.md | 105 -- scripts/ci/rust-coverage-changed.sh | 6 + scripts/feature-ids.json | 8 +- scripts/i18n-find-english.ts | 1 - scripts/test-rust-e2e.sh | 1 - src/core/all.rs | 7 - src/core/cli.rs | 4 - src/core/legacy_aliases.rs | 4 - src/openhuman/about_app/README.md | 4 +- src/openhuman/about_app/catalog_data.rs | 81 +- src/openhuman/about_app/catalog_tests.rs | 42 +- src/openhuman/about_app/types.rs | 26 +- src/openhuman/accessibility/README.md | 16 +- src/openhuman/accessibility/capture.rs | 336 ------ src/openhuman/accessibility/focus.rs | 300 +---- src/openhuman/accessibility/mod.rs | 22 +- src/openhuman/accessibility/permissions.rs | 25 +- .../accessibility/permissions_tests.rs | 22 +- src/openhuman/accessibility/types.rs | 177 +-- .../agent/harness/builtin_definitions.rs | 1 - .../agent/harness/definition_tests.rs | 1 - .../agent_registry/agents/help/prompt.md | 2 +- src/openhuman/agent_registry/agents/loader.rs | 40 +- src/openhuman/agent_registry/agents/mod.rs | 1 - .../agents/orchestrator/agent.toml | 5 +- .../agents/screen_awareness_agent/agent.toml | 37 - .../agents/screen_awareness_agent/mod.rs | 1 - .../agents/screen_awareness_agent/prompt.md | 14 - .../agents/screen_awareness_agent/prompt.rs | 38 - .../agents/vision_agent/agent.toml | 10 +- .../agents/vision_agent/prompt.md | 13 +- src/openhuman/app_state/README.md | 7 +- src/openhuman/app_state/ops.rs | 87 +- src/openhuman/app_state/ops_tests.rs | 1 - src/openhuman/config/README.md | 4 +- src/openhuman/config/mod.rs | 20 +- src/openhuman/config/ops/mod.rs | 14 +- src/openhuman/config/ops/ui.rs | 76 +- src/openhuman/config/ops_tests.rs | 28 - src/openhuman/config/schema/accessibility.rs | 97 -- src/openhuman/config/schema/mod.rs | 2 - src/openhuman/config/schema/types.rs | 23 +- src/openhuman/config/schemas/controllers.rs | 30 +- src/openhuman/config/schemas/helpers.rs | 14 - src/openhuman/config/schemas/schema_defs.rs | 40 - src/openhuman/config/schemas_tests.rs | 1 - src/openhuman/credentials/README.md | 4 +- src/openhuman/credentials/bus.rs | 2 +- src/openhuman/credentials/ops.rs | 51 +- src/openhuman/credentials/ops_tests.rs | 1 - src/openhuman/image/README.md | 19 +- src/openhuman/image/mod.rs | 2 +- src/openhuman/inference/README.md | 2 +- src/openhuman/mcp_server/resources.rs | 6 - src/openhuman/memory_store/client.rs | 6 +- .../memory_store/namespace_store/documents.rs | 4 +- src/openhuman/mod.rs | 1 - src/openhuman/overlay/README.md | 4 +- src/openhuman/overlay/mod.rs | 2 +- src/openhuman/overlay/types.rs | 2 +- src/openhuman/screen_intelligence/README.md | 93 -- src/openhuman/screen_intelligence/capture.rs | 7 - .../screen_intelligence/capture_worker.rs | 189 --- .../screen_intelligence/cli/capture.rs | 155 --- .../screen_intelligence/cli/doctor.rs | 107 -- src/openhuman/screen_intelligence/cli/mod.rs | 204 ---- .../screen_intelligence/cli/server.rs | 78 -- .../screen_intelligence/cli/session.rs | 152 --- src/openhuman/screen_intelligence/engine.rs | 629 ---------- .../screen_intelligence/engine_tests.rs | 34 - src/openhuman/screen_intelligence/helpers.rs | 300 ----- .../screen_intelligence/image_processing.rs | 370 ------ src/openhuman/screen_intelligence/input.rs | 120 -- src/openhuman/screen_intelligence/limits.rs | 7 - src/openhuman/screen_intelligence/mod.rs | 32 - src/openhuman/screen_intelligence/ops.rs | 310 ----- .../screen_intelligence/permissions.rs | 2 - .../screen_intelligence/processing_worker.rs | 432 ------- src/openhuman/screen_intelligence/schemas.rs | 408 ------- .../screen_intelligence/schemas_tests.rs | 176 --- src/openhuman/screen_intelligence/server.rs | 613 ---------- src/openhuman/screen_intelligence/state.rs | 80 -- src/openhuman/screen_intelligence/tests.rs | 924 -------------- src/openhuman/screen_intelligence/tools.rs | 391 ------ src/openhuman/screen_intelligence/types.rs | 196 --- src/openhuman/screen_intelligence/vision.rs | 129 -- src/openhuman/tools/README.md | 5 +- .../tools/impl/browser/action_parser.rs | 11 +- src/openhuman/tools/impl/browser/browser.rs | 42 +- .../tools/impl/browser/browser_tests.rs | 56 +- .../tools/impl/browser/image_output.rs | 112 -- src/openhuman/tools/impl/browser/mod.rs | 6 - .../tools/impl/browser/native_backend.rs | 26 - .../tools/impl/browser/playwright_backend.rs | 3 - .../tools/impl/browser/playwright_runner.mjs | 7 - .../tools/impl/browser/screenshot.rs | 567 --------- src/openhuman/tools/impl/browser/types.rs | 7 - src/openhuman/tools/local_cli.rs | 251 ---- src/openhuman/tools/mod.rs | 2 - src/openhuman/tools/ops.rs | 24 +- src/openhuman/tools/ops_tests.rs | 26 +- src/openhuman/tools/user_filter.rs | 13 - src/openhuman/voice/README.md | 2 +- src/openhuman/voice/bus.rs | 6 +- ...io_list_tools_stack_overflow_regression.rs | 1 - .../config_auth_app_state_connectivity_e2e.rs | 34 +- tests/json_rpc_e2e.rs | 225 +--- ...gent_credentials_state_raw_coverage_e2e.rs | 2 +- ...ools_approval_channels_raw_coverage_e2e.rs | 46 +- .../tools_channels_raw_coverage_e2e.rs | 29 +- tests/screen_intelligence_vision_e2e.rs | 810 ------------- 224 files changed, 1083 insertions(+), 18584 deletions(-) delete mode 100644 app/src-tauri/src/companion/pointing.rs delete mode 100644 app/src-tauri/src/companion/pointing_tests.rs delete mode 100644 app/src-tauri/src/screen_capture/mod.rs delete mode 100644 app/src/components/intelligence/ScreenIntelligenceDebugPanel.tsx delete mode 100644 app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx delete mode 100644 app/src/components/settings/panels/ScreenAwarenessDebugPanel.tsx delete mode 100644 app/src/components/settings/panels/ScreenIntelligencePanel.tsx delete mode 100644 app/src/components/settings/panels/__tests__/ScreenAwarenessDebugPanel.test.tsx delete mode 100644 app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx delete mode 100644 app/src/components/settings/panels/screen-intelligence/PermissionsSection.tsx delete mode 100644 app/src/components/settings/panels/screen-intelligence/__tests__/PermissionsSection.test.tsx create mode 100644 app/src/components/settings/settingsRouteElements.test.tsx delete mode 100644 app/src/components/skills/ScreenIntelligenceSetupModal.tsx delete mode 100644 app/src/components/skills/__tests__/ScreenIntelligenceSetupModal.test.tsx delete mode 100644 app/src/features/screen-intelligence/api.ts delete mode 100644 app/src/features/screen-intelligence/useScreenIntelligenceSkillStatus.ts delete mode 100644 app/src/features/screen-intelligence/useScreenIntelligenceState.ts delete mode 100644 app/src/overlay/CompanionPointer.tsx delete mode 100644 app/src/overlay/__tests__/CompanionPointer.test.tsx delete mode 100644 app/src/utils/tauriCommands/accessibility.ts delete mode 100644 app/test/e2e/specs/screen-intelligence.spec.ts delete mode 100644 app/test/playwright/specs/screen-intelligence.spec.ts create mode 100644 docs/plans/2026-07-24-remove-screen-awareness-implementation-plan.md create mode 100644 docs/specs/2026-07-24-remove-screen-awareness-design.md delete mode 100644 gitbooks/features/screen-intelligence.md delete mode 100644 src/openhuman/accessibility/capture.rs delete mode 100644 src/openhuman/agent_registry/agents/screen_awareness_agent/agent.toml delete mode 100644 src/openhuman/agent_registry/agents/screen_awareness_agent/mod.rs delete mode 100644 src/openhuman/agent_registry/agents/screen_awareness_agent/prompt.md delete mode 100644 src/openhuman/agent_registry/agents/screen_awareness_agent/prompt.rs delete mode 100644 src/openhuman/config/schema/accessibility.rs delete mode 100644 src/openhuman/screen_intelligence/README.md delete mode 100644 src/openhuman/screen_intelligence/capture.rs delete mode 100644 src/openhuman/screen_intelligence/capture_worker.rs delete mode 100644 src/openhuman/screen_intelligence/cli/capture.rs delete mode 100644 src/openhuman/screen_intelligence/cli/doctor.rs delete mode 100644 src/openhuman/screen_intelligence/cli/mod.rs delete mode 100644 src/openhuman/screen_intelligence/cli/server.rs delete mode 100644 src/openhuman/screen_intelligence/cli/session.rs delete mode 100644 src/openhuman/screen_intelligence/engine.rs delete mode 100644 src/openhuman/screen_intelligence/engine_tests.rs delete mode 100644 src/openhuman/screen_intelligence/helpers.rs delete mode 100644 src/openhuman/screen_intelligence/image_processing.rs delete mode 100644 src/openhuman/screen_intelligence/input.rs delete mode 100644 src/openhuman/screen_intelligence/limits.rs delete mode 100644 src/openhuman/screen_intelligence/mod.rs delete mode 100644 src/openhuman/screen_intelligence/ops.rs delete mode 100644 src/openhuman/screen_intelligence/permissions.rs delete mode 100644 src/openhuman/screen_intelligence/processing_worker.rs delete mode 100644 src/openhuman/screen_intelligence/schemas.rs delete mode 100644 src/openhuman/screen_intelligence/schemas_tests.rs delete mode 100644 src/openhuman/screen_intelligence/server.rs delete mode 100644 src/openhuman/screen_intelligence/state.rs delete mode 100644 src/openhuman/screen_intelligence/tests.rs delete mode 100644 src/openhuman/screen_intelligence/tools.rs delete mode 100644 src/openhuman/screen_intelligence/types.rs delete mode 100644 src/openhuman/screen_intelligence/vision.rs delete mode 100644 src/openhuman/tools/impl/browser/image_output.rs delete mode 100644 src/openhuman/tools/impl/browser/screenshot.rs delete mode 100644 src/openhuman/tools/local_cli.rs delete mode 100644 tests/screen_intelligence_vision_e2e.rs diff --git a/.github/tauri-cef-expected-sha b/.github/tauri-cef-expected-sha index 0809bcda4..9e09716ab 100644 --- a/.github/tauri-cef-expected-sha +++ b/.github/tauri-cef-expected-sha @@ -1 +1 @@ -f09d7e746a44a5187d2abc116e8a4cd7a9f94a67 +455b47debb9d1f5d0f1527266e48af1670ad4bc5 diff --git a/AGENTS.md b/AGENTS.md index 98e54bc3b..503409361 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -163,7 +163,7 @@ No `UserProvider`/`AIProvider`/`SkillProvider` — auth lives in `CoreStateProvi ## Tauri shell (`app/src-tauri/`) -Thin desktop host. Key modules: `core_process`, `core_rpc`, `cdp`, `cef_preflight`, `cef_profile`, `dictation_hotkeys`, `file_logging`, `mascot_native_window`, `screen_capture`, `window_state`, per-provider scanners (`discord_scanner`, `slack_scanner`, `telegram_scanner`, `whatsapp_scanner`, `wechat_scanner`, `gmessages_scanner`, `imessage_scanner`, `meet_scanner`), `meet_audio`/`meet_call`/`meet_video`, `fake_camera`, `webview_accounts`, `webview_apis`. +Thin desktop host. Key modules: `core_process`, `core_rpc`, `cdp`, `cef_preflight`, `cef_profile`, `dictation_hotkeys`, `file_logging`, `mascot_native_window`, `window_state`, per-provider scanners (`discord_scanner`, `slack_scanner`, `telegram_scanner`, `whatsapp_scanner`, `wechat_scanner`, `gmessages_scanner`, `imessage_scanner`, `meet_scanner`), `meet_audio`/`meet_call`/`meet_video`, `fake_camera`, `webview_accounts`, `webview_apis`. IPC commands (authoritative list: `generate_handler!` in `app/src-tauri/src/lib.rs`): `core_rpc::relay_http_rpc`, `core_rpc_url`, `core_rpc_token`, `start_core_process`/`restart_core_process`, update commands (`check_app_update`, `apply_core_update`, …), window commands (`activate_main_window`, `mascot_window_*`, `notch_window_*`), `webview_accounts::*`, `workspace_paths::*`, `artifact_commands::*`, hotkeys (dictation/PTT/companion), `meet_call::*`, `native_notifications::*`, `mcp_commands::*`, `loopback_oauth::*`. diff --git a/app/schema.json b/app/schema.json index 2bed0647b..7ec1cca42 100644 --- a/app/schema.json +++ b/app/schema.json @@ -834,70 +834,6 @@ } ] }, - { - "description": "Update screen intelligence runtime settings.", - "function": "update_screen_intelligence_settings", - "inputs": [ - { - "comment": "Enable screen intelligence.", - "name": "enabled", - "required": false, - "ty": { "Option": "Bool" } - }, - { - "comment": "Capture policy mode.", - "name": "capture_policy", - "required": false, - "ty": { "Option": "String" } - }, - { - "comment": "Policy mode override.", - "name": "policy_mode", - "required": false, - "ty": { "Option": "String" } - }, - { - "comment": "Baseline capture FPS.", - "name": "baseline_fps", - "required": false, - "ty": { "Option": "F64" } - }, - { - "comment": "Enable vision analysis.", - "name": "vision_enabled", - "required": false, - "ty": { "Option": "Bool" } - }, - { - "comment": "Enable autocomplete integration.", - "name": "autocomplete_enabled", - "required": false, - "ty": { "Option": "Bool" } - }, - { - "comment": "Allowed app list.", - "name": "allowlist", - "required": false, - "ty": { "Option": { "Array": "String" } } - }, - { - "comment": "Denied app list.", - "name": "denylist", - "required": false, - "ty": { "Option": { "Array": "String" } } - } - ], - "method": "openhuman.config_update_screen_intelligence_settings", - "namespace": "config", - "outputs": [ - { - "comment": "Updated config snapshot.", - "name": "snapshot", - "required": true, - "ty": "Json" - } - ] - }, { "description": "Replace tunnel settings with provided config payload.", "function": "update_tunnel_settings", @@ -1246,167 +1182,6 @@ } ] }, - { - "description": "Capture screenshot and return image ref.", - "function": "capture_image_ref", - "inputs": [], - "method": "openhuman.screen_intelligence_capture_image_ref", - "namespace": "screen_intelligence", - "outputs": [ - { - "comment": "Capture image_ref payload.", - "name": "capture", - "required": true, - "ty": "Json" - } - ] - }, - { - "description": "Trigger immediate screen capture.", - "function": "capture_now", - "inputs": [], - "method": "openhuman.screen_intelligence_capture_now", - "namespace": "screen_intelligence", - "outputs": [ - { "comment": "Capture result payload.", "name": "capture", "required": true, "ty": "Json" } - ] - }, - { - "description": "Perform input action through accessibility automation.", - "function": "input_action", - "inputs": [ - { - "comment": "Input action payload.", - "name": "action", - "required": true, - "ty": { "Ref": "InputActionParams" } - } - ], - "method": "openhuman.screen_intelligence_input_action", - "namespace": "screen_intelligence", - "outputs": [ - { - "comment": "Input action result payload.", - "name": "result", - "required": true, - "ty": "Json" - } - ] - }, - { - "description": "Request one accessibility permission.", - "function": "request_permission", - "inputs": [ - { "comment": "Permission name.", "name": "permission", "required": true, "ty": "String" } - ], - "method": "openhuman.screen_intelligence_request_permission", - "namespace": "screen_intelligence", - "outputs": [ - { - "comment": "Permission status payload.", - "name": "permissions", - "required": true, - "ty": "Json" - } - ] - }, - { - "description": "Request required accessibility permissions.", - "function": "request_permissions", - "inputs": [], - "method": "openhuman.screen_intelligence_request_permissions", - "namespace": "screen_intelligence", - "outputs": [ - { - "comment": "Permission status payload.", - "name": "permissions", - "required": true, - "ty": "Json" - } - ] - }, - { - "description": "Start screen intelligence session.", - "function": "start_session", - "inputs": [ - { - "comment": "Capture interval in milliseconds.", - "name": "sample_interval_ms", - "required": false, - "ty": { "Option": "U64" } - }, - { - "comment": "Capture policy mode.", - "name": "capture_policy", - "required": false, - "ty": { "Option": "String" } - } - ], - "method": "openhuman.screen_intelligence_start_session", - "namespace": "screen_intelligence", - "outputs": [ - { "comment": "Session status payload.", "name": "session", "required": true, "ty": "Json" } - ] - }, - { - "description": "Read screen intelligence accessibility status.", - "function": "status", - "inputs": [], - "method": "openhuman.screen_intelligence_status", - "namespace": "screen_intelligence", - "outputs": [ - { - "comment": "Accessibility status payload.", - "name": "status", - "required": true, - "ty": "Json" - } - ] - }, - { - "description": "Stop screen intelligence session.", - "function": "stop_session", - "inputs": [ - { - "comment": "Optional stop reason.", - "name": "reason", - "required": false, - "ty": { "Option": "String" } - } - ], - "method": "openhuman.screen_intelligence_stop_session", - "namespace": "screen_intelligence", - "outputs": [ - { "comment": "Session status payload.", "name": "session", "required": true, "ty": "Json" } - ] - }, - { - "description": "Flush stored vision summaries.", - "function": "vision_flush", - "inputs": [], - "method": "openhuman.screen_intelligence_vision_flush", - "namespace": "screen_intelligence", - "outputs": [ - { "comment": "Vision flush payload.", "name": "result", "required": true, "ty": "Json" } - ] - }, - { - "description": "Read recent vision summaries.", - "function": "vision_recent", - "inputs": [ - { - "comment": "Maximum number of summaries.", - "name": "limit", - "required": false, - "ty": { "Option": "U64" } - } - ], - "method": "openhuman.screen_intelligence_vision_recent", - "namespace": "screen_intelligence", - "outputs": [ - { "comment": "Vision recent payload.", "name": "result", "required": true, "ty": "Json" } - ] - }, { "description": "Manage desktop service lifecycle.", "function": "install", diff --git a/app/scripts/e2e-run-all-flows.sh b/app/scripts/e2e-run-all-flows.sh index 72541c2ae..37f50ab5f 100755 --- a/app/scripts/e2e-run-all-flows.sh +++ b/app/scripts/e2e-run-all-flows.sh @@ -403,7 +403,7 @@ if should_run_suite "settings"; then fi # --------------------------------------------------------------------------- -# System / AI / voice / screen / Tauri +# System / AI / voice / Tauri # linux-cef-deb-runtime.spec.ts is Linux-only (tests /usr/bin path resolution # for .deb package installs) — skipped on macOS/Windows. # --------------------------------------------------------------------------- @@ -412,7 +412,6 @@ if should_run_suite "system"; then echo "## Running suite: system" run "test/e2e/specs/local-model-runtime.spec.ts" "local-model" "system" run "test/e2e/specs/voice-mode.spec.ts" "voice-mode" "system" - run "test/e2e/specs/screen-intelligence.spec.ts" "screen-intelligence" "system" run "test/e2e/specs/audio-toolkit-flow.spec.ts" "audio-toolkit" "system" run "test/e2e/specs/tauri-commands.spec.ts" "tauri-commands" "system" # service-connectivity-flow tests the old sidecar service model removed in diff --git a/app/src-tauri/entitlements.sidecar.plist b/app/src-tauri/entitlements.sidecar.plist index cdad97f19..abeaaa8d7 100644 --- a/app/src-tauri/entitlements.sidecar.plist +++ b/app/src-tauri/entitlements.sidecar.plist @@ -22,14 +22,13 @@ (JSON-RPC server, Socket.IO) under the macOS Hardened Runtime. --> com.apple.security.network.server - + com.apple.security.automation.apple-events