From ee1fa76081bc03a9e18d7684e9150ecd0f186f69 Mon Sep 17 00:00:00 2001
From: Steven Enamakel <31011319+senamakel@users.noreply.github.com>
Date: Sat, 25 Jul 2026 09:18:42 +0300
Subject: [PATCH] refactor: remove screen awareness and capture surfaces
(#5186)
---
.github/tauri-cef-expected-sha | 2 +-
AGENTS.md | 2 +-
app/schema.json | 225 ----
app/scripts/e2e-run-all-flows.sh | 3 +-
app/src-tauri/entitlements.sidecar.plist | 15 +-
.../permissions/allow-core-process.toml | 9 +-
.../permissions/allow-webview-recipe.toml | 5 +-
app/src-tauri/src/cdp/session.rs | 12 -
app/src-tauri/src/companion/mod.rs | 51 +-
app/src-tauri/src/companion/pipeline.rs | 118 +-
app/src-tauri/src/companion/pipeline_tests.rs | 51 +-
app/src-tauri/src/companion/pointing.rs | 128 --
app/src-tauri/src/companion/pointing_tests.rs | 170 ---
app/src-tauri/src/companion/session.rs | 12 +-
app/src-tauri/src/companion/session_tests.rs | 25 -
app/src-tauri/src/companion/types.rs | 18 +-
app/src-tauri/src/lib.rs | 5 -
.../src/meet_audio/captions_bridge.js | 4 +-
app/src-tauri/src/screen_capture/mod.rs | 1068 -----------------
app/src-tauri/src/webview_accounts/runtime.js | 706 +----------
app/src-tauri/vendor/tauri-cef | 2 +-
.../ScreenIntelligenceDebugPanel.tsx | 266 ----
.../components/intelligence/WorkflowsTab.tsx | 3 +-
.../ScreenIntelligenceDebugPanel.test.tsx | 161 ---
.../__tests__/oauthAuthReadiness.test.ts | 4 +-
.../settings/hooks/useSettingsNavigation.ts | 2 -
.../settings/layout/settingsNavIcons.tsx | 1 -
.../settings/panels/CompanionPanel.tsx | 18 +-
.../settings/panels/DeveloperOptionsPanel.tsx | 20 +-
.../panels/ScreenAwarenessDebugPanel.tsx | 292 -----
.../panels/ScreenIntelligencePanel.tsx | 281 -----
.../panels/__tests__/CompanionPanel.test.tsx | 21 +-
.../__tests__/DeveloperOptionsPanel.test.tsx | 4 +-
.../panels/__tests__/EmbeddingsPanel.test.tsx | 2 +-
.../ScreenAwarenessDebugPanel.test.tsx | 310 -----
.../ScreenIntelligencePanel.test.tsx | 339 ------
.../PermissionsSection.tsx | 144 ---
.../__tests__/PermissionsSection.test.tsx | 130 --
.../settings/settingsRouteElements.test.tsx | 29 +
.../settings/settingsRouteElements.tsx | 15 -
.../settings/settingsRouteRegistry.ts | 25 +-
.../skills/ScreenIntelligenceSetupModal.tsx | 332 -----
.../ScreenIntelligenceSetupModal.test.tsx | 147 ---
app/src/components/skills/skillIcons.tsx | 2 -
app/src/features/human/Mascot/Ghosty.tsx | 12 -
app/src/features/human/Mascot/riveMaps.ts | 6 +-
app/src/features/human/useHumanMascot.test.ts | 11 +
app/src/features/human/useHumanMascot.ts | 6 +-
app/src/features/screen-intelligence/api.ts | 156 ---
.../useScreenIntelligenceSkillStatus.ts | 66 -
.../useScreenIntelligenceState.ts | 255 ----
app/src/lib/coreState/__tests__/store.test.ts | 2 +-
app/src/lib/coreState/store.ts | 4 +-
app/src/lib/i18n/ar.ts | 136 ---
app/src/lib/i18n/bn.ts | 137 ---
app/src/lib/i18n/de.ts | 143 ---
app/src/lib/i18n/en.ts | 148 +--
app/src/lib/i18n/es.ts | 140 ---
app/src/lib/i18n/fr.ts | 140 ---
app/src/lib/i18n/hi.ts | 139 +--
app/src/lib/i18n/id.ts | 138 ---
app/src/lib/i18n/it.ts | 143 +--
app/src/lib/i18n/ko.ts | 137 ---
app/src/lib/i18n/pl.ts | 140 ---
app/src/lib/i18n/pt.ts | 139 ---
app/src/lib/i18n/ru.ts | 140 ---
app/src/lib/i18n/zh-CN.ts | 132 --
app/src/overlay/CompanionPointer.tsx | 45 -
app/src/overlay/OverlayApp.tsx | 2 -
.../__tests__/CompanionPointer.test.tsx | 80 --
.../__tests__/companionStateLabel.test.ts | 1 -
app/src/pages/Skills.tsx | 73 +-
.../__tests__/OnboardingLayout.test.tsx | 2 +-
app/src/providers/CoreStateProvider.tsx | 6 +-
.../CoreStateProvider.identityFlip.test.tsx | 4 +-
.../__tests__/CoreStateProvider.test.tsx | 4 +-
.../__tests__/companionPayload.test.ts | 2 +-
.../services/__tests__/coreRpcClient.test.ts | 99 --
app/src/services/__tests__/rpcMethods.test.ts | 49 +-
app/src/services/companionEvents.ts | 1 -
app/src/services/coreStateApi.test.ts | 2 +-
app/src/services/coreStateApi.ts | 7 +-
app/src/services/rpcMethods.ts | 8 -
.../store/__tests__/socketSelectors.test.ts | 2 +-
app/src/store/companionSlice.test.ts | 24 +-
app/src/store/companionSlice.ts | 4 +-
app/src/test/mockDefaultSkillStatusHooks.ts | 10 +-
.../__tests__/toolTimelineFormatting.test.ts | 7 -
app/src/utils/tauriCommands/aboutApp.ts | 1 -
app/src/utils/tauriCommands/accessibility.ts | 331 -----
app/src/utils/tauriCommands/config.ts | 25 -
app/src/utils/tauriCommands/index.ts | 1 -
app/src/utils/toolDefinitions.ts | 10 +-
app/src/utils/toolTimelineFormatting.ts | 6 -
app/test/e2e/specs/mega-flow.spec.ts | 8 -
.../e2e/specs/screen-intelligence.spec.ts | 136 ---
.../settings-feature-preferences.spec.ts | 27 +-
app/test/e2e/specs/tool-browser-flow.spec.ts | 62 +-
.../playwright/specs/mcp-tab-flow.spec.ts | 1 -
.../specs/screen-intelligence.spec.ts | 33 -
.../settings-feature-preferences.spec.ts | 9 +-
docs/RELEASE-MANUAL-SMOKE.md | 3 -
docs/TEST-COVERAGE-MATRIX.md | 29 +-
docs/library-minimal-recipe.md | 4 +-
...ve-screen-awareness-implementation-plan.md | 364 ++++++
...26-07-24-remove-screen-awareness-design.md | 139 +++
docs/tinyagents-inference-migration-plan.md | 4 +-
gitbooks/SUMMARY.md | 1 -
.../developing/architecture/tauri-shell.md | 13 +-
gitbooks/developing/testing-strategy.md | 16 +-
gitbooks/features/native-tools/README.md | 2 +-
.../native-tools/browser-and-computer.md | 9 +-
gitbooks/features/native-tools/image-tools.md | 1 -
gitbooks/features/screen-intelligence.md | 105 --
scripts/ci/rust-coverage-changed.sh | 6 +
scripts/feature-ids.json | 8 +-
scripts/i18n-find-english.ts | 1 -
scripts/test-rust-e2e.sh | 1 -
src/core/all.rs | 7 -
src/core/cli.rs | 4 -
src/core/legacy_aliases.rs | 4 -
src/openhuman/about_app/README.md | 4 +-
src/openhuman/about_app/catalog_data.rs | 81 +-
src/openhuman/about_app/catalog_tests.rs | 42 +-
src/openhuman/about_app/types.rs | 26 +-
src/openhuman/accessibility/README.md | 16 +-
src/openhuman/accessibility/capture.rs | 336 ------
src/openhuman/accessibility/focus.rs | 300 +----
src/openhuman/accessibility/mod.rs | 22 +-
src/openhuman/accessibility/permissions.rs | 25 +-
.../accessibility/permissions_tests.rs | 22 +-
src/openhuman/accessibility/types.rs | 177 +--
.../agent/harness/builtin_definitions.rs | 1 -
.../agent/harness/definition_tests.rs | 1 -
.../agent_registry/agents/help/prompt.md | 2 +-
src/openhuman/agent_registry/agents/loader.rs | 40 +-
src/openhuman/agent_registry/agents/mod.rs | 1 -
.../agents/orchestrator/agent.toml | 5 +-
.../agents/screen_awareness_agent/agent.toml | 37 -
.../agents/screen_awareness_agent/mod.rs | 1 -
.../agents/screen_awareness_agent/prompt.md | 14 -
.../agents/screen_awareness_agent/prompt.rs | 38 -
.../agents/vision_agent/agent.toml | 10 +-
.../agents/vision_agent/prompt.md | 13 +-
src/openhuman/app_state/README.md | 7 +-
src/openhuman/app_state/ops.rs | 87 +-
src/openhuman/app_state/ops_tests.rs | 1 -
src/openhuman/config/README.md | 4 +-
src/openhuman/config/mod.rs | 20 +-
src/openhuman/config/ops/mod.rs | 14 +-
src/openhuman/config/ops/ui.rs | 76 +-
src/openhuman/config/ops_tests.rs | 28 -
src/openhuman/config/schema/accessibility.rs | 97 --
src/openhuman/config/schema/mod.rs | 2 -
src/openhuman/config/schema/types.rs | 23 +-
src/openhuman/config/schemas/controllers.rs | 30 +-
src/openhuman/config/schemas/helpers.rs | 14 -
src/openhuman/config/schemas/schema_defs.rs | 40 -
src/openhuman/config/schemas_tests.rs | 1 -
src/openhuman/credentials/README.md | 4 +-
src/openhuman/credentials/bus.rs | 2 +-
src/openhuman/credentials/ops.rs | 51 +-
src/openhuman/credentials/ops_tests.rs | 1 -
src/openhuman/image/README.md | 19 +-
src/openhuman/image/mod.rs | 2 +-
src/openhuman/inference/README.md | 2 +-
src/openhuman/mcp_server/resources.rs | 6 -
src/openhuman/memory_store/client.rs | 6 +-
.../memory_store/namespace_store/documents.rs | 4 +-
src/openhuman/mod.rs | 1 -
src/openhuman/overlay/README.md | 4 +-
src/openhuman/overlay/mod.rs | 2 +-
src/openhuman/overlay/types.rs | 2 +-
src/openhuman/screen_intelligence/README.md | 93 --
src/openhuman/screen_intelligence/capture.rs | 7 -
.../screen_intelligence/capture_worker.rs | 189 ---
.../screen_intelligence/cli/capture.rs | 155 ---
.../screen_intelligence/cli/doctor.rs | 107 --
src/openhuman/screen_intelligence/cli/mod.rs | 204 ----
.../screen_intelligence/cli/server.rs | 78 --
.../screen_intelligence/cli/session.rs | 152 ---
src/openhuman/screen_intelligence/engine.rs | 629 ----------
.../screen_intelligence/engine_tests.rs | 34 -
src/openhuman/screen_intelligence/helpers.rs | 300 -----
.../screen_intelligence/image_processing.rs | 370 ------
src/openhuman/screen_intelligence/input.rs | 120 --
src/openhuman/screen_intelligence/limits.rs | 7 -
src/openhuman/screen_intelligence/mod.rs | 32 -
src/openhuman/screen_intelligence/ops.rs | 310 -----
.../screen_intelligence/permissions.rs | 2 -
.../screen_intelligence/processing_worker.rs | 432 -------
src/openhuman/screen_intelligence/schemas.rs | 408 -------
.../screen_intelligence/schemas_tests.rs | 176 ---
src/openhuman/screen_intelligence/server.rs | 613 ----------
src/openhuman/screen_intelligence/state.rs | 80 --
src/openhuman/screen_intelligence/tests.rs | 924 --------------
src/openhuman/screen_intelligence/tools.rs | 391 ------
src/openhuman/screen_intelligence/types.rs | 196 ---
src/openhuman/screen_intelligence/vision.rs | 129 --
src/openhuman/tools/README.md | 5 +-
.../tools/impl/browser/action_parser.rs | 11 +-
src/openhuman/tools/impl/browser/browser.rs | 42 +-
.../tools/impl/browser/browser_tests.rs | 56 +-
.../tools/impl/browser/image_output.rs | 112 --
src/openhuman/tools/impl/browser/mod.rs | 6 -
.../tools/impl/browser/native_backend.rs | 26 -
.../tools/impl/browser/playwright_backend.rs | 3 -
.../tools/impl/browser/playwright_runner.mjs | 7 -
.../tools/impl/browser/screenshot.rs | 567 ---------
src/openhuman/tools/impl/browser/types.rs | 7 -
src/openhuman/tools/local_cli.rs | 251 ----
src/openhuman/tools/mod.rs | 2 -
src/openhuman/tools/ops.rs | 24 +-
src/openhuman/tools/ops_tests.rs | 26 +-
src/openhuman/tools/user_filter.rs | 13 -
src/openhuman/voice/README.md | 2 +-
src/openhuman/voice/bus.rs | 6 +-
...io_list_tools_stack_overflow_regression.rs | 1 -
.../config_auth_app_state_connectivity_e2e.rs | 34 +-
tests/json_rpc_e2e.rs | 225 +---
...gent_credentials_state_raw_coverage_e2e.rs | 2 +-
...ools_approval_channels_raw_coverage_e2e.rs | 46 +-
.../tools_channels_raw_coverage_e2e.rs | 29 +-
tests/screen_intelligence_vision_e2e.rs | 810 -------------
224 files changed, 1083 insertions(+), 18584 deletions(-)
delete mode 100644 app/src-tauri/src/companion/pointing.rs
delete mode 100644 app/src-tauri/src/companion/pointing_tests.rs
delete mode 100644 app/src-tauri/src/screen_capture/mod.rs
delete mode 100644 app/src/components/intelligence/ScreenIntelligenceDebugPanel.tsx
delete mode 100644 app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx
delete mode 100644 app/src/components/settings/panels/ScreenAwarenessDebugPanel.tsx
delete mode 100644 app/src/components/settings/panels/ScreenIntelligencePanel.tsx
delete mode 100644 app/src/components/settings/panels/__tests__/ScreenAwarenessDebugPanel.test.tsx
delete mode 100644 app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx
delete mode 100644 app/src/components/settings/panels/screen-intelligence/PermissionsSection.tsx
delete mode 100644 app/src/components/settings/panels/screen-intelligence/__tests__/PermissionsSection.test.tsx
create mode 100644 app/src/components/settings/settingsRouteElements.test.tsx
delete mode 100644 app/src/components/skills/ScreenIntelligenceSetupModal.tsx
delete mode 100644 app/src/components/skills/__tests__/ScreenIntelligenceSetupModal.test.tsx
delete mode 100644 app/src/features/screen-intelligence/api.ts
delete mode 100644 app/src/features/screen-intelligence/useScreenIntelligenceSkillStatus.ts
delete mode 100644 app/src/features/screen-intelligence/useScreenIntelligenceState.ts
delete mode 100644 app/src/overlay/CompanionPointer.tsx
delete mode 100644 app/src/overlay/__tests__/CompanionPointer.test.tsx
delete mode 100644 app/src/utils/tauriCommands/accessibility.ts
delete mode 100644 app/test/e2e/specs/screen-intelligence.spec.ts
delete mode 100644 app/test/playwright/specs/screen-intelligence.spec.ts
create mode 100644 docs/plans/2026-07-24-remove-screen-awareness-implementation-plan.md
create mode 100644 docs/specs/2026-07-24-remove-screen-awareness-design.md
delete mode 100644 gitbooks/features/screen-intelligence.md
delete mode 100644 src/openhuman/accessibility/capture.rs
delete mode 100644 src/openhuman/agent_registry/agents/screen_awareness_agent/agent.toml
delete mode 100644 src/openhuman/agent_registry/agents/screen_awareness_agent/mod.rs
delete mode 100644 src/openhuman/agent_registry/agents/screen_awareness_agent/prompt.md
delete mode 100644 src/openhuman/agent_registry/agents/screen_awareness_agent/prompt.rs
delete mode 100644 src/openhuman/config/schema/accessibility.rs
delete mode 100644 src/openhuman/screen_intelligence/README.md
delete mode 100644 src/openhuman/screen_intelligence/capture.rs
delete mode 100644 src/openhuman/screen_intelligence/capture_worker.rs
delete mode 100644 src/openhuman/screen_intelligence/cli/capture.rs
delete mode 100644 src/openhuman/screen_intelligence/cli/doctor.rs
delete mode 100644 src/openhuman/screen_intelligence/cli/mod.rs
delete mode 100644 src/openhuman/screen_intelligence/cli/server.rs
delete mode 100644 src/openhuman/screen_intelligence/cli/session.rs
delete mode 100644 src/openhuman/screen_intelligence/engine.rs
delete mode 100644 src/openhuman/screen_intelligence/engine_tests.rs
delete mode 100644 src/openhuman/screen_intelligence/helpers.rs
delete mode 100644 src/openhuman/screen_intelligence/image_processing.rs
delete mode 100644 src/openhuman/screen_intelligence/input.rs
delete mode 100644 src/openhuman/screen_intelligence/limits.rs
delete mode 100644 src/openhuman/screen_intelligence/mod.rs
delete mode 100644 src/openhuman/screen_intelligence/ops.rs
delete mode 100644 src/openhuman/screen_intelligence/permissions.rs
delete mode 100644 src/openhuman/screen_intelligence/processing_worker.rs
delete mode 100644 src/openhuman/screen_intelligence/schemas.rs
delete mode 100644 src/openhuman/screen_intelligence/schemas_tests.rs
delete mode 100644 src/openhuman/screen_intelligence/server.rs
delete mode 100644 src/openhuman/screen_intelligence/state.rs
delete mode 100644 src/openhuman/screen_intelligence/tests.rs
delete mode 100644 src/openhuman/screen_intelligence/tools.rs
delete mode 100644 src/openhuman/screen_intelligence/types.rs
delete mode 100644 src/openhuman/screen_intelligence/vision.rs
delete mode 100644 src/openhuman/tools/impl/browser/image_output.rs
delete mode 100644 src/openhuman/tools/impl/browser/screenshot.rs
delete mode 100644 src/openhuman/tools/local_cli.rs
delete mode 100644 tests/screen_intelligence_vision_e2e.rs
diff --git a/.github/tauri-cef-expected-sha b/.github/tauri-cef-expected-sha
index 0809bcda4..9e09716ab 100644
--- a/.github/tauri-cef-expected-sha
+++ b/.github/tauri-cef-expected-sha
@@ -1 +1 @@
-f09d7e746a44a5187d2abc116e8a4cd7a9f94a67
+455b47debb9d1f5d0f1527266e48af1670ad4bc5
diff --git a/AGENTS.md b/AGENTS.md
index 98e54bc3b..503409361 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -163,7 +163,7 @@ No `UserProvider`/`AIProvider`/`SkillProvider` — auth lives in `CoreStateProvi
## Tauri shell (`app/src-tauri/`)
-Thin desktop host. Key modules: `core_process`, `core_rpc`, `cdp`, `cef_preflight`, `cef_profile`, `dictation_hotkeys`, `file_logging`, `mascot_native_window`, `screen_capture`, `window_state`, per-provider scanners (`discord_scanner`, `slack_scanner`, `telegram_scanner`, `whatsapp_scanner`, `wechat_scanner`, `gmessages_scanner`, `imessage_scanner`, `meet_scanner`), `meet_audio`/`meet_call`/`meet_video`, `fake_camera`, `webview_accounts`, `webview_apis`.
+Thin desktop host. Key modules: `core_process`, `core_rpc`, `cdp`, `cef_preflight`, `cef_profile`, `dictation_hotkeys`, `file_logging`, `mascot_native_window`, `window_state`, per-provider scanners (`discord_scanner`, `slack_scanner`, `telegram_scanner`, `whatsapp_scanner`, `wechat_scanner`, `gmessages_scanner`, `imessage_scanner`, `meet_scanner`), `meet_audio`/`meet_call`/`meet_video`, `fake_camera`, `webview_accounts`, `webview_apis`.
IPC commands (authoritative list: `generate_handler!` in `app/src-tauri/src/lib.rs`): `core_rpc::relay_http_rpc`, `core_rpc_url`, `core_rpc_token`, `start_core_process`/`restart_core_process`, update commands (`check_app_update`, `apply_core_update`, …), window commands (`activate_main_window`, `mascot_window_*`, `notch_window_*`), `webview_accounts::*`, `workspace_paths::*`, `artifact_commands::*`, hotkeys (dictation/PTT/companion), `meet_call::*`, `native_notifications::*`, `mcp_commands::*`, `loopback_oauth::*`.
diff --git a/app/schema.json b/app/schema.json
index 2bed0647b..7ec1cca42 100644
--- a/app/schema.json
+++ b/app/schema.json
@@ -834,70 +834,6 @@
}
]
},
- {
- "description": "Update screen intelligence runtime settings.",
- "function": "update_screen_intelligence_settings",
- "inputs": [
- {
- "comment": "Enable screen intelligence.",
- "name": "enabled",
- "required": false,
- "ty": { "Option": "Bool" }
- },
- {
- "comment": "Capture policy mode.",
- "name": "capture_policy",
- "required": false,
- "ty": { "Option": "String" }
- },
- {
- "comment": "Policy mode override.",
- "name": "policy_mode",
- "required": false,
- "ty": { "Option": "String" }
- },
- {
- "comment": "Baseline capture FPS.",
- "name": "baseline_fps",
- "required": false,
- "ty": { "Option": "F64" }
- },
- {
- "comment": "Enable vision analysis.",
- "name": "vision_enabled",
- "required": false,
- "ty": { "Option": "Bool" }
- },
- {
- "comment": "Enable autocomplete integration.",
- "name": "autocomplete_enabled",
- "required": false,
- "ty": { "Option": "Bool" }
- },
- {
- "comment": "Allowed app list.",
- "name": "allowlist",
- "required": false,
- "ty": { "Option": { "Array": "String" } }
- },
- {
- "comment": "Denied app list.",
- "name": "denylist",
- "required": false,
- "ty": { "Option": { "Array": "String" } }
- }
- ],
- "method": "openhuman.config_update_screen_intelligence_settings",
- "namespace": "config",
- "outputs": [
- {
- "comment": "Updated config snapshot.",
- "name": "snapshot",
- "required": true,
- "ty": "Json"
- }
- ]
- },
{
"description": "Replace tunnel settings with provided config payload.",
"function": "update_tunnel_settings",
@@ -1246,167 +1182,6 @@
}
]
},
- {
- "description": "Capture screenshot and return image ref.",
- "function": "capture_image_ref",
- "inputs": [],
- "method": "openhuman.screen_intelligence_capture_image_ref",
- "namespace": "screen_intelligence",
- "outputs": [
- {
- "comment": "Capture image_ref payload.",
- "name": "capture",
- "required": true,
- "ty": "Json"
- }
- ]
- },
- {
- "description": "Trigger immediate screen capture.",
- "function": "capture_now",
- "inputs": [],
- "method": "openhuman.screen_intelligence_capture_now",
- "namespace": "screen_intelligence",
- "outputs": [
- { "comment": "Capture result payload.", "name": "capture", "required": true, "ty": "Json" }
- ]
- },
- {
- "description": "Perform input action through accessibility automation.",
- "function": "input_action",
- "inputs": [
- {
- "comment": "Input action payload.",
- "name": "action",
- "required": true,
- "ty": { "Ref": "InputActionParams" }
- }
- ],
- "method": "openhuman.screen_intelligence_input_action",
- "namespace": "screen_intelligence",
- "outputs": [
- {
- "comment": "Input action result payload.",
- "name": "result",
- "required": true,
- "ty": "Json"
- }
- ]
- },
- {
- "description": "Request one accessibility permission.",
- "function": "request_permission",
- "inputs": [
- { "comment": "Permission name.", "name": "permission", "required": true, "ty": "String" }
- ],
- "method": "openhuman.screen_intelligence_request_permission",
- "namespace": "screen_intelligence",
- "outputs": [
- {
- "comment": "Permission status payload.",
- "name": "permissions",
- "required": true,
- "ty": "Json"
- }
- ]
- },
- {
- "description": "Request required accessibility permissions.",
- "function": "request_permissions",
- "inputs": [],
- "method": "openhuman.screen_intelligence_request_permissions",
- "namespace": "screen_intelligence",
- "outputs": [
- {
- "comment": "Permission status payload.",
- "name": "permissions",
- "required": true,
- "ty": "Json"
- }
- ]
- },
- {
- "description": "Start screen intelligence session.",
- "function": "start_session",
- "inputs": [
- {
- "comment": "Capture interval in milliseconds.",
- "name": "sample_interval_ms",
- "required": false,
- "ty": { "Option": "U64" }
- },
- {
- "comment": "Capture policy mode.",
- "name": "capture_policy",
- "required": false,
- "ty": { "Option": "String" }
- }
- ],
- "method": "openhuman.screen_intelligence_start_session",
- "namespace": "screen_intelligence",
- "outputs": [
- { "comment": "Session status payload.", "name": "session", "required": true, "ty": "Json" }
- ]
- },
- {
- "description": "Read screen intelligence accessibility status.",
- "function": "status",
- "inputs": [],
- "method": "openhuman.screen_intelligence_status",
- "namespace": "screen_intelligence",
- "outputs": [
- {
- "comment": "Accessibility status payload.",
- "name": "status",
- "required": true,
- "ty": "Json"
- }
- ]
- },
- {
- "description": "Stop screen intelligence session.",
- "function": "stop_session",
- "inputs": [
- {
- "comment": "Optional stop reason.",
- "name": "reason",
- "required": false,
- "ty": { "Option": "String" }
- }
- ],
- "method": "openhuman.screen_intelligence_stop_session",
- "namespace": "screen_intelligence",
- "outputs": [
- { "comment": "Session status payload.", "name": "session", "required": true, "ty": "Json" }
- ]
- },
- {
- "description": "Flush stored vision summaries.",
- "function": "vision_flush",
- "inputs": [],
- "method": "openhuman.screen_intelligence_vision_flush",
- "namespace": "screen_intelligence",
- "outputs": [
- { "comment": "Vision flush payload.", "name": "result", "required": true, "ty": "Json" }
- ]
- },
- {
- "description": "Read recent vision summaries.",
- "function": "vision_recent",
- "inputs": [
- {
- "comment": "Maximum number of summaries.",
- "name": "limit",
- "required": false,
- "ty": { "Option": "U64" }
- }
- ],
- "method": "openhuman.screen_intelligence_vision_recent",
- "namespace": "screen_intelligence",
- "outputs": [
- { "comment": "Vision recent payload.", "name": "result", "required": true, "ty": "Json" }
- ]
- },
{
"description": "Manage desktop service lifecycle.",
"function": "install",
diff --git a/app/scripts/e2e-run-all-flows.sh b/app/scripts/e2e-run-all-flows.sh
index 72541c2ae..37f50ab5f 100755
--- a/app/scripts/e2e-run-all-flows.sh
+++ b/app/scripts/e2e-run-all-flows.sh
@@ -403,7 +403,7 @@ if should_run_suite "settings"; then
fi
# ---------------------------------------------------------------------------
-# System / AI / voice / screen / Tauri
+# System / AI / voice / Tauri
# linux-cef-deb-runtime.spec.ts is Linux-only (tests /usr/bin path resolution
# for .deb package installs) — skipped on macOS/Windows.
# ---------------------------------------------------------------------------
@@ -412,7 +412,6 @@ if should_run_suite "system"; then
echo "## Running suite: system"
run "test/e2e/specs/local-model-runtime.spec.ts" "local-model" "system"
run "test/e2e/specs/voice-mode.spec.ts" "voice-mode" "system"
- run "test/e2e/specs/screen-intelligence.spec.ts" "screen-intelligence" "system"
run "test/e2e/specs/audio-toolkit-flow.spec.ts" "audio-toolkit" "system"
run "test/e2e/specs/tauri-commands.spec.ts" "tauri-commands" "system"
# service-connectivity-flow tests the old sidecar service model removed in
diff --git a/app/src-tauri/entitlements.sidecar.plist b/app/src-tauri/entitlements.sidecar.plist
index cdad97f19..abeaaa8d7 100644
--- a/app/src-tauri/entitlements.sidecar.plist
+++ b/app/src-tauri/entitlements.sidecar.plist
@@ -22,14 +22,13 @@
(JSON-RPC server, Socket.IO) under the macOS Hardened Runtime. -->
com.apple.security.network.server
-
+
com.apple.security.automation.apple-events