feat: split screen intelligence from accessibility and add dedicated settings (#50)

* refactor(core_server): improve code formatting and readability - Reformatted code in `core_server.rs` for better readability, including consistent indentation and line breaks. - Enhanced the clarity of function calls and JSON handling by spreading parameters across multiple lines. - Improved overall structure and maintainability of the dispatch and settings view response functions. * feat(accessibility): refactor AccessibilityAutomationConfig to ScreenIntelligenceConfig - Renamed AccessibilityAutomationConfig to ScreenIntelligenceConfig across the codebase for improved clarity. - Updated related structures and functions to utilize the new configuration, including default values and additional fields for enhanced functionality. - Ensured backward compatibility by adjusting references in the AccessibilityEngine and related modules. - Enhanced the AppContext structure to include window bounds, improving state management for UI components. * feat(screen-intelligence): introduce Screen Intelligence settings and functionality - Added a new Screen Intelligence module to manage window capture policies, vision summaries, and memory ingestion. - Implemented settings for enabling/disabling features, configuring capture policies, and managing allowlists/deny lists. - Updated the accessibility panel to include Screen Intelligence options and integrated permission requests for screen recording and accessibility. - Enhanced the core server to handle updates to Screen Intelligence settings and ensure proper state management across the application. - Refactored related components and Redux state management to support the new features, improving user experience and functionality. * chore: apply rust fmt and fix blocking eslint warning
2026-07-27 21:08:00 +00:00 · 2026-03-28 12:20:34 -07:00
parent b6ec716b7c
commit a0c3089ee5
18 changed files with 2315 additions and 1555 deletions
@@ -484,6 +484,8 @@ Key updates from recent commits (cd9ebcd to current):
  4. `cargo fmt --manifest-path src-tauri/Cargo.toml` (Rust formatting, if Rust files were changed)
  5. `cargo check --manifest-path src-tauri/Cargo.toml` (Rust compilation, if Rust files were changed)
 - **Code Quality**: ESLint and Prettier enforce code standards with Husky hooks. Use type-only imports (`import type`) and consolidate imports from same modules.
+- **Frontend scope**: The frontend (`src/`) is primarily a REST API client to `rust-core/`; Rust code in `src-tauri/` manages the core process lifecycle and bridge surface.
+- **Core feature exposure**: Any new feature added in `rust-core/` must be exposed through both the CLI and a REST API so it can be integrated into the UI without backend rewrites.
 - **No dynamic imports**: All imports must be static `import` statements at the top of the file. Do not use `await import()` or `import().then()` inside functions or code blocks. Use try/catch around Tauri API calls for non-Tauri environments instead.
 - **No localStorage**: Avoid `localStorage` and `sessionStorage`; use Redux (and persist) for app state. Remove any direct usage when working on affected code.
 - **AI System Integration**: Use `src/lib/ai/` for memory management, constitution loading, entity queries, and session capture. AI providers abstracted through interface pattern.
@@ -24,11 +24,11 @@ use crate::openhuman::local_ai::{
 use crate::openhuman::security::{SecretStore, SecurityPolicy};
 use crate::openhuman::tools::{ScreenshotTool, Tool};
 use crate::openhuman::{
-    accessibility, doctor, hardware, integrations, migration, onboard, service,
+    doctor, hardware, integrations, migration, onboard, screen_intelligence, service,
 };
 use chrono::Utc;

-pub use crate::openhuman::accessibility::{
+pub use crate::openhuman::screen_intelligence::{
    AccessibilityStatus, AutocompleteCommitParams, AutocompleteCommitResult,
    AutocompleteSuggestParams, AutocompleteSuggestResult, CaptureImageRefResult, CaptureNowResult,
    InputActionParams, InputActionResult, PermissionRequestParams, PermissionState,
@@ -126,6 +126,18 @@ pub struct BrowserSettingsUpdate {
    pub enabled: Option<bool>,
 }

+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ScreenIntelligenceSettingsUpdate {
+    pub enabled: Option<bool>,
+    pub capture_policy: Option<String>,
+    pub policy_mode: Option<String>,
+    pub baseline_fps: Option<f32>,
+    pub vision_enabled: Option<bool>,
+    pub autocomplete_enabled: Option<bool>,
+    pub allowlist: Option<Vec<String>>,
+    pub denylist: Option<Vec<String>>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct RuntimeFlags {
    pub browser_allow_all: bool,
@@ -455,6 +467,50 @@ async fn dispatch(
            ))
        }

+        "openhuman.update_screen_intelligence_settings" => {
+            let update: ScreenIntelligenceSettingsUpdate = parse_params(params)?;
+            let mut config = load_openhuman_config().await?;
+
+            if let Some(enabled) = update.enabled {
+                config.screen_intelligence.enabled = enabled;
+            }
+            if let Some(capture_policy) = update.capture_policy {
+                config.screen_intelligence.capture_policy = capture_policy;
+            }
+            if let Some(policy_mode) = update.policy_mode {
+                config.screen_intelligence.policy_mode = policy_mode;
+            }
+            if let Some(baseline_fps) = update.baseline_fps {
+                config.screen_intelligence.baseline_fps = baseline_fps.clamp(0.2, 30.0);
+            }
+            if let Some(vision_enabled) = update.vision_enabled {
+                config.screen_intelligence.vision_enabled = vision_enabled;
+            }
+            if let Some(autocomplete_enabled) = update.autocomplete_enabled {
+                config.screen_intelligence.autocomplete_enabled = autocomplete_enabled;
+            }
+            if let Some(allowlist) = update.allowlist {
+                config.screen_intelligence.allowlist = allowlist;
+            }
+            if let Some(denylist) = update.denylist {
+                config.screen_intelligence.denylist = denylist;
+            }
+
+            config.save().await.map_err(|e| e.to_string())?;
+            let _ = screen_intelligence::global_engine()
+                .apply_config(config.screen_intelligence.clone())
+                .await;
+
+            let snapshot = snapshot_config(&config)?;
+            to_json_value(command_response(
+                snapshot,
+                vec![format!(
+                    "screen intelligence settings saved to {}",
+                    config.config_path.display()
+                )],
+            ))
+        }
+
        "openhuman.update_gateway_settings" => {
            let update: GatewaySettingsUpdate = parse_params(params)?;
            let mut config = load_openhuman_config().await?;
@@ -1019,15 +1075,22 @@ async fn dispatch(
        }

        "openhuman.accessibility_status" => {
-            let status = accessibility::global_engine().status().await;
+            if let Ok(config) = load_openhuman_config().await {
+                let _ = screen_intelligence::global_engine()
+                    .apply_config(config.screen_intelligence.clone())
+                    .await;
+            }
+            let status = screen_intelligence::global_engine().status().await;
            to_json_value(command_response(
                status,
-                vec!["accessibility status fetched".to_string()],
+                vec!["screen intelligence status fetched".to_string()],
            ))
        }

        "openhuman.accessibility_request_permissions" => {
-            let permissions = accessibility::global_engine().request_permissions().await?;
+            let permissions = screen_intelligence::global_engine()
+                .request_permissions()
+                .await?;
            to_json_value(command_response(
                permissions,
                vec!["accessibility permissions requested".to_string()],
@@ -1036,7 +1099,7 @@ async fn dispatch(

        "openhuman.accessibility_request_permission" => {
            let payload: PermissionRequestParams = parse_params(params)?;
-            let permissions = accessibility::global_engine()
+            let permissions = screen_intelligence::global_engine()
                .request_permission(payload.permission)
                .await?;
            to_json_value(command_response(
@@ -1046,29 +1109,27 @@ async fn dispatch(
        }

        "openhuman.accessibility_start_session" => {
-            let payload: StartSessionParams = parse_params(params)?;
-            let session = accessibility::global_engine()
-                .start_session(payload)
-                .await?;
+            let _payload: StartSessionParams = parse_params(params)?;
+            let session = screen_intelligence::global_engine().enable().await?;
            to_json_value(command_response(
                session,
-                vec!["accessibility session started".to_string()],
+                vec!["screen intelligence enabled".to_string()],
            ))
        }

        "openhuman.accessibility_stop_session" => {
            let payload: StopSessionParams = parse_params(params)?;
-            let session = accessibility::global_engine()
-                .stop_session(payload.reason)
+            let session = screen_intelligence::global_engine()
+                .disable(payload.reason)
                .await;
            to_json_value(command_response(
                session,
-                vec!["accessibility session stopped".to_string()],
+                vec!["screen intelligence stopped".to_string()],
            ))
        }

        "openhuman.accessibility_capture_now" => {
-            let result = accessibility::global_engine().capture_now().await?;
+            let result = screen_intelligence::global_engine().capture_now().await?;
            to_json_value(command_response(
                result,
                vec!["accessibility manual capture requested".to_string()],
@@ -1076,8 +1137,9 @@ async fn dispatch(
        }

        "openhuman.accessibility_capture_image_ref" => {
-            let result: CaptureImageRefResult =
-                accessibility::global_engine().capture_image_ref_test().await;
+            let result: CaptureImageRefResult = screen_intelligence::global_engine()
+                .capture_image_ref_test()
+                .await;
            to_json_value(command_response(
                result,
                vec!["accessibility direct image_ref capture requested".to_string()],
@@ -1086,51 +1148,54 @@ async fn dispatch(

        "openhuman.accessibility_input_action" => {
            let payload: InputActionParams = parse_params(params)?;
-            let result = accessibility::global_engine().input_action(payload).await?;
+            let result = screen_intelligence::global_engine()
+                .input_action(payload)
+                .await?;
            to_json_value(command_response(
                result,
-                vec!["accessibility input action processed".to_string()],
+                vec!["screen intelligence input action processed".to_string()],
            ))
        }

        "openhuman.accessibility_autocomplete_suggest" => {
            let payload: AutocompleteSuggestParams = parse_params(params)?;
-            let result = accessibility::global_engine()
+            let result = screen_intelligence::global_engine()
                .autocomplete_suggest(payload)
                .await?;
            to_json_value(command_response(
                result,
-                vec!["accessibility autocomplete suggestions generated".to_string()],
+                vec!["screen intelligence autocomplete suggestions generated".to_string()],
            ))
        }

        "openhuman.accessibility_autocomplete_commit" => {
            let payload: AutocompleteCommitParams = parse_params(params)?;
-            let result = accessibility::global_engine()
+            let result = screen_intelligence::global_engine()
                .autocomplete_commit(payload)
                .await?;
            to_json_value(command_response(
                result,
-                vec!["accessibility autocomplete suggestion committed".to_string()],
+                vec!["screen intelligence autocomplete suggestion committed".to_string()],
            ))
        }

        "openhuman.accessibility_vision_recent" => {
            let payload: AccessibilityVisionRecentParams = parse_params(params)?;
-            let result: VisionRecentResult = accessibility::global_engine()
+            let result: VisionRecentResult = screen_intelligence::global_engine()
                .vision_recent(payload.limit)
                .await;
            to_json_value(command_response(
                result,
-                vec!["accessibility vision summaries fetched".to_string()],
+                vec!["screen intelligence vision summaries fetched".to_string()],
            ))
        }

        "openhuman.accessibility_vision_flush" => {
-            let result: VisionFlushResult = accessibility::global_engine().vision_flush().await?;
+            let result: VisionFlushResult =
+                screen_intelligence::global_engine().vision_flush().await?;
            to_json_value(command_response(
                result,
-                vec!["accessibility vision flush completed".to_string()],
+                vec!["screen intelligence vision flush completed".to_string()],
            ))
        }

@@ -1418,9 +1483,9 @@ enum AccessibilityCommand {
    RequestPermissions,
    /// Request a specific permission kind
    RequestPermission(RequestPermissionArgs),
-    /// Start a bounded accessibility session
+    /// Start a bounded screen intelligence session
    StartSession(StartSessionCliArgs),
-    /// Stop the active accessibility session
+    /// Stop the active screen intelligence session
    StopSession(StopSessionCliArgs),
    /// Force an immediate capture sample
    CaptureNow,
@@ -1572,13 +1637,12 @@ fn ensure_non_empty_payload(payload: &serde_json::Map<String, serde_json::Value>
 }

 fn extract_data_url(raw: &str) -> Option<String> {
-    raw.lines()
-        .find_map(|line| {
-            let trimmed = line.trim();
-            trimmed
-                .starts_with("data:image/")
-                .then(|| trimmed.to_string())
-        })
+    raw.lines().find_map(|line| {
+        let trimmed = line.trim();
+        trimmed
+            .starts_with("data:image/")
+            .then(|| trimmed.to_string())
+    })
 }

 fn extract_saved_path(raw: &str) -> Option<PathBuf> {
@@ -1677,8 +1741,10 @@ async fn execute_tools_screenshot_ref(
    args: ToolsScreenshotRefArgs,
 ) -> Result<serde_json::Value, String> {
    let raw = call_method("openhuman.accessibility_capture_image_ref", json!({})).await?;
-    let payload: CommandResponse<CaptureImageRefResult> = serde_json::from_value(raw)
-        .map_err(|e| format!("failed to decode accessibility capture_image_ref response: {e}"))?;
+    let payload: CommandResponse<CaptureImageRefResult> =
+        serde_json::from_value(raw).map_err(|e| {
+            format!("failed to decode screen intelligence capture_image_ref response: {e}")
+        })?;

    let mut logs = payload.logs;
    logs.push("tools.screenshot-ref executed".to_string());
@@ -1693,7 +1759,9 @@ async fn execute_tools_screenshot_ref(
                output_path.display()
            ));
        } else {
-            return Err("accessibility capture_image_ref did not return image_ref".to_string());
+            return Err(
+                "screen intelligence capture_image_ref did not return image_ref".to_string(),
+            );
        }
    }

@@ -1729,11 +1797,26 @@ fn settings_view_response(
            "default_model": cfg.get("default_model"),
            "default_temperature": cfg.get("default_temperature"),
        }),
-        "memory" => cfg.get("memory").cloned().unwrap_or(serde_json::Value::Null),
-        "gateway" => cfg.get("gateway").cloned().unwrap_or(serde_json::Value::Null),
-        "tunnel" => cfg.get("tunnel").cloned().unwrap_or(serde_json::Value::Null),
-        "runtime" => cfg.get("runtime").cloned().unwrap_or(serde_json::Value::Null),
-        "browser" => cfg.get("browser").cloned().unwrap_or(serde_json::Value::Null),
+        "memory" => cfg
+            .get("memory")
+            .cloned()
+            .unwrap_or(serde_json::Value::Null),
+        "gateway" => cfg
+            .get("gateway")
+            .cloned()
+            .unwrap_or(serde_json::Value::Null),
+        "tunnel" => cfg
+            .get("tunnel")
+            .cloned()
+            .unwrap_or(serde_json::Value::Null),
+        "runtime" => cfg
+            .get("runtime")
+            .cloned()
+            .unwrap_or(serde_json::Value::Null),
+        "browser" => cfg
+            .get("browser")
+            .cloned()
+            .unwrap_or(serde_json::Value::Null),
        _ => serde_json::Value::Null,
    };

@@ -1761,7 +1844,9 @@ async fn execute_core_cli(cli: CoreCli) -> Result<serde_json::Value, String> {
        CoreCommand::SecurityPolicy => {
            call_method("openhuman.security_policy_info", json!({})).await
        }
-        CoreCommand::Call { method, params } => call_method(&method, parse_json_arg(&params)?).await,
+        CoreCommand::Call { method, params } => {
+            call_method(&method, parse_json_arg(&params)?).await
+        }
        CoreCommand::Completions { shell } => {
            let mut cmd = CoreCli::command();
            let bin_name = cmd.get_name().to_string();
@@ -1793,8 +1878,11 @@ async fn execute_core_cli(cli: CoreCli) -> Result<serde_json::Value, String> {
                        payload.insert("default_temperature".to_string(), json!(v));
                    }
                    ensure_non_empty_payload(&payload).map_err(|e| e.to_string())?;
-                    call_method("openhuman.update_model_settings", serde_json::Value::Object(payload))
-                        .await
+                    call_method(
+                        "openhuman.update_model_settings",
+                        serde_json::Value::Object(payload),
+                    )
+                    .await
                }
            },
            SettingsCommand::Memory { command } => match command {
@@ -1863,7 +1951,11 @@ async fn execute_core_cli(cli: CoreCli) -> Result<serde_json::Value, String> {
                        .map_err(|e| e.to_string())
                }
                TunnelSettingsCommand::Set(args) => {
-                    call_method("openhuman.update_tunnel_settings", parse_json_arg(&args.json)?).await
+                    call_method(
+                        "openhuman.update_tunnel_settings",
+                        parse_json_arg(&args.json)?,
+                    )
+                    .await
                }
            },
            SettingsCommand::Runtime { command } => match command {
@@ -1915,19 +2007,19 @@ async fn execute_core_cli(cli: CoreCli) -> Result<serde_json::Value, String> {
            AccessibilityCommand::Doctor => {
                let raw = call_method("openhuman.accessibility_status", json!({})).await?;
                let payload: CommandResponse<AccessibilityStatus> = serde_json::from_value(raw)
-                    .map_err(|e| format!("failed to decode accessibility status: {e}"))?;
+                    .map_err(|e| format!("failed to decode screen intelligence status: {e}"))?;
                let permissions = &payload.result.permissions;

                let screen_ready = permissions.screen_recording == PermissionState::Granted;
                let control_ready = permissions.accessibility == PermissionState::Granted;
                let monitoring_ready = permissions.input_monitoring == PermissionState::Granted;
-                let overall_ready = payload.result.platform_supported && screen_ready && control_ready;
+                let overall_ready =
+                    payload.result.platform_supported && screen_ready && control_ready;

                let mut recommendations: Vec<String> = Vec::new();
                if !payload.result.platform_supported {
                    recommendations.push(
-                        "Accessibility automation is macOS-only in this build/runtime."
-                            .to_string(),
+                        "Accessibility automation is macOS-only in this build/runtime.".to_string(),
                    );
                }
                if permissions.screen_recording != PermissionState::Granted {
@@ -1949,7 +2041,8 @@ async fn execute_core_cli(cli: CoreCli) -> Result<serde_json::Value, String> {
                    );
                }
                if recommendations.is_empty() {
-                    recommendations.push("No action required. Accessibility automation is ready.".to_string());
+                    recommendations
+                        .push("No action required. Accessibility automation is ready.".to_string());
                }

                Ok(json!({
@@ -2026,7 +2119,7 @@ async fn execute_core_cli(cli: CoreCli) -> Result<serde_json::Value, String> {
                        },
                        {
                            "name": "screenshot-ref",
-                            "description": "Capture data URL from accessibility capture_image_ref."
+                            "description": "Capture data URL from screen intelligence capture_image_ref."
                        }
                    ]
                },
@@ -2107,8 +2200,7 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> {
    let mut argv = Vec::with_capacity(args.len() + 1);
    argv.push("openhuman-core".to_string());
    argv.extend(args.iter().cloned());
-    let cli =
-        CoreCli::try_parse_from(argv).map_err(|e| anyhow::anyhow!(e.render().to_string()))?;
+    let cli = CoreCli::try_parse_from(argv).map_err(|e| anyhow::anyhow!(e.render().to_string()))?;

    let thread_stack_size = std::env::var("OPENHUMAN_CORE_THREAD_STACK_SIZE")
        .ok()
@@ -2119,7 +2211,9 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> {
        .thread_stack_size(thread_stack_size)
        .enable_all()
        .build()?;
-    let output = runtime.block_on(execute_core_cli(cli)).map_err(anyhow::Error::msg)?;
+    let output = runtime
+        .block_on(execute_core_cli(cli))
+        .map_err(anyhow::Error::msg)?;
    if !output.is_null() {
        println!(
            "{}",
@@ -8,18 +8,17 @@ pub use daemon::DaemonConfig;
 pub use schema::{
    apply_runtime_proxy_to_builder, build_runtime_proxy_client,
    build_runtime_proxy_client_with_timeouts, runtime_proxy_config, set_runtime_proxy_config,
-    AccessibilityAutomationConfig, AgentConfig, AuditConfig, AutonomyConfig,
-    BrowserComputerUseConfig, BrowserConfig, ChannelsConfig, ClassificationRule,
-    CloudflareTunnelConfig, ComposioConfig, Config, CostConfig, CronConfig, CustomTunnelConfig,
-    DelegateAgentConfig, DiscordConfig, DockerRuntimeConfig, EmbeddingRouteConfig, GatewayConfig,
-    HardwareConfig, HardwareTransport, HeartbeatConfig, HttpRequestConfig, IMessageConfig,
-    IdentityConfig, LarkConfig, LocalAiConfig, MatrixConfig, MemoryConfig, ModelRouteConfig,
-    MultimodalConfig, NgrokTunnelConfig, ObservabilityConfig, PeripheralBoardConfig,
-    PeripheralsConfig, ProxyConfig, ProxyScope, QueryClassificationConfig, ReliabilityConfig,
-    ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig,
-    SecretsConfig, SecurityConfig, SlackConfig, StorageConfig, StorageProviderConfig,
-    StorageProviderSection, StreamMode, TailscaleTunnelConfig, TelegramConfig, TunnelConfig,
-    WebSearchConfig, WebhookConfig,
+    AgentConfig, AuditConfig, AutonomyConfig, BrowserComputerUseConfig, BrowserConfig,
+    ChannelsConfig, ClassificationRule, CloudflareTunnelConfig, ComposioConfig, Config, CostConfig,
+    CronConfig, CustomTunnelConfig, DelegateAgentConfig, DiscordConfig, DockerRuntimeConfig,
+    EmbeddingRouteConfig, GatewayConfig, HardwareConfig, HardwareTransport, HeartbeatConfig,
+    HttpRequestConfig, IMessageConfig, IdentityConfig, LarkConfig, LocalAiConfig, MatrixConfig,
+    MemoryConfig, ModelRouteConfig, MultimodalConfig, NgrokTunnelConfig, ObservabilityConfig,
+    PeripheralBoardConfig, PeripheralsConfig, ProxyConfig, ProxyScope, QueryClassificationConfig,
+    ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig,
+    SchedulerConfig, ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, SlackConfig,
+    StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode,
+    TailscaleTunnelConfig, TelegramConfig, TunnelConfig, WebSearchConfig, WebhookConfig,
 };

 #[cfg(test)]
@@ -2,11 +2,17 @@ use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};

 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-pub struct AccessibilityAutomationConfig {
+pub struct ScreenIntelligenceConfig {
+    #[serde(default = "default_enabled")]
+    pub enabled: bool,
    #[serde(default = "default_capture_policy")]
    pub capture_policy: String,
+    #[serde(default = "default_policy_mode")]
+    pub policy_mode: String,
    #[serde(default = "default_baseline_fps")]
    pub baseline_fps: f32,
+    #[serde(default = "default_vision_enabled")]
+    pub vision_enabled: bool,
    #[serde(default = "default_session_ttl_secs")]
    pub session_ttl_secs: u64,
    #[serde(default = "default_panic_stop_hotkey")]
@@ -14,17 +20,31 @@ pub struct AccessibilityAutomationConfig {
    #[serde(default = "default_autocomplete_enabled")]
    pub autocomplete_enabled: bool,
    #[serde(default)]
+    pub allowlist: Vec<String>,
+    #[serde(default)]
    pub denylist: Vec<String>,
 }

+fn default_enabled() -> bool {
+    false
+}
+
 fn default_capture_policy() -> String {
    "hybrid".to_string()
 }

+fn default_policy_mode() -> String {
+    "all_except_blacklist".to_string()
+}
+
 fn default_baseline_fps() -> f32 {
    1.0
 }

+fn default_vision_enabled() -> bool {
+    true
+}
+
 fn default_session_ttl_secs() -> u64 {
    300
 }
@@ -37,14 +57,18 @@ fn default_autocomplete_enabled() -> bool {
    true
 }

-impl Default for AccessibilityAutomationConfig {
+impl Default for ScreenIntelligenceConfig {
    fn default() -> Self {
        Self {
+            enabled: default_enabled(),
            capture_policy: default_capture_policy(),
+            policy_mode: default_policy_mode(),
            baseline_fps: default_baseline_fps(),
+            vision_enabled: default_vision_enabled(),
            session_ttl_secs: default_session_ttl_secs(),
            panic_stop_hotkey: default_panic_stop_hotkey(),
            autocomplete_enabled: default_autocomplete_enabled(),
+            allowlist: vec![],
            denylist: vec![
                "1password".to_string(),
                "keychain".to_string(),
@@ -21,7 +21,7 @@ mod storage_memory;
 mod tools;
 mod tunnel;

-pub use accessibility::AccessibilityAutomationConfig;
+pub use accessibility::ScreenIntelligenceConfig;
 pub use agent::{AgentConfig, DelegateAgentConfig};
 pub use autonomy::AutonomyConfig;
 pub use channels::{
@@ -88,7 +88,7 @@ pub struct Config {
    pub runtime: RuntimeConfig,

    #[serde(default)]
-    pub accessibility: AccessibilityAutomationConfig,
+    pub screen_intelligence: ScreenIntelligenceConfig,

    #[serde(default)]
    pub reliability: ReliabilityConfig,
@@ -186,7 +186,7 @@ impl Default for Config {
            observability: ObservabilityConfig::default(),
            autonomy: AutonomyConfig::default(),
            runtime: RuntimeConfig::default(),
-            accessibility: AccessibilityAutomationConfig::default(),
+            screen_intelligence: ScreenIntelligenceConfig::default(),
            reliability: ReliabilityConfig::default(),
            scheduler: SchedulerConfig::default(),
            agent: AgentConfig::default(),
@@ -36,6 +36,7 @@ pub mod peripherals;
 pub mod providers;
 pub mod rag;
 pub mod runtime;
+pub mod screen_intelligence;
 pub mod security;
 pub mod service;
 pub mod skillforge;
@@ -5,8 +5,8 @@ use openhuman_core::core_server::{
    AutocompleteSuggestParams, AutocompleteSuggestResult, BrowserSettingsUpdate, CaptureNowResult,
    CommandResponse, ConfigSnapshot, GatewaySettingsUpdate, InputActionParams, InputActionResult,
    MemorySettingsUpdate, ModelSettingsUpdate, PermissionRequestParams, PermissionStatus,
-    RuntimeFlags, RuntimeSettingsUpdate, SessionStatus, StartSessionParams, StopSessionParams,
-    VisionFlushResult, VisionRecentResult,
+    RuntimeFlags, RuntimeSettingsUpdate, ScreenIntelligenceSettingsUpdate, SessionStatus,
+    StartSessionParams, StopSessionParams, VisionFlushResult, VisionRecentResult,
 };
 use openhuman_core::openhuman::{doctor, hardware, integrations, migration, onboard, service};
 use serde::de::DeserializeOwned;
@@ -281,6 +281,20 @@ pub async fn openhuman_update_memory_settings(
    .await
 }

+/// Update screen intelligence settings.
+#[tauri::command]
+pub async fn openhuman_update_screen_intelligence_settings(
+    app: tauri::AppHandle,
+    update: ScreenIntelligenceSettingsUpdate,
+) -> Result<CommandResponse<ConfigSnapshot>, String> {
+    call_core(
+        &app,
+        "openhuman.update_screen_intelligence_settings",
+        serde_json::json!(update),
+    )
+    .await
+}
+
 /// Update gateway settings.
 #[tauri::command]
 pub async fn openhuman_update_gateway_settings(
@@ -509,7 +523,7 @@ pub async fn openhuman_accessibility_request_permission(
    Ok(response)
 }

-/// Start a bounded accessibility session with explicit consent.
+/// Start a bounded screen intelligence session with explicit consent.
 #[tauri::command]
 pub async fn openhuman_accessibility_start_session(
    app: tauri::AppHandle,
@@ -524,7 +538,7 @@ pub async fn openhuman_accessibility_start_session(
    Ok(response)
 }

-/// Stop the active accessibility session.
+/// Stop the active screen intelligence session.
 #[tauri::command]
 pub async fn openhuman_accessibility_stop_session(
    app: tauri::AppHandle,
@@ -548,7 +562,7 @@ pub async fn openhuman_accessibility_capture_now(
    call_core_service_managed("openhuman.accessibility_capture_now", params_none()).await
 }

-/// Execute a validated input action in an active accessibility session.
+/// Execute a validated input action in an active screen intelligence session.
 #[tauri::command]
 pub async fn openhuman_accessibility_input_action(
    app: tauri::AppHandle,
@@ -1076,6 +1076,7 @@ pub fn run() {
                    openhuman_get_config,
                    openhuman_update_model_settings,
                    openhuman_update_memory_settings,
+                    openhuman_update_screen_intelligence_settings,
                    openhuman_update_gateway_settings,
                    openhuman_update_tunnel_settings,
                    openhuman_update_runtime_settings,
@@ -76,7 +76,7 @@ const SettingsHome = () => {
    {
      id: 'accessibility',
      title: 'Accessibility Automation',
-      description: 'Screen monitoring, device control, and predictive input',
+      description: 'Desktop permissions, assisted controls, and safety-bound sessions',
      icon: (
        <svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
          <path
@@ -90,6 +90,23 @@ const SettingsHome = () => {
      onClick: () => navigateToSettings('accessibility'),
      dangerous: false,
    },
+    {
+      id: 'screen-intelligence',
+      title: 'Screen Intelligence',
+      description: 'Window capture policy, vision summaries, and memory ingestion',
+      icon: (
+        <svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+          <path
+            strokeLinecap="round"
+            strokeLinejoin="round"
+            strokeWidth={2}
+            d="M3 5h18v12H3zM8 21h8m-4-4v4"
+          />
+        </svg>
+      ),
+      onClick: () => navigateToSettings('screen-intelligence'),
+      dangerous: false,
+    },
    {
      id: 'messaging',
      title: 'Messaging Channels',
@@ -6,6 +6,7 @@ export type SettingsRoute =
  | 'connections'
  | 'messaging'
  | 'cron-jobs'
+  | 'screen-intelligence'
  | 'privacy'
  | 'profile'
  | 'advanced'
@@ -47,6 +48,7 @@ export const useSettingsNavigation = (): SettingsNavigationHook => {
    if (path.includes('/settings/connections')) return 'connections';
    if (path.includes('/settings/messaging')) return 'messaging';
    if (path.includes('/settings/cron-jobs')) return 'cron-jobs';
+    if (path.includes('/settings/screen-intelligence')) return 'screen-intelligence';
    if (path.includes('/settings/privacy')) return 'privacy';
    if (path.includes('/settings/profile')) return 'profile';
    if (path.includes('/settings/advanced')) return 'advanced';
@@ -0,0 +1,418 @@
+import { useEffect, useMemo, useState } from 'react';
+
+import {
+  fetchAccessibilityStatus,
+  fetchAccessibilityVisionRecent,
+  flushAccessibilityVision,
+  requestAccessibilityPermission,
+  startAccessibilitySession,
+  stopAccessibilitySession,
+} from '../../../store/accessibilitySlice';
+import { useAppDispatch, useAppSelector } from '../../../store/hooks';
+import { isTauri, openhumanUpdateScreenIntelligenceSettings } from '../../../utils/tauriCommands';
+import SettingsHeader from '../components/SettingsHeader';
+import { useSettingsNavigation } from '../hooks/useSettingsNavigation';
+
+const formatRemaining = (remainingMs: number | null): string => {
+  if (remainingMs === null || remainingMs <= 0) {
+    return '00:00';
+  }
+
+  const totalSeconds = Math.floor(remainingMs / 1000);
+  const mins = Math.floor(totalSeconds / 60)
+    .toString()
+    .padStart(2, '0');
+  const secs = (totalSeconds % 60).toString().padStart(2, '0');
+  return `${mins}:${secs}`;
+};
+
+const PermissionBadge = ({ label, value }: { label: string; value: string }) => {
+  const colorClass =
+    value === 'granted'
+      ? 'bg-green-900/40 text-green-300 border-green-700/40'
+      : value === 'denied'
+        ? 'bg-red-900/40 text-red-300 border-red-700/40'
+        : 'bg-stone-800/60 text-stone-300 border-stone-700';
+
+  return (
+    <div className="flex items-center justify-between rounded-xl border border-stone-700 bg-stone-900/50 p-3">
+      <span className="text-sm text-stone-200">{label}</span>
+      <span className={`rounded-md border px-2 py-1 text-xs uppercase tracking-wide ${colorClass}`}>
+        {value}
+      </span>
+    </div>
+  );
+};
+
+const ScreenIntelligencePanel = () => {
+  const { navigateBack } = useSettingsNavigation();
+  const dispatch = useAppDispatch();
+  const {
+    status,
+    isLoading,
+    isRequestingPermissions,
+    isStartingSession,
+    isStoppingSession,
+    isLoadingVision,
+    isFlushingVision,
+    recentVisionSummaries,
+    lastError,
+  } = useAppSelector(state => state.accessibility);
+  const [featureOverrides, setFeatureOverrides] = useState<{
+    screen_monitoring?: boolean;
+    device_control?: boolean;
+    predictive_input?: boolean;
+  }>({});
+  const [enabled, setEnabled] = useState<boolean>(false);
+  const [policyMode, setPolicyMode] = useState<'all_except_blacklist' | 'whitelist_only'>(
+    'all_except_blacklist'
+  );
+  const [baselineFps, setBaselineFps] = useState<string>('1');
+  const [allowlistText, setAllowlistText] = useState('');
+  const [denylistText, setDenylistText] = useState('');
+  const [isSavingConfig, setIsSavingConfig] = useState(false);
+  const [configError, setConfigError] = useState<string | null>(null);
+
+  useEffect(() => {
+    void dispatch(fetchAccessibilityStatus());
+  }, [dispatch]);
+
+  useEffect(() => {
+    if (!status?.session.active) {
+      return;
+    }
+    const intervalId = window.setInterval(() => {
+      void dispatch(fetchAccessibilityStatus());
+      void dispatch(fetchAccessibilityVisionRecent(10));
+    }, 1000);
+    return () => window.clearInterval(intervalId);
+  }, [dispatch, status?.session.active]);
+
+  useEffect(() => {
+    void dispatch(fetchAccessibilityVisionRecent(10));
+  }, [dispatch]);
+
+  useEffect(() => {
+    if (!status?.config) {
+      return;
+    }
+    setEnabled(status.config.enabled ?? false);
+    setPolicyMode(
+      status.config.policy_mode === 'whitelist_only' ? 'whitelist_only' : 'all_except_blacklist'
+    );
+    setBaselineFps(String(status.config.baseline_fps ?? 1));
+    setAllowlistText((status.config.allowlist ?? []).join('\n'));
+    setDenylistText((status.config.denylist ?? []).join('\n'));
+  }, [status?.config]);
+
+  const screenMonitoring =
+    featureOverrides.screen_monitoring ?? status?.features.screen_monitoring ?? true;
+  const deviceControl = featureOverrides.device_control ?? status?.features.device_control ?? true;
+  const predictiveInput =
+    featureOverrides.predictive_input ?? status?.features.predictive_input ?? true;
+
+  const remaining = useMemo(
+    () => formatRemaining(status?.session.remaining_ms ?? null),
+    [status?.session.remaining_ms]
+  );
+
+  const startDisabled =
+    isStartingSession ||
+    isLoading ||
+    !status?.platform_supported ||
+    status.session.active ||
+    status.permissions.accessibility !== 'granted';
+  const stopDisabled = isStoppingSession || !status?.session.active;
+
+  const saveConfig = async () => {
+    if (!isTauri()) return;
+    setConfigError(null);
+    setIsSavingConfig(true);
+    try {
+      const fps = Number(baselineFps);
+      await openhumanUpdateScreenIntelligenceSettings({
+        enabled,
+        policy_mode: policyMode,
+        baseline_fps: Number.isFinite(fps) && fps > 0 ? fps : 1,
+        allowlist: allowlistText
+          .split('\n')
+          .map(v => v.trim())
+          .filter(Boolean),
+        denylist: denylistText
+          .split('\n')
+          .map(v => v.trim())
+          .filter(Boolean),
+      });
+      await dispatch(fetchAccessibilityStatus());
+    } catch (error) {
+      setConfigError(error instanceof Error ? error.message : 'Failed to save screen intelligence');
+    } finally {
+      setIsSavingConfig(false);
+    }
+  };
+
+  return (
+    <div className="overflow-hidden h-full flex flex-col z-10 relative">
+      <SettingsHeader title="Screen Intelligence" showBackButton={true} onBack={navigateBack} />
+
+      <div className="flex-1 overflow-y-auto max-w-2xl mx-auto w-full p-4 space-y-4">
+        <section className="rounded-2xl border border-stone-700 bg-black/30 p-4 space-y-3">
+          <h3 className="text-sm font-semibold text-white">Permissions</h3>
+          <PermissionBadge
+            label="Screen Recording"
+            value={status?.permissions.screen_recording ?? 'unknown'}
+          />
+          <PermissionBadge
+            label="Accessibility"
+            value={status?.permissions.accessibility ?? 'unknown'}
+          />
+          <PermissionBadge
+            label="Input Monitoring"
+            value={status?.permissions.input_monitoring ?? 'unknown'}
+          />
+
+          <button
+            type="button"
+            onClick={() => void dispatch(requestAccessibilityPermission('screen_recording'))}
+            disabled={isRequestingPermissions}
+            className="mt-1 rounded-lg border border-primary-500/60 bg-primary-500/20 px-3 py-2 text-sm text-primary-200 disabled:opacity-50">
+            {isRequestingPermissions ? 'Requesting…' : 'Request Screen Recording'}
+          </button>
+          <button
+            type="button"
+            onClick={() => void dispatch(requestAccessibilityPermission('accessibility'))}
+            disabled={isRequestingPermissions}
+            className="rounded-lg border border-primary-500/60 bg-primary-500/20 px-3 py-2 text-sm text-primary-200 disabled:opacity-50">
+            {isRequestingPermissions ? 'Requesting…' : 'Request Accessibility'}
+          </button>
+          <button
+            type="button"
+            onClick={() => void dispatch(requestAccessibilityPermission('input_monitoring'))}
+            disabled={isRequestingPermissions}
+            className="rounded-lg border border-primary-500/60 bg-primary-500/20 px-3 py-2 text-sm text-primary-200 disabled:opacity-50">
+            {isRequestingPermissions ? 'Requesting…' : 'Open Input Monitoring'}
+          </button>
+          <button
+            type="button"
+            onClick={() => void dispatch(fetchAccessibilityStatus())}
+            disabled={isLoading}
+            className="rounded-lg border border-stone-600 bg-stone-800/60 px-3 py-2 text-sm text-stone-200 disabled:opacity-50">
+            {isLoading ? 'Refreshing…' : 'Refresh Status'}
+          </button>
+        </section>
+
+        <section className="rounded-2xl border border-stone-700 bg-black/30 p-4 space-y-3">
+          <h3 className="text-sm font-semibold text-white">Screen Intelligence Policy</h3>
+
+          <label className="flex items-center justify-between rounded-xl border border-stone-700 bg-stone-900/50 px-3 py-2">
+            <span className="text-sm text-stone-200">Enabled</span>
+            <input
+              type="checkbox"
+              checked={enabled}
+              onChange={event => setEnabled(event.target.checked)}
+            />
+          </label>
+
+          <label className="flex items-center justify-between rounded-xl border border-stone-700 bg-stone-900/50 px-3 py-2">
+            <span className="text-sm text-stone-200">Mode</span>
+            <select
+              value={policyMode}
+              onChange={event =>
+                setPolicyMode(
+                  event.target.value === 'whitelist_only'
+                    ? 'whitelist_only'
+                    : 'all_except_blacklist'
+                )
+              }
+              className="rounded border border-stone-600 bg-stone-800 px-2 py-1 text-xs text-stone-200">
+              <option value="all_except_blacklist">All Except Blacklist</option>
+              <option value="whitelist_only">Whitelist Only</option>
+            </select>
+          </label>
+
+          <label className="flex items-center justify-between rounded-xl border border-stone-700 bg-stone-900/50 px-3 py-2">
+            <span className="text-sm text-stone-200">Baseline FPS</span>
+            <input
+              type="number"
+              min={0.2}
+              max={30}
+              step={0.1}
+              value={baselineFps}
+              onChange={event => setBaselineFps(event.target.value)}
+              className="w-24 rounded border border-stone-600 bg-stone-800 px-2 py-1 text-xs text-stone-200"
+            />
+          </label>
+
+          <div className="space-y-1">
+            <div className="text-xs text-stone-300">Allowlist (one rule per line)</div>
+            <textarea
+              value={allowlistText}
+              onChange={event => setAllowlistText(event.target.value)}
+              rows={3}
+              className="w-full rounded border border-stone-700 bg-stone-900/50 p-2 text-xs text-stone-200"
+            />
+          </div>
+
+          <div className="space-y-1">
+            <div className="text-xs text-stone-300">Denylist (one rule per line)</div>
+            <textarea
+              value={denylistText}
+              onChange={event => setDenylistText(event.target.value)}
+              rows={3}
+              className="w-full rounded border border-stone-700 bg-stone-900/50 p-2 text-xs text-stone-200"
+            />
+          </div>
+
+          <button
+            type="button"
+            onClick={() => void saveConfig()}
+            disabled={isSavingConfig}
+            className="rounded-lg border border-primary-500/60 bg-primary-500/20 px-3 py-2 text-sm text-primary-200 disabled:opacity-50">
+            {isSavingConfig ? 'Saving…' : 'Save Screen Intelligence Settings'}
+          </button>
+          {configError && <div className="text-xs text-red-300">{configError}</div>}
+
+          <label className="flex items-center justify-between rounded-xl border border-stone-700 bg-stone-900/50 px-3 py-2">
+            <span className="text-sm text-stone-200">Screen Monitoring</span>
+            <input
+              type="checkbox"
+              checked={screenMonitoring}
+              onChange={event =>
+                setFeatureOverrides(current => ({
+                  ...current,
+                  screen_monitoring: event.target.checked,
+                }))
+              }
+            />
+          </label>
+
+          <label className="flex items-center justify-between rounded-xl border border-stone-700 bg-stone-900/50 px-3 py-2">
+            <span className="text-sm text-stone-200">Device Control</span>
+            <input
+              type="checkbox"
+              checked={deviceControl}
+              onChange={event =>
+                setFeatureOverrides(current => ({
+                  ...current,
+                  device_control: event.target.checked,
+                }))
+              }
+            />
+          </label>
+
+          <label className="flex items-center justify-between rounded-xl border border-stone-700 bg-stone-900/50 px-3 py-2">
+            <span className="text-sm text-stone-200">Predictive Input</span>
+            <input
+              type="checkbox"
+              checked={predictiveInput}
+              onChange={event =>
+                setFeatureOverrides(current => ({
+                  ...current,
+                  predictive_input: event.target.checked,
+                }))
+              }
+            />
+          </label>
+        </section>
+
+        <section className="rounded-2xl border border-stone-700 bg-black/30 p-4 space-y-3">
+          <h3 className="text-sm font-semibold text-white">Session</h3>
+          <div className="text-sm text-stone-300 space-y-1">
+            <div>Status: {status?.session.active ? 'Active' : 'Stopped'}</div>
+            <div>Remaining: {remaining}</div>
+            <div>Frames (ephemeral): {status?.session.frames_in_memory ?? 0}</div>
+            <div>Panic stop: {status?.session.panic_hotkey ?? 'Cmd+Shift+.'}</div>
+            <div>Vision: {status?.session.vision_state ?? 'idle'}</div>
+            <div>Vision queue: {status?.session.vision_queue_depth ?? 0}</div>
+            <div>
+              Last vision:{' '}
+              {status?.session.last_vision_at_ms
+                ? new Date(status.session.last_vision_at_ms).toLocaleTimeString()
+                : 'n/a'}
+            </div>
+          </div>
+
+          <div className="flex gap-2">
+            <button
+              type="button"
+              onClick={() =>
+                void dispatch(
+                  startAccessibilitySession({
+                    consent: true,
+                    ttl_secs: status?.config.session_ttl_secs ?? 300,
+                    screen_monitoring: screenMonitoring,
+                    device_control: deviceControl,
+                    predictive_input: predictiveInput,
+                  })
+                )
+              }
+              disabled={startDisabled}
+              className="rounded-lg border border-green-500/60 bg-green-500/20 px-3 py-2 text-sm text-green-200 disabled:opacity-50">
+              {isStartingSession ? 'Starting…' : 'Start Session'}
+            </button>
+            <button
+              type="button"
+              onClick={() => void dispatch(stopAccessibilitySession('manual_stop'))}
+              disabled={stopDisabled}
+              className="rounded-lg border border-red-500/60 bg-red-500/20 px-3 py-2 text-sm text-red-200 disabled:opacity-50">
+              {isStoppingSession ? 'Stopping…' : 'Stop Session'}
+            </button>
+            <button
+              type="button"
+              onClick={() => void dispatch(flushAccessibilityVision())}
+              disabled={isFlushingVision || !status?.session.active}
+              className="rounded-lg border border-cyan-500/60 bg-cyan-500/20 px-3 py-2 text-sm text-cyan-200 disabled:opacity-50">
+              {isFlushingVision ? 'Analyzing…' : 'Analyze Now'}
+            </button>
+          </div>
+        </section>
+
+        <section className="rounded-2xl border border-stone-700 bg-black/30 p-4 space-y-3">
+          <div className="flex items-center justify-between">
+            <h3 className="text-sm font-semibold text-white">Vision Summaries</h3>
+            <button
+              type="button"
+              onClick={() => void dispatch(fetchAccessibilityVisionRecent(10))}
+              disabled={isLoadingVision}
+              className="rounded-lg border border-stone-600 bg-stone-800/60 px-3 py-1.5 text-xs text-stone-200 disabled:opacity-50">
+              {isLoadingVision ? 'Refreshing…' : 'Refresh'}
+            </button>
+          </div>
+
+          {recentVisionSummaries.length === 0 ? (
+            <div className="text-xs text-stone-400">No summaries yet.</div>
+          ) : (
+            <div className="space-y-2">
+              {recentVisionSummaries.map(summary => (
+                <div
+                  key={summary.id}
+                  className="rounded-xl border border-stone-700 bg-stone-900/50 p-3 text-xs text-stone-200">
+                  <div className="text-stone-400">
+                    {new Date(summary.captured_at_ms).toLocaleTimeString()} ·{' '}
+                    {summary.app_name ?? 'Unknown App'}
+                    {summary.window_title ? ` · ${summary.window_title}` : ''}
+                  </div>
+                  <div className="mt-1 text-stone-100">{summary.actionable_notes}</div>
+                </div>
+              ))}
+            </div>
+          )}
+        </section>
+
+        {!status?.platform_supported && (
+          <div className="rounded-xl border border-amber-700/40 bg-amber-900/20 p-3 text-sm text-amber-200">
+            Screen Intelligence V1 is currently supported on macOS only.
+          </div>
+        )}
+
+        {lastError && (
+          <div className="rounded-xl border border-red-700/40 bg-red-900/20 p-3 text-sm text-red-200">
+            {lastError}
+          </div>
+        )}
+      </div>
+    </div>
+  );
+};
+
+export default ScreenIntelligencePanel;
@@ -38,11 +38,15 @@ const status: AccessibilityStatus = {
    last_vision_summary: null,
  },
  config: {
+    enabled: true,
    capture_policy: 'hybrid',
+    policy_mode: 'all_except_blacklist',
    baseline_fps: 1,
+    vision_enabled: true,
    session_ttl_secs: 300,
    panic_stop_hotkey: 'Cmd+Shift+.',
    autocomplete_enabled: true,
+    allowlist: [],
    denylist: ['wallet'],
  },
  denylist: ['wallet'],
@@ -13,6 +13,7 @@ import MemoryDebugPanel from '../components/settings/panels/MemoryDebugPanel';
 import MessagingPanel from '../components/settings/panels/MessagingPanel';
 import PrivacyPanel from '../components/settings/panels/PrivacyPanel';
 import ProfilePanel from '../components/settings/panels/ProfilePanel';
+import ScreenIntelligencePanel from '../components/settings/panels/ScreenIntelligencePanel';
 import SkillsPanel from '../components/settings/panels/SkillsPanel';
 import TauriCommandsPanel from '../components/settings/panels/TauriCommandsPanel';
 import TeamInvitesPanel from '../components/settings/panels/TeamInvitesPanel';
@@ -29,6 +30,7 @@ const Settings = () => {
        <Route path="connections" element={<ConnectionsPanel />} />
        <Route path="messaging" element={<MessagingPanel />} />
        <Route path="cron-jobs" element={<CronJobsPanel />} />
+        <Route path="screen-intelligence" element={<ScreenIntelligencePanel />} />
        <Route path="privacy" element={<PrivacyPanel />} />
        <Route path="profile" element={<ProfilePanel />} />
        <Route path="advanced" element={<AdvancedPanel />} />
@@ -35,11 +35,15 @@ const sampleStatus: AccessibilityStatus = {
    last_vision_summary: null,
  },
  config: {
+    enabled: true,
    capture_policy: 'hybrid',
+    policy_mode: 'all_except_blacklist',
    baseline_fps: 1,
+    vision_enabled: true,
    session_ttl_secs: 300,
    panic_stop_hotkey: 'Cmd+Shift+.',
    autocomplete_enabled: true,
+    allowlist: [],
    denylist: ['wallet'],
  },
  denylist: ['wallet'],
@@ -465,11 +465,15 @@ export interface AccessibilitySessionStatus {
 }

 export interface AccessibilityConfig {
+  enabled: boolean;
  capture_policy: string;
+  policy_mode: 'all_except_blacklist' | 'whitelist_only' | string;
  baseline_fps: number;
+  vision_enabled: boolean;
  session_ttl_secs: number;
  panic_stop_hotkey: string;
  autocomplete_enabled: boolean;
+  allowlist: string[];
  denylist: string[];
 }

@@ -604,6 +608,17 @@ export interface BrowserSettingsUpdate {
  enabled?: boolean | null;
 }

+export interface ScreenIntelligenceSettingsUpdate {
+  enabled?: boolean | null;
+  capture_policy?: string | null;
+  policy_mode?: 'all_except_blacklist' | 'whitelist_only' | null;
+  baseline_fps?: number | null;
+  vision_enabled?: boolean | null;
+  autocomplete_enabled?: boolean | null;
+  allowlist?: string[] | null;
+  denylist?: string[] | null;
+}
+
 export interface RuntimeFlags {
  browser_allow_all: boolean;
  log_prompts: boolean;
@@ -825,6 +840,15 @@ export async function openhumanUpdateBrowserSettings(
  return await invoke('openhuman_update_browser_settings', { update });
 }

+export async function openhumanUpdateScreenIntelligenceSettings(
+  update: ScreenIntelligenceSettingsUpdate
+): Promise<CommandResponse<ConfigSnapshot>> {
+  if (!isTauri()) {
+    throw new Error('Not running in Tauri');
+  }
+  return await invoke('openhuman_update_screen_intelligence_settings', { update });
+}
+
 export async function openhumanGetRuntimeFlags(): Promise<CommandResponse<RuntimeFlags>> {
  if (!isTauri()) {
    throw new Error('Not running in Tauri');
@@ -1,4 +1,3 @@
-/* eslint-disable */
 // @ts-nocheck
 import { waitForApp, waitForAppReady } from '../helpers/app-helpers';
 import { triggerAuthDeepLink } from '../helpers/deep-link-helpers';