diff --git a/CLAUDE.md b/CLAUDE.md index 427e0f6f4..e2f279968 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -484,6 +484,8 @@ Key updates from recent commits (cd9ebcd to current): 4. `cargo fmt --manifest-path src-tauri/Cargo.toml` (Rust formatting, if Rust files were changed) 5. `cargo check --manifest-path src-tauri/Cargo.toml` (Rust compilation, if Rust files were changed) - **Code Quality**: ESLint and Prettier enforce code standards with Husky hooks. Use type-only imports (`import type`) and consolidate imports from same modules. +- **Frontend scope**: The frontend (`src/`) is primarily a REST API client to `rust-core/`; Rust code in `src-tauri/` manages the core process lifecycle and bridge surface. +- **Core feature exposure**: Any new feature added in `rust-core/` must be exposed through both the CLI and a REST API so it can be integrated into the UI without backend rewrites. - **No dynamic imports**: All imports must be static `import` statements at the top of the file. Do not use `await import()` or `import().then()` inside functions or code blocks. Use try/catch around Tauri API calls for non-Tauri environments instead. - **No localStorage**: Avoid `localStorage` and `sessionStorage`; use Redux (and persist) for app state. Remove any direct usage when working on affected code. - **AI System Integration**: Use `src/lib/ai/` for memory management, constitution loading, entity queries, and session capture. AI providers abstracted through interface pattern. diff --git a/rust-core/src/core_server.rs b/rust-core/src/core_server.rs index b813c9da7..cd66ccbea 100644 --- a/rust-core/src/core_server.rs +++ b/rust-core/src/core_server.rs @@ -24,11 +24,11 @@ use crate::openhuman::local_ai::{ use crate::openhuman::security::{SecretStore, SecurityPolicy}; use crate::openhuman::tools::{ScreenshotTool, Tool}; use crate::openhuman::{ - accessibility, doctor, hardware, integrations, migration, onboard, service, + doctor, hardware, integrations, migration, onboard, screen_intelligence, service, }; use chrono::Utc; -pub use crate::openhuman::accessibility::{ +pub use crate::openhuman::screen_intelligence::{ AccessibilityStatus, AutocompleteCommitParams, AutocompleteCommitResult, AutocompleteSuggestParams, AutocompleteSuggestResult, CaptureImageRefResult, CaptureNowResult, InputActionParams, InputActionResult, PermissionRequestParams, PermissionState, @@ -126,6 +126,18 @@ pub struct BrowserSettingsUpdate { pub enabled: Option, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScreenIntelligenceSettingsUpdate { + pub enabled: Option, + pub capture_policy: Option, + pub policy_mode: Option, + pub baseline_fps: Option, + pub vision_enabled: Option, + pub autocomplete_enabled: Option, + pub allowlist: Option>, + pub denylist: Option>, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RuntimeFlags { pub browser_allow_all: bool, @@ -455,6 +467,50 @@ async fn dispatch( )) } + "openhuman.update_screen_intelligence_settings" => { + let update: ScreenIntelligenceSettingsUpdate = parse_params(params)?; + let mut config = load_openhuman_config().await?; + + if let Some(enabled) = update.enabled { + config.screen_intelligence.enabled = enabled; + } + if let Some(capture_policy) = update.capture_policy { + config.screen_intelligence.capture_policy = capture_policy; + } + if let Some(policy_mode) = update.policy_mode { + config.screen_intelligence.policy_mode = policy_mode; + } + if let Some(baseline_fps) = update.baseline_fps { + config.screen_intelligence.baseline_fps = baseline_fps.clamp(0.2, 30.0); + } + if let Some(vision_enabled) = update.vision_enabled { + config.screen_intelligence.vision_enabled = vision_enabled; + } + if let Some(autocomplete_enabled) = update.autocomplete_enabled { + config.screen_intelligence.autocomplete_enabled = autocomplete_enabled; + } + if let Some(allowlist) = update.allowlist { + config.screen_intelligence.allowlist = allowlist; + } + if let Some(denylist) = update.denylist { + config.screen_intelligence.denylist = denylist; + } + + config.save().await.map_err(|e| e.to_string())?; + let _ = screen_intelligence::global_engine() + .apply_config(config.screen_intelligence.clone()) + .await; + + let snapshot = snapshot_config(&config)?; + to_json_value(command_response( + snapshot, + vec![format!( + "screen intelligence settings saved to {}", + config.config_path.display() + )], + )) + } + "openhuman.update_gateway_settings" => { let update: GatewaySettingsUpdate = parse_params(params)?; let mut config = load_openhuman_config().await?; @@ -1019,15 +1075,22 @@ async fn dispatch( } "openhuman.accessibility_status" => { - let status = accessibility::global_engine().status().await; + if let Ok(config) = load_openhuman_config().await { + let _ = screen_intelligence::global_engine() + .apply_config(config.screen_intelligence.clone()) + .await; + } + let status = screen_intelligence::global_engine().status().await; to_json_value(command_response( status, - vec!["accessibility status fetched".to_string()], + vec!["screen intelligence status fetched".to_string()], )) } "openhuman.accessibility_request_permissions" => { - let permissions = accessibility::global_engine().request_permissions().await?; + let permissions = screen_intelligence::global_engine() + .request_permissions() + .await?; to_json_value(command_response( permissions, vec!["accessibility permissions requested".to_string()], @@ -1036,7 +1099,7 @@ async fn dispatch( "openhuman.accessibility_request_permission" => { let payload: PermissionRequestParams = parse_params(params)?; - let permissions = accessibility::global_engine() + let permissions = screen_intelligence::global_engine() .request_permission(payload.permission) .await?; to_json_value(command_response( @@ -1046,29 +1109,27 @@ async fn dispatch( } "openhuman.accessibility_start_session" => { - let payload: StartSessionParams = parse_params(params)?; - let session = accessibility::global_engine() - .start_session(payload) - .await?; + let _payload: StartSessionParams = parse_params(params)?; + let session = screen_intelligence::global_engine().enable().await?; to_json_value(command_response( session, - vec!["accessibility session started".to_string()], + vec!["screen intelligence enabled".to_string()], )) } "openhuman.accessibility_stop_session" => { let payload: StopSessionParams = parse_params(params)?; - let session = accessibility::global_engine() - .stop_session(payload.reason) + let session = screen_intelligence::global_engine() + .disable(payload.reason) .await; to_json_value(command_response( session, - vec!["accessibility session stopped".to_string()], + vec!["screen intelligence stopped".to_string()], )) } "openhuman.accessibility_capture_now" => { - let result = accessibility::global_engine().capture_now().await?; + let result = screen_intelligence::global_engine().capture_now().await?; to_json_value(command_response( result, vec!["accessibility manual capture requested".to_string()], @@ -1076,8 +1137,9 @@ async fn dispatch( } "openhuman.accessibility_capture_image_ref" => { - let result: CaptureImageRefResult = - accessibility::global_engine().capture_image_ref_test().await; + let result: CaptureImageRefResult = screen_intelligence::global_engine() + .capture_image_ref_test() + .await; to_json_value(command_response( result, vec!["accessibility direct image_ref capture requested".to_string()], @@ -1086,51 +1148,54 @@ async fn dispatch( "openhuman.accessibility_input_action" => { let payload: InputActionParams = parse_params(params)?; - let result = accessibility::global_engine().input_action(payload).await?; + let result = screen_intelligence::global_engine() + .input_action(payload) + .await?; to_json_value(command_response( result, - vec!["accessibility input action processed".to_string()], + vec!["screen intelligence input action processed".to_string()], )) } "openhuman.accessibility_autocomplete_suggest" => { let payload: AutocompleteSuggestParams = parse_params(params)?; - let result = accessibility::global_engine() + let result = screen_intelligence::global_engine() .autocomplete_suggest(payload) .await?; to_json_value(command_response( result, - vec!["accessibility autocomplete suggestions generated".to_string()], + vec!["screen intelligence autocomplete suggestions generated".to_string()], )) } "openhuman.accessibility_autocomplete_commit" => { let payload: AutocompleteCommitParams = parse_params(params)?; - let result = accessibility::global_engine() + let result = screen_intelligence::global_engine() .autocomplete_commit(payload) .await?; to_json_value(command_response( result, - vec!["accessibility autocomplete suggestion committed".to_string()], + vec!["screen intelligence autocomplete suggestion committed".to_string()], )) } "openhuman.accessibility_vision_recent" => { let payload: AccessibilityVisionRecentParams = parse_params(params)?; - let result: VisionRecentResult = accessibility::global_engine() + let result: VisionRecentResult = screen_intelligence::global_engine() .vision_recent(payload.limit) .await; to_json_value(command_response( result, - vec!["accessibility vision summaries fetched".to_string()], + vec!["screen intelligence vision summaries fetched".to_string()], )) } "openhuman.accessibility_vision_flush" => { - let result: VisionFlushResult = accessibility::global_engine().vision_flush().await?; + let result: VisionFlushResult = + screen_intelligence::global_engine().vision_flush().await?; to_json_value(command_response( result, - vec!["accessibility vision flush completed".to_string()], + vec!["screen intelligence vision flush completed".to_string()], )) } @@ -1418,9 +1483,9 @@ enum AccessibilityCommand { RequestPermissions, /// Request a specific permission kind RequestPermission(RequestPermissionArgs), - /// Start a bounded accessibility session + /// Start a bounded screen intelligence session StartSession(StartSessionCliArgs), - /// Stop the active accessibility session + /// Stop the active screen intelligence session StopSession(StopSessionCliArgs), /// Force an immediate capture sample CaptureNow, @@ -1572,13 +1637,12 @@ fn ensure_non_empty_payload(payload: &serde_json::Map } fn extract_data_url(raw: &str) -> Option { - raw.lines() - .find_map(|line| { - let trimmed = line.trim(); - trimmed - .starts_with("data:image/") - .then(|| trimmed.to_string()) - }) + raw.lines().find_map(|line| { + let trimmed = line.trim(); + trimmed + .starts_with("data:image/") + .then(|| trimmed.to_string()) + }) } fn extract_saved_path(raw: &str) -> Option { @@ -1677,8 +1741,10 @@ async fn execute_tools_screenshot_ref( args: ToolsScreenshotRefArgs, ) -> Result { let raw = call_method("openhuman.accessibility_capture_image_ref", json!({})).await?; - let payload: CommandResponse = serde_json::from_value(raw) - .map_err(|e| format!("failed to decode accessibility capture_image_ref response: {e}"))?; + let payload: CommandResponse = + serde_json::from_value(raw).map_err(|e| { + format!("failed to decode screen intelligence capture_image_ref response: {e}") + })?; let mut logs = payload.logs; logs.push("tools.screenshot-ref executed".to_string()); @@ -1693,7 +1759,9 @@ async fn execute_tools_screenshot_ref( output_path.display() )); } else { - return Err("accessibility capture_image_ref did not return image_ref".to_string()); + return Err( + "screen intelligence capture_image_ref did not return image_ref".to_string(), + ); } } @@ -1729,11 +1797,26 @@ fn settings_view_response( "default_model": cfg.get("default_model"), "default_temperature": cfg.get("default_temperature"), }), - "memory" => cfg.get("memory").cloned().unwrap_or(serde_json::Value::Null), - "gateway" => cfg.get("gateway").cloned().unwrap_or(serde_json::Value::Null), - "tunnel" => cfg.get("tunnel").cloned().unwrap_or(serde_json::Value::Null), - "runtime" => cfg.get("runtime").cloned().unwrap_or(serde_json::Value::Null), - "browser" => cfg.get("browser").cloned().unwrap_or(serde_json::Value::Null), + "memory" => cfg + .get("memory") + .cloned() + .unwrap_or(serde_json::Value::Null), + "gateway" => cfg + .get("gateway") + .cloned() + .unwrap_or(serde_json::Value::Null), + "tunnel" => cfg + .get("tunnel") + .cloned() + .unwrap_or(serde_json::Value::Null), + "runtime" => cfg + .get("runtime") + .cloned() + .unwrap_or(serde_json::Value::Null), + "browser" => cfg + .get("browser") + .cloned() + .unwrap_or(serde_json::Value::Null), _ => serde_json::Value::Null, }; @@ -1761,7 +1844,9 @@ async fn execute_core_cli(cli: CoreCli) -> Result { CoreCommand::SecurityPolicy => { call_method("openhuman.security_policy_info", json!({})).await } - CoreCommand::Call { method, params } => call_method(&method, parse_json_arg(¶ms)?).await, + CoreCommand::Call { method, params } => { + call_method(&method, parse_json_arg(¶ms)?).await + } CoreCommand::Completions { shell } => { let mut cmd = CoreCli::command(); let bin_name = cmd.get_name().to_string(); @@ -1793,8 +1878,11 @@ async fn execute_core_cli(cli: CoreCli) -> Result { payload.insert("default_temperature".to_string(), json!(v)); } ensure_non_empty_payload(&payload).map_err(|e| e.to_string())?; - call_method("openhuman.update_model_settings", serde_json::Value::Object(payload)) - .await + call_method( + "openhuman.update_model_settings", + serde_json::Value::Object(payload), + ) + .await } }, SettingsCommand::Memory { command } => match command { @@ -1863,7 +1951,11 @@ async fn execute_core_cli(cli: CoreCli) -> Result { .map_err(|e| e.to_string()) } TunnelSettingsCommand::Set(args) => { - call_method("openhuman.update_tunnel_settings", parse_json_arg(&args.json)?).await + call_method( + "openhuman.update_tunnel_settings", + parse_json_arg(&args.json)?, + ) + .await } }, SettingsCommand::Runtime { command } => match command { @@ -1915,19 +2007,19 @@ async fn execute_core_cli(cli: CoreCli) -> Result { AccessibilityCommand::Doctor => { let raw = call_method("openhuman.accessibility_status", json!({})).await?; let payload: CommandResponse = serde_json::from_value(raw) - .map_err(|e| format!("failed to decode accessibility status: {e}"))?; + .map_err(|e| format!("failed to decode screen intelligence status: {e}"))?; let permissions = &payload.result.permissions; let screen_ready = permissions.screen_recording == PermissionState::Granted; let control_ready = permissions.accessibility == PermissionState::Granted; let monitoring_ready = permissions.input_monitoring == PermissionState::Granted; - let overall_ready = payload.result.platform_supported && screen_ready && control_ready; + let overall_ready = + payload.result.platform_supported && screen_ready && control_ready; let mut recommendations: Vec = Vec::new(); if !payload.result.platform_supported { recommendations.push( - "Accessibility automation is macOS-only in this build/runtime." - .to_string(), + "Accessibility automation is macOS-only in this build/runtime.".to_string(), ); } if permissions.screen_recording != PermissionState::Granted { @@ -1949,7 +2041,8 @@ async fn execute_core_cli(cli: CoreCli) -> Result { ); } if recommendations.is_empty() { - recommendations.push("No action required. Accessibility automation is ready.".to_string()); + recommendations + .push("No action required. Accessibility automation is ready.".to_string()); } Ok(json!({ @@ -2026,7 +2119,7 @@ async fn execute_core_cli(cli: CoreCli) -> Result { }, { "name": "screenshot-ref", - "description": "Capture data URL from accessibility capture_image_ref." + "description": "Capture data URL from screen intelligence capture_image_ref." } ] }, @@ -2107,8 +2200,7 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> { let mut argv = Vec::with_capacity(args.len() + 1); argv.push("openhuman-core".to_string()); argv.extend(args.iter().cloned()); - let cli = - CoreCli::try_parse_from(argv).map_err(|e| anyhow::anyhow!(e.render().to_string()))?; + let cli = CoreCli::try_parse_from(argv).map_err(|e| anyhow::anyhow!(e.render().to_string()))?; let thread_stack_size = std::env::var("OPENHUMAN_CORE_THREAD_STACK_SIZE") .ok() @@ -2119,7 +2211,9 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> { .thread_stack_size(thread_stack_size) .enable_all() .build()?; - let output = runtime.block_on(execute_core_cli(cli)).map_err(anyhow::Error::msg)?; + let output = runtime + .block_on(execute_core_cli(cli)) + .map_err(anyhow::Error::msg)?; if !output.is_null() { println!( "{}", diff --git a/rust-core/src/openhuman/accessibility/mod.rs b/rust-core/src/openhuman/accessibility/mod.rs index 32fe03cc3..18cd8bb50 100644 --- a/rust-core/src/openhuman/accessibility/mod.rs +++ b/rust-core/src/openhuman/accessibility/mod.rs @@ -1,1474 +1 @@ -use crate::openhuman::config::AccessibilityAutomationConfig; -use crate::openhuman::config::Config; -use crate::openhuman::local_ai; -use crate::openhuman::memory::{self, MemoryCategory}; -use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; -use chrono::Utc; -use once_cell::sync::Lazy; -use serde::{Deserialize, Serialize}; -use std::collections::VecDeque; -#[cfg(target_os = "macos")] -use std::ffi::c_void; -use std::sync::Arc; -use tokio::sync::Mutex; -use tokio::task::JoinHandle; -use tokio::time::{self, Duration}; -use uuid::Uuid; - -const MAX_EPHEMERAL_FRAMES: usize = 120; -const MAX_EPHEMERAL_VISION_SUMMARIES: usize = 120; -const MAX_SCREENSHOT_BYTES: usize = 1_500_000; -const MAX_CONTEXT_CHARS: usize = 256; -const MAX_SUGGESTION_CHARS: usize = 128; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum PermissionState { - Granted, - Denied, - Unknown, - Unsupported, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PermissionStatus { - pub screen_recording: PermissionState, - pub accessibility: PermissionState, - pub input_monitoring: PermissionState, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum PermissionKind { - ScreenRecording, - Accessibility, - InputMonitoring, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AccessibilityFeatures { - pub screen_monitoring: bool, - pub device_control: bool, - pub predictive_input: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SessionStatus { - pub active: bool, - pub started_at_ms: Option, - pub expires_at_ms: Option, - pub remaining_ms: Option, - pub ttl_secs: u64, - pub panic_hotkey: String, - pub stop_reason: Option, - pub frames_in_memory: usize, - pub last_capture_at_ms: Option, - pub last_context: Option, - pub vision_enabled: bool, - pub vision_state: String, - pub vision_queue_depth: usize, - pub last_vision_at_ms: Option, - pub last_vision_summary: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AccessibilityHealth { - pub last_error: Option, - pub last_event: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AccessibilityStatus { - pub platform_supported: bool, - pub permissions: PermissionStatus, - pub features: AccessibilityFeatures, - pub session: SessionStatus, - pub config: AccessibilityAutomationConfig, - pub denylist: Vec, - pub is_context_blocked: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StartSessionParams { - pub consent: bool, - pub ttl_secs: Option, - pub screen_monitoring: Option, - pub device_control: Option, - pub predictive_input: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PermissionRequestParams { - pub permission: PermissionKind, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StopSessionParams { - pub reason: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CaptureFrame { - pub captured_at_ms: i64, - pub reason: String, - pub app_name: Option, - pub window_title: Option, - pub image_ref: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CaptureNowResult { - pub accepted: bool, - pub frame: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CaptureImageRefResult { - pub ok: bool, - pub image_ref: Option, - pub mime_type: String, - pub bytes_estimate: Option, - pub message: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VisionSummary { - pub id: String, - pub captured_at_ms: i64, - pub app_name: Option, - pub window_title: Option, - pub ui_state: String, - pub key_text: String, - pub actionable_notes: String, - pub confidence: f32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VisionRecentResult { - pub summaries: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VisionFlushResult { - pub accepted: bool, - pub summary: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InputActionParams { - pub action: String, - pub x: Option, - pub y: Option, - pub button: Option, - pub text: Option, - pub key: Option, - pub modifiers: Option>, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InputActionResult { - pub accepted: bool, - pub blocked: bool, - pub reason: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteSuggestParams { - pub context: Option, - pub max_results: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteSuggestion { - pub value: String, - pub confidence: f32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteSuggestResult { - pub suggestions: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteCommitParams { - pub suggestion: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteCommitResult { - pub committed: bool, -} - -#[derive(Debug, Clone)] -struct AppContext { - app_name: Option, - window_title: Option, -} - -impl AppContext { - fn same_as(&self, other: &AppContext) -> bool { - self.app_name == other.app_name && self.window_title == other.window_title - } - - fn as_compound_text(&self) -> String { - format!( - "{} {}", - self.app_name.clone().unwrap_or_default(), - self.window_title.clone().unwrap_or_default() - ) - .to_lowercase() - } -} - -struct SessionRuntime { - started_at_ms: i64, - expires_at_ms: i64, - ttl_secs: u64, - panic_hotkey: String, - stop_reason: Option, - last_capture_at_ms: Option, - frames: VecDeque, - last_context: Option, - task: Option>, - vision_enabled: bool, - vision_state: String, - vision_queue_depth: usize, - last_vision_at_ms: Option, - last_vision_summary: Option, - vision_summaries: VecDeque, - vision_task: Option>, - vision_tx: Option>, -} - -struct EngineState { - config: AccessibilityAutomationConfig, - permissions: PermissionStatus, - features: AccessibilityFeatures, - session: Option, - last_error: Option, - last_event: Option, - autocomplete_context: String, -} - -impl EngineState { - fn new(config: AccessibilityAutomationConfig) -> Self { - Self { - permissions: PermissionStatus { - screen_recording: PermissionState::Unknown, - accessibility: PermissionState::Unknown, - input_monitoring: PermissionState::Unknown, - }, - features: AccessibilityFeatures { - screen_monitoring: true, - device_control: true, - predictive_input: config.autocomplete_enabled, - }, - config, - session: None, - last_error: None, - last_event: None, - autocomplete_context: String::new(), - } - } -} - -pub struct AccessibilityEngine { - inner: Mutex, -} - -static ACCESSIBILITY_ENGINE: Lazy> = Lazy::new(|| { - Arc::new(AccessibilityEngine { - inner: Mutex::new(EngineState::new(AccessibilityAutomationConfig::default())), - }) -}); - -pub fn global_engine() -> Arc { - ACCESSIBILITY_ENGINE.clone() -} - -impl AccessibilityEngine { - pub async fn status(&self) -> AccessibilityStatus { - let mut state = self.inner.lock().await; - state.permissions = detect_permissions(); - - let context = foreground_context(); - let blocked = context - .as_ref() - .map(|ctx| self.is_context_blocked_by(ctx, &state.config.denylist)) - .unwrap_or(false); - - let (session, denylist, config, permissions, features) = { - let now = now_ms(); - let session = match &state.session { - Some(session) => SessionStatus { - active: true, - started_at_ms: Some(session.started_at_ms), - expires_at_ms: Some(session.expires_at_ms), - remaining_ms: Some((session.expires_at_ms - now).max(0)), - ttl_secs: session.ttl_secs, - panic_hotkey: session.panic_hotkey.clone(), - stop_reason: session.stop_reason.clone(), - frames_in_memory: session.frames.len(), - last_capture_at_ms: session.last_capture_at_ms, - last_context: session - .last_context - .as_ref() - .and_then(|c| c.app_name.clone()), - vision_enabled: session.vision_enabled, - vision_state: session.vision_state.clone(), - vision_queue_depth: session.vision_queue_depth, - last_vision_at_ms: session.last_vision_at_ms, - last_vision_summary: session.last_vision_summary.clone(), - }, - None => SessionStatus { - active: false, - started_at_ms: None, - expires_at_ms: None, - remaining_ms: None, - ttl_secs: state.config.session_ttl_secs, - panic_hotkey: state.config.panic_stop_hotkey.clone(), - stop_reason: None, - frames_in_memory: 0, - last_capture_at_ms: None, - last_context: None, - vision_enabled: true, - vision_state: "idle".to_string(), - vision_queue_depth: 0, - last_vision_at_ms: None, - last_vision_summary: None, - }, - }; - - ( - session, - state.config.denylist.clone(), - state.config.clone(), - state.permissions.clone(), - state.features.clone(), - ) - }; - - AccessibilityStatus { - platform_supported: cfg!(target_os = "macos"), - permissions, - features, - session, - config, - denylist, - is_context_blocked: blocked, - } - } - - pub async fn request_permissions(&self) -> Result { - if !cfg!(target_os = "macos") { - return Ok(PermissionStatus { - screen_recording: PermissionState::Unsupported, - accessibility: PermissionState::Unsupported, - input_monitoring: PermissionState::Unsupported, - }); - } - - self.request_permission(PermissionKind::ScreenRecording) - .await?; - self.request_permission(PermissionKind::Accessibility) - .await?; - self.request_permission(PermissionKind::InputMonitoring) - .await?; - - let mut state = self.inner.lock().await; - state.permissions = detect_permissions(); - state.last_event = Some("permissions_requested".to_string()); - Ok(state.permissions.clone()) - } - - pub async fn request_permission( - &self, - permission: PermissionKind, - ) -> Result { - if !cfg!(target_os = "macos") { - return Ok(PermissionStatus { - screen_recording: PermissionState::Unsupported, - accessibility: PermissionState::Unsupported, - input_monitoring: PermissionState::Unsupported, - }); - } - - #[cfg(target_os = "macos")] - { - match permission { - PermissionKind::ScreenRecording => { - request_screen_recording_access(); - open_macos_privacy_pane("Privacy_ScreenCapture"); - } - PermissionKind::Accessibility => { - request_accessibility_access(); - open_macos_privacy_pane("Privacy_Accessibility"); - } - PermissionKind::InputMonitoring => { - open_macos_privacy_pane("Privacy_ListenEvent"); - } - } - } - - let mut state = self.inner.lock().await; - state.permissions = detect_permissions(); - state.last_event = Some(format!( - "permission_requested:{}", - permission_to_str(permission) - )); - Ok(state.permissions.clone()) - } - - pub async fn start_session( - self: &Arc, - params: StartSessionParams, - ) -> Result { - if !params.consent { - return Err("explicit consent is required to start accessibility session".to_string()); - } - - if !cfg!(target_os = "macos") { - return Err("accessibility automation is macOS-only in V1".to_string()); - } - - let ttl_secs = params - .ttl_secs - .unwrap_or(AccessibilityAutomationConfig::default().session_ttl_secs) - .clamp(30, 3600); - - { - let mut state = self.inner.lock().await; - if state.session.is_some() { - return Err("session already active".to_string()); - } - - state.permissions = detect_permissions(); - if state.permissions.accessibility != PermissionState::Granted { - return Err("accessibility permission is not granted".to_string()); - } - - let now = now_ms(); - let expires_at_ms = now + (ttl_secs as i64 * 1000); - state.features.screen_monitoring = params.screen_monitoring.unwrap_or(true); - state.features.device_control = params.device_control.unwrap_or(true); - state.features.predictive_input = params - .predictive_input - .unwrap_or(state.config.autocomplete_enabled); - - state.session = Some(SessionRuntime { - started_at_ms: now, - expires_at_ms, - ttl_secs, - panic_hotkey: state.config.panic_stop_hotkey.clone(), - stop_reason: None, - last_capture_at_ms: None, - frames: VecDeque::new(), - last_context: None, - task: None, - vision_enabled: true, - vision_state: "idle".to_string(), - vision_queue_depth: 0, - last_vision_at_ms: None, - last_vision_summary: None, - vision_summaries: VecDeque::new(), - vision_task: None, - vision_tx: None, - }); - state.last_event = Some("session_started".to_string()); - state.last_error = None; - } - - let (vision_tx, vision_rx) = tokio::sync::mpsc::unbounded_channel::(); - let engine = self.clone(); - let handle = tokio::spawn(async move { - engine.run_capture_worker().await; - }); - let vision_engine = self.clone(); - let vision_handle = tokio::spawn(async move { - vision_engine.run_vision_worker(vision_rx).await; - }); - - { - let mut state = self.inner.lock().await; - if let Some(session) = state.session.as_mut() { - session.task = Some(handle); - session.vision_task = Some(vision_handle); - session.vision_tx = Some(vision_tx); - } - } - - Ok(self.status().await.session) - } - - pub async fn stop_session(&self, reason: Option) -> SessionStatus { - self.stop_session_internal(reason.unwrap_or_else(|| "manual_stop".to_string())) - .await; - self.status().await.session - } - - pub async fn capture_now(&self) -> Result { - let mut state = self.inner.lock().await; - let reason = "manual_capture".to_string(); - let context = foreground_context(); - - let Some(session) = state.session.as_mut() else { - return Ok(CaptureNowResult { - accepted: false, - frame: None, - }); - }; - - let frame = CaptureFrame { - captured_at_ms: now_ms(), - reason, - app_name: context.as_ref().and_then(|c| c.app_name.clone()), - window_title: context.as_ref().and_then(|c| c.window_title.clone()), - image_ref: capture_screen_image_ref().ok(), - }; - - push_ephemeral_frame(&mut session.frames, frame.clone()); - session.last_capture_at_ms = Some(frame.captured_at_ms); - session.last_context = context; - if frame.image_ref.is_some() && session.vision_enabled { - if let Some(tx) = session.vision_tx.as_ref() { - if tx.send(frame.clone()).is_ok() { - session.vision_queue_depth = session.vision_queue_depth.saturating_add(1); - } - } - } - state.last_event = Some("capture_now".to_string()); - - Ok(CaptureNowResult { - accepted: true, - frame: Some(frame), - }) - } - - pub async fn capture_image_ref_test(&self) -> CaptureImageRefResult { - match capture_screen_image_ref() { - Ok(image_ref) => { - let bytes_estimate = image_ref - .strip_prefix("data:image/png;base64,") - .map(|payload| payload.len() * 3 / 4); - CaptureImageRefResult { - ok: true, - image_ref: Some(image_ref), - mime_type: "image/png".to_string(), - bytes_estimate, - message: "screen capture completed".to_string(), - } - } - Err(err) => CaptureImageRefResult { - ok: false, - image_ref: None, - mime_type: "image/png".to_string(), - bytes_estimate: None, - message: err, - }, - } - } - - pub async fn input_action( - &self, - action: InputActionParams, - ) -> Result { - let mut state = self.inner.lock().await; - - if action.action == "panic_stop" { - drop(state); - self.stop_session_internal("panic_stop".to_string()).await; - return Ok(InputActionResult { - accepted: true, - blocked: false, - reason: Some("panic stop executed".to_string()), - }); - } - - if state.session.is_none() { - return Ok(InputActionResult { - accepted: false, - blocked: true, - reason: Some("session is not active".to_string()), - }); - } - - if !state.features.device_control { - return Ok(InputActionResult { - accepted: false, - blocked: true, - reason: Some("device control is disabled".to_string()), - }); - } - - let context = foreground_context(); - if let Some(ctx) = &context { - if self.is_context_blocked_by(ctx, &state.config.denylist) { - return Ok(InputActionResult { - accepted: false, - blocked: true, - reason: Some("action blocked by denylisted context".to_string()), - }); - } - } - - validate_input_action(&action)?; - - if let Some(text) = action.text.as_ref() { - if !text.is_empty() { - if !state.autocomplete_context.is_empty() { - state.autocomplete_context.push(' '); - } - state.autocomplete_context.push_str(text); - state.autocomplete_context = - truncate_tail(&state.autocomplete_context, MAX_CONTEXT_CHARS); - } - } - - let action_name = action.action.clone(); - state.last_event = Some(format!("input_action:{action_name}")); - - Ok(InputActionResult { - accepted: true, - blocked: false, - reason: None, - }) - } - - pub async fn autocomplete_suggest( - &self, - params: AutocompleteSuggestParams, - ) -> Result { - let state = self.inner.lock().await; - - if !state.features.predictive_input { - return Ok(AutocompleteSuggestResult { - suggestions: Vec::new(), - }); - } - - let mut context = params.context.unwrap_or_default(); - if context.trim().is_empty() { - context = state.autocomplete_context.clone(); - } - drop(state); - - let max_results = params.max_results.unwrap_or(3).clamp(1, 8); - let suggestions = generate_suggestions(&context, max_results); - - Ok(AutocompleteSuggestResult { suggestions }) - } - - pub async fn autocomplete_commit( - &self, - params: AutocompleteCommitParams, - ) -> Result { - let cleaned = params.suggestion.trim(); - if cleaned.is_empty() { - return Err("suggestion cannot be empty".to_string()); - } - if cleaned.len() > MAX_SUGGESTION_CHARS { - return Err("suggestion exceeds maximum length".to_string()); - } - - let mut state = self.inner.lock().await; - if !state.features.predictive_input { - return Ok(AutocompleteCommitResult { committed: false }); - } - if !state.autocomplete_context.is_empty() { - state.autocomplete_context.push(' '); - } - state.autocomplete_context.push_str(cleaned); - state.autocomplete_context = truncate_tail(&state.autocomplete_context, MAX_CONTEXT_CHARS); - state.last_event = Some("autocomplete_commit".to_string()); - - Ok(AutocompleteCommitResult { committed: true }) - } - - pub async fn vision_recent(&self, limit: Option) -> VisionRecentResult { - let state = self.inner.lock().await; - let max_items = limit.unwrap_or(10).clamp(1, 120); - - let summaries = state - .session - .as_ref() - .map(|session| { - session - .vision_summaries - .iter() - .rev() - .take(max_items) - .cloned() - .collect::>() - }) - .unwrap_or_default(); - - VisionRecentResult { summaries } - } - - pub async fn vision_flush(&self) -> Result { - let candidate = { - let mut state = self.inner.lock().await; - let Some(session) = state.session.as_mut() else { - return Ok(VisionFlushResult { - accepted: false, - summary: None, - }); - }; - - let latest = session - .frames - .iter() - .rev() - .find(|f| f.image_ref.is_some()) - .cloned(); - if let Some(frame) = latest.clone() { - session.vision_state = "queued".to_string(); - session.vision_queue_depth = session.vision_queue_depth.saturating_add(1); - Some(frame) - } else { - None - } - }; - - let Some(frame) = candidate else { - return Ok(VisionFlushResult { - accepted: false, - summary: None, - }); - }; - - let summary = self - .analyze_frame_with_vision(frame) - .await - .map_err(|e| format!("vision flush failed: {e}"))?; - Ok(VisionFlushResult { - accepted: true, - summary: Some(summary), - }) - } - - async fn run_capture_worker(self: Arc) { - let mut tick = time::interval(Duration::from_millis(250)); - - loop { - tick.tick().await; - - let should_stop = { - let state = self.inner.lock().await; - match &state.session { - Some(session) => now_ms() >= session.expires_at_ms, - None => return, - } - }; - if should_stop { - self.stop_session_internal("ttl_expired".to_string()).await; - return; - } - - let context = foreground_context(); - let now = now_ms(); - let mut state = self.inner.lock().await; - let baseline_ms = (1000.0 / state.config.baseline_fps.max(0.2)).round() as i64; - let denylist = state.config.denylist.clone(); - let screen_monitoring = state.features.screen_monitoring; - - let Some(session) = state.session.as_mut() else { - return; - }; - if !screen_monitoring { - continue; - } - - let is_blocked = context - .as_ref() - .map(|ctx| self.is_context_blocked_by(ctx, &denylist)) - .unwrap_or(false); - if is_blocked { - continue; - } - - let context_changed = match (&session.last_context, &context) { - (Some(prev), Some(curr)) => !prev.same_as(curr), - (None, Some(_)) => true, - _ => false, - }; - - let baseline_due = session - .last_capture_at_ms - .map(|last| now - last >= baseline_ms) - .unwrap_or(true); - - if context_changed || baseline_due { - let reason = if context_changed { - "event:foreground_changed" - } else { - "baseline" - }; - - let frame = CaptureFrame { - captured_at_ms: now, - reason: reason.to_string(), - app_name: context.as_ref().and_then(|c| c.app_name.clone()), - window_title: context.as_ref().and_then(|c| c.window_title.clone()), - image_ref: capture_screen_image_ref().ok(), - }; - push_ephemeral_frame(&mut session.frames, frame.clone()); - session.last_capture_at_ms = Some(now); - session.last_context = context; - if frame.image_ref.is_some() && session.vision_enabled { - if let Some(tx) = session.vision_tx.as_ref() { - if tx.send(frame).is_ok() { - session.vision_queue_depth = - session.vision_queue_depth.saturating_add(1); - session.vision_state = "queued".to_string(); - } - } - } - state.last_event = Some(reason.to_string()); - } - } - } - - async fn run_vision_worker( - self: Arc, - mut rx: tokio::sync::mpsc::UnboundedReceiver, - ) { - while let Some(frame) = rx.recv().await { - { - let mut state = self.inner.lock().await; - if let Some(session) = state.session.as_mut() { - session.vision_state = "processing".to_string(); - } else { - break; - } - } - - let result = self.analyze_frame_with_vision(frame).await; - - let mut state = self.inner.lock().await; - let Some(session) = state.session.as_mut() else { - break; - }; - session.vision_queue_depth = session.vision_queue_depth.saturating_sub(1); - match result { - Ok(summary) => { - push_ephemeral_vision_summary(&mut session.vision_summaries, summary.clone()); - session.last_vision_at_ms = Some(summary.captured_at_ms); - session.last_vision_summary = Some(summary.actionable_notes.clone()); - session.vision_state = "ready".to_string(); - let summary_for_store = summary.clone(); - tokio::spawn(async move { - persist_vision_summary(summary_for_store).await; - }); - } - Err(err) => { - session.vision_state = "error".to_string(); - state.last_error = Some(err); - } - } - } - } - - async fn analyze_frame_with_vision( - &self, - frame: CaptureFrame, - ) -> Result { - let image_ref = frame - .image_ref - .clone() - .ok_or_else(|| "frame has no image payload".to_string())?; - let config = Config::load_or_init() - .await - .map_err(|e| format!("failed to load config: {e}"))?; - let service = local_ai::global(&config); - let prompt = "Analyze this UI screenshot. Return strict JSON with keys: ui_state, key_text, actionable_notes, confidence (0..1). Keep actionable_notes concise."; - let raw = service - .vision_prompt(&config, prompt, &[image_ref], Some(180)) - .await?; - Ok(parse_vision_summary_output(frame, &raw)) - } - - async fn stop_session_internal(&self, reason: String) { - let mut state = self.inner.lock().await; - - let Some(mut session) = state.session.take() else { - return; - }; - - session.stop_reason = Some(reason.clone()); - if let Some(task) = session.task.take() { - task.abort(); - } - if let Some(task) = session.vision_task.take() { - task.abort(); - } - session.vision_tx = None; - - state.last_event = Some(format!("session_stopped:{reason}")); - } - - fn is_context_blocked_by(&self, ctx: &AppContext, denylist: &[String]) -> bool { - let compound = ctx.as_compound_text(); - denylist - .iter() - .any(|d| !d.trim().is_empty() && compound.contains(&d.to_lowercase())) - } -} - -fn validate_input_action(action: &InputActionParams) -> Result<(), String> { - match action.action.as_str() { - "mouse_move" | "mouse_click" | "mouse_drag" => { - let x = action - .x - .ok_or_else(|| "x coordinate is required".to_string())?; - let y = action - .y - .ok_or_else(|| "y coordinate is required".to_string())?; - if !(0..=10000).contains(&x) || !(0..=10000).contains(&y) { - return Err("coordinates must be between 0 and 10000".to_string()); - } - } - "key_type" => { - let text = action - .text - .as_ref() - .ok_or_else(|| "text is required for key_type".to_string())?; - if text.is_empty() || text.len() > MAX_CONTEXT_CHARS { - return Err("text length must be between 1 and 256".to_string()); - } - } - "key_press" => { - let key = action - .key - .as_ref() - .ok_or_else(|| "key is required for key_press".to_string())?; - if key.trim().is_empty() { - return Err("key cannot be empty".to_string()); - } - } - other => { - return Err(format!("unsupported input action: {other}")); - } - } - - Ok(()) -} - -fn push_ephemeral_frame(frames: &mut VecDeque, frame: CaptureFrame) { - frames.push_back(frame); - while frames.len() > MAX_EPHEMERAL_FRAMES { - let _ = frames.pop_front(); - } -} - -fn push_ephemeral_vision_summary(summaries: &mut VecDeque, summary: VisionSummary) { - summaries.push_back(summary); - while summaries.len() > MAX_EPHEMERAL_VISION_SUMMARIES { - let _ = summaries.pop_front(); - } -} - -fn parse_vision_summary_output(frame: CaptureFrame, raw: &str) -> VisionSummary { - let fallback = truncate_tail(raw.trim(), 512); - let value = serde_json::from_str::(raw).ok(); - let ui_state = value - .as_ref() - .and_then(|v| v.get("ui_state")) - .and_then(|v| v.as_str()) - .map(str::trim) - .filter(|v| !v.is_empty()) - .unwrap_or("UI state unavailable"); - let key_text = value - .as_ref() - .and_then(|v| v.get("key_text")) - .and_then(|v| v.as_str()) - .map(str::trim) - .unwrap_or(""); - let actionable_notes = value - .as_ref() - .and_then(|v| v.get("actionable_notes")) - .and_then(|v| v.as_str()) - .map(str::trim) - .filter(|v| !v.is_empty()) - .unwrap_or(&fallback); - let confidence = value - .as_ref() - .and_then(|v| v.get("confidence")) - .and_then(|v| v.as_f64()) - .map(|v| v as f32) - .unwrap_or(0.66) - .clamp(0.0, 1.0); - - VisionSummary { - id: format!("vision-{}-{}", frame.captured_at_ms, Uuid::new_v4()), - captured_at_ms: frame.captured_at_ms, - app_name: frame.app_name, - window_title: frame.window_title, - ui_state: truncate_tail(ui_state, 220), - key_text: truncate_tail(key_text, 280), - actionable_notes: truncate_tail(actionable_notes, 560), - confidence, - } -} - -async fn persist_vision_summary(summary: VisionSummary) { - let config = match Config::load_or_init().await { - Ok(cfg) => cfg, - Err(err) => { - tracing::debug!("vision summary persistence skipped: config load failed: {err}"); - return; - } - }; - - let mem = match memory::create_memory_with_storage_and_routes( - &config.memory, - &config.embedding_routes, - Some(&config.storage.provider.config), - &config.workspace_dir, - config.api_key.as_deref(), - ) { - Ok(mem) => mem, - Err(err) => { - tracing::debug!("vision summary persistence skipped: memory init failed: {err}"); - return; - } - }; - - let content = match serde_json::to_string(&summary) { - Ok(content) => content, - Err(err) => { - tracing::debug!("vision summary persistence skipped: serialization failed: {err}"); - return; - } - }; - - let key = format!("accessibility_vision_{}", summary.id); - let _ = mem - .store( - &key, - &content, - MemoryCategory::Custom("accessibility_vision".to_string()), - None, - ) - .await; -} - -fn truncate_tail(text: &str, max_chars: usize) -> String { - let chars: Vec = text.chars().collect(); - if chars.len() <= max_chars { - return text.to_string(); - } - chars[chars.len() - max_chars..].iter().collect() -} - -fn generate_suggestions(context: &str, max_results: usize) -> Vec { - let trimmed = context.trim(); - let lower = trimmed.to_lowercase(); - - let mut out = Vec::new(); - if lower.ends_with("thanks") || lower.ends_with("thank you") { - out.push(AutocompleteSuggestion { - value: " for your help!".to_string(), - confidence: 0.89, - }); - } - if lower.contains("meeting") { - out.push(AutocompleteSuggestion { - value: " tomorrow at 10am works for me.".to_string(), - confidence: 0.84, - }); - } - if lower.contains("ship") || lower.contains("release") { - out.push(AutocompleteSuggestion { - value: " after we pass QA and smoke tests.".to_string(), - confidence: 0.81, - }); - } - - if out.is_empty() { - out.push(AutocompleteSuggestion { - value: " Please share any constraints and I can refine this.".to_string(), - confidence: 0.55, - }); - } - - out.truncate(max_results); - out -} - -fn now_ms() -> i64 { - Utc::now().timestamp_millis() -} - -fn capture_screen_image_ref() -> Result { - #[cfg(target_os = "macos")] - { - let tmp_path = - std::env::temp_dir().join(format!("openhuman_accessibility_{}.png", Uuid::new_v4())); - let status = std::process::Command::new("screencapture") - .arg("-x") - .arg("-t") - .arg("png") - .arg(&tmp_path) - .status() - .map_err(|e| format!("screencapture failed to start: {e}"))?; - if !status.success() { - return Err("screencapture returned non-zero status".to_string()); - } - let bytes = - std::fs::read(&tmp_path).map_err(|e| format!("failed to read screenshot: {e}"))?; - let _ = std::fs::remove_file(&tmp_path); - if bytes.len() > MAX_SCREENSHOT_BYTES { - return Err("captured screenshot exceeds size limit".to_string()); - } - let encoded = BASE64_STANDARD.encode(bytes); - return Ok(format!("data:image/png;base64,{encoded}")); - } - - #[cfg(not(target_os = "macos"))] - { - Err("screen capture is unsupported on this platform".to_string()) - } -} - -#[cfg(target_os = "macos")] -fn foreground_context() -> Option { - let script = r#" - tell application "System Events" - set frontApp to name of first application process whose frontmost is true - set frontWindow to "" - try - tell process frontApp - if (count of windows) > 0 then - set frontWindow to name of front window - end if - end tell - end try - return frontApp & "\n" & frontWindow - end tell - "#; - - let output = std::process::Command::new("osascript") - .arg("-e") - .arg(script) - .output() - .ok()?; - - if !output.status.success() { - return None; - } - - let text = String::from_utf8_lossy(&output.stdout); - let mut lines = text.lines(); - let app = lines.next().map(|s| s.trim().to_string()); - let title = lines.next().map(|s| s.trim().to_string()); - - Some(AppContext { - app_name: app.filter(|s| !s.is_empty()), - window_title: title.filter(|s| !s.is_empty()), - }) -} - -#[cfg(not(target_os = "macos"))] -fn foreground_context() -> Option { - None -} - -#[cfg(target_os = "macos")] -type CFAllocatorRef = *const c_void; -#[cfg(target_os = "macos")] -type CFDictionaryRef = *const c_void; -#[cfg(target_os = "macos")] -type CFBooleanRef = *const c_void; -#[cfg(target_os = "macos")] -type CFStringRef = *const c_void; - -#[cfg(target_os = "macos")] -#[link(name = "ApplicationServices", kind = "framework")] -extern "C" { - fn AXIsProcessTrusted() -> bool; - fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> bool; - static kAXTrustedCheckOptionPrompt: CFStringRef; - fn CGPreflightScreenCaptureAccess() -> bool; - fn CGRequestScreenCaptureAccess() -> bool; -} - -#[cfg(target_os = "macos")] -#[link(name = "CoreFoundation", kind = "framework")] -extern "C" { - static kCFAllocatorDefault: CFAllocatorRef; - static kCFBooleanTrue: CFBooleanRef; - fn CFDictionaryCreate( - allocator: CFAllocatorRef, - keys: *const *const c_void, - values: *const *const c_void, - num_values: isize, - key_callbacks: *const c_void, - value_callbacks: *const c_void, - ) -> CFDictionaryRef; - fn CFRelease(cf: *const c_void); -} - -#[cfg(target_os = "macos")] -#[link(name = "IOKit", kind = "framework")] -extern "C" { - fn IOHIDCheckAccess(request_type: i32) -> isize; -} - -#[cfg(target_os = "macos")] -const IOHID_REQUEST_TYPE_LISTEN_EVENT: i32 = 1; -#[cfg(target_os = "macos")] -const IOHID_ACCESS_GRANTED: isize = 0; -#[cfg(target_os = "macos")] -const IOHID_ACCESS_DENIED: isize = 1; -#[cfg(target_os = "macos")] -const IOHID_ACCESS_UNKNOWN: isize = 2; - -fn permission_to_str(permission: PermissionKind) -> &'static str { - match permission { - PermissionKind::ScreenRecording => "screen_recording", - PermissionKind::Accessibility => "accessibility", - PermissionKind::InputMonitoring => "input_monitoring", - } -} - -#[cfg(target_os = "macos")] -fn open_macos_privacy_pane(pane: &str) { - let url = format!("x-apple.systempreferences:com.apple.preference.security?{pane}"); - let _ = std::process::Command::new("open").arg(url).status(); -} - -#[cfg(target_os = "macos")] -fn request_accessibility_access() { - unsafe { - let keys = [kAXTrustedCheckOptionPrompt as *const c_void]; - let values = [kCFBooleanTrue as *const c_void]; - let options = CFDictionaryCreate( - kCFAllocatorDefault, - keys.as_ptr(), - values.as_ptr(), - 1, - std::ptr::null(), - std::ptr::null(), - ); - let _ = AXIsProcessTrustedWithOptions(options); - if !options.is_null() { - CFRelease(options); - } - } -} - -#[cfg(target_os = "macos")] -fn request_screen_recording_access() { - unsafe { - let _ = CGRequestScreenCaptureAccess(); - } -} - -#[cfg(target_os = "macos")] -fn detect_accessibility_permission() -> PermissionState { - unsafe { - if AXIsProcessTrusted() { - PermissionState::Granted - } else { - PermissionState::Denied - } - } -} - -#[cfg(target_os = "macos")] -fn detect_screen_recording_permission() -> PermissionState { - unsafe { - if CGPreflightScreenCaptureAccess() { - PermissionState::Granted - } else { - PermissionState::Denied - } - } -} - -#[cfg(target_os = "macos")] -fn detect_input_monitoring_permission() -> PermissionState { - let access = unsafe { IOHIDCheckAccess(IOHID_REQUEST_TYPE_LISTEN_EVENT) }; - match access { - IOHID_ACCESS_GRANTED => PermissionState::Granted, - IOHID_ACCESS_DENIED => PermissionState::Denied, - IOHID_ACCESS_UNKNOWN => PermissionState::Unknown, - _ => PermissionState::Unknown, - } -} - -#[cfg(target_os = "macos")] -fn detect_permissions() -> PermissionStatus { - PermissionStatus { - screen_recording: detect_screen_recording_permission(), - accessibility: detect_accessibility_permission(), - input_monitoring: detect_input_monitoring_permission(), - } -} - -#[cfg(not(target_os = "macos"))] -fn detect_permissions() -> PermissionStatus { - PermissionStatus { - screen_recording: PermissionState::Unsupported, - accessibility: PermissionState::Unsupported, - input_monitoring: PermissionState::Unsupported, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn validates_coordinates_and_actions() { - let ok = InputActionParams { - action: "mouse_move".to_string(), - x: Some(10), - y: Some(20), - button: None, - text: None, - key: None, - modifiers: None, - }; - assert!(validate_input_action(&ok).is_ok()); - - let bad = InputActionParams { - action: "mouse_click".to_string(), - x: Some(-1), - y: Some(20), - button: None, - text: None, - key: None, - modifiers: None, - }; - assert!(validate_input_action(&bad).is_err()); - - let unsupported = InputActionParams { - action: "open_portal".to_string(), - x: None, - y: None, - button: None, - text: None, - key: None, - modifiers: None, - }; - assert!(validate_input_action(&unsupported).is_err()); - } - - #[tokio::test] - async fn session_lifecycle_transitions_and_ttl_expiry() { - let engine = Arc::new(AccessibilityEngine { - inner: Mutex::new(EngineState::new(AccessibilityAutomationConfig { - capture_policy: "hybrid".to_string(), - baseline_fps: 8.0, - session_ttl_secs: 1, - panic_stop_hotkey: "Cmd+Shift+.".to_string(), - autocomplete_enabled: true, - denylist: vec!["1password".to_string()], - })), - }); - - let start = engine - .start_session(StartSessionParams { - consent: true, - ttl_secs: Some(1), - screen_monitoring: Some(true), - device_control: Some(true), - predictive_input: Some(true), - }) - .await; - - if cfg!(target_os = "macos") { - if start.is_ok() { - let active = engine.status().await; - assert!(active.session.active); - - time::sleep(Duration::from_millis(1400)).await; - - let ended = engine.status().await; - assert!(!ended.session.active); - } - } else { - assert!(start.is_err()); - } - } - - #[tokio::test] - async fn panic_stop_behavior_stops_session() { - if !cfg!(target_os = "macos") { - return; - } - - let engine = global_engine(); - - let started = engine - .start_session(StartSessionParams { - consent: true, - ttl_secs: Some(60), - screen_monitoring: Some(true), - device_control: Some(true), - predictive_input: Some(true), - }) - .await; - - if started.is_err() { - return; - } - - let result = engine - .input_action(InputActionParams { - action: "panic_stop".to_string(), - x: None, - y: None, - button: None, - text: None, - key: None, - modifiers: None, - }) - .await - .expect("panic action should return"); - - assert!(result.accepted); - assert!(!engine.status().await.session.active); - } - - #[tokio::test] - async fn capture_scheduler_adds_baseline_frames() { - if !cfg!(target_os = "macos") { - return; - } - - let engine = Arc::new(AccessibilityEngine { - inner: Mutex::new(EngineState::new(AccessibilityAutomationConfig { - capture_policy: "hybrid".to_string(), - baseline_fps: 6.0, - session_ttl_secs: 2, - panic_stop_hotkey: "Cmd+Shift+.".to_string(), - autocomplete_enabled: true, - denylist: vec![], - })), - }); - - let started = engine - .start_session(StartSessionParams { - consent: true, - ttl_secs: Some(2), - screen_monitoring: Some(true), - device_control: Some(true), - predictive_input: Some(true), - }) - .await; - - if started.is_err() { - return; - } - - time::sleep(Duration::from_millis(700)).await; - - let status = engine.status().await; - assert!(status.session.frames_in_memory >= 1); - - let _ = engine.stop_session(Some("test_end".to_string())).await; - } -} +pub use crate::openhuman::screen_intelligence::*; diff --git a/rust-core/src/openhuman/config/mod.rs b/rust-core/src/openhuman/config/mod.rs index ff3d4a980..0748f54a9 100644 --- a/rust-core/src/openhuman/config/mod.rs +++ b/rust-core/src/openhuman/config/mod.rs @@ -8,18 +8,17 @@ pub use daemon::DaemonConfig; pub use schema::{ apply_runtime_proxy_to_builder, build_runtime_proxy_client, build_runtime_proxy_client_with_timeouts, runtime_proxy_config, set_runtime_proxy_config, - AccessibilityAutomationConfig, AgentConfig, AuditConfig, AutonomyConfig, - BrowserComputerUseConfig, BrowserConfig, ChannelsConfig, ClassificationRule, - CloudflareTunnelConfig, ComposioConfig, Config, CostConfig, CronConfig, CustomTunnelConfig, - DelegateAgentConfig, DiscordConfig, DockerRuntimeConfig, EmbeddingRouteConfig, GatewayConfig, - HardwareConfig, HardwareTransport, HeartbeatConfig, HttpRequestConfig, IMessageConfig, - IdentityConfig, LarkConfig, LocalAiConfig, MatrixConfig, MemoryConfig, ModelRouteConfig, - MultimodalConfig, NgrokTunnelConfig, ObservabilityConfig, PeripheralBoardConfig, - PeripheralsConfig, ProxyConfig, ProxyScope, QueryClassificationConfig, ReliabilityConfig, - ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig, - SecretsConfig, SecurityConfig, SlackConfig, StorageConfig, StorageProviderConfig, - StorageProviderSection, StreamMode, TailscaleTunnelConfig, TelegramConfig, TunnelConfig, - WebSearchConfig, WebhookConfig, + AgentConfig, AuditConfig, AutonomyConfig, BrowserComputerUseConfig, BrowserConfig, + ChannelsConfig, ClassificationRule, CloudflareTunnelConfig, ComposioConfig, Config, CostConfig, + CronConfig, CustomTunnelConfig, DelegateAgentConfig, DiscordConfig, DockerRuntimeConfig, + EmbeddingRouteConfig, GatewayConfig, HardwareConfig, HardwareTransport, HeartbeatConfig, + HttpRequestConfig, IMessageConfig, IdentityConfig, LarkConfig, LocalAiConfig, MatrixConfig, + MemoryConfig, ModelRouteConfig, MultimodalConfig, NgrokTunnelConfig, ObservabilityConfig, + PeripheralBoardConfig, PeripheralsConfig, ProxyConfig, ProxyScope, QueryClassificationConfig, + ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, + SchedulerConfig, ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, SlackConfig, + StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode, + TailscaleTunnelConfig, TelegramConfig, TunnelConfig, WebSearchConfig, WebhookConfig, }; #[cfg(test)] diff --git a/rust-core/src/openhuman/config/schema/accessibility.rs b/rust-core/src/openhuman/config/schema/accessibility.rs index 447f55fef..367c7c274 100644 --- a/rust-core/src/openhuman/config/schema/accessibility.rs +++ b/rust-core/src/openhuman/config/schema/accessibility.rs @@ -2,11 +2,17 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] -pub struct AccessibilityAutomationConfig { +pub struct ScreenIntelligenceConfig { + #[serde(default = "default_enabled")] + pub enabled: bool, #[serde(default = "default_capture_policy")] pub capture_policy: String, + #[serde(default = "default_policy_mode")] + pub policy_mode: String, #[serde(default = "default_baseline_fps")] pub baseline_fps: f32, + #[serde(default = "default_vision_enabled")] + pub vision_enabled: bool, #[serde(default = "default_session_ttl_secs")] pub session_ttl_secs: u64, #[serde(default = "default_panic_stop_hotkey")] @@ -14,17 +20,31 @@ pub struct AccessibilityAutomationConfig { #[serde(default = "default_autocomplete_enabled")] pub autocomplete_enabled: bool, #[serde(default)] + pub allowlist: Vec, + #[serde(default)] pub denylist: Vec, } +fn default_enabled() -> bool { + false +} + fn default_capture_policy() -> String { "hybrid".to_string() } +fn default_policy_mode() -> String { + "all_except_blacklist".to_string() +} + fn default_baseline_fps() -> f32 { 1.0 } +fn default_vision_enabled() -> bool { + true +} + fn default_session_ttl_secs() -> u64 { 300 } @@ -37,14 +57,18 @@ fn default_autocomplete_enabled() -> bool { true } -impl Default for AccessibilityAutomationConfig { +impl Default for ScreenIntelligenceConfig { fn default() -> Self { Self { + enabled: default_enabled(), capture_policy: default_capture_policy(), + policy_mode: default_policy_mode(), baseline_fps: default_baseline_fps(), + vision_enabled: default_vision_enabled(), session_ttl_secs: default_session_ttl_secs(), panic_stop_hotkey: default_panic_stop_hotkey(), autocomplete_enabled: default_autocomplete_enabled(), + allowlist: vec![], denylist: vec![ "1password".to_string(), "keychain".to_string(), diff --git a/rust-core/src/openhuman/config/schema/mod.rs b/rust-core/src/openhuman/config/schema/mod.rs index 67124fe57..95ba61cef 100644 --- a/rust-core/src/openhuman/config/schema/mod.rs +++ b/rust-core/src/openhuman/config/schema/mod.rs @@ -21,7 +21,7 @@ mod storage_memory; mod tools; mod tunnel; -pub use accessibility::AccessibilityAutomationConfig; +pub use accessibility::ScreenIntelligenceConfig; pub use agent::{AgentConfig, DelegateAgentConfig}; pub use autonomy::AutonomyConfig; pub use channels::{ @@ -88,7 +88,7 @@ pub struct Config { pub runtime: RuntimeConfig, #[serde(default)] - pub accessibility: AccessibilityAutomationConfig, + pub screen_intelligence: ScreenIntelligenceConfig, #[serde(default)] pub reliability: ReliabilityConfig, @@ -186,7 +186,7 @@ impl Default for Config { observability: ObservabilityConfig::default(), autonomy: AutonomyConfig::default(), runtime: RuntimeConfig::default(), - accessibility: AccessibilityAutomationConfig::default(), + screen_intelligence: ScreenIntelligenceConfig::default(), reliability: ReliabilityConfig::default(), scheduler: SchedulerConfig::default(), agent: AgentConfig::default(), diff --git a/rust-core/src/openhuman/mod.rs b/rust-core/src/openhuman/mod.rs index 1832034d5..d9562325a 100644 --- a/rust-core/src/openhuman/mod.rs +++ b/rust-core/src/openhuman/mod.rs @@ -36,6 +36,7 @@ pub mod peripherals; pub mod providers; pub mod rag; pub mod runtime; +pub mod screen_intelligence; pub mod security; pub mod service; pub mod skillforge; diff --git a/rust-core/src/openhuman/screen_intelligence/mod.rs b/rust-core/src/openhuman/screen_intelligence/mod.rs new file mode 100644 index 000000000..d001546e4 --- /dev/null +++ b/rust-core/src/openhuman/screen_intelligence/mod.rs @@ -0,0 +1,1628 @@ +use crate::openhuman::config::Config; +use crate::openhuman::config::ScreenIntelligenceConfig; +use crate::openhuman::local_ai; +use crate::openhuman::memory::{self, MemoryCategory}; +use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; +use chrono::Utc; +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use std::collections::VecDeque; +#[cfg(target_os = "macos")] +use std::ffi::c_void; +use std::sync::Arc; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use tokio::time::{self, Duration}; +use uuid::Uuid; + +const MAX_EPHEMERAL_FRAMES: usize = 120; +const MAX_EPHEMERAL_VISION_SUMMARIES: usize = 120; +const MAX_SCREENSHOT_BYTES: usize = 1_500_000; +const MAX_CONTEXT_CHARS: usize = 256; +const MAX_SUGGESTION_CHARS: usize = 128; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PermissionState { + Granted, + Denied, + Unknown, + Unsupported, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PermissionStatus { + pub screen_recording: PermissionState, + pub accessibility: PermissionState, + pub input_monitoring: PermissionState, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PermissionKind { + ScreenRecording, + Accessibility, + InputMonitoring, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessibilityFeatures { + pub screen_monitoring: bool, + pub device_control: bool, + pub predictive_input: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SessionStatus { + pub active: bool, + pub started_at_ms: Option, + pub expires_at_ms: Option, + pub remaining_ms: Option, + pub ttl_secs: u64, + pub panic_hotkey: String, + pub stop_reason: Option, + pub frames_in_memory: usize, + pub last_capture_at_ms: Option, + pub last_context: Option, + pub vision_enabled: bool, + pub vision_state: String, + pub vision_queue_depth: usize, + pub last_vision_at_ms: Option, + pub last_vision_summary: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessibilityHealth { + pub last_error: Option, + pub last_event: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessibilityStatus { + pub platform_supported: bool, + pub permissions: PermissionStatus, + pub features: AccessibilityFeatures, + pub session: SessionStatus, + pub config: ScreenIntelligenceConfig, + pub denylist: Vec, + pub is_context_blocked: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StartSessionParams { + pub consent: bool, + pub ttl_secs: Option, + pub screen_monitoring: Option, + pub device_control: Option, + pub predictive_input: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PermissionRequestParams { + pub permission: PermissionKind, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StopSessionParams { + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CaptureFrame { + pub captured_at_ms: i64, + pub reason: String, + pub app_name: Option, + pub window_title: Option, + pub image_ref: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CaptureNowResult { + pub accepted: bool, + pub frame: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CaptureImageRefResult { + pub ok: bool, + pub image_ref: Option, + pub mime_type: String, + pub bytes_estimate: Option, + pub message: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VisionSummary { + pub id: String, + pub captured_at_ms: i64, + pub app_name: Option, + pub window_title: Option, + pub ui_state: String, + pub key_text: String, + pub actionable_notes: String, + pub confidence: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VisionRecentResult { + pub summaries: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VisionFlushResult { + pub accepted: bool, + pub summary: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InputActionParams { + pub action: String, + pub x: Option, + pub y: Option, + pub button: Option, + pub text: Option, + pub key: Option, + pub modifiers: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InputActionResult { + pub accepted: bool, + pub blocked: bool, + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSuggestParams { + pub context: Option, + pub max_results: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSuggestion { + pub value: String, + pub confidence: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSuggestResult { + pub suggestions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteCommitParams { + pub suggestion: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteCommitResult { + pub committed: bool, +} + +#[derive(Debug, Clone)] +struct WindowBounds { + x: i32, + y: i32, + width: i32, + height: i32, +} + +#[derive(Debug, Clone)] +struct AppContext { + app_name: Option, + window_title: Option, + bounds: Option, +} + +impl AppContext { + fn same_as(&self, other: &AppContext) -> bool { + self.app_name == other.app_name + && self.window_title == other.window_title + && self.bounds.as_ref().map(|b| (b.x, b.y, b.width, b.height)) + == other.bounds.as_ref().map(|b| (b.x, b.y, b.width, b.height)) + } + + fn as_compound_text(&self) -> String { + format!( + "{} {}", + self.app_name.clone().unwrap_or_default(), + self.window_title.clone().unwrap_or_default() + ) + .to_lowercase() + } +} + +struct SessionRuntime { + started_at_ms: i64, + expires_at_ms: i64, + ttl_secs: u64, + panic_hotkey: String, + stop_reason: Option, + last_capture_at_ms: Option, + frames: VecDeque, + last_context: Option, + task: Option>, + vision_enabled: bool, + vision_state: String, + vision_queue_depth: usize, + last_vision_at_ms: Option, + last_vision_summary: Option, + vision_summaries: VecDeque, + vision_task: Option>, + vision_tx: Option>, +} + +struct EngineState { + config: ScreenIntelligenceConfig, + permissions: PermissionStatus, + features: AccessibilityFeatures, + session: Option, + last_error: Option, + last_event: Option, + autocomplete_context: String, +} + +impl EngineState { + fn new(config: ScreenIntelligenceConfig) -> Self { + Self { + permissions: PermissionStatus { + screen_recording: PermissionState::Unknown, + accessibility: PermissionState::Unknown, + input_monitoring: PermissionState::Unknown, + }, + features: AccessibilityFeatures { + screen_monitoring: true, + device_control: true, + predictive_input: config.autocomplete_enabled, + }, + config, + session: None, + last_error: None, + last_event: None, + autocomplete_context: String::new(), + } + } +} + +pub struct AccessibilityEngine { + inner: Mutex, +} + +static ACCESSIBILITY_ENGINE: Lazy> = Lazy::new(|| { + Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(ScreenIntelligenceConfig::default())), + }) +}); + +pub fn global_engine() -> Arc { + ACCESSIBILITY_ENGINE.clone() +} + +impl AccessibilityEngine { + pub async fn apply_config( + self: &Arc, + config: ScreenIntelligenceConfig, + ) -> Result { + { + let mut state = self.inner.lock().await; + state.config = config.clone(); + state.features.predictive_input = state.config.autocomplete_enabled; + } + + if config.enabled { + let _ = self.enable().await; + } else { + let _ = self.disable(Some("disabled_by_config".to_string())).await; + } + + Ok(self.status().await) + } + + pub async fn enable(self: &Arc) -> Result { + if !cfg!(target_os = "macos") { + return Err("screen intelligence is macOS-only in V1".to_string()); + } + + { + let mut state = self.inner.lock().await; + if state.session.is_some() { + return Ok(self.status().await.session); + } + state.permissions = detect_permissions(); + if state.permissions.screen_recording != PermissionState::Granted { + return Err("screen recording permission is not granted".to_string()); + } + + let now = now_ms(); + state.features.screen_monitoring = true; + state.features.predictive_input = state.config.autocomplete_enabled; + state.session = Some(SessionRuntime { + started_at_ms: now, + expires_at_ms: i64::MAX, + ttl_secs: 0, + panic_hotkey: state.config.panic_stop_hotkey.clone(), + stop_reason: None, + last_capture_at_ms: None, + frames: VecDeque::new(), + last_context: None, + task: None, + vision_enabled: state.config.vision_enabled, + vision_state: "idle".to_string(), + vision_queue_depth: 0, + last_vision_at_ms: None, + last_vision_summary: None, + vision_summaries: VecDeque::new(), + vision_task: None, + vision_tx: None, + }); + state.last_event = Some("screen_intelligence_enabled".to_string()); + state.last_error = None; + } + + let (vision_tx, vision_rx) = tokio::sync::mpsc::unbounded_channel::(); + let engine = self.clone(); + let handle = tokio::spawn(async move { + engine.run_capture_worker().await; + }); + let vision_engine = self.clone(); + let vision_handle = tokio::spawn(async move { + vision_engine.run_vision_worker(vision_rx).await; + }); + + { + let mut state = self.inner.lock().await; + if let Some(session) = state.session.as_mut() { + session.task = Some(handle); + session.vision_task = Some(vision_handle); + session.vision_tx = Some(vision_tx); + } + } + + Ok(self.status().await.session) + } + + pub async fn disable(&self, reason: Option) -> SessionStatus { + self.stop_session_internal(reason.unwrap_or_else(|| "manual_stop".to_string())) + .await; + self.status().await.session + } + + pub async fn status(&self) -> AccessibilityStatus { + let mut state = self.inner.lock().await; + state.permissions = detect_permissions(); + + let context = foreground_context(); + let blocked = context + .as_ref() + .map(|ctx| !self.should_capture_context(ctx, &state.config)) + .unwrap_or(false); + + let (session, denylist, config, permissions, features) = { + let now = now_ms(); + let session = match &state.session { + Some(session) => SessionStatus { + active: true, + started_at_ms: Some(session.started_at_ms), + expires_at_ms: Some(session.expires_at_ms), + remaining_ms: Some((session.expires_at_ms - now).max(0)), + ttl_secs: session.ttl_secs, + panic_hotkey: session.panic_hotkey.clone(), + stop_reason: session.stop_reason.clone(), + frames_in_memory: session.frames.len(), + last_capture_at_ms: session.last_capture_at_ms, + last_context: session + .last_context + .as_ref() + .and_then(|c| c.app_name.clone()), + vision_enabled: session.vision_enabled, + vision_state: session.vision_state.clone(), + vision_queue_depth: session.vision_queue_depth, + last_vision_at_ms: session.last_vision_at_ms, + last_vision_summary: session.last_vision_summary.clone(), + }, + None => SessionStatus { + active: false, + started_at_ms: None, + expires_at_ms: None, + remaining_ms: None, + ttl_secs: state.config.session_ttl_secs, + panic_hotkey: state.config.panic_stop_hotkey.clone(), + stop_reason: None, + frames_in_memory: 0, + last_capture_at_ms: None, + last_context: None, + vision_enabled: state.config.vision_enabled, + vision_state: "idle".to_string(), + vision_queue_depth: 0, + last_vision_at_ms: None, + last_vision_summary: None, + }, + }; + + ( + session, + state.config.denylist.clone(), + state.config.clone(), + state.permissions.clone(), + state.features.clone(), + ) + }; + + AccessibilityStatus { + platform_supported: cfg!(target_os = "macos"), + permissions, + features, + session, + config, + denylist, + is_context_blocked: blocked, + } + } + + pub async fn request_permissions(&self) -> Result { + if !cfg!(target_os = "macos") { + return Ok(PermissionStatus { + screen_recording: PermissionState::Unsupported, + accessibility: PermissionState::Unsupported, + input_monitoring: PermissionState::Unsupported, + }); + } + + self.request_permission(PermissionKind::ScreenRecording) + .await?; + self.request_permission(PermissionKind::Accessibility) + .await?; + self.request_permission(PermissionKind::InputMonitoring) + .await?; + + let mut state = self.inner.lock().await; + state.permissions = detect_permissions(); + state.last_event = Some("permissions_requested".to_string()); + Ok(state.permissions.clone()) + } + + pub async fn request_permission( + &self, + permission: PermissionKind, + ) -> Result { + if !cfg!(target_os = "macos") { + return Ok(PermissionStatus { + screen_recording: PermissionState::Unsupported, + accessibility: PermissionState::Unsupported, + input_monitoring: PermissionState::Unsupported, + }); + } + + #[cfg(target_os = "macos")] + { + match permission { + PermissionKind::ScreenRecording => { + request_screen_recording_access(); + open_macos_privacy_pane("Privacy_ScreenCapture"); + } + PermissionKind::Accessibility => { + request_accessibility_access(); + open_macos_privacy_pane("Privacy_Accessibility"); + } + PermissionKind::InputMonitoring => { + open_macos_privacy_pane("Privacy_ListenEvent"); + } + } + } + + let mut state = self.inner.lock().await; + state.permissions = detect_permissions(); + state.last_event = Some(format!( + "permission_requested:{}", + permission_to_str(permission) + )); + Ok(state.permissions.clone()) + } + + pub async fn start_session( + self: &Arc, + params: StartSessionParams, + ) -> Result { + if !params.consent { + return Err("explicit consent is required to start accessibility session".to_string()); + } + + if !cfg!(target_os = "macos") { + return Err("accessibility automation is macOS-only in V1".to_string()); + } + + let ttl_secs = params + .ttl_secs + .unwrap_or(ScreenIntelligenceConfig::default().session_ttl_secs) + .clamp(30, 3600); + + { + let mut state = self.inner.lock().await; + if state.session.is_some() { + return Err("session already active".to_string()); + } + + state.permissions = detect_permissions(); + if state.permissions.accessibility != PermissionState::Granted { + return Err("accessibility permission is not granted".to_string()); + } + + let now = now_ms(); + let expires_at_ms = now + (ttl_secs as i64 * 1000); + state.features.screen_monitoring = params.screen_monitoring.unwrap_or(true); + state.features.device_control = params.device_control.unwrap_or(true); + state.features.predictive_input = params + .predictive_input + .unwrap_or(state.config.autocomplete_enabled); + + state.session = Some(SessionRuntime { + started_at_ms: now, + expires_at_ms, + ttl_secs, + panic_hotkey: state.config.panic_stop_hotkey.clone(), + stop_reason: None, + last_capture_at_ms: None, + frames: VecDeque::new(), + last_context: None, + task: None, + vision_enabled: true, + vision_state: "idle".to_string(), + vision_queue_depth: 0, + last_vision_at_ms: None, + last_vision_summary: None, + vision_summaries: VecDeque::new(), + vision_task: None, + vision_tx: None, + }); + state.last_event = Some("session_started".to_string()); + state.last_error = None; + } + + let (vision_tx, vision_rx) = tokio::sync::mpsc::unbounded_channel::(); + let engine = self.clone(); + let handle = tokio::spawn(async move { + engine.run_capture_worker().await; + }); + let vision_engine = self.clone(); + let vision_handle = tokio::spawn(async move { + vision_engine.run_vision_worker(vision_rx).await; + }); + + { + let mut state = self.inner.lock().await; + if let Some(session) = state.session.as_mut() { + session.task = Some(handle); + session.vision_task = Some(vision_handle); + session.vision_tx = Some(vision_tx); + } + } + + Ok(self.status().await.session) + } + + pub async fn stop_session(&self, reason: Option) -> SessionStatus { + self.disable(reason).await + } + + pub async fn capture_now(&self) -> Result { + let mut state = self.inner.lock().await; + let reason = "manual_capture".to_string(); + let context = foreground_context(); + + let Some(session) = state.session.as_mut() else { + return Ok(CaptureNowResult { + accepted: false, + frame: None, + }); + }; + + let frame = CaptureFrame { + captured_at_ms: now_ms(), + reason, + app_name: context.as_ref().and_then(|c| c.app_name.clone()), + window_title: context.as_ref().and_then(|c| c.window_title.clone()), + image_ref: capture_screen_image_ref_for_context(context.as_ref()).ok(), + }; + + push_ephemeral_frame(&mut session.frames, frame.clone()); + session.last_capture_at_ms = Some(frame.captured_at_ms); + session.last_context = context; + if frame.image_ref.is_some() && session.vision_enabled { + if let Some(tx) = session.vision_tx.as_ref() { + if tx.send(frame.clone()).is_ok() { + session.vision_queue_depth = session.vision_queue_depth.saturating_add(1); + } + } + } + state.last_event = Some("capture_now".to_string()); + + Ok(CaptureNowResult { + accepted: true, + frame: Some(frame), + }) + } + + pub async fn capture_image_ref_test(&self) -> CaptureImageRefResult { + let context = foreground_context(); + match capture_screen_image_ref_for_context(context.as_ref()) { + Ok(image_ref) => { + let bytes_estimate = image_ref + .strip_prefix("data:image/png;base64,") + .map(|payload| payload.len() * 3 / 4); + CaptureImageRefResult { + ok: true, + image_ref: Some(image_ref), + mime_type: "image/png".to_string(), + bytes_estimate, + message: "screen capture completed".to_string(), + } + } + Err(err) => CaptureImageRefResult { + ok: false, + image_ref: None, + mime_type: "image/png".to_string(), + bytes_estimate: None, + message: err, + }, + } + } + + pub async fn input_action( + &self, + action: InputActionParams, + ) -> Result { + let mut state = self.inner.lock().await; + + if action.action == "panic_stop" { + drop(state); + self.stop_session_internal("panic_stop".to_string()).await; + return Ok(InputActionResult { + accepted: true, + blocked: false, + reason: Some("panic stop executed".to_string()), + }); + } + + if state.session.is_none() { + return Ok(InputActionResult { + accepted: false, + blocked: true, + reason: Some("session is not active".to_string()), + }); + } + + if !state.features.device_control { + return Ok(InputActionResult { + accepted: false, + blocked: true, + reason: Some("device control is disabled".to_string()), + }); + } + + let context = foreground_context(); + if let Some(ctx) = &context { + if !self.should_capture_context(ctx, &state.config) { + return Ok(InputActionResult { + accepted: false, + blocked: true, + reason: Some("action blocked by denylisted context".to_string()), + }); + } + } + + validate_input_action(&action)?; + + if let Some(text) = action.text.as_ref() { + if !text.is_empty() { + if !state.autocomplete_context.is_empty() { + state.autocomplete_context.push(' '); + } + state.autocomplete_context.push_str(text); + state.autocomplete_context = + truncate_tail(&state.autocomplete_context, MAX_CONTEXT_CHARS); + } + } + + let action_name = action.action.clone(); + state.last_event = Some(format!("input_action:{action_name}")); + + Ok(InputActionResult { + accepted: true, + blocked: false, + reason: None, + }) + } + + pub async fn autocomplete_suggest( + &self, + params: AutocompleteSuggestParams, + ) -> Result { + let state = self.inner.lock().await; + + if !state.features.predictive_input { + return Ok(AutocompleteSuggestResult { + suggestions: Vec::new(), + }); + } + + let mut context = params.context.unwrap_or_default(); + if context.trim().is_empty() { + context = state.autocomplete_context.clone(); + } + drop(state); + + let max_results = params.max_results.unwrap_or(3).clamp(1, 8); + let suggestions = generate_suggestions(&context, max_results); + + Ok(AutocompleteSuggestResult { suggestions }) + } + + pub async fn autocomplete_commit( + &self, + params: AutocompleteCommitParams, + ) -> Result { + let cleaned = params.suggestion.trim(); + if cleaned.is_empty() { + return Err("suggestion cannot be empty".to_string()); + } + if cleaned.len() > MAX_SUGGESTION_CHARS { + return Err("suggestion exceeds maximum length".to_string()); + } + + let mut state = self.inner.lock().await; + if !state.features.predictive_input { + return Ok(AutocompleteCommitResult { committed: false }); + } + if !state.autocomplete_context.is_empty() { + state.autocomplete_context.push(' '); + } + state.autocomplete_context.push_str(cleaned); + state.autocomplete_context = truncate_tail(&state.autocomplete_context, MAX_CONTEXT_CHARS); + state.last_event = Some("autocomplete_commit".to_string()); + + Ok(AutocompleteCommitResult { committed: true }) + } + + pub async fn vision_recent(&self, limit: Option) -> VisionRecentResult { + let state = self.inner.lock().await; + let max_items = limit.unwrap_or(10).clamp(1, 120); + + let summaries = state + .session + .as_ref() + .map(|session| { + session + .vision_summaries + .iter() + .rev() + .take(max_items) + .cloned() + .collect::>() + }) + .unwrap_or_default(); + + VisionRecentResult { summaries } + } + + pub async fn vision_flush(&self) -> Result { + let candidate = { + let mut state = self.inner.lock().await; + let Some(session) = state.session.as_mut() else { + return Ok(VisionFlushResult { + accepted: false, + summary: None, + }); + }; + + let latest = session + .frames + .iter() + .rev() + .find(|f| f.image_ref.is_some()) + .cloned(); + if let Some(frame) = latest.clone() { + session.vision_state = "queued".to_string(); + session.vision_queue_depth = session.vision_queue_depth.saturating_add(1); + Some(frame) + } else { + None + } + }; + + let Some(frame) = candidate else { + return Ok(VisionFlushResult { + accepted: false, + summary: None, + }); + }; + + let summary = self + .analyze_frame_with_vision(frame) + .await + .map_err(|e| format!("vision flush failed: {e}"))?; + Ok(VisionFlushResult { + accepted: true, + summary: Some(summary), + }) + } + + async fn run_capture_worker(self: Arc) { + let mut tick = time::interval(Duration::from_millis(250)); + + loop { + tick.tick().await; + + let should_stop = { + let state = self.inner.lock().await; + match &state.session { + Some(session) => now_ms() >= session.expires_at_ms, + None => return, + } + }; + if should_stop { + self.stop_session_internal("ttl_expired".to_string()).await; + return; + } + + let context = foreground_context(); + let now = now_ms(); + let mut state = self.inner.lock().await; + let baseline_ms = (1000.0 / state.config.baseline_fps.max(0.2)).round() as i64; + let screen_monitoring = state.features.screen_monitoring; + let config = state.config.clone(); + + let Some(session) = state.session.as_mut() else { + return; + }; + if !screen_monitoring { + continue; + } + + let is_allowed = context + .as_ref() + .map(|ctx| self.should_capture_context(ctx, &config)) + .unwrap_or(false); + if !is_allowed { + continue; + } + + let context_changed = match (&session.last_context, &context) { + (Some(prev), Some(curr)) => !prev.same_as(curr), + (None, Some(_)) => true, + _ => false, + }; + + let baseline_due = session + .last_capture_at_ms + .map(|last| now - last >= baseline_ms) + .unwrap_or(true); + + if context_changed || baseline_due { + let reason = if context_changed { + "event:foreground_changed" + } else { + "baseline" + }; + + let frame = CaptureFrame { + captured_at_ms: now, + reason: reason.to_string(), + app_name: context.as_ref().and_then(|c| c.app_name.clone()), + window_title: context.as_ref().and_then(|c| c.window_title.clone()), + image_ref: capture_screen_image_ref_for_context(context.as_ref()).ok(), + }; + push_ephemeral_frame(&mut session.frames, frame.clone()); + session.last_capture_at_ms = Some(now); + session.last_context = context; + if frame.image_ref.is_some() && session.vision_enabled { + if let Some(tx) = session.vision_tx.as_ref() { + if tx.send(frame).is_ok() { + session.vision_queue_depth = + session.vision_queue_depth.saturating_add(1); + session.vision_state = "queued".to_string(); + } + } + } + state.last_event = Some(reason.to_string()); + } + } + } + + async fn run_vision_worker( + self: Arc, + mut rx: tokio::sync::mpsc::UnboundedReceiver, + ) { + while let Some(frame) = rx.recv().await { + { + let mut state = self.inner.lock().await; + if let Some(session) = state.session.as_mut() { + session.vision_state = "processing".to_string(); + } else { + break; + } + } + + let result = self.analyze_frame_with_vision(frame).await; + + let mut state = self.inner.lock().await; + let Some(session) = state.session.as_mut() else { + break; + }; + session.vision_queue_depth = session.vision_queue_depth.saturating_sub(1); + match result { + Ok(summary) => { + push_ephemeral_vision_summary(&mut session.vision_summaries, summary.clone()); + session.last_vision_at_ms = Some(summary.captured_at_ms); + session.last_vision_summary = Some(summary.actionable_notes.clone()); + session.vision_state = "ready".to_string(); + let summary_for_store = summary.clone(); + tokio::spawn(async move { + persist_vision_summary(summary_for_store).await; + }); + } + Err(err) => { + session.vision_state = "error".to_string(); + state.last_error = Some(err); + } + } + } + } + + async fn analyze_frame_with_vision( + &self, + frame: CaptureFrame, + ) -> Result { + let image_ref = frame + .image_ref + .clone() + .ok_or_else(|| "frame has no image payload".to_string())?; + let config = Config::load_or_init() + .await + .map_err(|e| format!("failed to load config: {e}"))?; + let service = local_ai::global(&config); + let prompt = "Analyze this UI screenshot. Return strict JSON with keys: ui_state, key_text, actionable_notes, confidence (0..1). Keep actionable_notes concise."; + let raw = service + .vision_prompt(&config, prompt, &[image_ref], Some(180)) + .await?; + Ok(parse_vision_summary_output(frame, &raw)) + } + + async fn stop_session_internal(&self, reason: String) { + let mut state = self.inner.lock().await; + + let Some(mut session) = state.session.take() else { + return; + }; + + session.stop_reason = Some(reason.clone()); + if let Some(task) = session.task.take() { + task.abort(); + } + if let Some(task) = session.vision_task.take() { + task.abort(); + } + session.vision_tx = None; + + state.last_event = Some(format!("session_stopped:{reason}")); + } + + fn rule_matches_context(&self, ctx: &AppContext, rules: &[String]) -> bool { + let compound = ctx.as_compound_text(); + rules + .iter() + .any(|d| !d.trim().is_empty() && compound.contains(&d.to_lowercase())) + } + + fn should_capture_context(&self, ctx: &AppContext, config: &ScreenIntelligenceConfig) -> bool { + let blacklisted = self.rule_matches_context(ctx, &config.denylist); + let whitelisted = self.rule_matches_context(ctx, &config.allowlist); + + match config.policy_mode.as_str() { + "whitelist_only" => whitelisted && !blacklisted, + _ => !blacklisted, + } + } +} + +fn validate_input_action(action: &InputActionParams) -> Result<(), String> { + match action.action.as_str() { + "mouse_move" | "mouse_click" | "mouse_drag" => { + let x = action + .x + .ok_or_else(|| "x coordinate is required".to_string())?; + let y = action + .y + .ok_or_else(|| "y coordinate is required".to_string())?; + if !(0..=10000).contains(&x) || !(0..=10000).contains(&y) { + return Err("coordinates must be between 0 and 10000".to_string()); + } + } + "key_type" => { + let text = action + .text + .as_ref() + .ok_or_else(|| "text is required for key_type".to_string())?; + if text.is_empty() || text.len() > MAX_CONTEXT_CHARS { + return Err("text length must be between 1 and 256".to_string()); + } + } + "key_press" => { + let key = action + .key + .as_ref() + .ok_or_else(|| "key is required for key_press".to_string())?; + if key.trim().is_empty() { + return Err("key cannot be empty".to_string()); + } + } + other => { + return Err(format!("unsupported input action: {other}")); + } + } + + Ok(()) +} + +fn push_ephemeral_frame(frames: &mut VecDeque, frame: CaptureFrame) { + frames.push_back(frame); + while frames.len() > MAX_EPHEMERAL_FRAMES { + let _ = frames.pop_front(); + } +} + +fn push_ephemeral_vision_summary(summaries: &mut VecDeque, summary: VisionSummary) { + summaries.push_back(summary); + while summaries.len() > MAX_EPHEMERAL_VISION_SUMMARIES { + let _ = summaries.pop_front(); + } +} + +fn parse_vision_summary_output(frame: CaptureFrame, raw: &str) -> VisionSummary { + let fallback = truncate_tail(raw.trim(), 512); + let value = serde_json::from_str::(raw).ok(); + let ui_state = value + .as_ref() + .and_then(|v| v.get("ui_state")) + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|v| !v.is_empty()) + .unwrap_or("UI state unavailable"); + let key_text = value + .as_ref() + .and_then(|v| v.get("key_text")) + .and_then(|v| v.as_str()) + .map(str::trim) + .unwrap_or(""); + let actionable_notes = value + .as_ref() + .and_then(|v| v.get("actionable_notes")) + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|v| !v.is_empty()) + .unwrap_or(&fallback); + let confidence = value + .as_ref() + .and_then(|v| v.get("confidence")) + .and_then(|v| v.as_f64()) + .map(|v| v as f32) + .unwrap_or(0.66) + .clamp(0.0, 1.0); + + VisionSummary { + id: format!("vision-{}-{}", frame.captured_at_ms, Uuid::new_v4()), + captured_at_ms: frame.captured_at_ms, + app_name: frame.app_name, + window_title: frame.window_title, + ui_state: truncate_tail(ui_state, 220), + key_text: truncate_tail(key_text, 280), + actionable_notes: truncate_tail(actionable_notes, 560), + confidence, + } +} + +async fn persist_vision_summary(summary: VisionSummary) { + let config = match Config::load_or_init().await { + Ok(cfg) => cfg, + Err(err) => { + tracing::debug!("vision summary persistence skipped: config load failed: {err}"); + return; + } + }; + + let mem = match memory::create_memory_with_storage_and_routes( + &config.memory, + &config.embedding_routes, + Some(&config.storage.provider.config), + &config.workspace_dir, + config.api_key.as_deref(), + ) { + Ok(mem) => mem, + Err(err) => { + tracing::debug!("vision summary persistence skipped: memory init failed: {err}"); + return; + } + }; + + let content = match serde_json::to_string(&summary) { + Ok(content) => content, + Err(err) => { + tracing::debug!("vision summary persistence skipped: serialization failed: {err}"); + return; + } + }; + + let key = format!("screen_intelligence_{}", summary.id); + let _ = mem + .store( + &key, + &content, + MemoryCategory::Custom("screen_intelligence".to_string()), + None, + ) + .await; +} + +fn truncate_tail(text: &str, max_chars: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= max_chars { + return text.to_string(); + } + chars[chars.len() - max_chars..].iter().collect() +} + +fn generate_suggestions(context: &str, max_results: usize) -> Vec { + let trimmed = context.trim(); + let lower = trimmed.to_lowercase(); + + let mut out = Vec::new(); + if lower.ends_with("thanks") || lower.ends_with("thank you") { + out.push(AutocompleteSuggestion { + value: " for your help!".to_string(), + confidence: 0.89, + }); + } + if lower.contains("meeting") { + out.push(AutocompleteSuggestion { + value: " tomorrow at 10am works for me.".to_string(), + confidence: 0.84, + }); + } + if lower.contains("ship") || lower.contains("release") { + out.push(AutocompleteSuggestion { + value: " after we pass QA and smoke tests.".to_string(), + confidence: 0.81, + }); + } + + if out.is_empty() { + out.push(AutocompleteSuggestion { + value: " Please share any constraints and I can refine this.".to_string(), + confidence: 0.55, + }); + } + + out.truncate(max_results); + out +} + +fn now_ms() -> i64 { + Utc::now().timestamp_millis() +} + +fn capture_screen_image_ref_for_context(context: Option<&AppContext>) -> Result { + #[cfg(target_os = "macos")] + { + let tmp_path = std::env::temp_dir().join(format!( + "openhuman_screen_intelligence_{}.png", + Uuid::new_v4() + )); + + let bounds = context + .and_then(|ctx| ctx.bounds.clone()) + .ok_or_else(|| "active window bounds unavailable".to_string())?; + if bounds.width <= 0 || bounds.height <= 0 { + return Err("active window bounds are invalid".to_string()); + } + + let rect = format!( + "{},{},{},{}", + bounds.x, bounds.y, bounds.width, bounds.height + ); + + let status = std::process::Command::new("screencapture") + .arg("-x") + .arg("-t") + .arg("png") + .arg("-R") + .arg(&rect) + .arg(&tmp_path) + .status() + .map_err(|e| format!("screencapture failed to start: {e}"))?; + if !status.success() { + return Err("screencapture returned non-zero status".to_string()); + } + let bytes = + std::fs::read(&tmp_path).map_err(|e| format!("failed to read screenshot: {e}"))?; + let _ = std::fs::remove_file(&tmp_path); + if bytes.len() > MAX_SCREENSHOT_BYTES { + return Err("captured screenshot exceeds size limit".to_string()); + } + let encoded = BASE64_STANDARD.encode(bytes); + return Ok(format!("data:image/png;base64,{encoded}")); + } + + #[cfg(not(target_os = "macos"))] + { + Err("screen capture is unsupported on this platform".to_string()) + } +} + +#[cfg(target_os = "macos")] +fn foreground_context() -> Option { + let script = r#" + tell application "System Events" + set frontApp to name of first application process whose frontmost is true + set frontWindow to "" + set windowX to "" + set windowY to "" + set windowW to "" + set windowH to "" + try + tell process frontApp + if (count of windows) > 0 then + set w to front window + set frontWindow to name of w + set p to position of w + set s to size of w + set windowX to item 1 of p as text + set windowY to item 2 of p as text + set windowW to item 1 of s as text + set windowH to item 2 of s as text + end if + end tell + end try + return frontApp & "\n" & frontWindow & "\n" & windowX & "\n" & windowY & "\n" & windowW & "\n" & windowH + end tell + "#; + + let output = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .ok()?; + + if !output.status.success() { + return None; + } + + let text = String::from_utf8_lossy(&output.stdout); + let mut lines = text.lines(); + let app = lines.next().map(|s| s.trim().to_string()); + let title = lines.next().map(|s| s.trim().to_string()); + let x = lines.next().and_then(|s| s.trim().parse::().ok()); + let y = lines.next().and_then(|s| s.trim().parse::().ok()); + let width = lines.next().and_then(|s| s.trim().parse::().ok()); + let height = lines.next().and_then(|s| s.trim().parse::().ok()); + + let bounds = match (x, y, width, height) { + (Some(x), Some(y), Some(width), Some(height)) if width > 0 && height > 0 => { + Some(WindowBounds { + x, + y, + width, + height, + }) + } + _ => None, + }; + + Some(AppContext { + app_name: app.filter(|s| !s.is_empty()), + window_title: title.filter(|s| !s.is_empty()), + bounds, + }) +} + +#[cfg(not(target_os = "macos"))] +fn foreground_context() -> Option { + None +} + +#[cfg(target_os = "macos")] +type CFAllocatorRef = *const c_void; +#[cfg(target_os = "macos")] +type CFDictionaryRef = *const c_void; +#[cfg(target_os = "macos")] +type CFBooleanRef = *const c_void; +#[cfg(target_os = "macos")] +type CFStringRef = *const c_void; + +#[cfg(target_os = "macos")] +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + fn AXIsProcessTrusted() -> bool; + fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> bool; + static kAXTrustedCheckOptionPrompt: CFStringRef; + fn CGPreflightScreenCaptureAccess() -> bool; + fn CGRequestScreenCaptureAccess() -> bool; +} + +#[cfg(target_os = "macos")] +#[link(name = "CoreFoundation", kind = "framework")] +extern "C" { + static kCFAllocatorDefault: CFAllocatorRef; + static kCFBooleanTrue: CFBooleanRef; + fn CFDictionaryCreate( + allocator: CFAllocatorRef, + keys: *const *const c_void, + values: *const *const c_void, + num_values: isize, + key_callbacks: *const c_void, + value_callbacks: *const c_void, + ) -> CFDictionaryRef; + fn CFRelease(cf: *const c_void); +} + +#[cfg(target_os = "macos")] +#[link(name = "IOKit", kind = "framework")] +extern "C" { + fn IOHIDCheckAccess(request_type: i32) -> isize; +} + +#[cfg(target_os = "macos")] +const IOHID_REQUEST_TYPE_LISTEN_EVENT: i32 = 1; +#[cfg(target_os = "macos")] +const IOHID_ACCESS_GRANTED: isize = 0; +#[cfg(target_os = "macos")] +const IOHID_ACCESS_DENIED: isize = 1; +#[cfg(target_os = "macos")] +const IOHID_ACCESS_UNKNOWN: isize = 2; + +fn permission_to_str(permission: PermissionKind) -> &'static str { + match permission { + PermissionKind::ScreenRecording => "screen_recording", + PermissionKind::Accessibility => "accessibility", + PermissionKind::InputMonitoring => "input_monitoring", + } +} + +#[cfg(target_os = "macos")] +fn open_macos_privacy_pane(pane: &str) { + let url = format!("x-apple.systempreferences:com.apple.preference.security?{pane}"); + let _ = std::process::Command::new("open").arg(url).status(); +} + +#[cfg(target_os = "macos")] +fn request_accessibility_access() { + unsafe { + let keys = [kAXTrustedCheckOptionPrompt as *const c_void]; + let values = [kCFBooleanTrue as *const c_void]; + let options = CFDictionaryCreate( + kCFAllocatorDefault, + keys.as_ptr(), + values.as_ptr(), + 1, + std::ptr::null(), + std::ptr::null(), + ); + let _ = AXIsProcessTrustedWithOptions(options); + if !options.is_null() { + CFRelease(options); + } + } +} + +#[cfg(target_os = "macos")] +fn request_screen_recording_access() { + unsafe { + let _ = CGRequestScreenCaptureAccess(); + } +} + +#[cfg(target_os = "macos")] +fn detect_accessibility_permission() -> PermissionState { + unsafe { + if AXIsProcessTrusted() { + PermissionState::Granted + } else { + PermissionState::Denied + } + } +} + +#[cfg(target_os = "macos")] +fn detect_screen_recording_permission() -> PermissionState { + unsafe { + if CGPreflightScreenCaptureAccess() { + PermissionState::Granted + } else { + PermissionState::Denied + } + } +} + +#[cfg(target_os = "macos")] +fn detect_input_monitoring_permission() -> PermissionState { + let access = unsafe { IOHIDCheckAccess(IOHID_REQUEST_TYPE_LISTEN_EVENT) }; + match access { + IOHID_ACCESS_GRANTED => PermissionState::Granted, + IOHID_ACCESS_DENIED => PermissionState::Denied, + IOHID_ACCESS_UNKNOWN => PermissionState::Unknown, + _ => PermissionState::Unknown, + } +} + +#[cfg(target_os = "macos")] +fn detect_permissions() -> PermissionStatus { + PermissionStatus { + screen_recording: detect_screen_recording_permission(), + accessibility: detect_accessibility_permission(), + input_monitoring: detect_input_monitoring_permission(), + } +} + +#[cfg(not(target_os = "macos"))] +fn detect_permissions() -> PermissionStatus { + PermissionStatus { + screen_recording: PermissionState::Unsupported, + accessibility: PermissionState::Unsupported, + input_monitoring: PermissionState::Unsupported, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_coordinates_and_actions() { + let ok = InputActionParams { + action: "mouse_move".to_string(), + x: Some(10), + y: Some(20), + button: None, + text: None, + key: None, + modifiers: None, + }; + assert!(validate_input_action(&ok).is_ok()); + + let bad = InputActionParams { + action: "mouse_click".to_string(), + x: Some(-1), + y: Some(20), + button: None, + text: None, + key: None, + modifiers: None, + }; + assert!(validate_input_action(&bad).is_err()); + + let unsupported = InputActionParams { + action: "open_portal".to_string(), + x: None, + y: None, + button: None, + text: None, + key: None, + modifiers: None, + }; + assert!(validate_input_action(&unsupported).is_err()); + } + + #[tokio::test] + async fn session_lifecycle_transitions_and_ttl_expiry() { + let engine = Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(ScreenIntelligenceConfig { + capture_policy: "hybrid".to_string(), + baseline_fps: 8.0, + session_ttl_secs: 1, + panic_stop_hotkey: "Cmd+Shift+.".to_string(), + autocomplete_enabled: true, + denylist: vec!["1password".to_string()], + })), + }); + + let start = engine + .start_session(StartSessionParams { + consent: true, + ttl_secs: Some(1), + screen_monitoring: Some(true), + device_control: Some(true), + predictive_input: Some(true), + }) + .await; + + if cfg!(target_os = "macos") { + if start.is_ok() { + let active = engine.status().await; + assert!(active.session.active); + + time::sleep(Duration::from_millis(1400)).await; + + let ended = engine.status().await; + assert!(!ended.session.active); + } + } else { + assert!(start.is_err()); + } + } + + #[tokio::test] + async fn panic_stop_behavior_stops_session() { + if !cfg!(target_os = "macos") { + return; + } + + let engine = global_engine(); + + let started = engine + .start_session(StartSessionParams { + consent: true, + ttl_secs: Some(60), + screen_monitoring: Some(true), + device_control: Some(true), + predictive_input: Some(true), + }) + .await; + + if started.is_err() { + return; + } + + let result = engine + .input_action(InputActionParams { + action: "panic_stop".to_string(), + x: None, + y: None, + button: None, + text: None, + key: None, + modifiers: None, + }) + .await + .expect("panic action should return"); + + assert!(result.accepted); + assert!(!engine.status().await.session.active); + } + + #[tokio::test] + async fn capture_scheduler_adds_baseline_frames() { + if !cfg!(target_os = "macos") { + return; + } + + let engine = Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(ScreenIntelligenceConfig { + capture_policy: "hybrid".to_string(), + baseline_fps: 6.0, + session_ttl_secs: 2, + panic_stop_hotkey: "Cmd+Shift+.".to_string(), + autocomplete_enabled: true, + denylist: vec![], + })), + }); + + let started = engine + .start_session(StartSessionParams { + consent: true, + ttl_secs: Some(2), + screen_monitoring: Some(true), + device_control: Some(true), + predictive_input: Some(true), + }) + .await; + + if started.is_err() { + return; + } + + time::sleep(Duration::from_millis(700)).await; + + let status = engine.status().await; + assert!(status.session.frames_in_memory >= 1); + + let _ = engine.stop_session(Some("test_end".to_string())).await; + } +} diff --git a/src-tauri/src/commands/openhuman.rs b/src-tauri/src/commands/openhuman.rs index 19a3a41ce..2a18c47ee 100644 --- a/src-tauri/src/commands/openhuman.rs +++ b/src-tauri/src/commands/openhuman.rs @@ -5,8 +5,8 @@ use openhuman_core::core_server::{ AutocompleteSuggestParams, AutocompleteSuggestResult, BrowserSettingsUpdate, CaptureNowResult, CommandResponse, ConfigSnapshot, GatewaySettingsUpdate, InputActionParams, InputActionResult, MemorySettingsUpdate, ModelSettingsUpdate, PermissionRequestParams, PermissionStatus, - RuntimeFlags, RuntimeSettingsUpdate, SessionStatus, StartSessionParams, StopSessionParams, - VisionFlushResult, VisionRecentResult, + RuntimeFlags, RuntimeSettingsUpdate, ScreenIntelligenceSettingsUpdate, SessionStatus, + StartSessionParams, StopSessionParams, VisionFlushResult, VisionRecentResult, }; use openhuman_core::openhuman::{doctor, hardware, integrations, migration, onboard, service}; use serde::de::DeserializeOwned; @@ -281,6 +281,20 @@ pub async fn openhuman_update_memory_settings( .await } +/// Update screen intelligence settings. +#[tauri::command] +pub async fn openhuman_update_screen_intelligence_settings( + app: tauri::AppHandle, + update: ScreenIntelligenceSettingsUpdate, +) -> Result, String> { + call_core( + &app, + "openhuman.update_screen_intelligence_settings", + serde_json::json!(update), + ) + .await +} + /// Update gateway settings. #[tauri::command] pub async fn openhuman_update_gateway_settings( @@ -509,7 +523,7 @@ pub async fn openhuman_accessibility_request_permission( Ok(response) } -/// Start a bounded accessibility session with explicit consent. +/// Start a bounded screen intelligence session with explicit consent. #[tauri::command] pub async fn openhuman_accessibility_start_session( app: tauri::AppHandle, @@ -524,7 +538,7 @@ pub async fn openhuman_accessibility_start_session( Ok(response) } -/// Stop the active accessibility session. +/// Stop the active screen intelligence session. #[tauri::command] pub async fn openhuman_accessibility_stop_session( app: tauri::AppHandle, @@ -548,7 +562,7 @@ pub async fn openhuman_accessibility_capture_now( call_core_service_managed("openhuman.accessibility_capture_now", params_none()).await } -/// Execute a validated input action in an active accessibility session. +/// Execute a validated input action in an active screen intelligence session. #[tauri::command] pub async fn openhuman_accessibility_input_action( app: tauri::AppHandle, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index b85fd960a..3e6f21511 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1076,6 +1076,7 @@ pub fn run() { openhuman_get_config, openhuman_update_model_settings, openhuman_update_memory_settings, + openhuman_update_screen_intelligence_settings, openhuman_update_gateway_settings, openhuman_update_tunnel_settings, openhuman_update_runtime_settings, diff --git a/src/components/settings/SettingsHome.tsx b/src/components/settings/SettingsHome.tsx index 71387fb8b..758a83e4f 100644 --- a/src/components/settings/SettingsHome.tsx +++ b/src/components/settings/SettingsHome.tsx @@ -76,7 +76,7 @@ const SettingsHome = () => { { id: 'accessibility', title: 'Accessibility Automation', - description: 'Screen monitoring, device control, and predictive input', + description: 'Desktop permissions, assisted controls, and safety-bound sessions', icon: ( { onClick: () => navigateToSettings('accessibility'), dangerous: false, }, + { + id: 'screen-intelligence', + title: 'Screen Intelligence', + description: 'Window capture policy, vision summaries, and memory ingestion', + icon: ( + + + + ), + onClick: () => navigateToSettings('screen-intelligence'), + dangerous: false, + }, { id: 'messaging', title: 'Messaging Channels', diff --git a/src/components/settings/hooks/useSettingsNavigation.ts b/src/components/settings/hooks/useSettingsNavigation.ts index 0adf4c165..70ee65bc6 100644 --- a/src/components/settings/hooks/useSettingsNavigation.ts +++ b/src/components/settings/hooks/useSettingsNavigation.ts @@ -6,6 +6,7 @@ export type SettingsRoute = | 'connections' | 'messaging' | 'cron-jobs' + | 'screen-intelligence' | 'privacy' | 'profile' | 'advanced' @@ -47,6 +48,7 @@ export const useSettingsNavigation = (): SettingsNavigationHook => { if (path.includes('/settings/connections')) return 'connections'; if (path.includes('/settings/messaging')) return 'messaging'; if (path.includes('/settings/cron-jobs')) return 'cron-jobs'; + if (path.includes('/settings/screen-intelligence')) return 'screen-intelligence'; if (path.includes('/settings/privacy')) return 'privacy'; if (path.includes('/settings/profile')) return 'profile'; if (path.includes('/settings/advanced')) return 'advanced'; diff --git a/src/components/settings/panels/ScreenIntelligencePanel.tsx b/src/components/settings/panels/ScreenIntelligencePanel.tsx new file mode 100644 index 000000000..001d8b617 --- /dev/null +++ b/src/components/settings/panels/ScreenIntelligencePanel.tsx @@ -0,0 +1,418 @@ +import { useEffect, useMemo, useState } from 'react'; + +import { + fetchAccessibilityStatus, + fetchAccessibilityVisionRecent, + flushAccessibilityVision, + requestAccessibilityPermission, + startAccessibilitySession, + stopAccessibilitySession, +} from '../../../store/accessibilitySlice'; +import { useAppDispatch, useAppSelector } from '../../../store/hooks'; +import { isTauri, openhumanUpdateScreenIntelligenceSettings } from '../../../utils/tauriCommands'; +import SettingsHeader from '../components/SettingsHeader'; +import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; + +const formatRemaining = (remainingMs: number | null): string => { + if (remainingMs === null || remainingMs <= 0) { + return '00:00'; + } + + const totalSeconds = Math.floor(remainingMs / 1000); + const mins = Math.floor(totalSeconds / 60) + .toString() + .padStart(2, '0'); + const secs = (totalSeconds % 60).toString().padStart(2, '0'); + return `${mins}:${secs}`; +}; + +const PermissionBadge = ({ label, value }: { label: string; value: string }) => { + const colorClass = + value === 'granted' + ? 'bg-green-900/40 text-green-300 border-green-700/40' + : value === 'denied' + ? 'bg-red-900/40 text-red-300 border-red-700/40' + : 'bg-stone-800/60 text-stone-300 border-stone-700'; + + return ( +
+ {label} + + {value} + +
+ ); +}; + +const ScreenIntelligencePanel = () => { + const { navigateBack } = useSettingsNavigation(); + const dispatch = useAppDispatch(); + const { + status, + isLoading, + isRequestingPermissions, + isStartingSession, + isStoppingSession, + isLoadingVision, + isFlushingVision, + recentVisionSummaries, + lastError, + } = useAppSelector(state => state.accessibility); + const [featureOverrides, setFeatureOverrides] = useState<{ + screen_monitoring?: boolean; + device_control?: boolean; + predictive_input?: boolean; + }>({}); + const [enabled, setEnabled] = useState(false); + const [policyMode, setPolicyMode] = useState<'all_except_blacklist' | 'whitelist_only'>( + 'all_except_blacklist' + ); + const [baselineFps, setBaselineFps] = useState('1'); + const [allowlistText, setAllowlistText] = useState(''); + const [denylistText, setDenylistText] = useState(''); + const [isSavingConfig, setIsSavingConfig] = useState(false); + const [configError, setConfigError] = useState(null); + + useEffect(() => { + void dispatch(fetchAccessibilityStatus()); + }, [dispatch]); + + useEffect(() => { + if (!status?.session.active) { + return; + } + const intervalId = window.setInterval(() => { + void dispatch(fetchAccessibilityStatus()); + void dispatch(fetchAccessibilityVisionRecent(10)); + }, 1000); + return () => window.clearInterval(intervalId); + }, [dispatch, status?.session.active]); + + useEffect(() => { + void dispatch(fetchAccessibilityVisionRecent(10)); + }, [dispatch]); + + useEffect(() => { + if (!status?.config) { + return; + } + setEnabled(status.config.enabled ?? false); + setPolicyMode( + status.config.policy_mode === 'whitelist_only' ? 'whitelist_only' : 'all_except_blacklist' + ); + setBaselineFps(String(status.config.baseline_fps ?? 1)); + setAllowlistText((status.config.allowlist ?? []).join('\n')); + setDenylistText((status.config.denylist ?? []).join('\n')); + }, [status?.config]); + + const screenMonitoring = + featureOverrides.screen_monitoring ?? status?.features.screen_monitoring ?? true; + const deviceControl = featureOverrides.device_control ?? status?.features.device_control ?? true; + const predictiveInput = + featureOverrides.predictive_input ?? status?.features.predictive_input ?? true; + + const remaining = useMemo( + () => formatRemaining(status?.session.remaining_ms ?? null), + [status?.session.remaining_ms] + ); + + const startDisabled = + isStartingSession || + isLoading || + !status?.platform_supported || + status.session.active || + status.permissions.accessibility !== 'granted'; + const stopDisabled = isStoppingSession || !status?.session.active; + + const saveConfig = async () => { + if (!isTauri()) return; + setConfigError(null); + setIsSavingConfig(true); + try { + const fps = Number(baselineFps); + await openhumanUpdateScreenIntelligenceSettings({ + enabled, + policy_mode: policyMode, + baseline_fps: Number.isFinite(fps) && fps > 0 ? fps : 1, + allowlist: allowlistText + .split('\n') + .map(v => v.trim()) + .filter(Boolean), + denylist: denylistText + .split('\n') + .map(v => v.trim()) + .filter(Boolean), + }); + await dispatch(fetchAccessibilityStatus()); + } catch (error) { + setConfigError(error instanceof Error ? error.message : 'Failed to save screen intelligence'); + } finally { + setIsSavingConfig(false); + } + }; + + return ( +
+ + +
+
+

Permissions

+ + + + + + + + +
+ +
+

Screen Intelligence Policy

+ + + + + + + +
+
Allowlist (one rule per line)
+