From 23c4e11ffe1343d51b1fe73a7c3eaff35e174c85 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Fri, 27 Mar 2026 17:28:13 -0700 Subject: [PATCH] feat: accessibility automation v1 (core + tauri + settings) (#47) * feat(accessibility): add core automation RPC + settings panel * added accessibility code --- rust-core/src/core_server.rs | 134 ++- rust-core/src/openhuman/accessibility/mod.rs | 956 ++++++++++++++++++ rust-core/src/openhuman/config/mod.rs | 23 +- .../openhuman/config/schema/accessibility.rs | 56 + rust-core/src/openhuman/config/schema/mod.rs | 6 + rust-core/src/openhuman/mod.rs | 1 + skills | 2 +- src-tauri/src/commands/openhuman.rs | 156 ++- src-tauri/src/lib.rs | 8 + src/components/settings/SettingsHome.tsx | 17 + .../settings/hooks/useSettingsNavigation.ts | 2 + .../settings/panels/AccessibilityPanel.tsx | 226 +++++ .../__tests__/AccessibilityPanel.test.tsx | 85 ++ src/pages/Settings.tsx | 2 + .../__tests__/accessibilitySlice.test.ts | 94 ++ src/store/accessibilitySlice.ts | 187 ++++ src/store/index.ts | 2 + src/utils/tauriCommands.ts | 180 ++++ 18 files changed, 2121 insertions(+), 16 deletions(-) create mode 100644 rust-core/src/openhuman/accessibility/mod.rs create mode 100644 rust-core/src/openhuman/config/schema/accessibility.rs create mode 100644 src/components/settings/panels/AccessibilityPanel.tsx create mode 100644 src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx create mode 100644 src/store/__tests__/accessibilitySlice.test.ts create mode 100644 src/store/accessibilitySlice.ts diff --git a/rust-core/src/core_server.rs b/rust-core/src/core_server.rs index 4a32edf79..643ab8db0 100644 --- a/rust-core/src/core_server.rs +++ b/rust-core/src/core_server.rs @@ -12,7 +12,15 @@ use crate::openhuman::config::Config; use crate::openhuman::health; use crate::openhuman::local_ai::{self, Suggestion}; use crate::openhuman::security::{SecretStore, SecurityPolicy}; -use crate::openhuman::{doctor, hardware, integrations, migration, onboard, service}; +use crate::openhuman::{ + accessibility, doctor, hardware, integrations, migration, onboard, service, +}; + +pub use crate::openhuman::accessibility::{ + AccessibilityStatus, AutocompleteCommitParams, AutocompleteCommitResult, + AutocompleteSuggestParams, AutocompleteSuggestResult, CaptureNowResult, InputActionParams, + InputActionResult, PermissionStatus, SessionStatus, StartSessionParams, StopSessionParams, +}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CommandResponse { @@ -728,6 +736,83 @@ async fn dispatch( )) } + "openhuman.accessibility_status" => { + let status = accessibility::global_engine().status().await; + to_json_value(command_response( + status, + vec!["accessibility status fetched".to_string()], + )) + } + + "openhuman.accessibility_request_permissions" => { + let permissions = accessibility::global_engine().request_permissions().await?; + to_json_value(command_response( + permissions, + vec!["accessibility permissions requested".to_string()], + )) + } + + "openhuman.accessibility_start_session" => { + let payload: StartSessionParams = parse_params(params)?; + let session = accessibility::global_engine() + .start_session(payload) + .await?; + to_json_value(command_response( + session, + vec!["accessibility session started".to_string()], + )) + } + + "openhuman.accessibility_stop_session" => { + let payload: StopSessionParams = parse_params(params)?; + let session = accessibility::global_engine() + .stop_session(payload.reason) + .await; + to_json_value(command_response( + session, + vec!["accessibility session stopped".to_string()], + )) + } + + "openhuman.accessibility_capture_now" => { + let result = accessibility::global_engine().capture_now().await?; + to_json_value(command_response( + result, + vec!["accessibility manual capture requested".to_string()], + )) + } + + "openhuman.accessibility_input_action" => { + let payload: InputActionParams = parse_params(params)?; + let result = accessibility::global_engine().input_action(payload).await?; + to_json_value(command_response( + result, + vec!["accessibility input action processed".to_string()], + )) + } + + "openhuman.accessibility_autocomplete_suggest" => { + let payload: AutocompleteSuggestParams = parse_params(params)?; + let result = accessibility::global_engine() + .autocomplete_suggest(payload) + .await?; + to_json_value(command_response( + result, + vec!["accessibility autocomplete suggestions generated".to_string()], + )) + } + + "openhuman.accessibility_autocomplete_commit" => { + let payload: AutocompleteCommitParams = parse_params(params)?; + let result = accessibility::global_engine() + .autocomplete_commit(payload) + .await?; + to_json_value(command_response( + result, + vec!["accessibility autocomplete suggestion committed".to_string()], + )) + } + _ => Err(format!("unknown method: {method}")), } } @@ -846,3 +931,50 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> { .build()?; runtime.block_on(run_server(port)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn accessibility_status_rpc_returns_valid_schema() { + let raw = call_method("openhuman.accessibility_status", json!({})) + .await + .expect("status rpc should return"); + let payload: CommandResponse = + serde_json::from_value(raw).expect("status payload should decode"); + + assert!(!payload.logs.is_empty()); + assert!(!payload.result.config.capture_policy.is_empty()); + } + + #[tokio::test] + async fn accessibility_start_session_requires_consent() { + let err = call_method( + "openhuman.accessibility_start_session", + json!({ + "consent": false, + "ttl_secs": 60 + }), + ) + .await + .expect_err("session start without consent should fail"); + + assert!(err.contains("consent")); + } + + #[tokio::test] + async fn accessibility_input_action_rejects_invalid_envelope() { + let err = call_method( + "openhuman.accessibility_input_action", + json!({ + "x": 10, + "y": 20 + }), + ) + .await + .expect_err("missing action should fail envelope validation"); + + assert!(err.contains("invalid params")); + } +} diff --git a/rust-core/src/openhuman/accessibility/mod.rs b/rust-core/src/openhuman/accessibility/mod.rs new file mode 100644 index 000000000..8c686fc8b --- /dev/null +++ b/rust-core/src/openhuman/accessibility/mod.rs @@ -0,0 +1,956 @@ +use crate::openhuman::config::AccessibilityAutomationConfig; +use chrono::Utc; +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use std::collections::VecDeque; +use std::sync::Arc; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use tokio::time::{self, Duration}; + +const MAX_EPHEMERAL_FRAMES: usize = 120; +const MAX_CONTEXT_CHARS: usize = 256; +const MAX_SUGGESTION_CHARS: usize = 128; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PermissionState { + Granted, + Denied, + Unknown, + Unsupported, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PermissionStatus { + pub screen_recording: PermissionState, + pub accessibility: PermissionState, + pub input_monitoring: PermissionState, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessibilityFeatures { + pub screen_monitoring: bool, + pub device_control: bool, + pub predictive_input: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SessionStatus { + pub active: bool, + pub started_at_ms: Option, + pub expires_at_ms: Option, + pub remaining_ms: Option, + pub ttl_secs: u64, + pub panic_hotkey: String, + pub stop_reason: Option, + pub frames_in_memory: usize, + pub last_capture_at_ms: Option, + pub last_context: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessibilityHealth { + pub last_error: Option, + pub last_event: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessibilityStatus { + pub platform_supported: bool, + pub permissions: PermissionStatus, + pub features: AccessibilityFeatures, + pub session: SessionStatus, + pub config: AccessibilityAutomationConfig, + pub denylist: Vec, + pub is_context_blocked: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StartSessionParams { + pub consent: bool, + pub ttl_secs: Option, + pub screen_monitoring: Option, + pub device_control: Option, + pub predictive_input: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StopSessionParams { + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CaptureFrame { + pub captured_at_ms: i64, + pub reason: String, + pub app_name: Option, + pub window_title: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CaptureNowResult { + pub accepted: bool, + pub frame: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InputActionParams { + pub action: String, + pub x: Option, + pub y: Option, + pub button: Option, + pub text: Option, + pub key: Option, + pub modifiers: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InputActionResult { + pub accepted: bool, + pub blocked: bool, + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSuggestParams { + pub context: Option, + pub max_results: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSuggestion { + pub value: String, + pub confidence: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSuggestResult { + pub suggestions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteCommitParams { + pub suggestion: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteCommitResult { + pub committed: bool, +} + +#[derive(Debug, Clone)] +struct AppContext { + app_name: Option, + window_title: Option, +} + +impl AppContext { + fn same_as(&self, other: &AppContext) -> bool { + self.app_name == other.app_name && self.window_title == other.window_title + } + + fn as_compound_text(&self) -> String { + format!( + "{} {}", + self.app_name.clone().unwrap_or_default(), + self.window_title.clone().unwrap_or_default() + ) + .to_lowercase() + } +} + +struct SessionRuntime { + started_at_ms: i64, + expires_at_ms: i64, + ttl_secs: u64, + panic_hotkey: String, + stop_reason: Option, + last_capture_at_ms: Option, + frames: VecDeque, + last_context: Option, + task: Option>, +} + +struct EngineState { + config: AccessibilityAutomationConfig, + permissions: PermissionStatus, + features: AccessibilityFeatures, + session: Option, + last_error: Option, + last_event: Option, + autocomplete_context: String, +} + +impl EngineState { + fn new(config: AccessibilityAutomationConfig) -> Self { + Self { + permissions: PermissionStatus { + screen_recording: PermissionState::Unknown, + accessibility: PermissionState::Unknown, + input_monitoring: PermissionState::Unknown, + }, + features: AccessibilityFeatures { + screen_monitoring: true, + device_control: true, + predictive_input: config.autocomplete_enabled, + }, + config, + session: None, + last_error: None, + last_event: None, + autocomplete_context: String::new(), + } + } +} + +pub struct AccessibilityEngine { + inner: Mutex, +} + +static ACCESSIBILITY_ENGINE: Lazy> = Lazy::new(|| { + Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(AccessibilityAutomationConfig::default())), + }) +}); + +pub fn global_engine() -> Arc { + ACCESSIBILITY_ENGINE.clone() +} + +impl AccessibilityEngine { + pub async fn status(&self) -> AccessibilityStatus { + let mut state = self.inner.lock().await; + state.permissions = detect_permissions(); + + let context = foreground_context(); + let blocked = context + .as_ref() + .map(|ctx| self.is_context_blocked_by(ctx, &state.config.denylist)) + .unwrap_or(false); + + let (session, denylist, config, permissions, features) = { + let now = now_ms(); + let session = match &state.session { + Some(session) => SessionStatus { + active: true, + started_at_ms: Some(session.started_at_ms), + expires_at_ms: Some(session.expires_at_ms), + remaining_ms: Some((session.expires_at_ms - now).max(0)), + ttl_secs: session.ttl_secs, + panic_hotkey: session.panic_hotkey.clone(), + stop_reason: session.stop_reason.clone(), + frames_in_memory: session.frames.len(), + last_capture_at_ms: session.last_capture_at_ms, + last_context: session + .last_context + .as_ref() + .and_then(|c| c.app_name.clone()), + }, + None => SessionStatus { + active: false, + started_at_ms: None, + expires_at_ms: None, + remaining_ms: None, + ttl_secs: state.config.session_ttl_secs, + panic_hotkey: state.config.panic_stop_hotkey.clone(), + stop_reason: None, + frames_in_memory: 0, + last_capture_at_ms: None, + last_context: None, + }, + }; + + ( + session, + state.config.denylist.clone(), + state.config.clone(), + state.permissions.clone(), + state.features.clone(), + ) + }; + + AccessibilityStatus { + platform_supported: cfg!(target_os = "macos"), + permissions, + features, + session, + config, + denylist, + is_context_blocked: blocked, + } + } + + pub async fn request_permissions(&self) -> Result { + if !cfg!(target_os = "macos") { + return Ok(PermissionStatus { + screen_recording: PermissionState::Unsupported, + accessibility: PermissionState::Unsupported, + input_monitoring: PermissionState::Unsupported, + }); + } + + #[cfg(target_os = "macos")] + { + let _ = std::process::Command::new("open") + .arg( + "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture", + ) + .status(); + let _ = std::process::Command::new("open") + .arg( + "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility", + ) + .status(); + let _ = std::process::Command::new("open") + .arg("x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent") + .status(); + } + + let mut state = self.inner.lock().await; + state.permissions = detect_permissions(); + state.last_event = Some("permissions_requested".to_string()); + Ok(state.permissions.clone()) + } + + pub async fn start_session( + self: &Arc, + params: StartSessionParams, + ) -> Result { + if !params.consent { + return Err("explicit consent is required to start accessibility session".to_string()); + } + + if !cfg!(target_os = "macos") { + return Err("accessibility automation is macOS-only in V1".to_string()); + } + + let ttl_secs = params + .ttl_secs + .unwrap_or(AccessibilityAutomationConfig::default().session_ttl_secs) + .clamp(30, 3600); + + { + let mut state = self.inner.lock().await; + if state.session.is_some() { + return Err("session already active".to_string()); + } + + state.permissions = detect_permissions(); + if state.permissions.accessibility != PermissionState::Granted { + return Err("accessibility permission is not granted".to_string()); + } + + let now = now_ms(); + let expires_at_ms = now + (ttl_secs as i64 * 1000); + state.features.screen_monitoring = params.screen_monitoring.unwrap_or(true); + state.features.device_control = params.device_control.unwrap_or(true); + state.features.predictive_input = params + .predictive_input + .unwrap_or(state.config.autocomplete_enabled); + + state.session = Some(SessionRuntime { + started_at_ms: now, + expires_at_ms, + ttl_secs, + panic_hotkey: state.config.panic_stop_hotkey.clone(), + stop_reason: None, + last_capture_at_ms: None, + frames: VecDeque::new(), + last_context: None, + task: None, + }); + state.last_event = Some("session_started".to_string()); + state.last_error = None; + } + + let engine = self.clone(); + let handle = tokio::spawn(async move { + engine.run_capture_worker().await; + }); + + { + let mut state = self.inner.lock().await; + if let Some(session) = state.session.as_mut() { + session.task = Some(handle); + } + } + + Ok(self.status().await.session) + } + + pub async fn stop_session(&self, reason: Option) -> SessionStatus { + self.stop_session_internal(reason.unwrap_or_else(|| "manual_stop".to_string())) + .await; + self.status().await.session + } + + pub async fn capture_now(&self) -> Result { + let mut state = self.inner.lock().await; + let reason = "manual_capture".to_string(); + let context = foreground_context(); + + let Some(session) = state.session.as_mut() else { + return Ok(CaptureNowResult { + accepted: false, + frame: None, + }); + }; + + let frame = CaptureFrame { + captured_at_ms: now_ms(), + reason, + app_name: context.as_ref().and_then(|c| c.app_name.clone()), + window_title: context.as_ref().and_then(|c| c.window_title.clone()), + }; + + push_ephemeral_frame(&mut session.frames, frame.clone()); + session.last_capture_at_ms = Some(frame.captured_at_ms); + session.last_context = context; + state.last_event = Some("capture_now".to_string()); + + Ok(CaptureNowResult { + accepted: true, + frame: Some(frame), + }) + } + + pub async fn input_action( + &self, + action: InputActionParams, + ) -> Result { + let mut state = self.inner.lock().await; + + if action.action == "panic_stop" { + drop(state); + self.stop_session_internal("panic_stop".to_string()).await; + return Ok(InputActionResult { + accepted: true, + blocked: false, + reason: Some("panic stop executed".to_string()), + }); + } + + if state.session.is_none() { + return Ok(InputActionResult { + accepted: false, + blocked: true, + reason: Some("session is not active".to_string()), + }); + } + + if !state.features.device_control { + return Ok(InputActionResult { + accepted: false, + blocked: true, + reason: Some("device control is disabled".to_string()), + }); + } + + let context = foreground_context(); + if let Some(ctx) = &context { + if self.is_context_blocked_by(ctx, &state.config.denylist) { + return Ok(InputActionResult { + accepted: false, + blocked: true, + reason: Some("action blocked by denylisted context".to_string()), + }); + } + } + + validate_input_action(&action)?; + + if let Some(text) = action.text.as_ref() { + if !text.is_empty() { + if !state.autocomplete_context.is_empty() { + state.autocomplete_context.push(' '); + } + state.autocomplete_context.push_str(text); + state.autocomplete_context = + truncate_tail(&state.autocomplete_context, MAX_CONTEXT_CHARS); + } + } + + let action_name = action.action.clone(); + state.last_event = Some(format!("input_action:{action_name}")); + + Ok(InputActionResult { + accepted: true, + blocked: false, + reason: None, + }) + } + + pub async fn autocomplete_suggest( + &self, + params: AutocompleteSuggestParams, + ) -> Result { + let state = self.inner.lock().await; + + if !state.features.predictive_input { + return Ok(AutocompleteSuggestResult { + suggestions: Vec::new(), + }); + } + + let mut context = params.context.unwrap_or_default(); + if context.trim().is_empty() { + context = state.autocomplete_context.clone(); + } + drop(state); + + let max_results = params.max_results.unwrap_or(3).clamp(1, 8); + let suggestions = generate_suggestions(&context, max_results); + + Ok(AutocompleteSuggestResult { suggestions }) + } + + pub async fn autocomplete_commit( + &self, + params: AutocompleteCommitParams, + ) -> Result { + let cleaned = params.suggestion.trim(); + if cleaned.is_empty() { + return Err("suggestion cannot be empty".to_string()); + } + if cleaned.len() > MAX_SUGGESTION_CHARS { + return Err("suggestion exceeds maximum length".to_string()); + } + + let mut state = self.inner.lock().await; + if !state.features.predictive_input { + return Ok(AutocompleteCommitResult { committed: false }); + } + if !state.autocomplete_context.is_empty() { + state.autocomplete_context.push(' '); + } + state.autocomplete_context.push_str(cleaned); + state.autocomplete_context = truncate_tail(&state.autocomplete_context, MAX_CONTEXT_CHARS); + state.last_event = Some("autocomplete_commit".to_string()); + + Ok(AutocompleteCommitResult { committed: true }) + } + + async fn run_capture_worker(self: Arc) { + let mut tick = time::interval(Duration::from_millis(250)); + + loop { + tick.tick().await; + + let should_stop = { + let state = self.inner.lock().await; + match &state.session { + Some(session) => now_ms() >= session.expires_at_ms, + None => return, + } + }; + if should_stop { + self.stop_session_internal("ttl_expired".to_string()).await; + return; + } + + let context = foreground_context(); + let now = now_ms(); + let mut state = self.inner.lock().await; + let baseline_ms = (1000.0 / state.config.baseline_fps.max(0.2)).round() as i64; + let denylist = state.config.denylist.clone(); + let screen_monitoring = state.features.screen_monitoring; + + let Some(session) = state.session.as_mut() else { + return; + }; + if !screen_monitoring { + continue; + } + + let is_blocked = context + .as_ref() + .map(|ctx| self.is_context_blocked_by(ctx, &denylist)) + .unwrap_or(false); + if is_blocked { + continue; + } + + let context_changed = match (&session.last_context, &context) { + (Some(prev), Some(curr)) => !prev.same_as(curr), + (None, Some(_)) => true, + _ => false, + }; + + let baseline_due = session + .last_capture_at_ms + .map(|last| now - last >= baseline_ms) + .unwrap_or(true); + + if context_changed || baseline_due { + let reason = if context_changed { + "event:foreground_changed" + } else { + "baseline" + }; + + let frame = CaptureFrame { + captured_at_ms: now, + reason: reason.to_string(), + app_name: context.as_ref().and_then(|c| c.app_name.clone()), + window_title: context.as_ref().and_then(|c| c.window_title.clone()), + }; + push_ephemeral_frame(&mut session.frames, frame); + session.last_capture_at_ms = Some(now); + session.last_context = context; + state.last_event = Some(reason.to_string()); + } + } + } + + async fn stop_session_internal(&self, reason: String) { + let mut state = self.inner.lock().await; + + let Some(mut session) = state.session.take() else { + return; + }; + + session.stop_reason = Some(reason.clone()); + if let Some(task) = session.task.take() { + task.abort(); + } + + state.last_event = Some(format!("session_stopped:{reason}")); + } + + fn is_context_blocked_by(&self, ctx: &AppContext, denylist: &[String]) -> bool { + let compound = ctx.as_compound_text(); + denylist + .iter() + .any(|d| !d.trim().is_empty() && compound.contains(&d.to_lowercase())) + } +} + +fn validate_input_action(action: &InputActionParams) -> Result<(), String> { + match action.action.as_str() { + "mouse_move" | "mouse_click" | "mouse_drag" => { + let x = action + .x + .ok_or_else(|| "x coordinate is required".to_string())?; + let y = action + .y + .ok_or_else(|| "y coordinate is required".to_string())?; + if !(0..=10000).contains(&x) || !(0..=10000).contains(&y) { + return Err("coordinates must be between 0 and 10000".to_string()); + } + } + "key_type" => { + let text = action + .text + .as_ref() + .ok_or_else(|| "text is required for key_type".to_string())?; + if text.is_empty() || text.len() > MAX_CONTEXT_CHARS { + return Err("text length must be between 1 and 256".to_string()); + } + } + "key_press" => { + let key = action + .key + .as_ref() + .ok_or_else(|| "key is required for key_press".to_string())?; + if key.trim().is_empty() { + return Err("key cannot be empty".to_string()); + } + } + other => { + return Err(format!("unsupported input action: {other}")); + } + } + + Ok(()) +} + +fn push_ephemeral_frame(frames: &mut VecDeque, frame: CaptureFrame) { + frames.push_back(frame); + while frames.len() > MAX_EPHEMERAL_FRAMES { + let _ = frames.pop_front(); + } +} + +fn truncate_tail(text: &str, max_chars: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= max_chars { + return text.to_string(); + } + chars[chars.len() - max_chars..].iter().collect() +} + +fn generate_suggestions(context: &str, max_results: usize) -> Vec { + let trimmed = context.trim(); + let lower = trimmed.to_lowercase(); + + let mut out = Vec::new(); + if lower.ends_with("thanks") || lower.ends_with("thank you") { + out.push(AutocompleteSuggestion { + value: " for your help!".to_string(), + confidence: 0.89, + }); + } + if lower.contains("meeting") { + out.push(AutocompleteSuggestion { + value: " tomorrow at 10am works for me.".to_string(), + confidence: 0.84, + }); + } + if lower.contains("ship") || lower.contains("release") { + out.push(AutocompleteSuggestion { + value: " after we pass QA and smoke tests.".to_string(), + confidence: 0.81, + }); + } + + if out.is_empty() { + out.push(AutocompleteSuggestion { + value: " Please share any constraints and I can refine this.".to_string(), + confidence: 0.55, + }); + } + + out.truncate(max_results); + out +} + +fn now_ms() -> i64 { + Utc::now().timestamp_millis() +} + +#[cfg(target_os = "macos")] +fn foreground_context() -> Option { + let script = r#" + tell application "System Events" + set frontApp to name of first application process whose frontmost is true + set frontWindow to "" + try + tell process frontApp + if (count of windows) > 0 then + set frontWindow to name of front window + end if + end tell + end try + return frontApp & "\n" & frontWindow + end tell + "#; + + let output = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .ok()?; + + if !output.status.success() { + return None; + } + + let text = String::from_utf8_lossy(&output.stdout); + let mut lines = text.lines(); + let app = lines.next().map(|s| s.trim().to_string()); + let title = lines.next().map(|s| s.trim().to_string()); + + Some(AppContext { + app_name: app.filter(|s| !s.is_empty()), + window_title: title.filter(|s| !s.is_empty()), + }) +} + +#[cfg(not(target_os = "macos"))] +fn foreground_context() -> Option { + None +} + +#[cfg(target_os = "macos")] +fn detect_permissions() -> PermissionStatus { + let accessibility = std::process::Command::new("osascript") + .arg("-e") + .arg("tell application \"System Events\" to return UI elements enabled") + .output() + .map(|o| { + if o.status.success() { + let value = String::from_utf8_lossy(&o.stdout).to_lowercase(); + if value.contains("true") { + PermissionState::Granted + } else { + PermissionState::Denied + } + } else { + PermissionState::Denied + } + }) + .unwrap_or(PermissionState::Unknown); + + PermissionStatus { + screen_recording: PermissionState::Unknown, + accessibility, + input_monitoring: PermissionState::Unknown, + } +} + +#[cfg(not(target_os = "macos"))] +fn detect_permissions() -> PermissionStatus { + PermissionStatus { + screen_recording: PermissionState::Unsupported, + accessibility: PermissionState::Unsupported, + input_monitoring: PermissionState::Unsupported, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_coordinates_and_actions() { + let ok = InputActionParams { + action: "mouse_move".to_string(), + x: Some(10), + y: Some(20), + button: None, + text: None, + key: None, + modifiers: None, + }; + assert!(validate_input_action(&ok).is_ok()); + + let bad = InputActionParams { + action: "mouse_click".to_string(), + x: Some(-1), + y: Some(20), + button: None, + text: None, + key: None, + modifiers: None, + }; + assert!(validate_input_action(&bad).is_err()); + + let unsupported = InputActionParams { + action: "open_portal".to_string(), + x: None, + y: None, + button: None, + text: None, + key: None, + modifiers: None, + }; + assert!(validate_input_action(&unsupported).is_err()); + } + + #[tokio::test] + async fn session_lifecycle_transitions_and_ttl_expiry() { + let engine = Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(AccessibilityAutomationConfig { + capture_policy: "hybrid".to_string(), + baseline_fps: 8.0, + session_ttl_secs: 1, + panic_stop_hotkey: "Cmd+Shift+.".to_string(), + autocomplete_enabled: true, + denylist: vec!["1password".to_string()], + })), + }); + + let start = engine + .start_session(StartSessionParams { + consent: true, + ttl_secs: Some(1), + screen_monitoring: Some(true), + device_control: Some(true), + predictive_input: Some(true), + }) + .await; + + if cfg!(target_os = "macos") { + if start.is_ok() { + let active = engine.status().await; + assert!(active.session.active); + + time::sleep(Duration::from_millis(1400)).await; + + let ended = engine.status().await; + assert!(!ended.session.active); + } + } else { + assert!(start.is_err()); + } + } + + #[tokio::test] + async fn panic_stop_behavior_stops_session() { + if !cfg!(target_os = "macos") { + return; + } + + let engine = global_engine(); + + let started = engine + .start_session(StartSessionParams { + consent: true, + ttl_secs: Some(60), + screen_monitoring: Some(true), + device_control: Some(true), + predictive_input: Some(true), + }) + .await; + + if started.is_err() { + return; + } + + let result = engine + .input_action(InputActionParams { + action: "panic_stop".to_string(), + x: None, + y: None, + button: None, + text: None, + key: None, + modifiers: None, + }) + .await + .expect("panic action should return"); + + assert!(result.accepted); + assert!(!engine.status().await.session.active); + } + + #[tokio::test] + async fn capture_scheduler_adds_baseline_frames() { + if !cfg!(target_os = "macos") { + return; + } + + let engine = Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(AccessibilityAutomationConfig { + capture_policy: "hybrid".to_string(), + baseline_fps: 6.0, + session_ttl_secs: 2, + panic_stop_hotkey: "Cmd+Shift+.".to_string(), + autocomplete_enabled: true, + denylist: vec![], + })), + }); + + let started = engine + .start_session(StartSessionParams { + consent: true, + ttl_secs: Some(2), + screen_monitoring: Some(true), + device_control: Some(true), + predictive_input: Some(true), + }) + .await; + + if started.is_err() { + return; + } + + time::sleep(Duration::from_millis(700)).await; + + let status = engine.status().await; + assert!(status.session.frames_in_memory >= 1); + + let _ = engine.stop_session(Some("test_end".to_string())).await; + } +} diff --git a/rust-core/src/openhuman/config/mod.rs b/rust-core/src/openhuman/config/mod.rs index c0822d32a..ff3d4a980 100644 --- a/rust-core/src/openhuman/config/mod.rs +++ b/rust-core/src/openhuman/config/mod.rs @@ -8,17 +8,18 @@ pub use daemon::DaemonConfig; pub use schema::{ apply_runtime_proxy_to_builder, build_runtime_proxy_client, build_runtime_proxy_client_with_timeouts, runtime_proxy_config, set_runtime_proxy_config, - AgentConfig, AuditConfig, AutonomyConfig, BrowserComputerUseConfig, BrowserConfig, - ChannelsConfig, ClassificationRule, CloudflareTunnelConfig, ComposioConfig, Config, CostConfig, - CronConfig, CustomTunnelConfig, DelegateAgentConfig, DiscordConfig, DockerRuntimeConfig, - EmbeddingRouteConfig, GatewayConfig, HardwareConfig, HardwareTransport, HeartbeatConfig, - HttpRequestConfig, IMessageConfig, IdentityConfig, LarkConfig, LocalAiConfig, MatrixConfig, - MemoryConfig, ModelRouteConfig, MultimodalConfig, NgrokTunnelConfig, ObservabilityConfig, - PeripheralBoardConfig, PeripheralsConfig, ProxyConfig, ProxyScope, QueryClassificationConfig, - ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, - SchedulerConfig, SecretsConfig, SecurityConfig, SlackConfig, StorageConfig, - StorageProviderConfig, StorageProviderSection, StreamMode, TailscaleTunnelConfig, - TelegramConfig, TunnelConfig, WebSearchConfig, WebhookConfig, + AccessibilityAutomationConfig, AgentConfig, AuditConfig, AutonomyConfig, + BrowserComputerUseConfig, BrowserConfig, ChannelsConfig, ClassificationRule, + CloudflareTunnelConfig, ComposioConfig, Config, CostConfig, CronConfig, CustomTunnelConfig, + DelegateAgentConfig, DiscordConfig, DockerRuntimeConfig, EmbeddingRouteConfig, GatewayConfig, + HardwareConfig, HardwareTransport, HeartbeatConfig, HttpRequestConfig, IMessageConfig, + IdentityConfig, LarkConfig, LocalAiConfig, MatrixConfig, MemoryConfig, ModelRouteConfig, + MultimodalConfig, NgrokTunnelConfig, ObservabilityConfig, PeripheralBoardConfig, + PeripheralsConfig, ProxyConfig, ProxyScope, QueryClassificationConfig, ReliabilityConfig, + ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig, + SecretsConfig, SecurityConfig, SlackConfig, StorageConfig, StorageProviderConfig, + StorageProviderSection, StreamMode, TailscaleTunnelConfig, TelegramConfig, TunnelConfig, + WebSearchConfig, WebhookConfig, }; #[cfg(test)] diff --git a/rust-core/src/openhuman/config/schema/accessibility.rs b/rust-core/src/openhuman/config/schema/accessibility.rs new file mode 100644 index 000000000..447f55fef --- /dev/null +++ b/rust-core/src/openhuman/config/schema/accessibility.rs @@ -0,0 +1,56 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct AccessibilityAutomationConfig { + #[serde(default = "default_capture_policy")] + pub capture_policy: String, + #[serde(default = "default_baseline_fps")] + pub baseline_fps: f32, + #[serde(default = "default_session_ttl_secs")] + pub session_ttl_secs: u64, + #[serde(default = "default_panic_stop_hotkey")] + pub panic_stop_hotkey: String, + #[serde(default = "default_autocomplete_enabled")] + pub autocomplete_enabled: bool, + #[serde(default)] + pub denylist: Vec, +} + +fn default_capture_policy() -> String { + "hybrid".to_string() +} + +fn default_baseline_fps() -> f32 { + 1.0 +} + +fn default_session_ttl_secs() -> u64 { + 300 +} + +fn default_panic_stop_hotkey() -> String { + "Cmd+Shift+.".to_string() +} + +fn default_autocomplete_enabled() -> bool { + true +} + +impl Default for AccessibilityAutomationConfig { + fn default() -> Self { + Self { + capture_policy: default_capture_policy(), + baseline_fps: default_baseline_fps(), + session_ttl_secs: default_session_ttl_secs(), + panic_stop_hotkey: default_panic_stop_hotkey(), + autocomplete_enabled: default_autocomplete_enabled(), + denylist: vec![ + "1password".to_string(), + "keychain".to_string(), + "wallet".to_string(), + "seed phrase".to_string(), + ], + } + } +} diff --git a/rust-core/src/openhuman/config/schema/mod.rs b/rust-core/src/openhuman/config/schema/mod.rs index 9d015b9d7..67124fe57 100644 --- a/rust-core/src/openhuman/config/schema/mod.rs +++ b/rust-core/src/openhuman/config/schema/mod.rs @@ -2,6 +2,7 @@ //! //! Split into submodules; this module re-exports the main `Config` and all public types. +mod accessibility; mod agent; mod autonomy; mod channels; @@ -20,6 +21,7 @@ mod storage_memory; mod tools; mod tunnel; +pub use accessibility::AccessibilityAutomationConfig; pub use agent::{AgentConfig, DelegateAgentConfig}; pub use autonomy::AutonomyConfig; pub use channels::{ @@ -85,6 +87,9 @@ pub struct Config { #[serde(default)] pub runtime: RuntimeConfig, + #[serde(default)] + pub accessibility: AccessibilityAutomationConfig, + #[serde(default)] pub reliability: ReliabilityConfig, @@ -181,6 +186,7 @@ impl Default for Config { observability: ObservabilityConfig::default(), autonomy: AutonomyConfig::default(), runtime: RuntimeConfig::default(), + accessibility: AccessibilityAutomationConfig::default(), reliability: ReliabilityConfig::default(), scheduler: SchedulerConfig::default(), agent: AgentConfig::default(), diff --git a/rust-core/src/openhuman/mod.rs b/rust-core/src/openhuman/mod.rs index 901914362..1832034d5 100644 --- a/rust-core/src/openhuman/mod.rs +++ b/rust-core/src/openhuman/mod.rs @@ -11,6 +11,7 @@ // Many types/functions are not yet consumed but are intentionally exported. #![allow(dead_code)] +pub mod accessibility; pub mod agent; pub mod approval; pub mod channels; diff --git a/skills b/skills index 0685a6af4..ce54fea60 160000 --- a/skills +++ b/skills @@ -1 +1 @@ -Subproject commit 0685a6af4ef3ddea4fb26364dd72ea574f307314 +Subproject commit ce54fea606cd5e382c36f5b83ae0a9be003b2210 diff --git a/src-tauri/src/commands/openhuman.rs b/src-tauri/src/commands/openhuman.rs index f765d4ab0..38f4c47e8 100644 --- a/src-tauri/src/commands/openhuman.rs +++ b/src-tauri/src/commands/openhuman.rs @@ -1,16 +1,50 @@ //! Tauri command proxies for the standalone openhuman core process. use openhuman_core::core_server::{ - BrowserSettingsUpdate, CommandResponse, ConfigSnapshot, GatewaySettingsUpdate, - MemorySettingsUpdate, ModelSettingsUpdate, RuntimeFlags, RuntimeSettingsUpdate, + AccessibilityStatus, AutocompleteCommitParams, AutocompleteCommitResult, + AutocompleteSuggestParams, AutocompleteSuggestResult, BrowserSettingsUpdate, CaptureNowResult, + CommandResponse, ConfigSnapshot, GatewaySettingsUpdate, InputActionParams, InputActionResult, + MemorySettingsUpdate, ModelSettingsUpdate, PermissionStatus, RuntimeFlags, + RuntimeSettingsUpdate, SessionStatus, StartSessionParams, StopSessionParams, }; use openhuman_core::openhuman::{doctor, hardware, integrations, migration, onboard, service}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use tauri::Manager; +use tauri::{Emitter, Manager}; + +#[cfg(desktop)] +use crate::services::notification_service::NotificationService; const DEFAULT_CORE_RPC_URL: &str = "http://127.0.0.1:7788/rpc"; +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct AccessibilityBridgeEvent { + event: String, + timestamp_ms: i64, + details: serde_json::Value, +} + +fn emit_accessibility_event(app: &tauri::AppHandle, event: &str, details: serde_json::Value) { + let payload = AccessibilityBridgeEvent { + event: event.to_string(), + timestamp_ms: chrono::Utc::now().timestamp_millis(), + details, + }; + let _ = app.emit("openhuman:accessibility", &payload); + + #[cfg(desktop)] + { + let body = match event { + "session_started" => "Accessibility session started", + "session_stopped" => "Accessibility session stopped", + "permissions_requested" => "Accessibility permission request opened", + _ => "Accessibility automation event", + }; + let _ = NotificationService::show(app, "Accessibility Automation", body); + } +} + fn params_none() -> serde_json::Value { serde_json::json!({}) } @@ -252,6 +286,122 @@ pub async fn openhuman_agent_chat( .await } +/// Fetch accessibility automation status. +#[tauri::command] +pub async fn openhuman_accessibility_status( + app: tauri::AppHandle, +) -> Result, String> { + call_core(&app, "openhuman.accessibility_status", params_none()).await +} + +/// Request accessibility-related permissions on macOS. +#[tauri::command] +pub async fn openhuman_accessibility_request_permissions( + app: tauri::AppHandle, +) -> Result, String> { + let response: CommandResponse = call_core( + &app, + "openhuman.accessibility_request_permissions", + params_none(), + ) + .await?; + emit_accessibility_event( + &app, + "permissions_requested", + serde_json::json!(response.result), + ); + Ok(response) +} + +/// Start a bounded accessibility session with explicit consent. +#[tauri::command] +pub async fn openhuman_accessibility_start_session( + app: tauri::AppHandle, + params: StartSessionParams, +) -> Result, String> { + let response: CommandResponse = call_core( + &app, + "openhuman.accessibility_start_session", + serde_json::json!(params), + ) + .await?; + emit_accessibility_event(&app, "session_started", serde_json::json!(response.result)); + Ok(response) +} + +/// Stop the active accessibility session. +#[tauri::command] +pub async fn openhuman_accessibility_stop_session( + app: tauri::AppHandle, + params: Option, +) -> Result, String> { + let response: CommandResponse = call_core( + &app, + "openhuman.accessibility_stop_session", + serde_json::json!(params.unwrap_or(StopSessionParams { reason: None })), + ) + .await?; + emit_accessibility_event(&app, "session_stopped", serde_json::json!(response.result)); + Ok(response) +} + +/// Force an immediate capture sample from the accessibility runtime. +#[tauri::command] +pub async fn openhuman_accessibility_capture_now( + app: tauri::AppHandle, +) -> Result, String> { + call_core(&app, "openhuman.accessibility_capture_now", params_none()).await +} + +/// Execute a validated input action in an active accessibility session. +#[tauri::command] +pub async fn openhuman_accessibility_input_action( + app: tauri::AppHandle, + params: InputActionParams, +) -> Result, String> { + let response: CommandResponse = call_core( + &app, + "openhuman.accessibility_input_action", + serde_json::json!(params), + ) + .await?; + if response.result.accepted { + emit_accessibility_event(&app, "input_action", serde_json::json!(response.result)); + } + Ok(response) +} + +/// Generate autocomplete suggestions from captured typing context. +#[tauri::command] +pub async fn openhuman_accessibility_autocomplete_suggest( + app: tauri::AppHandle, + params: Option, +) -> Result, String> { + call_core( + &app, + "openhuman.accessibility_autocomplete_suggest", + serde_json::json!(params.unwrap_or(AutocompleteSuggestParams { + context: None, + max_results: None, + })), + ) + .await +} + +/// Commit an autocomplete suggestion into the runtime context. +#[tauri::command] +pub async fn openhuman_accessibility_autocomplete_commit( + app: tauri::AppHandle, + params: AutocompleteCommitParams, +) -> Result, String> { + call_core( + &app, + "openhuman.accessibility_autocomplete_commit", + serde_json::json!(params), + ) + .await +} + #[tauri::command] pub async fn openhuman_local_ai_status( app: tauri::AppHandle, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 0468498e7..69081309d 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1068,6 +1068,14 @@ pub fn run() { openhuman_get_runtime_flags, openhuman_set_browser_allow_all, openhuman_agent_chat, + openhuman_accessibility_status, + openhuman_accessibility_request_permissions, + openhuman_accessibility_start_session, + openhuman_accessibility_stop_session, + openhuman_accessibility_capture_now, + openhuman_accessibility_input_action, + openhuman_accessibility_autocomplete_suggest, + openhuman_accessibility_autocomplete_commit, openhuman_local_ai_status, openhuman_local_ai_download, openhuman_local_ai_summarize, diff --git a/src/components/settings/SettingsHome.tsx b/src/components/settings/SettingsHome.tsx index e3544e908..ab8b5215a 100644 --- a/src/components/settings/SettingsHome.tsx +++ b/src/components/settings/SettingsHome.tsx @@ -73,6 +73,23 @@ const SettingsHome = () => { // Main settings menu items const mainMenuItems = [ + { + id: 'accessibility', + title: 'Accessibility Automation', + description: 'Screen monitoring, device control, and predictive input', + icon: ( + + + + ), + onClick: () => navigateToSettings('accessibility'), + dangerous: false, + }, // { // id: "messaging", // title: "Messaging", diff --git a/src/components/settings/hooks/useSettingsNavigation.ts b/src/components/settings/hooks/useSettingsNavigation.ts index 8aa288a58..2983c83ee 100644 --- a/src/components/settings/hooks/useSettingsNavigation.ts +++ b/src/components/settings/hooks/useSettingsNavigation.ts @@ -13,6 +13,7 @@ export type SettingsRoute = | 'team-members' | 'team-invites' | 'developer-options' + | 'accessibility' | 'skills' | 'ai' | 'local-model' @@ -49,6 +50,7 @@ export const useSettingsNavigation = (): SettingsNavigationHook => { if (path.includes('/settings/advanced')) return 'advanced'; if (path.includes('/settings/billing')) return 'billing'; if (path.includes('/settings/developer-options')) return 'developer-options'; + if (path.includes('/settings/accessibility')) return 'accessibility'; if (path.includes('/settings/skills')) return 'skills'; if (path.includes('/settings/ai')) return 'ai'; if (path.includes('/settings/local-model')) return 'local-model'; diff --git a/src/components/settings/panels/AccessibilityPanel.tsx b/src/components/settings/panels/AccessibilityPanel.tsx new file mode 100644 index 000000000..9253d05fa --- /dev/null +++ b/src/components/settings/panels/AccessibilityPanel.tsx @@ -0,0 +1,226 @@ +import { useEffect, useMemo, useState } from 'react'; + +import { + fetchAccessibilityStatus, + requestAccessibilityPermissions, + startAccessibilitySession, + stopAccessibilitySession, +} from '../../../store/accessibilitySlice'; +import { useAppDispatch, useAppSelector } from '../../../store/hooks'; +import SettingsHeader from '../components/SettingsHeader'; +import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; + +const formatRemaining = (remainingMs: number | null): string => { + if (remainingMs === null || remainingMs <= 0) { + return '00:00'; + } + + const totalSeconds = Math.floor(remainingMs / 1000); + const mins = Math.floor(totalSeconds / 60) + .toString() + .padStart(2, '0'); + const secs = (totalSeconds % 60).toString().padStart(2, '0'); + return `${mins}:${secs}`; +}; + +const PermissionBadge = ({ label, value }: { label: string; value: string }) => { + const colorClass = + value === 'granted' + ? 'bg-green-900/40 text-green-300 border-green-700/40' + : value === 'denied' + ? 'bg-red-900/40 text-red-300 border-red-700/40' + : 'bg-stone-800/60 text-stone-300 border-stone-700'; + + return ( +
+ {label} + + {value} + +
+ ); +}; + +const AccessibilityPanel = () => { + const { navigateBack } = useSettingsNavigation(); + const dispatch = useAppDispatch(); + const { + status, + isLoading, + isRequestingPermissions, + isStartingSession, + isStoppingSession, + lastError, + } = useAppSelector(state => state.accessibility); + const [featureOverrides, setFeatureOverrides] = useState<{ + screen_monitoring?: boolean; + device_control?: boolean; + predictive_input?: boolean; + }>({}); + + useEffect(() => { + void dispatch(fetchAccessibilityStatus()); + }, [dispatch]); + + useEffect(() => { + if (!status?.session.active) { + return; + } + const intervalId = window.setInterval(() => { + void dispatch(fetchAccessibilityStatus()); + }, 1000); + return () => window.clearInterval(intervalId); + }, [dispatch, status?.session.active]); + + const screenMonitoring = + featureOverrides.screen_monitoring ?? status?.features.screen_monitoring ?? true; + const deviceControl = featureOverrides.device_control ?? status?.features.device_control ?? true; + const predictiveInput = + featureOverrides.predictive_input ?? status?.features.predictive_input ?? true; + + const remaining = useMemo( + () => formatRemaining(status?.session.remaining_ms ?? null), + [status?.session.remaining_ms] + ); + + const startDisabled = + isStartingSession || + isLoading || + !status?.platform_supported || + status.session.active || + status.permissions.accessibility !== 'granted'; + const stopDisabled = isStoppingSession || !status?.session.active; + + return ( +
+ + +
+
+

Permissions

+ + + + + +
+ +
+

Features

+ + + + + + +
+ +
+

Session

+
+
Status: {status?.session.active ? 'Active' : 'Stopped'}
+
Remaining: {remaining}
+
Frames (ephemeral): {status?.session.frames_in_memory ?? 0}
+
Panic stop: {status?.session.panic_hotkey ?? 'Cmd+Shift+.'}
+
+ +
+ + +
+
+ + {!status?.platform_supported && ( +
+ Accessibility Automation V1 is currently supported on macOS only. +
+ )} + + {lastError && ( +
+ {lastError} +
+ )} +
+
+ ); +}; + +export default AccessibilityPanel; diff --git a/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx b/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx new file mode 100644 index 000000000..282d42f78 --- /dev/null +++ b/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx @@ -0,0 +1,85 @@ +import { configureStore } from '@reduxjs/toolkit'; +import { render, screen } from '@testing-library/react'; +import { Provider } from 'react-redux'; +import { MemoryRouter } from 'react-router-dom'; +import { describe, expect, it } from 'vitest'; + +import accessibilityReducer from '../../../../store/accessibilitySlice'; +import authReducer from '../../../../store/authSlice'; +import socketReducer from '../../../../store/socketSlice'; +import teamReducer from '../../../../store/teamSlice'; +import userReducer from '../../../../store/userSlice'; +import type { AccessibilityStatus } from '../../../../utils/tauriCommands'; +import AccessibilityPanel from '../AccessibilityPanel'; + +const status: AccessibilityStatus = { + platform_supported: true, + permissions: { + screen_recording: 'unknown', + accessibility: 'granted', + input_monitoring: 'unknown', + }, + features: { screen_monitoring: true, device_control: true, predictive_input: true }, + session: { + active: false, + started_at_ms: null, + expires_at_ms: null, + remaining_ms: null, + ttl_secs: 300, + panic_hotkey: 'Cmd+Shift+.', + stop_reason: null, + frames_in_memory: 0, + last_capture_at_ms: null, + last_context: null, + }, + config: { + capture_policy: 'hybrid', + baseline_fps: 1, + session_ttl_secs: 300, + panic_stop_hotkey: 'Cmd+Shift+.', + autocomplete_enabled: true, + denylist: ['wallet'], + }, + denylist: ['wallet'], + is_context_blocked: false, +}; + +const createStore = () => + configureStore({ + reducer: { + auth: authReducer, + socket: socketReducer, + user: userReducer, + team: teamReducer, + accessibility: accessibilityReducer, + }, + preloadedState: { + accessibility: { + status, + isLoading: false, + isRequestingPermissions: false, + isStartingSession: false, + isStoppingSession: false, + lastError: null, + }, + }, + }); + +describe('AccessibilityPanel', () => { + it('renders permission and session sections', () => { + const store = createStore(); + + render( + + + + + + ); + + expect(screen.getByText('Accessibility Automation')).toBeInTheDocument(); + expect(screen.getByText('Permissions')).toBeInTheDocument(); + expect(screen.getByText('Session')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Start Session' })).toBeInTheDocument(); + }); +}); diff --git a/src/pages/Settings.tsx b/src/pages/Settings.tsx index d39714e7f..4775c9f72 100644 --- a/src/pages/Settings.tsx +++ b/src/pages/Settings.tsx @@ -1,5 +1,6 @@ import { Route, Routes } from 'react-router-dom'; +import AccessibilityPanel from '../components/settings/panels/AccessibilityPanel'; import AdvancedPanel from '../components/settings/panels/AdvancedPanel'; import AgentChatPanel from '../components/settings/panels/AgentChatPanel'; import AIPanel from '../components/settings/panels/AIPanel'; @@ -31,6 +32,7 @@ const Settings = () => { } /> } /> } /> + } /> } /> } /> } /> diff --git a/src/store/__tests__/accessibilitySlice.test.ts b/src/store/__tests__/accessibilitySlice.test.ts new file mode 100644 index 000000000..4ef876ba0 --- /dev/null +++ b/src/store/__tests__/accessibilitySlice.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from 'vitest'; + +import type { AccessibilityStatus } from '../../utils/tauriCommands'; +import reducer, { + clearAccessibilityError, + fetchAccessibilityStatus, + setAccessibilityStatus, + startAccessibilitySession, + stopAccessibilitySession, +} from '../accessibilitySlice'; + +const sampleStatus: AccessibilityStatus = { + platform_supported: true, + permissions: { + screen_recording: 'granted', + accessibility: 'granted', + input_monitoring: 'unknown', + }, + features: { screen_monitoring: true, device_control: true, predictive_input: true }, + session: { + active: false, + started_at_ms: null, + expires_at_ms: null, + remaining_ms: null, + ttl_secs: 300, + panic_hotkey: 'Cmd+Shift+.', + stop_reason: null, + frames_in_memory: 0, + last_capture_at_ms: null, + last_context: null, + }, + config: { + capture_policy: 'hybrid', + baseline_fps: 1, + session_ttl_secs: 300, + panic_stop_hotkey: 'Cmd+Shift+.', + autocomplete_enabled: true, + denylist: ['wallet'], + }, + denylist: ['wallet'], + is_context_blocked: false, +}; + +describe('accessibilitySlice', () => { + it('has expected initial state', () => { + const state = reducer(undefined, { type: '@@INIT' }); + expect(state.status).toBeNull(); + expect(state.isLoading).toBe(false); + expect(state.lastError).toBeNull(); + }); + + it('stores status payload', () => { + const state = reducer(undefined, setAccessibilityStatus(sampleStatus)); + expect(state.status?.platform_supported).toBe(true); + expect(state.status?.config.capture_policy).toBe('hybrid'); + }); + + it('tracks fetch lifecycle', () => { + const pending = reducer(undefined, { type: fetchAccessibilityStatus.pending.type }); + expect(pending.isLoading).toBe(true); + + const fulfilled = reducer( + pending, + fetchAccessibilityStatus.fulfilled(sampleStatus, 'req-1', undefined) + ); + expect(fulfilled.isLoading).toBe(false); + expect(fulfilled.status?.permissions.accessibility).toBe('granted'); + }); + + it('tracks session start/stop async flags', () => { + const starting = reducer(undefined, { type: startAccessibilitySession.pending.type }); + expect(starting.isStartingSession).toBe(true); + + const started = reducer( + starting, + startAccessibilitySession.fulfilled(sampleStatus, 'req-2', { consent: true }) + ); + expect(started.isStartingSession).toBe(false); + + const stopping = reducer(started, { type: stopAccessibilitySession.pending.type }); + expect(stopping.isStoppingSession).toBe(true); + }); + + it('clears errors', () => { + const errored = reducer(undefined, { + type: fetchAccessibilityStatus.rejected.type, + payload: 'boom', + }); + expect(errored.lastError).toBe('boom'); + + const cleared = reducer(errored, clearAccessibilityError()); + expect(cleared.lastError).toBeNull(); + }); +}); diff --git a/src/store/accessibilitySlice.ts b/src/store/accessibilitySlice.ts new file mode 100644 index 000000000..bcfa5152e --- /dev/null +++ b/src/store/accessibilitySlice.ts @@ -0,0 +1,187 @@ +import { createAsyncThunk, createSlice, type PayloadAction } from '@reduxjs/toolkit'; + +import { + type AccessibilityInputActionParams, + type AccessibilitySessionStatus, + type AccessibilityStatus, + openhumanAccessibilityInputAction, + openhumanAccessibilityRequestPermissions, + openhumanAccessibilityStartSession, + openhumanAccessibilityStatus, + openhumanAccessibilityStopSession, +} from '../utils/tauriCommands'; + +interface AccessibilityState { + status: AccessibilityStatus | null; + isLoading: boolean; + isRequestingPermissions: boolean; + isStartingSession: boolean; + isStoppingSession: boolean; + lastError: string | null; +} + +const initialState: AccessibilityState = { + status: null, + isLoading: false, + isRequestingPermissions: false, + isStartingSession: false, + isStoppingSession: false, + lastError: null, +}; + +const extractError = (error: unknown, fallback: string): string => { + if (error instanceof Error && error.message.trim()) { + return error.message; + } + return fallback; +}; + +export const fetchAccessibilityStatus = createAsyncThunk( + 'accessibility/fetchStatus', + async (_, { rejectWithValue }) => { + try { + const response = await openhumanAccessibilityStatus(); + return response.result; + } catch (error) { + return rejectWithValue(extractError(error, 'Failed to fetch accessibility status')); + } + } +); + +export const requestAccessibilityPermissions = createAsyncThunk( + 'accessibility/requestPermissions', + async (_, { rejectWithValue }) => { + try { + await openhumanAccessibilityRequestPermissions(); + const response = await openhumanAccessibilityStatus(); + return response.result; + } catch (error) { + return rejectWithValue(extractError(error, 'Failed to request accessibility permissions')); + } + } +); + +export const startAccessibilitySession = createAsyncThunk( + 'accessibility/startSession', + async ( + params: { + consent: boolean; + ttl_secs?: number; + screen_monitoring?: boolean; + device_control?: boolean; + predictive_input?: boolean; + }, + { rejectWithValue } + ) => { + try { + await openhumanAccessibilityStartSession(params); + const response = await openhumanAccessibilityStatus(); + return response.result; + } catch (error) { + return rejectWithValue(extractError(error, 'Failed to start accessibility session')); + } + } +); + +export const stopAccessibilitySession = createAsyncThunk( + 'accessibility/stopSession', + async (reason: string | undefined, { rejectWithValue }) => { + try { + await openhumanAccessibilityStopSession(reason ? { reason } : undefined); + const response = await openhumanAccessibilityStatus(); + return response.result; + } catch (error) { + return rejectWithValue(extractError(error, 'Failed to stop accessibility session')); + } + } +); + +export const executeAccessibilityInputAction = createAsyncThunk( + 'accessibility/inputAction', + async (params: AccessibilityInputActionParams, { rejectWithValue }) => { + try { + const response = await openhumanAccessibilityInputAction(params); + return response.result; + } catch (error) { + return rejectWithValue(extractError(error, 'Failed to execute input action')); + } + } +); + +const accessibilitySlice = createSlice({ + name: 'accessibility', + initialState, + reducers: { + clearAccessibilityError(state) { + state.lastError = null; + }, + setAccessibilityStatus(state, action: PayloadAction) { + state.status = action.payload; + }, + setAccessibilitySessionFeatures(state, action: PayloadAction) { + if (state.status) { + state.status.session = action.payload; + } + }, + }, + extraReducers: builder => { + builder + .addCase(fetchAccessibilityStatus.pending, state => { + state.isLoading = true; + state.lastError = null; + }) + .addCase(fetchAccessibilityStatus.fulfilled, (state, action) => { + state.isLoading = false; + state.status = action.payload; + }) + .addCase(fetchAccessibilityStatus.rejected, (state, action) => { + state.isLoading = false; + state.lastError = (action.payload as string) ?? 'Failed to fetch accessibility status'; + }) + .addCase(requestAccessibilityPermissions.pending, state => { + state.isRequestingPermissions = true; + state.lastError = null; + }) + .addCase(requestAccessibilityPermissions.fulfilled, (state, action) => { + state.isRequestingPermissions = false; + state.status = action.payload; + }) + .addCase(requestAccessibilityPermissions.rejected, (state, action) => { + state.isRequestingPermissions = false; + state.lastError = + (action.payload as string) ?? 'Failed to request accessibility permissions'; + }) + .addCase(startAccessibilitySession.pending, state => { + state.isStartingSession = true; + state.lastError = null; + }) + .addCase(startAccessibilitySession.fulfilled, (state, action) => { + state.isStartingSession = false; + state.status = action.payload; + }) + .addCase(startAccessibilitySession.rejected, (state, action) => { + state.isStartingSession = false; + state.lastError = (action.payload as string) ?? 'Failed to start accessibility session'; + }) + .addCase(stopAccessibilitySession.pending, state => { + state.isStoppingSession = true; + state.lastError = null; + }) + .addCase(stopAccessibilitySession.fulfilled, (state, action) => { + state.isStoppingSession = false; + state.status = action.payload; + }) + .addCase(stopAccessibilitySession.rejected, (state, action) => { + state.isStoppingSession = false; + state.lastError = (action.payload as string) ?? 'Failed to stop accessibility session'; + }) + .addCase(executeAccessibilityInputAction.rejected, (state, action) => { + state.lastError = (action.payload as string) ?? 'Failed to execute accessibility action'; + }); + }, +}); + +export const { clearAccessibilityError, setAccessibilitySessionFeatures, setAccessibilityStatus } = + accessibilitySlice.actions; + +export default accessibilitySlice.reducer; diff --git a/src/store/index.ts b/src/store/index.ts index d4a549476..ceaa99e1d 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -15,6 +15,7 @@ import storage from 'redux-persist/lib/storage'; import { setStoreForApiClient } from '../services/apiClient'; import { IS_DEV } from '../utils/config'; import { storeSession, syncMemoryClientToken } from '../utils/tauriCommands'; +import accessibilityReducer from './accessibilitySlice'; import aiReducer from './aiSlice'; import authReducer, { setOnboardedForUser, setToken } from './authSlice'; import daemonReducer from './daemonSlice'; @@ -108,6 +109,7 @@ export const store = configureStore({ intelligence: intelligenceReducer, invite: inviteReducer, notion: notionReducer, + accessibility: accessibilityReducer, }, middleware: getDefaultMiddleware => { const middleware = getDefaultMiddleware({ diff --git a/src/utils/tauriCommands.ts b/src/utils/tauriCommands.ts index 24f9fa8e4..41c6d81a5 100644 --- a/src/utils/tauriCommands.ts +++ b/src/utils/tauriCommands.ts @@ -383,6 +383,114 @@ export interface AgentServerStatus { url: string; } +export type AccessibilityPermissionState = 'granted' | 'denied' | 'unknown' | 'unsupported'; + +export interface AccessibilityPermissionStatus { + screen_recording: AccessibilityPermissionState; + accessibility: AccessibilityPermissionState; + input_monitoring: AccessibilityPermissionState; +} + +export interface AccessibilityFeatures { + screen_monitoring: boolean; + device_control: boolean; + predictive_input: boolean; +} + +export interface AccessibilitySessionStatus { + active: boolean; + started_at_ms: number | null; + expires_at_ms: number | null; + remaining_ms: number | null; + ttl_secs: number; + panic_hotkey: string; + stop_reason: string | null; + frames_in_memory: number; + last_capture_at_ms: number | null; + last_context: string | null; +} + +export interface AccessibilityConfig { + capture_policy: string; + baseline_fps: number; + session_ttl_secs: number; + panic_stop_hotkey: string; + autocomplete_enabled: boolean; + denylist: string[]; +} + +export interface AccessibilityStatus { + platform_supported: boolean; + permissions: AccessibilityPermissionStatus; + features: AccessibilityFeatures; + session: AccessibilitySessionStatus; + config: AccessibilityConfig; + denylist: string[]; + is_context_blocked: boolean; +} + +export interface AccessibilityStartSessionParams { + consent: boolean; + ttl_secs?: number; + screen_monitoring?: boolean; + device_control?: boolean; + predictive_input?: boolean; +} + +export interface AccessibilityStopSessionParams { + reason?: string; +} + +export interface AccessibilityCaptureFrame { + captured_at_ms: number; + reason: string; + app_name: string | null; + window_title: string | null; +} + +export interface AccessibilityCaptureNowResult { + accepted: boolean; + frame: AccessibilityCaptureFrame | null; +} + +export interface AccessibilityInputActionParams { + action: string; + x?: number; + y?: number; + button?: string; + text?: string; + key?: string; + modifiers?: string[]; +} + +export interface AccessibilityInputActionResult { + accepted: boolean; + blocked: boolean; + reason: string | null; +} + +export interface AccessibilityAutocompleteSuggestion { + value: string; + confidence: number; +} + +export interface AccessibilityAutocompleteSuggestParams { + context?: string; + max_results?: number; +} + +export interface AccessibilityAutocompleteSuggestResult { + suggestions: AccessibilityAutocompleteSuggestion[]; +} + +export interface AccessibilityAutocompleteCommitParams { + suggestion: string; +} + +export interface AccessibilityAutocompleteCommitResult { + committed: boolean; +} + export interface ConfigSnapshot { config: Record; workspace_dir: string; @@ -764,6 +872,78 @@ export async function openhumanAgentServerStatus(): Promise +> { + if (!isTauri()) { + throw new Error('Not running in Tauri'); + } + return await invoke('openhuman_accessibility_status'); +} + +export async function openhumanAccessibilityRequestPermissions(): Promise< + CommandResponse +> { + if (!isTauri()) { + throw new Error('Not running in Tauri'); + } + return await invoke('openhuman_accessibility_request_permissions'); +} + +export async function openhumanAccessibilityStartSession( + params: AccessibilityStartSessionParams +): Promise> { + if (!isTauri()) { + throw new Error('Not running in Tauri'); + } + return await invoke('openhuman_accessibility_start_session', { params }); +} + +export async function openhumanAccessibilityStopSession( + params?: AccessibilityStopSessionParams +): Promise> { + if (!isTauri()) { + throw new Error('Not running in Tauri'); + } + return await invoke('openhuman_accessibility_stop_session', { params: params ?? null }); +} + +export async function openhumanAccessibilityCaptureNow(): Promise< + CommandResponse +> { + if (!isTauri()) { + throw new Error('Not running in Tauri'); + } + return await invoke('openhuman_accessibility_capture_now'); +} + +export async function openhumanAccessibilityInputAction( + params: AccessibilityInputActionParams +): Promise> { + if (!isTauri()) { + throw new Error('Not running in Tauri'); + } + return await invoke('openhuman_accessibility_input_action', { params }); +} + +export async function openhumanAccessibilityAutocompleteSuggest( + params?: AccessibilityAutocompleteSuggestParams +): Promise> { + if (!isTauri()) { + throw new Error('Not running in Tauri'); + } + return await invoke('openhuman_accessibility_autocomplete_suggest', { params: params ?? null }); +} + +export async function openhumanAccessibilityAutocompleteCommit( + params: AccessibilityAutocompleteCommitParams +): Promise> { + if (!isTauri()) { + throw new Error('Not running in Tauri'); + } + return await invoke('openhuman_accessibility_autocomplete_commit', { params }); +} + export async function runtimeListSkills(): Promise { if (!isTauri()) { throw new Error('Not running in Tauri');