diff --git a/Cargo.lock b/Cargo.lock index 7e780fc38..4c30638a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -694,6 +694,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.11.1" @@ -2057,6 +2063,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -2975,6 +2990,21 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "365a784774bb381e8c19edb91190a90d7f2625e057b55de2bc0f6b57bc779ff2" +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + [[package]] name = "imap-proto" version = "0.16.6" @@ -4081,6 +4111,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "multimap" version = "0.10.1" @@ -4600,6 +4640,7 @@ dependencies = [ "hex", "hmac 0.12.1", "hostname", + "image", "landlock", "lettre", "log", @@ -5068,6 +5109,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.11.0", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "polling" version = "3.11.0" @@ -5429,6 +5483,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "pxfm" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a041e753da8b807c9255f28de81879c78c876392ff2469cde94799b2896b9d" + [[package]] name = "quinn" version = "0.11.9" @@ -9336,3 +9396,18 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index fd344194c..cd83d96ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,7 @@ tokio-stream = { version = "0.1.18", features = ["full"] } url = "2" socketioxide = { version = "0.15", features = ["extensions"] } whisper-rs = "0.16" +image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } matrix-sdk = { version = "0.16", optional = true, default-features = false, features = ["e2e-encryption", "rustls-tls", "markdown"] } fantoccini = { version = "0.22.0", optional = true, default-features = false, features = ["rustls-tls"] } diff --git a/app/package.json b/app/package.json index 049a2a501..4e2ac7558 100644 --- a/app/package.json +++ b/app/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "vite", "dev:web": "vite", - "dev:app": "source ../scripts/load-dotenv.sh && tauri dev", + "dev:app": "yarn core:stage && source ../scripts/load-dotenv.sh && tauri dev", "core:stage": "node ../scripts/stage-core-sidecar.mjs", "build": "tsc && vite build", "build:app": "tsc && vite build", diff --git a/src/openhuman/accessibility/capture.rs b/src/openhuman/accessibility/capture.rs new file mode 100644 index 000000000..d25c5d46c --- /dev/null +++ b/src/openhuman/accessibility/capture.rs @@ -0,0 +1,154 @@ +//! Timestamp helper and screen capture via platform-native tools. + +use super::types::AppContext; + +/// Maximum screenshot size in bytes before downscaling is attempted. +pub const MAX_SCREENSHOT_BYTES: usize = 1_500_000; + +/// Capture mode used for a screenshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CaptureMode { + Windowed, + Fullscreen, +} + +impl std::fmt::Display for CaptureMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CaptureMode::Windowed => write!(f, "windowed"), + CaptureMode::Fullscreen => write!(f, "fullscreen"), + } + } +} + +pub fn capture_screen_image_ref_for_context( + context: Option<&AppContext>, +) -> Result { + #[cfg(target_os = "macos")] + { + use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; + use uuid::Uuid; + + let tmp_path = std::env::temp_dir().join(format!( + "openhuman_screen_intelligence_{}.png", + Uuid::new_v4() + )); + + let bounds = context.and_then(|ctx| ctx.bounds); + + // Determine capture mode: windowed if we have valid bounds, fullscreen otherwise. + let capture_mode = match &bounds { + Some(b) if b.width > 0 && b.height > 0 => CaptureMode::Windowed, + Some(b) => { + tracing::debug!( + "[accessibility] invalid bounds ({}x{}), falling back to fullscreen", + b.width, + b.height + ); + CaptureMode::Fullscreen + } + None => { + tracing::debug!( + "[accessibility] no window bounds available, falling back to fullscreen" + ); + CaptureMode::Fullscreen + } + }; + + let mut cmd = std::process::Command::new("screencapture"); + cmd.arg("-x").arg("-t").arg("png"); + + if capture_mode == CaptureMode::Windowed { + let b = bounds.as_ref().unwrap(); + let rect = format!("{},{},{},{}", b.x, b.y, b.width, b.height); + tracing::debug!( + "[accessibility] capture mode=windowed rect={rect} app={:?}", + context.and_then(|c| c.app_name.as_deref()) + ); + cmd.arg("-R").arg(&rect); + } else { + tracing::debug!("[accessibility] capture mode=fullscreen (primary display)"); + } + + cmd.arg(&tmp_path); + + let status = cmd + .status() + .map_err(|e| format!("screencapture failed to start: {e}"))?; + if !status.success() { + tracing::debug!( + "[accessibility] screencapture exited with status: {:?}", + status.code() + ); + return Err("screencapture returned non-zero status".to_string()); + } + + let bytes = + std::fs::read(&tmp_path).map_err(|e| format!("failed to read screenshot: {e}"))?; + tracing::debug!( + "[accessibility] captured {} bytes (mode={})", + bytes.len(), + capture_mode + ); + + // If fullscreen capture is too large, try downscaling with sips. + if bytes.len() > MAX_SCREENSHOT_BYTES && capture_mode == CaptureMode::Fullscreen { + tracing::debug!( + "[accessibility] fullscreen capture {} bytes exceeds limit, downscaling with sips", + bytes.len() + ); + let sips_status = std::process::Command::new("sips") + .arg("--resampleWidth") + .arg("1280") + .arg(&tmp_path) + .status(); + match sips_status { + Ok(s) if s.success() => { + let resized = match std::fs::read(&tmp_path) { + Ok(bytes) => bytes, + Err(e) => { + let _ = std::fs::remove_file(&tmp_path); + return Err(format!("failed to read resized screenshot: {e}")); + } + }; + let _ = std::fs::remove_file(&tmp_path); + tracing::debug!("[accessibility] resized to {} bytes", resized.len()); + if resized.len() > MAX_SCREENSHOT_BYTES { + return Err( + "captured screenshot exceeds size limit after downscale".to_string() + ); + } + let encoded = BASE64_STANDARD.encode(resized); + return Ok(format!("data:image/png;base64,{encoded}")); + } + Ok(s) => { + let _ = std::fs::remove_file(&tmp_path); + tracing::debug!("[accessibility] sips failed with status: {:?}", s.code()); + return Err( + "captured screenshot exceeds size limit and downscale failed".to_string(), + ); + } + Err(e) => { + let _ = std::fs::remove_file(&tmp_path); + tracing::debug!("[accessibility] sips not available: {e}"); + return Err("captured screenshot exceeds size limit".to_string()); + } + } + } + + let _ = std::fs::remove_file(&tmp_path); + + if bytes.len() > MAX_SCREENSHOT_BYTES { + return Err("captured screenshot exceeds size limit".to_string()); + } + + let encoded = BASE64_STANDARD.encode(bytes); + Ok(format!("data:image/png;base64,{encoded}")) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = context; + Err("screen capture is unsupported on this platform".to_string()) + } +} diff --git a/src/openhuman/accessibility/focus.rs b/src/openhuman/accessibility/focus.rs new file mode 100644 index 000000000..e356a1200 --- /dev/null +++ b/src/openhuman/accessibility/focus.rs @@ -0,0 +1,473 @@ +//! Accessibility focus queries and foreground app context. +//! +//! Primary path: unified Swift helper (native AX API, fast, persistent process). +//! Fallback: osascript subprocess (slower, but works without compiled helper). + +use super::terminal::{is_terminal_app, is_text_role}; +use super::text_util::{normalize_ax_value, parse_ax_number}; +use super::types::{AppContext, ElementBounds, FocusedTextContext}; + +// --------------------------------------------------------------------------- +// Focus query: unified helper → osascript fallback +// --------------------------------------------------------------------------- + +#[cfg(target_os = "macos")] +pub fn focused_text_context() -> Result { + let ctx = focused_text_context_verbose()?; + if let Some(err) = ctx.raw_error.as_ref() { + return Err(format!( + "focused text unavailable via accessibility api: {err}" + )); + } + Ok(ctx) +} + +/// Query the focused text element. Tries the unified Swift helper first (native AX, ~5-15ms), +/// falls back to osascript (~50-100ms) if the helper is unavailable. +#[cfg(target_os = "macos")] +pub fn focused_text_context_verbose() -> Result { + match focused_text_via_helper() { + Ok(ctx) if ctx.raw_error.is_some() => { + log::debug!( + "[accessibility] helper returned raw_error={:?}, falling back to osascript", + ctx.raw_error + ); + focused_text_via_osascript() + } + Ok(ctx) => Ok(ctx), + Err(helper_err) => { + log::debug!( + "[accessibility] helper focus query failed ({}), falling back to osascript", + helper_err + ); + focused_text_via_osascript() + } + } +} + +/// Focus query via the unified Swift helper. +#[cfg(target_os = "macos")] +fn focused_text_via_helper() -> Result { + let request = serde_json::json!({"type": "focus"}); + let resp = super::helper::helper_send_receive(&request)?; + + let app_name = resp + .get("app_name") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let role = resp + .get("role") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let text = resp + .get("text") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let selected_text = resp + .get("selected_text") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let raw_error = resp + .get("error") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + let x = resp.get("x").and_then(|v| v.as_i64()).map(|v| v as i32); + let y = resp.get("y").and_then(|v| v.as_i64()).map(|v| v as i32); + let w = resp.get("w").and_then(|v| v.as_i64()).map(|v| v as i32); + let h = resp.get("h").and_then(|v| v.as_i64()).map(|v| v as i32); + + Ok(FocusedTextContext { + app_name, + role, + text, + selected_text, + raw_error, + bounds: match (x, y, w, h) { + (Some(x), Some(y), Some(width), Some(height)) if width > 0 && height > 0 => { + Some(ElementBounds { + x, + y, + width, + height, + }) + } + _ => None, + }, + }) +} + +/// Focus query via osascript (fallback when helper is unavailable). +#[cfg(target_os = "macos")] +fn focused_text_via_osascript() -> Result { + let script = r##" + tell application "System Events" + set sep to character id 31 + set frontApp to first application process whose frontmost is true + set appName to name of frontApp + set roleValue to "unknown" + set textValue to "" + set selectedValue to "" + set errValue to "" + set posX to "" + set posY to "" + set sizeW to "" + set sizeH to "" + set targetRoles to {"AXTextArea", "AXTextField", "AXSearchField", "AXComboBox", "AXEditableText"} + + try + set value of attribute "AXEnhancedUserInterface" of frontApp to true + end try + + try + set focusedElement to value of attribute "AXFocusedUIElement" of frontApp + try + set roleValue to value of attribute "AXRole" of focusedElement as text + end try + try + set textValue to value of attribute "AXValue" of focusedElement as text + end try + try + set p to value of attribute "AXPosition" of focusedElement + set posX to item 1 of p as text + set posY to item 2 of p as text + end try + try + set s to value of attribute "AXSize" of focusedElement + set sizeW to item 1 of s as text + set sizeH to item 2 of s as text + end try + if textValue is "missing value" then set textValue to "" + if textValue is "" then + try + set selectedValue to value of attribute "AXSelectedText" of focusedElement as text + end try + if selectedValue is "missing value" then set selectedValue to "" + if selectedValue is not "" then set textValue to selectedValue + end if + if textValue is "" then + try + set textValue to value of attribute "AXTitle" of focusedElement as text + end try + if textValue is "missing value" then set textValue to "" + end if + on error errMsg number errNum + set errValue to "ERROR:" & errNum & ":" & errMsg + end try + + if textValue is "" then + try + set focusedWindow to value of attribute "AXFocusedWindow" of frontApp + set childElems to entire contents of focusedWindow + set staticPromptValue to "" + set staticFallbackValue to "" + repeat with childElem in childElems + set childRole to "" + set childValue to "" + set childSelectedValue to "" + try + set childRole to value of attribute "AXRole" of childElem as text + end try + if childRole is in targetRoles then + try + set childValue to value of attribute "AXValue" of childElem as text + end try + set childPosX to "" + set childPosY to "" + set childSizeW to "" + set childSizeH to "" + try + set cp to value of attribute "AXPosition" of childElem + set childPosX to item 1 of cp as text + set childPosY to item 2 of cp as text + end try + try + set cs to value of attribute "AXSize" of childElem + set childSizeW to item 1 of cs as text + set childSizeH to item 2 of cs as text + end try + if childValue is "missing value" then set childValue to "" + if childValue is "" then + try + set childSelectedValue to value of attribute "AXSelectedText" of childElem as text + end try + if childSelectedValue is "missing value" then set childSelectedValue to "" + if childSelectedValue is not "" then set childValue to childSelectedValue + end if + if childValue is not "" then + set roleValue to childRole + set textValue to childValue + if childPosX is not "" then set posX to childPosX + if childPosY is not "" then set posY to childPosY + if childSizeW is not "" then set sizeW to childSizeW + if childSizeH is not "" then set sizeH to childSizeH + exit repeat + end if + end if + end repeat + if textValue is "" then + repeat with childElem in childElems + set childRole to "" + set childValue to "" + try + set childRole to value of attribute "AXRole" of childElem as text + end try + if childRole is "AXStaticText" then + try + set childValue to value of attribute "AXValue" of childElem as text + end try + if childValue is "missing value" then set childValue to "" + if childValue is not "" then + set staticFallbackValue to childValue + if childValue contains "$ " or childValue contains "# " or childValue contains "> " then + set staticPromptValue to childValue + end if + end if + end if + end repeat + if staticPromptValue is not "" then + set roleValue to "AXStaticText" + set textValue to staticPromptValue + else if staticFallbackValue is not "" then + set roleValue to "AXStaticText" + set textValue to staticFallbackValue + end if + end if + on error errMsg2 number errNum2 + if errValue is "" then set errValue to "ERROR:" & errNum2 & ":" & errMsg2 + end try + end if + + if textValue is "" and errValue is "" then + set errValue to "ERROR:no_text_candidate_found" + end if + + return appName & sep & roleValue & sep & textValue & sep & selectedValue & sep & errValue & sep & posX & sep & posY & sep & sizeW & sep & sizeH + end tell + "##; + + let output = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .map_err(|e| format!("failed to run osascript: {e}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.is_empty() { + return Err("unable to query focused text context".to_string()); + } + return Err(format!("unable to query focused text context: {stderr}")); + } + + let text = String::from_utf8_lossy(&output.stdout); + let trimmed = text.trim_end_matches(['\r', '\n']); + let mut segments = trimmed.splitn(9, '\u{1f}'); + let app_name = segments + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + let role = segments + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + let mut value = segments.next().map(normalize_ax_value).unwrap_or_default(); + let mut selected_text = segments + .next() + .map(normalize_ax_value) + .filter(|s| !s.is_empty()); + let mut raw_error = segments + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + let pos_x = segments.next().and_then(parse_ax_number); + let pos_y = segments.next().and_then(parse_ax_number); + let size_w = segments.next().and_then(parse_ax_number); + let size_h = segments.next().and_then(parse_ax_number); + + let allow_terminal_text_value = + is_terminal_app(app_name.as_deref()) && !value.trim().is_empty(); + if !is_text_role(role.as_deref()) && !allow_terminal_text_value { + value.clear(); + selected_text = None; + if raw_error.is_none() { + raw_error = Some("ERROR:no_text_candidate_found".to_string()); + } + } + + Ok(FocusedTextContext { + app_name, + role, + text: value, + selected_text, + raw_error, + bounds: match (pos_x, pos_y, size_w, size_h) { + (Some(x), Some(y), Some(width), Some(height)) if width > 0 && height > 0 => { + Some(ElementBounds { + x, + y, + width, + height, + }) + } + _ => None, + }, + }) +} + +#[cfg(not(target_os = "macos"))] +pub fn focused_text_context() -> Result { + Err("accessibility focus queries are only supported on macOS".to_string()) +} + +#[cfg(not(target_os = "macos"))] +pub fn focused_text_context_verbose() -> Result { + Err("accessibility focus queries are only supported on macOS".to_string()) +} + +// --------------------------------------------------------------------------- +// Focus target validation +// --------------------------------------------------------------------------- + +/// Validate that the currently focused element still matches the target we generated the +/// suggestion for. Returns Ok if it matches or if validation is inconclusive. +#[cfg(target_os = "macos")] +pub fn validate_focused_target( + expected_app: Option<&str>, + expected_role: Option<&str>, +) -> Result<(), String> { + if expected_app.is_none() { + return Ok(()); + } + let current = focused_text_context_verbose(); + match current { + Ok(ctx) => { + if let (Some(expected), Some(actual)) = (expected_app, ctx.app_name.as_deref()) { + if expected.to_lowercase() != actual.to_lowercase() { + return Err(format!( + "focus shifted from '{}' to '{}', aborting insertion", + expected, actual + )); + } + } + if let (Some(expected), Some(actual)) = (expected_role, ctx.role.as_deref()) { + if expected != actual { + return Err(format!( + "target role changed from '{}' to '{}', aborting insertion", + expected, actual + )); + } + } + Ok(()) + } + Err(_) => Ok(()), + } +} + +// --------------------------------------------------------------------------- +// Foreground app context (from screen_intelligence) +// --------------------------------------------------------------------------- + +/// Parse the raw stdout from the AppleScript foreground-context query. +/// +/// Expected format: 6 lines — app_name, window_title, x, y, width, height. +/// This is a pure function, fully testable without macOS. +pub fn parse_foreground_output(stdout: &str) -> Option { + let mut lines = stdout.lines(); + let app = lines.next().map(|s| s.trim().to_string()); + let title = lines.next().map(|s| s.trim().to_string()); + let x = lines.next().and_then(|s| s.trim().parse::().ok()); + let y = lines.next().and_then(|s| s.trim().parse::().ok()); + let width = lines.next().and_then(|s| s.trim().parse::().ok()); + let height = lines.next().and_then(|s| s.trim().parse::().ok()); + + let bounds = match (x, y, width, height) { + (Some(x), Some(y), Some(width), Some(height)) if width > 0 && height > 0 => { + Some(ElementBounds { + x, + y, + width, + height, + }) + } + _ => None, + }; + + let app = app.filter(|s| !s.is_empty()); + let title = title.filter(|s| !s.is_empty()); + if app.is_none() && title.is_none() && bounds.is_none() { + return None; + } + Some(AppContext { + app_name: app, + window_title: title, + bounds, + }) +} + +#[cfg(target_os = "macos")] +pub fn foreground_context() -> Option { + let script = r#" + tell application "System Events" + set frontApp to name of first application process whose frontmost is true + set frontWindow to "" + set windowX to "" + set windowY to "" + set windowW to "" + set windowH to "" + try + tell process frontApp + if (count of windows) > 0 then + set w to front window + set frontWindow to name of w + set p to position of w + set s to size of w + set windowX to item 1 of p as text + set windowY to item 2 of p as text + set windowW to item 1 of s as text + set windowH to item 2 of s as text + end if + end tell + end try + return frontApp & "\n" & frontWindow & "\n" & windowX & "\n" & windowY & "\n" & windowW & "\n" & windowH + end tell + "#; + + let output = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .ok()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + tracing::debug!( + "[accessibility] osascript failed: status={:?} stderr={}", + output.status.code(), + stderr.trim() + ); + return None; + } + + let text = String::from_utf8_lossy(&output.stdout); + let result = parse_foreground_output(&text); + tracing::debug!( + "[accessibility] foreground_context: app={:?} bounds_present={}", + result.as_ref().and_then(|c| c.app_name.as_deref()), + result.as_ref().map(|c| c.bounds.is_some()).unwrap_or(false) + ); + result +} + +#[cfg(not(target_os = "macos"))] +pub fn validate_focused_target( + _expected_app: Option<&str>, + _expected_role: Option<&str>, +) -> Result<(), String> { + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +pub fn foreground_context() -> Option { + None +} diff --git a/src/openhuman/accessibility/helper.rs b/src/openhuman/accessibility/helper.rs new file mode 100644 index 000000000..9fcfd86ed --- /dev/null +++ b/src/openhuman/accessibility/helper.rs @@ -0,0 +1,629 @@ +//! Unified Swift helper process: focus queries, paste, and overlay in one native binary. +//! +//! Replaces the separate osascript subprocess spawns and standalone overlay binary +//! with a single persistent Swift process communicating via stdin/stdout JSON. + +#[cfg(target_os = "macos")] +use once_cell::sync::Lazy; +#[cfg(target_os = "macos")] +use std::io::{BufRead, BufReader, Write}; +#[cfg(target_os = "macos")] +use std::sync::Mutex as StdMutex; +#[cfg(target_os = "macos")] +use std::{ + fs, + path::PathBuf, + process::{Child, ChildStdin, ChildStdout, Command, Stdio}, +}; + +#[cfg(target_os = "macos")] +struct UnifiedHelperProcess { + child: Child, + stdin: ChildStdin, + stdout: BufReader, +} + +#[cfg(target_os = "macos")] +static UNIFIED_HELPER: Lazy>> = + Lazy::new(|| StdMutex::new(None)); + +/// Send a JSON request and read a JSON response (one line each). +/// Used for `focus` and `paste` commands that produce a response. +#[cfg(target_os = "macos")] +pub(super) fn helper_send_receive( + request: &serde_json::Value, +) -> Result { + ensure_helper_running()?; + let mut guard = UNIFIED_HELPER + .lock() + .map_err(|_| "unified helper lock poisoned".to_string())?; + let helper = guard + .as_mut() + .ok_or_else(|| "unified helper unavailable".to_string())?; + + // Write request + let line = request.to_string(); + helper + .stdin + .write_all(line.as_bytes()) + .and_then(|_| helper.stdin.write_all(b"\n")) + .and_then(|_| helper.stdin.flush()) + .map_err(|e| format!("failed to write to helper stdin: {e}"))?; + + // Read response (one line) + let mut response_line = String::new(); + helper + .stdout + .read_line(&mut response_line) + .map_err(|e| format!("failed to read helper stdout: {e}"))?; + + if response_line.trim().is_empty() { + return Err("helper returned empty response".to_string()); + } + + serde_json::from_str(response_line.trim()) + .map_err(|e| format!("failed to parse helper response: {e}")) +} + +/// Send a JSON request without waiting for a response. +/// Used for `show`, `hide`, and `quit` commands. +#[cfg(target_os = "macos")] +pub(super) fn helper_send_fire_and_forget(request: &serde_json::Value) -> Result<(), String> { + ensure_helper_running()?; + let mut guard = UNIFIED_HELPER + .lock() + .map_err(|_| "unified helper lock poisoned".to_string())?; + let helper = guard + .as_mut() + .ok_or_else(|| "unified helper unavailable".to_string())?; + + let line = request.to_string(); + helper + .stdin + .write_all(line.as_bytes()) + .and_then(|_| helper.stdin.write_all(b"\n")) + .and_then(|_| helper.stdin.flush()) + .map_err(|e| format!("failed to write to helper stdin: {e}"))?; + Ok(()) +} + +/// Quit and clean up the helper process. +#[cfg(target_os = "macos")] +pub(super) fn helper_quit() -> Result<(), String> { + let mut guard = UNIFIED_HELPER + .lock() + .map_err(|_| "unified helper lock poisoned".to_string())?; + if let Some(mut helper) = guard.take() { + let _ = helper.stdin.write_all(br#"{"type":"quit"}"#); + let _ = helper.stdin.write_all(b"\n"); + let _ = helper.stdin.flush(); + let _ = helper.child.kill(); + let _ = helper.child.wait(); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn ensure_helper_running() -> Result<(), String> { + let mut guard = UNIFIED_HELPER + .lock() + .map_err(|_| "unified helper lock poisoned".to_string())?; + + if let Some(helper) = guard.as_mut() { + if helper + .child + .try_wait() + .map_err(|e| format!("failed to query helper state: {e}"))? + .is_none() + { + return Ok(()); // Still running + } + log::debug!("[accessibility] unified helper exited, restarting"); + *guard = None; + } + + let binary_path = ensure_helper_binary()?; + let mut child = Command::new(&binary_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| format!("failed to spawn unified helper: {e}"))?; + + let stdin = child + .stdin + .take() + .ok_or_else(|| "failed to capture helper stdin".to_string())?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "failed to capture helper stdout".to_string())?; + + *guard = Some(UnifiedHelperProcess { + child, + stdin, + stdout: BufReader::new(stdout), + }); + log::debug!("[accessibility] unified helper started"); + Ok(()) +} + +#[cfg(target_os = "macos")] +fn ensure_helper_binary() -> Result { + let cache_dir = std::env::temp_dir().join("openhuman-accessibility-helper"); + fs::create_dir_all(&cache_dir).map_err(|e| format!("failed to create cache dir: {e}"))?; + let source_path = cache_dir.join("unified_helper.swift"); + let binary_path = cache_dir.join("unified_helper_bin"); + let source = unified_swift_source(); + + let needs_write = match fs::read_to_string(&source_path) { + Ok(existing) => existing != source, + Err(_) => true, + }; + if needs_write { + fs::write(&source_path, &source) + .map_err(|e| format!("failed to write helper source: {e}"))?; + } + + let needs_compile = needs_write || !binary_path.exists(); + if needs_compile { + log::debug!("[accessibility] compiling unified Swift helper"); + let output = Command::new("xcrun") + .args([ + "swiftc", + "-O", + "-framework", + "Cocoa", + "-framework", + "ApplicationServices", + ]) + .arg(&source_path) + .arg("-o") + .arg(&binary_path) + .output() + .or_else(|_| { + Command::new("swiftc") + .args([ + "-O", + "-framework", + "Cocoa", + "-framework", + "ApplicationServices", + ]) + .arg(&source_path) + .arg("-o") + .arg(&binary_path) + .output() + }) + .map_err(|e| format!("failed to invoke swiftc: {e}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return Err(format!( + "failed to compile unified helper: {}", + if stderr.is_empty() { + "swiftc returned non-zero exit status".to_string() + } else { + stderr + } + )); + } + log::debug!("[accessibility] unified helper compiled successfully"); + } + + Ok(binary_path) +} + +#[cfg(target_os = "macos")] +fn unified_swift_source() -> String { + r##"import Cocoa +import Foundation +import ApplicationServices + +// MARK: - Thread-safe stdout writer + +let stdoutLock = NSLock() + +func writeResponse(_ dict: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: dict), + let line = String(data: data, encoding: .utf8) else { return } + stdoutLock.lock() + print(line) + fflush(stdout) + stdoutLock.unlock() +} + +// MARK: - Accessibility Focus Query + +let textRoles: Set = ["AXTextArea", "AXTextField", "AXSearchField", "AXComboBox", "AXEditableText"] + +// Apps that need AXEnhancedUserInterface to expose focused text elements properly. +let chromiumAppPatterns = ["chrom", "electron", "code", "slack", "discord", "brave", "edge", "opera", "vivaldi", "arc"] + +func isChromiumApp(_ name: String) -> Bool { + let lower = name.lowercased() + return chromiumAppPatterns.contains(where: { lower.contains($0) }) +} + +func getAXStringAttr(_ element: AXUIElement, _ attr: String) -> String? { + var value: AnyObject? + let err = AXUIElementCopyAttributeValue(element, attr as CFString, &value) + guard err == .success, let str = value as? String, str != "missing value" else { return nil } + return str +} + +func getAXPosition(_ element: AXUIElement) -> (x: Int, y: Int)? { + var value: AnyObject? + let err = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as String as CFString, &value) + guard err == .success else { return nil } + var point = CGPoint.zero + AXValueGetValue(value as! AXValue, .cgPoint, &point) + return (Int(point.x), Int(point.y)) +} + +func getAXSize(_ element: AXUIElement) -> (w: Int, h: Int)? { + var value: AnyObject? + let err = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as String as CFString, &value) + guard err == .success else { return nil } + var size = CGSize.zero + AXValueGetValue(value as! AXValue, .cgSize, &size) + return (Int(size.width), Int(size.height)) +} + +func scanChildrenForText(_ parent: AXUIElement, depth: Int = 0) -> (role: String, text: String, pos: (Int, Int)?, size: (Int, Int)?)? { + if depth > 5 { return nil } + var childrenRef: AnyObject? + let err = AXUIElementCopyAttributeValue(parent, kAXChildrenAttribute as String as CFString, &childrenRef) + guard err == .success, let children = childrenRef as? [AXUIElement] else { return nil } + + // First pass: look for text-role elements with content + for child in children.prefix(200) { + let role = getAXStringAttr(child, kAXRoleAttribute as String) ?? "" + if textRoles.contains(role) { + var text = getAXStringAttr(child, kAXValueAttribute as String) ?? "" + if text.isEmpty { + text = getAXStringAttr(child, kAXSelectedTextAttribute as String) ?? "" + } + if !text.isEmpty { + return (role, text, getAXPosition(child), getAXSize(child)) + } + } + } + + // Second pass: look for AXStaticText with prompt patterns (terminal support) + var staticFallback: (role: String, text: String, pos: (Int, Int)?, size: (Int, Int)?)? + for child in children.prefix(200) { + let role = getAXStringAttr(child, kAXRoleAttribute as String) ?? "" + if role == "AXStaticText" { + let text = getAXStringAttr(child, kAXValueAttribute as String) ?? "" + if !text.isEmpty { + if text.contains("$ ") || text.contains("# ") || text.contains("> ") { + return (role, text, getAXPosition(child), getAXSize(child)) + } + if staticFallback == nil { + staticFallback = (role, text, getAXPosition(child), getAXSize(child)) + } + } + } + } + if let fb = staticFallback { return fb } + + // Recurse into children + for child in children.prefix(50) { + if let result = scanChildrenForText(child, depth: depth + 1) { + return result + } + } + return nil +} + +func queryFocusedElement(id: String?) -> [String: Any] { + var result: [String: Any] = [ + "type": "focus", + "app_name": NSNull(), + "role": NSNull(), + "text": "", + "selected_text": NSNull(), + "x": NSNull(), "y": NSNull(), "w": NSNull(), "h": NSNull(), + "error": NSNull(), + "ax_trusted": AXIsProcessTrusted(), + ] + if let id = id { result["id"] = id } + + let systemWide = AXUIElementCreateSystemWide() + + // Get focused application + var appRef: AnyObject? + var appErr = AXUIElementCopyAttributeValue(systemWide, kAXFocusedApplicationAttribute as String as CFString, &appRef) + guard appErr == .success, let appElement = appRef else { + result["error"] = "ERROR:no_focused_application" + return result + } + + let appName = getAXStringAttr(appElement as! AXUIElement, kAXTitleAttribute as String) ?? "unknown" + result["app_name"] = appName + + // Enable AXEnhancedUserInterface for Chromium apps + if isChromiumApp(appName) { + AXUIElementSetAttributeValue(appElement as! AXUIElement, "AXEnhancedUserInterface" as CFString, true as CFBoolean) + } + + // Get focused element + var focusedRef: AnyObject? + let focusErr = AXUIElementCopyAttributeValue(appElement as! AXUIElement, kAXFocusedUIElementAttribute as String as CFString, &focusedRef) + + if focusErr == .success, let focused = focusedRef { + let focusedElement = focused as! AXUIElement + let role = getAXStringAttr(focusedElement, kAXRoleAttribute as String) ?? "unknown" + result["role"] = role + + var text = getAXStringAttr(focusedElement, kAXValueAttribute as String) ?? "" + let selectedText = getAXStringAttr(focusedElement, kAXSelectedTextAttribute as String) + result["selected_text"] = selectedText ?? NSNull() + + if text.isEmpty, let sel = selectedText, !sel.isEmpty { + text = sel + } + if text.isEmpty { + text = getAXStringAttr(focusedElement, kAXTitleAttribute as String) ?? "" + } + + if let pos = getAXPosition(focusedElement) { + result["x"] = pos.x + result["y"] = pos.y + } + if let size = getAXSize(focusedElement) { + result["w"] = size.w + result["h"] = size.h + } + + // If we got text from a text-role element, we're done + if !text.isEmpty && textRoles.contains(role) { + result["text"] = text + return result + } + + // If role is not a text role, still return text if it looks terminal-like + let terminalApps = ["terminal", "iterm", "wezterm", "warp", "alacritty", "kitty", "ghostty", "hyper", "rio"] + let isTerminal = terminalApps.contains(where: { appName.lowercased().contains($0) }) + if isTerminal && !text.isEmpty { + result["text"] = text + return result + } + + // Text is empty or not from a text role — scan window children + if text.isEmpty || !textRoles.contains(role) { + // Try scanning focused window's children + var windowRef: AnyObject? + let winErr = AXUIElementCopyAttributeValue(appElement as! AXUIElement, kAXFocusedWindowAttribute as String as CFString, &windowRef) + if winErr == .success, let window = windowRef { + if let found = scanChildrenForText(window as! AXUIElement) { + result["role"] = found.role + result["text"] = found.text + if let pos = found.pos { result["x"] = pos.0; result["y"] = pos.1 } + if let size = found.size { result["w"] = size.0; result["h"] = size.1 } + return result + } + } + + if text.isEmpty { + result["error"] = "ERROR:no_text_candidate_found" + } else { + // Got text but from non-text role and not terminal — still return it + result["text"] = text + } + } else { + result["text"] = text + } + } else { + // No focused element found — try window scanning + var windowRef: AnyObject? + let winErr = AXUIElementCopyAttributeValue(appElement as! AXUIElement, kAXFocusedWindowAttribute as String as CFString, &windowRef) + if winErr == .success, let window = windowRef { + if let found = scanChildrenForText(window as! AXUIElement) { + result["role"] = found.role + result["text"] = found.text + if let pos = found.pos { result["x"] = pos.0; result["y"] = pos.1 } + if let size = found.size { result["w"] = size.0; result["h"] = size.1 } + return result + } + } + result["error"] = "ERROR:-1728:no_focused_element" + } + + return result +} + +// MARK: - Paste Helper + +func pasteText(id: String?, text: String) -> [String: Any] { + var result: [String: Any] = ["type": "paste", "ok": true, "error": NSNull()] + if let id = id { result["id"] = id } + + let pb = NSPasteboard.general + let originalContents = pb.string(forType: .string) + + // Set clipboard to new text + pb.clearContents() + pb.setString(text, forType: .string) + + // Brief delay for clipboard to settle + usleep(10_000) // 10ms + + // Simulate Cmd+V via CGEvent + guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: 0x09, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: 0x09, keyDown: false) else { + result["ok"] = false + result["error"] = "failed to create CGEvent" + return result + } + keyDown.flags = .maskCommand + keyUp.flags = .maskCommand + keyDown.post(tap: .cgSessionEventTap) + usleep(8_000) // 8ms between key down/up + keyUp.post(tap: .cgSessionEventTap) + + // Restore clipboard after delay + if let original = originalContents { + DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + .milliseconds(250)) { + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(original, forType: .string) + } + } + + return result +} + +// MARK: - Overlay Controller + +final class OverlayController { + private var panel: NSPanel? + private var textField: NSTextField? + private var hideWorkItem: DispatchWorkItem? + + func show(x: CGFloat, yTop: CGFloat, width: CGFloat, height: CGFloat, text: String, ttlMs: Int) { + let panelWidth = min(420, max(140, CGFloat(text.count) * 7 + 26)) + let panelHeight: CGFloat = 26 + + // Multi-monitor: find the screen containing the target or mouse cursor. + let screen: NSScreen? = { + let mainHeight = NSScreen.screens.first?.frame.height ?? 900 + if width > 0 && height > 0 { + let cocoaPoint = NSPoint(x: x + width / 2, y: mainHeight - (yTop + height / 2)) + if let s = NSScreen.screens.first(where: { $0.frame.contains(cocoaPoint) }) { + return s + } + } + let mouseLocation = NSEvent.mouseLocation + if let s = NSScreen.screens.first(where: { $0.frame.contains(mouseLocation) }) { + return s + } + return NSScreen.main ?? NSScreen.screens.first + }() + let screenFrame = screen?.frame ?? NSRect(x: 0, y: 0, width: 1440, height: 900) + let screenHeight = screenFrame.height + screenFrame.origin.y + + var originX: CGFloat + var originYCocoa: CGFloat + + if width > 0 && height > 0 { + originX = x + max(8, min(width - panelWidth - 8, 28)) + let originYTop = yTop + max(5, min(height - panelHeight - 4, 10)) + originYCocoa = max(6, screenHeight - originYTop - panelHeight) + } else { + let mouseLocation = NSEvent.mouseLocation + originX = mouseLocation.x + 8 + originYCocoa = mouseLocation.y - panelHeight - 8 + } + + // Clamp to screen bounds + originX = max(screenFrame.origin.x + 4, min(originX, screenFrame.origin.x + screenFrame.width - panelWidth - 4)) + originYCocoa = max(screenFrame.origin.y + 4, min(originYCocoa, screenFrame.origin.y + screenFrame.height - panelHeight - 4)) + + if panel == nil { + let p = NSPanel( + contentRect: NSRect(x: originX, y: originYCocoa, width: panelWidth, height: panelHeight), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + p.level = .statusBar + p.hasShadow = false + p.isOpaque = false + p.backgroundColor = .clear + p.ignoresMouseEvents = true + p.collectionBehavior = [.canJoinAllSpaces, .transient] + + let content = NSView(frame: NSRect(x: 0, y: 0, width: panelWidth, height: panelHeight)) + content.wantsLayer = true + content.layer?.cornerRadius = 6 + content.layer?.backgroundColor = NSColor(white: 0.08, alpha: 0.35).cgColor + p.contentView = content + + let label = NSTextField(labelWithString: text) + label.frame = NSRect(x: 8, y: 4, width: panelWidth - 12, height: 18) + label.textColor = NSColor(white: 1.0, alpha: 0.46) + label.font = NSFont.systemFont(ofSize: 13) + label.lineBreakMode = .byTruncatingTail + content.addSubview(label) + + panel = p + textField = label + } + + panel?.setFrame(NSRect(x: originX, y: originYCocoa, width: panelWidth, height: panelHeight), display: true) + panel?.contentView?.frame = NSRect(x: 0, y: 0, width: panelWidth, height: panelHeight) + textField?.frame = NSRect(x: 8, y: 4, width: panelWidth - 12, height: 18) + textField?.stringValue = text + panel?.orderFrontRegardless() + + hideWorkItem?.cancel() + let work = DispatchWorkItem { [weak self] in + self?.hide() + } + hideWorkItem = work + DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(max(120, ttlMs)), execute: work) + } + + func hide() { + panel?.orderOut(nil) + } +} + +// MARK: - Main Entry Point + +let app = NSApplication.shared +app.setActivationPolicy(.accessory) +let controller = OverlayController() + +DispatchQueue.global(qos: .userInitiated).async { + while let line = readLine() { + guard let data = line.data(using: .utf8), + let payload = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let kind = payload["type"] as? String else { + continue + } + let id = payload["id"] as? String + + switch kind { + case "focus": + let response = queryFocusedElement(id: id) + writeResponse(response) + + case "paste": + let text = (payload["text"] as? String) ?? "" + let response = pasteText(id: id, text: text) + writeResponse(response) + + case "show": + let x = CGFloat((payload["x"] as? NSNumber)?.doubleValue ?? 0) + let y = CGFloat((payload["y"] as? NSNumber)?.doubleValue ?? 0) + let w = CGFloat((payload["w"] as? NSNumber)?.doubleValue ?? 0) + let h = CGFloat((payload["h"] as? NSNumber)?.doubleValue ?? 0) + let text = (payload["text"] as? String) ?? "" + let ttl = (payload["ttl_ms"] as? NSNumber)?.intValue ?? 900 + DispatchQueue.main.async { + controller.show(x: x, yTop: y, width: w, height: h, text: text, ttlMs: ttl) + } + + case "hide": + DispatchQueue.main.async { + controller.hide() + } + + case "quit": + DispatchQueue.main.async { + controller.hide() + NSApplication.shared.terminate(nil) + } + return + + default: + break + } + } +} + +app.run() +"##.to_string() +} diff --git a/src/openhuman/accessibility/keys.rs b/src/openhuman/accessibility/keys.rs new file mode 100644 index 000000000..125aed9ce --- /dev/null +++ b/src/openhuman/accessibility/keys.rs @@ -0,0 +1,38 @@ +//! Key state probes via direct FFI (lightweight, no helper needed). + +#[cfg(target_os = "macos")] +pub fn is_tab_key_down() -> bool { + unsafe { CGEventSourceKeyState(KCG_EVENT_SOURCE_STATE_COMBINED_SESSION_STATE, KVK_TAB) } +} + +#[cfg(not(target_os = "macos"))] +pub fn is_tab_key_down() -> bool { + false +} + +#[cfg(target_os = "macos")] +pub fn is_escape_key_down() -> bool { + unsafe { CGEventSourceKeyState(KCG_EVENT_SOURCE_STATE_COMBINED_SESSION_STATE, KVK_ESCAPE) } +} + +#[cfg(not(target_os = "macos"))] +pub fn is_escape_key_down() -> bool { + false +} + +// --------------------------------------------------------------------------- +// macOS FFI declarations +// --------------------------------------------------------------------------- + +#[cfg(target_os = "macos")] +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + fn CGEventSourceKeyState(state_id: i32, key: u16) -> bool; +} + +#[cfg(target_os = "macos")] +const KCG_EVENT_SOURCE_STATE_COMBINED_SESSION_STATE: i32 = 0; +#[cfg(target_os = "macos")] +const KVK_TAB: u16 = 48; +#[cfg(target_os = "macos")] +const KVK_ESCAPE: u16 = 53; diff --git a/src/openhuman/accessibility/mod.rs b/src/openhuman/accessibility/mod.rs new file mode 100644 index 000000000..292e12c45 --- /dev/null +++ b/src/openhuman/accessibility/mod.rs @@ -0,0 +1,41 @@ +//! Platform accessibility middleware: focus queries, text insertion, key state, +//! overlays, screen capture, and permission management. +//! +//! Centralises all macOS AX/CGEvent/IOKit FFI and the unified Swift helper process. +//! Consumer modules (autocomplete, screen_intelligence, voice) call into this module +//! instead of owning platform-specific code directly. + +mod capture; +mod focus; +mod helper; +mod keys; +mod overlay; +mod paste; +mod permissions; +mod terminal; +mod text_util; +mod types; + +pub use capture::{capture_screen_image_ref_for_context, CaptureMode, MAX_SCREENSHOT_BYTES}; +pub use focus::{ + focused_text_context, focused_text_context_verbose, foreground_context, + parse_foreground_output, validate_focused_target, +}; +pub use keys::{is_escape_key_down, is_tab_key_down}; +pub use overlay::{hide_overlay, quit_overlay, show_overlay}; +pub use paste::apply_text_to_focused_field; +#[cfg(target_os = "macos")] +pub use permissions::{ + detect_accessibility_permission, detect_input_monitoring_permission, + detect_screen_recording_permission, open_macos_privacy_pane, request_accessibility_access, + request_screen_recording_access, +}; +pub use permissions::{detect_permissions, permission_to_str}; +pub use terminal::{ + extract_terminal_input_context, is_terminal_app, is_text_role, looks_like_terminal_buffer, +}; +pub use text_util::{normalize_ax_value, parse_ax_number, truncate_tail}; +pub use types::{ + AppContext, ElementBounds, FocusedTextContext, PermissionKind, PermissionState, + PermissionStatus, +}; diff --git a/src/openhuman/accessibility/overlay.rs b/src/openhuman/accessibility/overlay.rs new file mode 100644 index 000000000..758db2034 --- /dev/null +++ b/src/openhuman/accessibility/overlay.rs @@ -0,0 +1,47 @@ +//! Overlay display via the unified Swift helper process. + +use super::text_util::truncate_tail; +use super::types::ElementBounds; + +/// Show an overlay badge near the given element bounds. +#[cfg(target_os = "macos")] +pub fn show_overlay(bounds: &ElementBounds, text: &str, ttl_ms: u32) -> Result<(), String> { + let message = serde_json::json!({ + "type": "show", + "x": bounds.x, + "y": bounds.y, + "w": bounds.width, + "h": bounds.height, + "text": truncate_tail(text, 96), + "ttl_ms": ttl_ms + }); + super::helper::helper_send_fire_and_forget(&message) +} + +/// Hide the overlay badge. +#[cfg(target_os = "macos")] +pub fn hide_overlay() -> Result<(), String> { + let message = serde_json::json!({"type": "hide"}); + super::helper::helper_send_fire_and_forget(&message) +} + +/// Quit the unified helper process (cleanup on shutdown). +#[cfg(target_os = "macos")] +pub fn quit_overlay() -> Result<(), String> { + super::helper::helper_quit() +} + +#[cfg(not(target_os = "macos"))] +pub fn show_overlay(_bounds: &ElementBounds, _text: &str, _ttl_ms: u32) -> Result<(), String> { + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +pub fn hide_overlay() -> Result<(), String> { + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +pub fn quit_overlay() -> Result<(), String> { + Ok(()) +} diff --git a/src/openhuman/accessibility/paste.rs b/src/openhuman/accessibility/paste.rs new file mode 100644 index 000000000..91ff2eca3 --- /dev/null +++ b/src/openhuman/accessibility/paste.rs @@ -0,0 +1,229 @@ +//! Text insertion into focused fields via accessibility APIs. +//! +//! Three-tier strategy: (1) Swift helper paste, (2) osascript clipboard + CGEvent, (3) AXValue write. + +use super::text_util::truncate_tail; + +/// Apply suggestion text to the focused field. +/// Tries: (1) helper paste, (2) osascript clipboard+CGEvent, (3) AXValue write. +#[cfg(target_os = "macos")] +pub fn apply_text_to_focused_field(text: &str) -> Result<(), String> { + log::debug!( + "[accessibility] applying text: {:?}", + truncate_tail(text, 40) + ); + + // Try 1: unified Swift helper (handles clipboard save/set/paste/restore internally) + match paste_text_via_helper(text) { + Ok(()) => return Ok(()), + Err(e) => { + log::debug!( + "[accessibility] helper paste failed ({}), trying osascript+CGEvent", + e + ); + } + } + + // Try 2: osascript clipboard + CGEvent Cmd+V + match paste_text_via_osascript_cgevent(text) { + Ok(()) => return Ok(()), + Err(e) => { + log::debug!( + "[accessibility] osascript+CGEvent paste failed ({}), trying AXValue write", + e + ); + } + } + + // Try 3: direct AXValue write (last resort) + apply_text_via_axvalue(text) +} + +/// Paste via the unified Swift helper. +#[cfg(target_os = "macos")] +fn paste_text_via_helper(text: &str) -> Result<(), String> { + let request = serde_json::json!({"type": "paste", "text": text}); + let resp = super::helper::helper_send_receive(&request)?; + let ok = resp.get("ok").and_then(|v| v.as_bool()).unwrap_or(false); + if ok { + Ok(()) + } else { + let err = resp + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("unknown paste error"); + Err(err.to_string()) + } +} + +/// Paste via osascript (clipboard set) + CGEvent (Cmd+V simulation). +#[cfg(target_os = "macos")] +fn paste_text_via_osascript_cgevent(text: &str) -> Result<(), String> { + let original_clipboard = clipboard_save_osascript(); + + // Set clipboard via osascript — preserve multi-line text using AppleScript linefeed. + let script = { + let lines: Vec = text + .split('\n') + .map(|line| { + let escaped = line.replace('\\', "\\\\").replace('\"', "\\\""); + format!("\"{}\"", escaped) + }) + .collect(); + let joined = lines.join(" & linefeed & "); + format!("set the clipboard to ({})", joined) + }; + let output = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .map_err(|e| format!("failed to set clipboard: {e}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return Err(format!("failed to set clipboard: {stderr}")); + } + + std::thread::sleep(std::time::Duration::from_millis(10)); + + // Cmd+V via CGEvent + unsafe { + let key_down = CGEventCreateKeyboardEvent(std::ptr::null(), KVK_V, true); + let key_up = CGEventCreateKeyboardEvent(std::ptr::null(), KVK_V, false); + if key_down.is_null() || key_up.is_null() { + if !key_down.is_null() { + CFRelease(key_down as *const _); + } + if !key_up.is_null() { + CFRelease(key_up as *const _); + } + return Err("failed to create CGEvent for paste".to_string()); + } + CGEventSetFlags(key_down, KCG_EVENT_FLAG_MASK_COMMAND); + CGEventSetFlags(key_up, KCG_EVENT_FLAG_MASK_COMMAND); + CGEventPost(KCG_HID_EVENT_TAP, key_down); + std::thread::sleep(std::time::Duration::from_millis(8)); + CGEventPost(KCG_HID_EVENT_TAP, key_up); + CFRelease(key_down as *const _); + CFRelease(key_up as *const _); + } + + // Restore clipboard + if let Some(original) = original_clipboard { + std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(250)); + let lines: Vec = original + .split('\n') + .map(|line| { + let escaped = line.replace('\\', "\\\\").replace('\"', "\\\""); + format!("\"{}\"", escaped) + }) + .collect(); + let joined = lines.join(" & linefeed & "); + let script = format!("set the clipboard to ({})", joined); + let _ = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output(); + }); + } + + Ok(()) +} + +#[cfg(target_os = "macos")] +fn clipboard_save_osascript() -> Option { + let output = std::process::Command::new("osascript") + .arg("-e") + .arg("the clipboard as text") + .output() + .ok()?; + if output.status.success() { + let text = String::from_utf8_lossy(&output.stdout) + .trim_end() + .to_string(); + if text.is_empty() || text == "missing value" { + None + } else { + Some(text) + } + } else { + None + } +} + +/// Fallback insertion: direct AXValue write via AppleScript. +#[cfg(target_os = "macos")] +fn apply_text_via_axvalue(text: &str) -> Result<(), String> { + let escaped = text + .replace('\\', "\\\\") + .replace('\"', "\\\"") + .replace('\n', " "); + let script = format!( + r##" +tell application "System Events" + set frontApp to first application process whose frontmost is true + set focusedElement to value of attribute "AXFocusedUIElement" of frontApp + set currentValue to "" + try + set currentValue to value of attribute "AXValue" of focusedElement as text + end try + if currentValue is "missing value" then set currentValue to "" + if currentValue is "" then + try + set currentValue to value of attribute "AXSelectedText" of focusedElement as text + end try + if currentValue is "missing value" then set currentValue to "" + end if + set value of attribute "AXValue" of focusedElement to (currentValue & "{}") +end tell +"##, + escaped + ); + let output = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .map_err(|e| format!("failed to run osascript: {e}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.is_empty() { + return Err("failed to apply text to focused field".to_string()); + } + return Err(format!("failed to apply text to focused field: {stderr}")); + } + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +pub fn apply_text_to_focused_field(_text: &str) -> Result<(), String> { + Err("text insertion is only supported on macOS".to_string()) +} + +// --------------------------------------------------------------------------- +// macOS FFI declarations for paste +// --------------------------------------------------------------------------- + +#[cfg(target_os = "macos")] +#[link(name = "CoreGraphics", kind = "framework")] +extern "C" { + fn CGEventCreateKeyboardEvent( + source: *const std::ffi::c_void, + virtual_key: u16, + key_down: bool, + ) -> *mut std::ffi::c_void; + fn CGEventSetFlags(event: *mut std::ffi::c_void, flags: u64); + fn CGEventPost(tap: u32, event: *mut std::ffi::c_void); +} + +#[cfg(target_os = "macos")] +#[link(name = "CoreFoundation", kind = "framework")] +extern "C" { + fn CFRelease(cf: *const std::ffi::c_void); +} + +#[cfg(target_os = "macos")] +const KVK_V: u16 = 9; +#[cfg(target_os = "macos")] +const KCG_HID_EVENT_TAP: u32 = 0; +#[cfg(target_os = "macos")] +const KCG_EVENT_FLAG_MASK_COMMAND: u64 = 0x00100000; diff --git a/src/openhuman/accessibility/permissions.rs b/src/openhuman/accessibility/permissions.rs new file mode 100644 index 000000000..98c9528e4 --- /dev/null +++ b/src/openhuman/accessibility/permissions.rs @@ -0,0 +1,148 @@ +//! Platform permission detection and requests for accessibility, screen recording, input monitoring. + +use super::types::{PermissionKind, PermissionState, PermissionStatus}; + +#[cfg(target_os = "macos")] +use std::ffi::c_void; + +#[cfg(target_os = "macos")] +type CFAllocatorRef = *const c_void; +#[cfg(target_os = "macos")] +type CFDictionaryRef = *const c_void; +#[cfg(target_os = "macos")] +type CFBooleanRef = *const c_void; +#[cfg(target_os = "macos")] +type CFStringRef = *const c_void; + +#[cfg(target_os = "macos")] +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + fn AXIsProcessTrusted() -> bool; + fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> bool; + static kAXTrustedCheckOptionPrompt: CFStringRef; + fn CGPreflightScreenCaptureAccess() -> bool; + fn CGRequestScreenCaptureAccess() -> bool; +} + +#[cfg(target_os = "macos")] +#[link(name = "CoreFoundation", kind = "framework")] +extern "C" { + static kCFAllocatorDefault: CFAllocatorRef; + static kCFBooleanTrue: CFBooleanRef; + fn CFDictionaryCreate( + allocator: CFAllocatorRef, + keys: *const *const c_void, + values: *const *const c_void, + num_values: isize, + key_callbacks: *const c_void, + value_callbacks: *const c_void, + ) -> CFDictionaryRef; + fn CFRelease(cf: *const c_void); +} + +#[cfg(target_os = "macos")] +#[link(name = "IOKit", kind = "framework")] +extern "C" { + fn IOHIDCheckAccess(request_type: i32) -> isize; +} + +#[cfg(target_os = "macos")] +const IOHID_REQUEST_TYPE_LISTEN_EVENT: i32 = 1; +#[cfg(target_os = "macos")] +const IOHID_ACCESS_GRANTED: isize = 0; +#[cfg(target_os = "macos")] +const IOHID_ACCESS_DENIED: isize = 1; +#[cfg(target_os = "macos")] +const IOHID_ACCESS_UNKNOWN: isize = 2; + +pub fn permission_to_str(permission: PermissionKind) -> &'static str { + match permission { + PermissionKind::ScreenRecording => "screen_recording", + PermissionKind::Accessibility => "accessibility", + PermissionKind::InputMonitoring => "input_monitoring", + } +} + +#[cfg(target_os = "macos")] +pub fn open_macos_privacy_pane(pane: &str) { + let url = format!("x-apple.systempreferences:com.apple.preference.security?{pane}"); + let _ = std::process::Command::new("open").arg(url).status(); +} + +#[cfg(target_os = "macos")] +pub fn request_accessibility_access() { + unsafe { + let keys = [kAXTrustedCheckOptionPrompt as *const c_void]; + let values = [kCFBooleanTrue as *const c_void]; + let options = CFDictionaryCreate( + kCFAllocatorDefault, + keys.as_ptr(), + values.as_ptr(), + 1, + std::ptr::null(), + std::ptr::null(), + ); + let _ = AXIsProcessTrustedWithOptions(options); + if !options.is_null() { + CFRelease(options); + } + } +} + +#[cfg(target_os = "macos")] +pub fn request_screen_recording_access() { + unsafe { + let _ = CGRequestScreenCaptureAccess(); + } +} + +#[cfg(target_os = "macos")] +pub fn detect_accessibility_permission() -> PermissionState { + unsafe { + if AXIsProcessTrusted() { + PermissionState::Granted + } else { + PermissionState::Denied + } + } +} + +#[cfg(target_os = "macos")] +pub fn detect_screen_recording_permission() -> PermissionState { + unsafe { + if CGPreflightScreenCaptureAccess() { + PermissionState::Granted + } else { + PermissionState::Denied + } + } +} + +#[cfg(target_os = "macos")] +pub fn detect_input_monitoring_permission() -> PermissionState { + let access = unsafe { IOHIDCheckAccess(IOHID_REQUEST_TYPE_LISTEN_EVENT) }; + match access { + IOHID_ACCESS_GRANTED => PermissionState::Granted, + IOHID_ACCESS_DENIED => PermissionState::Denied, + IOHID_ACCESS_UNKNOWN => PermissionState::Unknown, + _ => PermissionState::Unknown, + } +} + +#[cfg(target_os = "macos")] +pub fn detect_permissions() -> PermissionStatus { + PermissionStatus { + screen_recording: detect_screen_recording_permission(), + accessibility: detect_accessibility_permission(), + input_monitoring: detect_input_monitoring_permission(), + } +} + +#[cfg(not(target_os = "macos"))] +pub fn detect_permissions() -> PermissionStatus { + PermissionStatus { + screen_recording: PermissionState::Unsupported, + accessibility: PermissionState::Unsupported, + input_monitoring: PermissionState::Unsupported, + } +} diff --git a/src/openhuman/accessibility/terminal.rs b/src/openhuman/accessibility/terminal.rs new file mode 100644 index 000000000..7516891ae --- /dev/null +++ b/src/openhuman/accessibility/terminal.rs @@ -0,0 +1,82 @@ +//! Terminal app detection and context extraction. + +/// Known terminal application name substrings (lowercase). +/// Extend this list to support additional terminal emulators. +pub const TERMINAL_NAMES: &[&str] = &[ + "terminal", + "iterm", + "wezterm", + "warp", + "alacritty", + "kitty", + "ghostty", + "hyper", + "rio", + "tabby", + "wave", + "contour", + "foot", +]; + +pub fn is_text_role(role: Option<&str>) -> bool { + matches!( + role.unwrap_or_default(), + "AXTextArea" | "AXTextField" | "AXSearchField" | "AXComboBox" | "AXEditableText" + ) +} + +pub fn is_terminal_app(app_name: Option<&str>) -> bool { + let app = app_name.unwrap_or_default().to_ascii_lowercase(); + TERMINAL_NAMES.iter().any(|needle| app.contains(needle)) +} + +pub fn looks_like_terminal_buffer(text: &str) -> bool { + let lower = text.to_ascii_lowercase(); + let line_count = text.lines().count(); + line_count >= 5 + && (lower.contains("$ ") + || lower.contains("# ") + || lower.contains("❯") + || lower.contains("[1] 0:") + || lower.contains("tmux") + || lower.contains("cargo run") + || lower.contains("git status")) +} + +fn is_terminal_noise_line(line: &str) -> bool { + let trimmed = line.trim(); + if trimmed.is_empty() { + return true; + } + trimmed.starts_with('•') + || trimmed.starts_with('└') + || trimmed.starts_with('─') + || trimmed.starts_with('│') + || (trimmed.starts_with('[') + && (trimmed.contains(" 0:") || trimmed.contains("[tmux]") || trimmed.contains("\"⠙"))) +} + +pub fn extract_terminal_input_context(text: &str) -> String { + let mut fallback = String::new(); + for raw_line in text.lines().rev().take(40) { + let line = raw_line.trim(); + if line.is_empty() { + continue; + } + if fallback.is_empty() && !is_terminal_noise_line(line) { + fallback = line.to_string(); + } + if is_terminal_noise_line(line) { + continue; + } + if line.contains("$ ") + || line.contains("# ") + || line.contains("❯") + || line.contains("➜") + || line.contains("λ") + { + return line.to_string(); + } + } + fallback +} diff --git a/src/openhuman/accessibility/text_util.rs b/src/openhuman/accessibility/text_util.rs new file mode 100644 index 000000000..63f8bbbc0 --- /dev/null +++ b/src/openhuman/accessibility/text_util.rs @@ -0,0 +1,36 @@ +//! Shared text utilities for accessibility value parsing. + +pub fn truncate_tail(text: &str, max_chars: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= max_chars { + return text.to_string(); + } + chars[chars.len() - max_chars..].iter().collect() +} + +pub fn normalize_ax_value(raw: &str) -> String { + let v = raw.trim(); + if v.eq_ignore_ascii_case("missing value") { + String::new() + } else { + v.to_string() + } +} + +pub fn parse_ax_number(raw: &str) -> Option { + let trimmed = normalize_ax_value(raw); + if trimmed.is_empty() { + return None; + } + let cleaned = trimmed.replace(',', "."); + cleaned.parse::().ok().and_then(|v| { + if !v.is_finite() { + return None; + } + let rounded = v.round(); + if rounded < i32::MIN as f64 || rounded > i32::MAX as f64 { + return None; + } + Some(rounded as i32) + }) +} diff --git a/src/openhuman/accessibility/types.rs b/src/openhuman/accessibility/types.rs new file mode 100644 index 000000000..8cb018588 --- /dev/null +++ b/src/openhuman/accessibility/types.rs @@ -0,0 +1,73 @@ +//! Shared platform types for accessibility, focus, and permissions. + +use serde::{Deserialize, Serialize}; + +/// Unified element bounds — used by both autocomplete and screen intelligence. +#[derive(Debug, Clone, Copy)] +pub struct ElementBounds { + pub x: i32, + pub y: i32, + pub width: i32, + pub height: i32, +} + +/// Context returned by an accessibility focus query. +#[derive(Debug, Clone)] +pub struct FocusedTextContext { + pub app_name: Option, + pub role: Option, + pub text: String, + pub selected_text: Option, + pub raw_error: Option, + pub bounds: Option, +} + +/// Foreground application context for capture and policy decisions. +#[derive(Debug, Clone)] +pub struct AppContext { + pub app_name: Option, + pub window_title: Option, + pub bounds: Option, +} + +impl AppContext { + pub fn same_as(&self, other: &AppContext) -> bool { + self.app_name == other.app_name + && self.window_title == other.window_title + && self.bounds.as_ref().map(|b| (b.x, b.y, b.width, b.height)) + == other.bounds.as_ref().map(|b| (b.x, b.y, b.width, b.height)) + } + + pub fn as_compound_text(&self) -> String { + format!( + "{} {}", + self.app_name.clone().unwrap_or_default(), + self.window_title.clone().unwrap_or_default() + ) + .to_lowercase() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PermissionState { + Granted, + Denied, + Unknown, + Unsupported, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PermissionStatus { + pub screen_recording: PermissionState, + pub accessibility: PermissionState, + pub input_monitoring: PermissionState, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PermissionKind { + ScreenRecording, + Accessibility, + InputMonitoring, +} diff --git a/src/openhuman/autocomplete/core.rs b/src/openhuman/autocomplete/core.rs deleted file mode 100644 index bddd127f0..000000000 --- a/src/openhuman/autocomplete/core.rs +++ /dev/null @@ -1,1601 +0,0 @@ -use crate::openhuman::config::{AutocompleteConfig, Config}; -use crate::openhuman::local_ai; -use chrono::Utc; -use once_cell::sync::Lazy; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -#[cfg(target_os = "macos")] -use std::sync::Mutex as StdMutex; -#[cfg(target_os = "macos")] -use std::{ - fs, - io::Write, - path::PathBuf, - process::{Child, ChildStdin, Command, Stdio}, -}; -use tokio::sync::Mutex; -use tokio::task::JoinHandle; -use tokio::time::{self, Duration, Instant}; - -const MAX_SUGGESTION_CHARS: usize = 64; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteSuggestion { - pub value: String, - pub confidence: f32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteStatus { - pub platform_supported: bool, - pub enabled: bool, - pub running: bool, - pub phase: String, - pub debounce_ms: u64, - pub model_id: String, - pub app_name: Option, - pub last_error: Option, - pub updated_at_ms: Option, - pub suggestion: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteStartParams { - pub debounce_ms: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteStartResult { - pub started: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteStopParams { - pub reason: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteStopResult { - pub stopped: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteCurrentParams { - pub context: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteCurrentResult { - pub app_name: Option, - pub context: String, - pub suggestion: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteDebugFocusResult { - pub app_name: Option, - pub role: Option, - pub context: String, - pub selected_text: Option, - pub raw_error: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteAcceptParams { - pub suggestion: Option, - /// When true, skip applying text via accessibility (caller already inserted it). - pub skip_apply: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteAcceptResult { - pub accepted: bool, - pub applied: bool, - pub value: Option, - pub reason: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteSetStyleParams { - pub enabled: Option, - pub debounce_ms: Option, - pub max_chars: Option, - pub style_preset: Option, - pub style_instructions: Option, - pub style_examples: Option>, - pub disabled_apps: Option>, - pub accept_with_tab: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AutocompleteSetStyleResult { - pub config: AutocompleteConfig, -} - -#[derive(Debug, Clone)] -struct FocusedTextContext { - app_name: Option, - role: Option, - text: String, - selected_text: Option, - raw_error: Option, - bounds: Option, -} - -#[derive(Debug, Clone, Copy)] -struct FocusedElementBounds { - x: i32, - y: i32, - width: i32, - height: i32, -} - -fn is_text_role(role: Option<&str>) -> bool { - matches!( - role.unwrap_or_default(), - "AXTextArea" | "AXTextField" | "AXSearchField" | "AXComboBox" | "AXEditableText" - ) -} - -fn is_terminal_app(app_name: Option<&str>) -> bool { - let app = app_name.unwrap_or_default().to_ascii_lowercase(); - [ - "terminal", - "iterm", - "wezterm", - "warp", - "alacritty", - "kitty", - "ghostty", - "hyper", - "rio", - ] - .iter() - .any(|needle| app.contains(needle)) -} - -fn looks_like_terminal_buffer(text: &str) -> bool { - let lower = text.to_ascii_lowercase(); - let line_count = text.lines().count(); - line_count >= 5 - && (lower.contains("$ ") - || lower.contains("# ") - || lower.contains("❯") - || lower.contains("[1] 0:") - || lower.contains("tmux") - || lower.contains("cargo run") - || lower.contains("git status")) -} - -fn is_terminal_noise_line(line: &str) -> bool { - let trimmed = line.trim(); - if trimmed.is_empty() { - return true; - } - trimmed.starts_with('•') - || trimmed.starts_with('└') - || trimmed.starts_with('─') - || trimmed.starts_with('│') - || (trimmed.starts_with('[') - && (trimmed.contains(" 0:") || trimmed.contains("[tmux]") || trimmed.contains("\"⠙"))) -} - -fn extract_terminal_input_context(text: &str) -> String { - let mut fallback = String::new(); - for raw_line in text.lines().rev().take(40) { - let line = raw_line.trim(); - if line.is_empty() { - continue; - } - if fallback.is_empty() && !is_terminal_noise_line(line) { - fallback = line.to_string(); - } - if is_terminal_noise_line(line) { - continue; - } - if line.contains("$ ") - || line.contains("# ") - || line.contains("❯") - || line.contains("➜") - || line.contains("λ") - { - return line.to_string(); - } - } - fallback -} - -struct EngineState { - running: bool, - phase: String, - debounce_ms: u64, - app_name: Option, - context: String, - suggestion: Option, - last_error: Option, - updated_at_ms: Option, - last_tab_down: bool, - last_escape_down: bool, - last_overlay_signature: Option, - task: Option>, -} - -impl Default for EngineState { - fn default() -> Self { - Self { - running: false, - phase: "idle".to_string(), - debounce_ms: 120, - app_name: None, - context: String::new(), - suggestion: None, - last_error: None, - updated_at_ms: None, - last_tab_down: false, - last_escape_down: false, - last_overlay_signature: None, - task: None, - } - } -} - -pub struct AutocompleteEngine { - inner: Mutex, -} - -impl Default for AutocompleteEngine { - fn default() -> Self { - Self::new() - } -} - -impl AutocompleteEngine { - pub fn new() -> Self { - Self { - inner: Mutex::new(EngineState::default()), - } - } - - pub async fn status(&self) -> AutocompleteStatus { - let config = Config::load_or_init() - .await - .unwrap_or_else(|_| Config::default()); - let state = self.inner.lock().await; - - AutocompleteStatus { - platform_supported: cfg!(target_os = "macos"), - enabled: config.autocomplete.enabled, - running: state.running, - phase: state.phase.clone(), - debounce_ms: state.debounce_ms, - model_id: config.local_ai.chat_model_id, - app_name: state.app_name.clone(), - last_error: state.last_error.clone(), - updated_at_ms: state.updated_at_ms, - suggestion: state.suggestion.clone(), - } - } - - pub async fn start( - &self, - params: AutocompleteStartParams, - ) -> Result { - if !cfg!(target_os = "macos") { - return Err("autocomplete is only supported on macOS".to_string()); - } - - let config = Config::load_or_init() - .await - .map_err(|e| format!("failed to load config: {e}"))?; - if !config.autocomplete.enabled { - return Ok(AutocompleteStartResult { started: false }); - } - - let debounce_ms = params - .debounce_ms - .unwrap_or(config.autocomplete.debounce_ms) - .clamp(50, 2000); - - let mut state = self.inner.lock().await; - if state.running { - return Ok(AutocompleteStartResult { started: false }); - } - state.running = true; - state.phase = "idle".to_string(); - state.debounce_ms = debounce_ms; - state.last_error = None; - - let engine = global_engine(); - state.task = Some(tokio::spawn(async move { - let mut last_refresh = Instant::now() - Duration::from_millis(debounce_ms); - loop { - { - let state = engine.inner.lock().await; - if !state.running { - break; - } - } - let _ = engine.try_reject_via_escape().await; - let _ = engine.try_accept_via_tab().await; - if last_refresh.elapsed() >= Duration::from_millis(debounce_ms) { - let refresh_result = - time::timeout(Duration::from_secs(15), engine.refresh(None)).await; - match refresh_result { - Ok(Err(err)) => { - let error_message = { - let mut state = engine.inner.lock().await; - state.phase = "error".to_string(); - state.last_error = Some(err); - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - state.last_error.clone() - }; - if let Some(error_message) = error_message { - let app_lower = engine - .inner - .lock() - .await - .app_name - .clone() - .unwrap_or_default() - .to_lowercase(); - if !app_lower.contains("openhuman") { - show_overflow_badge( - "error", - None, - Some(&error_message), - None, - None, - ); - } - } - } - Ok(Ok(())) => { - let mut state = engine.inner.lock().await; - if state.phase == "error" { - state.phase = "idle".to_string(); - } - state.last_error = None; - } - Err(_elapsed) => { - log::warn!("[autocomplete] refresh timed out after 15s, skipping"); - let mut state = engine.inner.lock().await; - state.phase = "idle".to_string(); - state.suggestion = None; - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - } - } - last_refresh = Instant::now(); - } - time::sleep(Duration::from_millis(24)).await; - } - })); - - Ok(AutocompleteStartResult { started: true }) - } - - pub async fn stop(&self, _params: Option) -> AutocompleteStopResult { - let mut state = self.inner.lock().await; - state.running = false; - state.phase = "idle".to_string(); - state.last_escape_down = false; - state.last_overlay_signature = None; - state.last_error = None; - state.suggestion = None; - state.context = String::new(); - state.app_name = None; - if let Some(task) = state.task.take() { - task.abort(); - } - #[cfg(target_os = "macos")] - let _ = overlay_helper_quit(); - AutocompleteStopResult { stopped: true } - } - - pub async fn current( - &self, - params: Option, - ) -> Result { - let context_override = params - .and_then(|p| p.context) - .filter(|c| !c.trim().is_empty()); - self.refresh(context_override).await?; - let state = self.inner.lock().await; - Ok(AutocompleteCurrentResult { - app_name: state.app_name.clone(), - context: state.context.clone(), - suggestion: state.suggestion.clone(), - }) - } - - pub async fn debug_focus(&self) -> Result { - let focused = focused_text_context_verbose()?; - Ok(AutocompleteDebugFocusResult { - app_name: focused.app_name, - role: focused.role, - context: focused.text, - selected_text: focused.selected_text, - raw_error: focused.raw_error, - }) - } - - pub async fn accept( - &self, - params: AutocompleteAcceptParams, - ) -> Result { - let value = if let Some(value) = params.suggestion { - value - } else { - let state = self.inner.lock().await; - state - .suggestion - .as_ref() - .map(|s| s.value.clone()) - .unwrap_or_default() - }; - - let cleaned = sanitize_suggestion(&value); - if cleaned.is_empty() { - return Ok(AutocompleteAcceptResult { - accepted: false, - applied: false, - value: None, - reason: Some("no suggestion available".to_string()), - }); - } - - let should_apply = !params.skip_apply.unwrap_or(false); - - { - let mut state = self.inner.lock().await; - state.phase = "accepting".to_string(); - } - if should_apply { - apply_text_to_focused_field(&cleaned)?; - } - { - let mut state = self.inner.lock().await; - state.suggestion = None; - state.phase = "idle".to_string(); - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - state.last_overlay_signature = None; - } - if should_apply { - show_overflow_badge("accepted", Some(&cleaned), None, None, None); - } - - // Persist acceptance for personalisation (fire-and-forget). - // Dual-write: KV (UI list) + local docs (semantic search). - { - let (ctx, app) = { - let s = self.inner.lock().await; - (s.context.clone(), s.app_name.clone()) - }; - let sug = cleaned.clone(); - tokio::spawn(async move { - crate::openhuman::autocomplete::history::save_accepted_completion( - &ctx, - &sug, - app.as_deref(), - ) - .await; - crate::openhuman::autocomplete::history::save_completion_to_local_docs( - &ctx, - &sug, - app.as_deref(), - ) - .await; - }); - } - - Ok(AutocompleteAcceptResult { - accepted: true, - applied: should_apply, - value: Some(cleaned), - reason: None, - }) - } - - pub async fn set_style( - &self, - params: AutocompleteSetStyleParams, - ) -> Result { - let mut config = Config::load_or_init() - .await - .map_err(|e| format!("failed to load config: {e}"))?; - if let Some(enabled) = params.enabled { - config.autocomplete.enabled = enabled; - } - if let Some(debounce_ms) = params.debounce_ms { - config.autocomplete.debounce_ms = debounce_ms.clamp(50, 2000); - } - if let Some(max_chars) = params.max_chars { - config.autocomplete.max_chars = max_chars.clamp(64, 2048); - } - if let Some(style_preset) = params.style_preset { - config.autocomplete.style_preset = style_preset.trim().to_string(); - } - if let Some(style_instructions) = params.style_instructions { - config.autocomplete.style_instructions = if style_instructions.trim().is_empty() { - None - } else { - Some(style_instructions.trim().to_string()) - }; - } - if let Some(style_examples) = params.style_examples { - config.autocomplete.style_examples = style_examples - .into_iter() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .take(8) - .collect(); - } - if let Some(disabled_apps) = params.disabled_apps { - config.autocomplete.disabled_apps = disabled_apps - .into_iter() - .map(|s| s.trim().to_lowercase()) - .filter(|s| !s.is_empty()) - .collect(); - } - if let Some(accept_with_tab) = params.accept_with_tab { - config.autocomplete.accept_with_tab = accept_with_tab; - } - config.save().await.map_err(|e| e.to_string())?; - - let mut state = self.inner.lock().await; - state.debounce_ms = config.autocomplete.debounce_ms; - state.last_tab_down = false; - state.last_escape_down = false; - if !config.autocomplete.enabled { - state.running = false; - if let Some(task) = state.task.take() { - task.abort(); - } - state.suggestion = None; - state.last_overlay_signature = None; - #[cfg(target_os = "macos")] - let _ = overlay_helper_quit(); - } - - Ok(AutocompleteSetStyleResult { - config: config.autocomplete, - }) - } - - async fn refresh(&self, context_override: Option) -> Result<(), String> { - let is_in_app = context_override.is_some(); - let config = Config::load_or_init() - .await - .map_err(|e| format!("failed to load config: {e}"))?; - if !config.autocomplete.enabled { - let mut state = self.inner.lock().await; - state.suggestion = None; - state.phase = "disabled".to_string(); - return Ok(()); - } - { - let mut state = self.inner.lock().await; - state.phase = "capturing_context".to_string(); - } - - let focused = if let Some(context) = context_override { - FocusedTextContext { - app_name: Some("OpenHuman".to_string()), - role: None, - text: context, - selected_text: None, - raw_error: None, - bounds: None, - } - } else { - let focused = focused_text_context_verbose()?; - if let Some(err) = focused.raw_error.as_deref() { - if is_no_text_candidate_error(err) || err.contains("ERROR:-1728") { - let mut state = self.inner.lock().await; - state.app_name = focused.app_name; - state.context = String::new(); - state.suggestion = None; - state.phase = "idle".to_string(); - state.last_error = None; - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - return Ok(()); - } - return Err(format!( - "focused text unavailable via accessibility api: {err}" - )); - } - focused - }; - - let app_lower = focused.app_name.clone().unwrap_or_default().to_lowercase(); - - // When OpenHuman itself is focused AND this is the background engine loop, - // skip AX-based refresh — the in-app React polling handles suggestions. - // When is_in_app (context_override provided), we still want inference to run. - if !is_in_app && app_lower.contains("openhuman") { - let mut state = self.inner.lock().await; - state.app_name = focused.app_name; - state.phase = "idle".to_string(); - state.last_error = None; - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - return Ok(()); - } - - let is_terminalish = is_terminal_app(focused.app_name.as_deref()) - || looks_like_terminal_buffer(&focused.text); - let focused_text = if is_terminalish { - extract_terminal_input_context(&focused.text) - } else { - focused.text.clone() - }; - if config - .autocomplete - .disabled_apps - .iter() - .any(|needle| !needle.trim().is_empty() && app_lower.contains(needle)) - { - let mut state = self.inner.lock().await; - state.app_name = focused.app_name; - state.context = truncate_tail(&focused_text, config.autocomplete.max_chars); - state.suggestion = None; - state.phase = "blocked_app".to_string(); - state.last_error = None; - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - return Ok(()); - } - - let context = truncate_tail(&focused_text, config.autocomplete.max_chars); - if context.trim().is_empty() { - let mut state = self.inner.lock().await; - state.app_name = focused.app_name; - state.context = context; - state.suggestion = None; - state.phase = "idle".to_string(); - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - return Ok(()); - } - - // Short-circuit: if context AND frontmost app unchanged and we already have a suggestion, skip inference. - { - let mut state = self.inner.lock().await; - if state.context == context - && state.app_name == focused.app_name - && state.suggestion.is_some() - { - log::debug!("[autocomplete] context unchanged, returning cached suggestion"); - return Ok(()); - } - // Refresh metadata so try_accept_via_tab() sees current values - state.app_name = focused.app_name.clone(); - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - } - - { - let mut state = self.inner.lock().await; - state.phase = "generating".to_string(); - } - let service = local_ai::global(&config); - - // Build personalised style examples from three sources: - // 1. Semantically relevant past completions (local doc query) - // 2. Most recent past completions (KV recency signal / fallback) - // 3. Static user-configured examples - // Deduplicated and capped at 8 total. - let relevant_examples = - crate::openhuman::autocomplete::history::query_relevant_examples(&context, 4).await; - let recent_examples = - crate::openhuman::autocomplete::history::load_recent_examples(4).await; - let static_examples = config.autocomplete.style_examples.clone(); - - let merged_examples: Vec = { - let mut seen = std::collections::HashSet::new(); - let mut v = Vec::new(); - for ex in relevant_examples - .into_iter() - .chain(recent_examples) - .chain(static_examples) - { - if seen.insert(ex.clone()) { - v.push(ex); - } - if v.len() >= 8 { - break; - } - } - v - }; - - let generated = service - .inline_complete( - &config, - &context, - &config.autocomplete.style_preset, - config.autocomplete.style_instructions.as_deref(), - &merged_examples, - Some(36), - ) - .await - .unwrap_or_default(); - - let suggestion = sanitize_suggestion(&generated); - let app_name = focused.app_name.clone(); - let mut state = self.inner.lock().await; - state.app_name = app_name.clone(); - state.context = context; - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - if suggestion.is_empty() { - state.suggestion = None; - state.phase = "idle".to_string(); - state.last_error = None; - state.last_overlay_signature = None; - return Ok(()); - } - state.suggestion = Some(AutocompleteSuggestion { - value: suggestion.clone(), - confidence: 0.72, - }); - state.phase = "ready".to_string(); - state.last_error = None; - let ready_signature = format!( - "ready:{}:{}", - app_name.as_deref().unwrap_or_default(), - suggestion - ); - if !is_in_app && state.last_overlay_signature.as_deref() != Some(ready_signature.as_str()) { - state.last_overlay_signature = Some(ready_signature); - drop(state); - show_overflow_badge( - "ready", - Some(&suggestion), - None, - app_name.as_deref(), - focused.bounds.as_ref(), - ); - return Ok(()); - } - Ok(()) - } - - async fn try_accept_via_tab(&self) -> Result<(), String> { - let accept_with_tab = Config::load_or_init() - .await - .map(|cfg| cfg.autocomplete.accept_with_tab) - .unwrap_or(true); - if !accept_with_tab { - let mut state = self.inner.lock().await; - state.last_tab_down = false; - return Ok(()); - } - - // Skip AX-based Tab accept when OpenHuman itself is focused — - // the in-app React handler manages insertion directly. - { - let state = self.inner.lock().await; - let app = state.app_name.as_deref().unwrap_or_default().to_lowercase(); - if app.contains("openhuman") { - return Ok(()); - } - } - - let is_down = is_tab_key_down(); - let pending = { - let mut state = self.inner.lock().await; - let edge = is_down && !state.last_tab_down; - state.last_tab_down = is_down; - if !edge { - None - } else { - state.suggestion.as_ref().map(|s| s.value.clone()) - } - }; - - if let Some(suggestion) = pending { - let cleaned = sanitize_suggestion(&suggestion); - if !cleaned.is_empty() { - { - let mut state = self.inner.lock().await; - state.phase = "accepting".to_string(); - } - apply_text_to_focused_field(&cleaned)?; - { - let mut state = self.inner.lock().await; - state.suggestion = None; - state.phase = "idle".to_string(); - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - state.last_overlay_signature = None; - } - { - let app_lower = self - .inner - .lock() - .await - .app_name - .clone() - .unwrap_or_default() - .to_lowercase(); - if !app_lower.contains("openhuman") { - show_overflow_badge("accepted", Some(&cleaned), None, None, None); - } - } - - // Persist acceptance for personalisation (fire-and-forget). - // Dual-write: KV (UI list) + local docs (semantic search). - { - let (ctx, app) = { - let s = self.inner.lock().await; - (s.context.clone(), s.app_name.clone()) - }; - let sug = cleaned.clone(); - tokio::spawn(async move { - crate::openhuman::autocomplete::history::save_accepted_completion( - &ctx, - &sug, - app.as_deref(), - ) - .await; - crate::openhuman::autocomplete::history::save_completion_to_local_docs( - &ctx, - &sug, - app.as_deref(), - ) - .await; - }); - } - } - } - - Ok(()) - } - - async fn try_reject_via_escape(&self) -> Result<(), String> { - let is_down = is_escape_key_down(); - let rejected = { - let mut state = self.inner.lock().await; - let edge = is_down && !state.last_escape_down; - state.last_escape_down = is_down; - if !edge || state.suggestion.is_none() { - None - } else { - let value = state.suggestion.as_ref().map(|s| s.value.clone()); - state.suggestion = None; - state.phase = "idle".to_string(); - state.updated_at_ms = Some(Utc::now().timestamp_millis()); - state.last_overlay_signature = None; - value - } - }; - if let Some(value) = rejected { - let app_lower = self - .inner - .lock() - .await - .app_name - .clone() - .unwrap_or_default() - .to_lowercase(); - if !app_lower.contains("openhuman") { - show_overflow_badge("rejected", Some(&value), None, None, None); - } - } - Ok(()) - } -} - -pub static AUTOCOMPLETE_ENGINE: Lazy> = - Lazy::new(|| Arc::new(AutocompleteEngine::new())); - -pub fn global_engine() -> Arc { - AUTOCOMPLETE_ENGINE.clone() -} - -#[cfg(target_os = "macos")] -static LAST_OVERFLOW_BADGE: Lazy>> = - Lazy::new(|| StdMutex::new(None)); - -#[cfg(target_os = "macos")] -struct OverlayHelperProcess { - child: Child, - stdin: ChildStdin, -} - -#[cfg(target_os = "macos")] -static OVERLAY_HELPER_PROCESS: Lazy>> = - Lazy::new(|| StdMutex::new(None)); - -fn truncate_tail(text: &str, max_chars: usize) -> String { - let chars: Vec = text.chars().collect(); - if chars.len() <= max_chars { - return text.to_string(); - } - chars[chars.len() - max_chars..].iter().collect() -} - -fn sanitize_suggestion(text: &str) -> String { - let first_line = text.lines().next().unwrap_or_default().trim(); - let cleaned = first_line - .trim_matches('"') - .replace('\t', " ") - .replace('\r', "") - .trim() - .to_string(); - if cleaned.is_empty() { - return String::new(); - } - truncate_tail(&cleaned, MAX_SUGGESTION_CHARS) -} - -#[cfg_attr(not(target_os = "macos"), allow(unused_variables))] -fn show_overflow_badge( - kind: &str, - suggestion: Option<&str>, - error: Option<&str>, - app_name: Option<&str>, - anchor_bounds: Option<&FocusedElementBounds>, -) { - #[cfg(target_os = "macos")] - { - const READY_THROTTLE_MS: i64 = 1_200; - let now_ms = Utc::now().timestamp_millis(); - let signature = format!( - "{}:{}:{}:{}", - kind, - app_name.unwrap_or_default(), - suggestion.unwrap_or_default(), - error.unwrap_or_default() - ); - - if let Ok(mut guard) = LAST_OVERFLOW_BADGE.lock() { - if let Some((last_signature, last_ms)) = guard.as_ref() { - if *last_signature == signature { - return; - } - if kind == "ready" && (now_ms - *last_ms) < READY_THROTTLE_MS { - return; - } - } - *guard = Some((signature, now_ms)); - } - - if kind == "ready" { - if let (Some(bounds), Some(suggestion_text)) = (anchor_bounds, suggestion) { - if overlay_helper_show(bounds, suggestion_text).is_ok() { - return; - } - } - } else { - let _ = overlay_helper_hide(); - } - - let title = match kind { - "ready" => "OpenHuman suggestion", - "accepted" => "OpenHuman applied", - "rejected" => "OpenHuman dismissed", - "error" => "OpenHuman autocomplete error", - _ => "OpenHuman autocomplete", - }; - - let mut body = match kind { - "ready" => suggestion.unwrap_or_default().to_string(), - "accepted" => format!("Inserted: {}", suggestion.unwrap_or_default()), - "rejected" => "Suggestion dismissed.".to_string(), - "error" => error.unwrap_or("Autocomplete failed").to_string(), - _ => suggestion.unwrap_or_default().to_string(), - }; - if body.trim().is_empty() { - body = "No suggestion".to_string(); - } - body = truncate_tail(&body, 140); - - let subtitle = app_name.unwrap_or_default().trim().to_string(); - let escaped_title = escape_osascript_text(title); - let escaped_body = escape_osascript_text(&body); - let escaped_subtitle = escape_osascript_text(&subtitle); - - let script = if subtitle.is_empty() { - format!( - r#"display notification "{}" with title "{}""#, - escaped_body, escaped_title - ) - } else { - format!( - r#"display notification "{}" with title "{}" subtitle "{}""#, - escaped_body, escaped_title, escaped_subtitle - ) - }; - - std::thread::spawn(move || { - let _ = std::process::Command::new("osascript") - .arg("-e") - .arg(script) - .output(); - }); - } -} - -#[cfg(target_os = "macos")] -fn escape_osascript_text(raw: &str) -> String { - raw.replace('\\', "\\\\") - .replace('\"', "\\\"") - .replace(['\n', '\r'], " ") -} - -#[cfg(target_os = "macos")] -fn overlay_helper_show(bounds: &FocusedElementBounds, text: &str) -> Result<(), String> { - let message = serde_json::json!({ - "type": "show", - "x": bounds.x, - "y": bounds.y, - "w": bounds.width, - "h": bounds.height, - "text": truncate_tail(text, 96), - "ttl_ms": 1100 - }) - .to_string(); - overlay_helper_send_line(&message) -} - -#[cfg(target_os = "macos")] -fn overlay_helper_hide() -> Result<(), String> { - overlay_helper_send_line(r#"{"type":"hide"}"#) -} - -#[cfg(target_os = "macos")] -fn overlay_helper_quit() -> Result<(), String> { - let mut guard = OVERLAY_HELPER_PROCESS - .lock() - .map_err(|_| "overlay helper lock poisoned".to_string())?; - if let Some(mut helper) = guard.take() { - let _ = helper.stdin.write_all(br#"{"type":"quit"}"#); - let _ = helper.stdin.write_all(b"\n"); - let _ = helper.stdin.flush(); - let _ = helper.child.kill(); - let _ = helper.child.wait(); - } - Ok(()) -} - -#[cfg(target_os = "macos")] -fn overlay_helper_send_line(line: &str) -> Result<(), String> { - ensure_overlay_helper_running()?; - let mut guard = OVERLAY_HELPER_PROCESS - .lock() - .map_err(|_| "overlay helper lock poisoned".to_string())?; - let Some(helper) = guard.as_mut() else { - return Err("overlay helper unavailable".to_string()); - }; - helper - .stdin - .write_all(line.as_bytes()) - .and_then(|_| helper.stdin.write_all(b"\n")) - .and_then(|_| helper.stdin.flush()) - .map_err(|e| format!("failed to write overlay helper stdin: {e}"))?; - Ok(()) -} - -#[cfg(target_os = "macos")] -fn ensure_overlay_helper_running() -> Result<(), String> { - let mut guard = OVERLAY_HELPER_PROCESS - .lock() - .map_err(|_| "overlay helper lock poisoned".to_string())?; - - if let Some(helper) = guard.as_mut() { - if helper - .child - .try_wait() - .map_err(|e| format!("failed to query overlay helper state: {e}"))? - .is_none() - { - return Ok(()); - } - *guard = None; - } - - let binary_path = ensure_overlay_helper_binary()?; - let mut child = Command::new(&binary_path) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .map_err(|e| format!("failed to spawn overlay helper: {e}"))?; - let stdin = child - .stdin - .take() - .ok_or_else(|| "failed to capture overlay helper stdin".to_string())?; - *guard = Some(OverlayHelperProcess { child, stdin }); - Ok(()) -} - -#[cfg(target_os = "macos")] -fn ensure_overlay_helper_binary() -> Result { - let cache_dir = std::env::temp_dir().join("openhuman-autocomplete-overlay"); - fs::create_dir_all(&cache_dir).map_err(|e| format!("failed to create cache dir: {e}"))?; - let source_path = cache_dir.join("overlay_helper.swift"); - let binary_path = cache_dir.join("overlay_helper_bin"); - let source = overlay_helper_swift_source(); - - let needs_write = match fs::read_to_string(&source_path) { - Ok(existing) => existing != source, - Err(_) => true, - }; - if needs_write { - fs::write(&source_path, source) - .map_err(|e| format!("failed to write overlay helper source: {e}"))?; - } - - let needs_compile = needs_write || !binary_path.exists(); - if needs_compile { - let output = Command::new("xcrun") - .arg("swiftc") - .arg("-O") - .arg(&source_path) - .arg("-o") - .arg(&binary_path) - .output() - .or_else(|_| { - Command::new("swiftc") - .arg("-O") - .arg(&source_path) - .arg("-o") - .arg(&binary_path) - .output() - }) - .map_err(|e| format!("failed to invoke swiftc: {e}"))?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - return Err(format!( - "failed to compile overlay helper: {}", - if stderr.is_empty() { - "swiftc returned non-zero exit status".to_string() - } else { - stderr - } - )); - } - } - - Ok(binary_path) -} - -#[cfg(target_os = "macos")] -fn overlay_helper_swift_source() -> &'static str { - r#"import Cocoa -import Foundation - -final class OverlayController { - private var panel: NSPanel? - private var textField: NSTextField? - private var hideWorkItem: DispatchWorkItem? - - func show(x: CGFloat, yTop: CGFloat, width: CGFloat, height: CGFloat, text: String, ttlMs: Int) { - let screen = NSScreen.main ?? NSScreen.screens.first - let screenHeight = screen?.frame.height ?? 900 - let panelWidth = min(420, max(140, CGFloat(text.count) * 7 + 26)) - let panelHeight: CGFloat = 26 - let originX = x + max(8, min(width - panelWidth - 8, 28)) - let originYTop = yTop + max(5, min(height - panelHeight - 4, 10)) - let originYCocoa = max(6, screenHeight - originYTop - panelHeight) - - if panel == nil { - let p = NSPanel( - contentRect: NSRect(x: originX, y: originYCocoa, width: panelWidth, height: panelHeight), - styleMask: [.borderless, .nonactivatingPanel], - backing: .buffered, - defer: false - ) - p.level = .statusBar - p.hasShadow = false - p.isOpaque = false - p.backgroundColor = .clear - p.ignoresMouseEvents = true - p.collectionBehavior = [.canJoinAllSpaces, .transient] - - let content = NSView(frame: NSRect(x: 0, y: 0, width: panelWidth, height: panelHeight)) - content.wantsLayer = true - content.layer?.cornerRadius = 6 - content.layer?.backgroundColor = NSColor(white: 0.08, alpha: 0.35).cgColor - p.contentView = content - - let label = NSTextField(labelWithString: text) - label.frame = NSRect(x: 8, y: 4, width: panelWidth - 12, height: 18) - label.textColor = NSColor(white: 1.0, alpha: 0.46) - label.font = NSFont.systemFont(ofSize: 13) - label.lineBreakMode = .byTruncatingTail - content.addSubview(label) - - panel = p - textField = label - } - - panel?.setFrame(NSRect(x: originX, y: originYCocoa, width: panelWidth, height: panelHeight), display: true) - panel?.contentView?.frame = NSRect(x: 0, y: 0, width: panelWidth, height: panelHeight) - textField?.frame = NSRect(x: 8, y: 4, width: panelWidth - 12, height: 18) - textField?.stringValue = text - panel?.orderFrontRegardless() - - hideWorkItem?.cancel() - let work = DispatchWorkItem { [weak self] in - self?.hide() - } - hideWorkItem = work - DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(max(120, ttlMs)), execute: work) - } - - func hide() { - panel?.orderOut(nil) - } -} - -let app = NSApplication.shared -app.setActivationPolicy(.accessory) -let controller = OverlayController() - -DispatchQueue.global(qos: .utility).async { - while let line = readLine() { - guard let data = line.data(using: .utf8), - let payload = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let kind = payload["type"] as? String else { - continue - } - if kind == "show" { - let x = CGFloat((payload["x"] as? NSNumber)?.doubleValue ?? 0) - let y = CGFloat((payload["y"] as? NSNumber)?.doubleValue ?? 0) - let w = CGFloat((payload["w"] as? NSNumber)?.doubleValue ?? 0) - let h = CGFloat((payload["h"] as? NSNumber)?.doubleValue ?? 0) - let text = (payload["text"] as? String) ?? "" - let ttl = (payload["ttl_ms"] as? NSNumber)?.intValue ?? 900 - DispatchQueue.main.async { - controller.show(x: x, yTop: y, width: w, height: h, text: text, ttlMs: ttl) - } - } else if kind == "hide" { - DispatchQueue.main.async { - controller.hide() - } - } else if kind == "quit" { - DispatchQueue.main.async { - controller.hide() - NSApplication.shared.terminate(nil) - } - break - } - } -} - -app.run() -"# -} - -fn is_no_text_candidate_error(err: &str) -> bool { - err.contains("ERROR:no_text_candidate_found") -} - -#[cfg(target_os = "macos")] -fn focused_text_context() -> Result { - let ctx = focused_text_context_verbose()?; - if let Some(err) = ctx.raw_error.as_ref() { - return Err(format!( - "focused text unavailable via accessibility api: {err}" - )); - } - Ok(ctx) -} - -#[cfg(target_os = "macos")] -fn focused_text_context_verbose() -> Result { - let script = r##" - tell application "System Events" - set sep to character id 31 - set frontApp to first application process whose frontmost is true - set appName to name of frontApp - set roleValue to "unknown" - set textValue to "" - set selectedValue to "" - set errValue to "" - set posX to "" - set posY to "" - set sizeW to "" - set sizeH to "" - set targetRoles to {"AXTextArea", "AXTextField", "AXSearchField", "AXComboBox", "AXEditableText"} - - try - set focusedElement to value of attribute "AXFocusedUIElement" of frontApp - try - set roleValue to value of attribute "AXRole" of focusedElement as text - end try - try - set textValue to value of attribute "AXValue" of focusedElement as text - end try - try - set p to value of attribute "AXPosition" of focusedElement - set posX to item 1 of p as text - set posY to item 2 of p as text - end try - try - set s to value of attribute "AXSize" of focusedElement - set sizeW to item 1 of s as text - set sizeH to item 2 of s as text - end try - if textValue is "missing value" then set textValue to "" - if textValue is "" then - try - set selectedValue to value of attribute "AXSelectedText" of focusedElement as text - end try - if selectedValue is "missing value" then set selectedValue to "" - if selectedValue is not "" then set textValue to selectedValue - end if - if textValue is "" then - try - set textValue to value of attribute "AXTitle" of focusedElement as text - end try - if textValue is "missing value" then set textValue to "" - end if - on error errMsg number errNum - set errValue to "ERROR:" & errNum & ":" & errMsg - end try - - if textValue is "" then - try - set focusedWindow to value of attribute "AXFocusedWindow" of frontApp - set childElems to entire contents of focusedWindow - set staticPromptValue to "" - set staticFallbackValue to "" - repeat with childElem in childElems - set childRole to "" - set childValue to "" - set childSelectedValue to "" - try - set childRole to value of attribute "AXRole" of childElem as text - end try - if childRole is in targetRoles then - try - set childValue to value of attribute "AXValue" of childElem as text - end try - set childPosX to "" - set childPosY to "" - set childSizeW to "" - set childSizeH to "" - try - set cp to value of attribute "AXPosition" of childElem - set childPosX to item 1 of cp as text - set childPosY to item 2 of cp as text - end try - try - set cs to value of attribute "AXSize" of childElem - set childSizeW to item 1 of cs as text - set childSizeH to item 2 of cs as text - end try - if childValue is "missing value" then set childValue to "" - if childValue is "" then - try - set childSelectedValue to value of attribute "AXSelectedText" of childElem as text - end try - if childSelectedValue is "missing value" then set childSelectedValue to "" - if childSelectedValue is not "" then set childValue to childSelectedValue - end if - if childValue is not "" then - set roleValue to childRole - set textValue to childValue - if childPosX is not "" then set posX to childPosX - if childPosY is not "" then set posY to childPosY - if childSizeW is not "" then set sizeW to childSizeW - if childSizeH is not "" then set sizeH to childSizeH - exit repeat - end if - end if - end repeat - if textValue is "" then - repeat with childElem in childElems - set childRole to "" - set childValue to "" - try - set childRole to value of attribute "AXRole" of childElem as text - end try - if childRole is "AXStaticText" then - try - set childValue to value of attribute "AXValue" of childElem as text - end try - if childValue is "missing value" then set childValue to "" - if childValue is not "" then - set staticFallbackValue to childValue - if childValue contains "$ " or childValue contains "# " or childValue contains "> " then - set staticPromptValue to childValue - end if - end if - end if - end repeat - if staticPromptValue is not "" then - set roleValue to "AXStaticText" - set textValue to staticPromptValue - else if staticFallbackValue is not "" then - set roleValue to "AXStaticText" - set textValue to staticFallbackValue - end if - end if - on error errMsg2 number errNum2 - if errValue is "" then set errValue to "ERROR:" & errNum2 & ":" & errMsg2 - end try - end if - - if textValue is "" and errValue is "" then - set errValue to "ERROR:no_text_candidate_found" - end if - - return appName & sep & roleValue & sep & textValue & sep & selectedValue & sep & errValue & sep & posX & sep & posY & sep & sizeW & sep & sizeH - end tell - "##; - - let output = std::process::Command::new("osascript") - .arg("-e") - .arg(script) - .output() - .map_err(|e| format!("failed to run osascript: {e}"))?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if stderr.is_empty() { - return Err("unable to query focused text context".to_string()); - } - return Err(format!("unable to query focused text context: {stderr}")); - } - - let text = String::from_utf8_lossy(&output.stdout); - let trimmed = text.trim_end_matches(['\r', '\n']); - let mut segments = trimmed.splitn(9, '\u{1f}'); - let app_name = segments - .next() - .map(|s| normalize_ax_value(s.trim())) - .filter(|s| !s.is_empty()); - let role = segments - .next() - .map(|s| normalize_ax_value(s.trim())) - .filter(|s| !s.is_empty()); - let mut value = segments.next().map(normalize_ax_value).unwrap_or_default(); - let mut selected_text = segments - .next() - .map(normalize_ax_value) - .filter(|s| !s.is_empty()); - let mut raw_error = segments - .next() - .map(|s| normalize_ax_value(s.trim())) - .filter(|s| !s.is_empty()); - let pos_x = segments.next().and_then(parse_ax_number); - let pos_y = segments.next().and_then(parse_ax_number); - let size_w = segments.next().and_then(parse_ax_number); - let size_h = segments.next().and_then(parse_ax_number); - - let allow_terminal_text_value = - is_terminal_app(app_name.as_deref()) && !value.trim().is_empty(); - if !is_text_role(role.as_deref()) && !allow_terminal_text_value { - value.clear(); - selected_text = None; - if raw_error.is_none() { - raw_error = Some("ERROR:no_text_candidate_found".to_string()); - } - } - - Ok(FocusedTextContext { - app_name, - role, - text: value, - selected_text, - raw_error, - bounds: match (pos_x, pos_y, size_w, size_h) { - (Some(x), Some(y), Some(width), Some(height)) if width > 0 && height > 0 => { - Some(FocusedElementBounds { - x, - y, - width, - height, - }) - } - _ => None, - }, - }) -} - -#[cfg(not(target_os = "macos"))] -fn focused_text_context() -> Result { - Err("autocomplete is only supported on macOS".to_string()) -} - -#[cfg(not(target_os = "macos"))] -fn focused_text_context_verbose() -> Result { - Err("autocomplete is only supported on macOS".to_string()) -} - -fn normalize_ax_value(raw: &str) -> String { - let v = raw.trim(); - if v.eq_ignore_ascii_case("missing value") { - String::new() - } else { - v.to_string() - } -} - -fn parse_ax_number(raw: &str) -> Option { - let trimmed = normalize_ax_value(raw); - if trimmed.is_empty() { - return None; - } - let cleaned = trimmed.replace(',', "."); - cleaned.parse::().ok().map(|v| v.round() as i32) -} - -#[cfg(target_os = "macos")] -fn apply_text_to_focused_field(text: &str) -> Result<(), String> { - let escaped = text - .replace('\\', "\\\\") - .replace('\"', "\\\"") - .replace('\n', " "); - let script = format!( - r##" -tell application "System Events" - set frontApp to first application process whose frontmost is true - set focusedElement to value of attribute "AXFocusedUIElement" of frontApp - set currentValue to "" - try - set currentValue to value of attribute "AXValue" of focusedElement as text - end try - if currentValue is "missing value" then set currentValue to "" - if currentValue is "" then - try - set currentValue to value of attribute "AXSelectedText" of focusedElement as text - end try - if currentValue is "missing value" then set currentValue to "" - end if - set value of attribute "AXValue" of focusedElement to (currentValue & "{}") -end tell -"##, - escaped - ); - let output = std::process::Command::new("osascript") - .arg("-e") - .arg(script) - .output() - .map_err(|e| format!("failed to run osascript: {e}"))?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if stderr.is_empty() { - return Err("failed to apply suggestion to focused text field".to_string()); - } - return Err(format!( - "failed to apply suggestion to focused text field: {stderr}" - )); - } - Ok(()) -} - -#[cfg(not(target_os = "macos"))] -fn apply_text_to_focused_field(_text: &str) -> Result<(), String> { - Err("autocomplete is only supported on macOS".to_string()) -} - -#[cfg(target_os = "macos")] -fn is_tab_key_down() -> bool { - unsafe { CGEventSourceKeyState(KCG_EVENT_SOURCE_STATE_COMBINED_SESSION_STATE, KVK_TAB) } -} - -#[cfg(not(target_os = "macos"))] -fn is_tab_key_down() -> bool { - false -} - -#[cfg(target_os = "macos")] -fn is_escape_key_down() -> bool { - unsafe { CGEventSourceKeyState(KCG_EVENT_SOURCE_STATE_COMBINED_SESSION_STATE, KVK_ESCAPE) } -} - -#[cfg(not(target_os = "macos"))] -fn is_escape_key_down() -> bool { - false -} - -#[cfg(target_os = "macos")] -#[link(name = "ApplicationServices", kind = "framework")] -extern "C" { - fn CGEventSourceKeyState(state_id: i32, key: u16) -> bool; -} - -#[cfg(target_os = "macos")] -const KCG_EVENT_SOURCE_STATE_COMBINED_SESSION_STATE: i32 = 0; -#[cfg(target_os = "macos")] -const KVK_TAB: u16 = 48; -#[cfg(target_os = "macos")] -const KVK_ESCAPE: u16 = 53; diff --git a/src/openhuman/autocomplete/core/engine.rs b/src/openhuman/autocomplete/core/engine.rs new file mode 100644 index 000000000..1aef35761 --- /dev/null +++ b/src/openhuman/autocomplete/core/engine.rs @@ -0,0 +1,747 @@ +use crate::openhuman::config::Config; +use crate::openhuman::local_ai; +use chrono::Utc; +use once_cell::sync::Lazy; +use std::sync::Arc; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use tokio::time::{self, Duration, Instant}; + +use super::focus::{ + apply_text_to_focused_field, focused_text_context_verbose, is_escape_key_down, is_tab_key_down, + validate_focused_target, +}; +use super::overlay::{overlay_helper_quit, show_overflow_badge}; +use super::terminal::{ + extract_terminal_input_context, is_terminal_app, looks_like_terminal_buffer, +}; +use super::text::{is_no_text_candidate_error, sanitize_suggestion, truncate_tail}; +use super::types::{ + AutocompleteAcceptParams, AutocompleteAcceptResult, AutocompleteCurrentParams, + AutocompleteCurrentResult, AutocompleteDebugFocusResult, AutocompleteSetStyleParams, + AutocompleteSetStyleResult, AutocompleteStartParams, AutocompleteStartResult, + AutocompleteStatus, AutocompleteStopParams, AutocompleteStopResult, AutocompleteSuggestion, + FocusedTextContext, +}; + +struct EngineState { + running: bool, + phase: String, + debounce_ms: u64, + app_name: Option, + /// AXRole of the text element when the suggestion was generated. + target_role: Option, + context: String, + suggestion: Option, + last_error: Option, + updated_at_ms: Option, + last_tab_down: bool, + last_escape_down: bool, + last_overlay_signature: Option, + task: Option>, +} + +impl Default for EngineState { + fn default() -> Self { + Self { + running: false, + phase: "idle".to_string(), + debounce_ms: 120, + app_name: None, + target_role: None, + context: String::new(), + suggestion: None, + last_error: None, + updated_at_ms: None, + last_tab_down: false, + last_escape_down: false, + last_overlay_signature: None, + task: None, + } + } +} + +pub struct AutocompleteEngine { + inner: Mutex, +} + +impl Default for AutocompleteEngine { + fn default() -> Self { + Self::new() + } +} +impl AutocompleteEngine { + pub fn new() -> Self { + Self { + inner: Mutex::new(EngineState::default()), + } + } + + pub async fn status(&self) -> AutocompleteStatus { + let config = Config::load_or_init() + .await + .unwrap_or_else(|_| Config::default()); + let state = self.inner.lock().await; + + AutocompleteStatus { + platform_supported: cfg!(target_os = "macos"), + enabled: config.autocomplete.enabled, + running: state.running, + phase: state.phase.clone(), + debounce_ms: state.debounce_ms, + model_id: config.local_ai.chat_model_id, + app_name: state.app_name.clone(), + last_error: state.last_error.clone(), + updated_at_ms: state.updated_at_ms, + suggestion: state.suggestion.clone(), + } + } + + pub async fn start( + &self, + params: AutocompleteStartParams, + ) -> Result { + if !cfg!(target_os = "macos") { + return Err("autocomplete is only supported on macOS".to_string()); + } + + let config = Config::load_or_init() + .await + .map_err(|e| format!("failed to load config: {e}"))?; + if !config.autocomplete.enabled { + return Ok(AutocompleteStartResult { started: false }); + } + + let debounce_ms = params + .debounce_ms + .unwrap_or(config.autocomplete.debounce_ms) + .clamp(50, 2000); + + let mut state = self.inner.lock().await; + if state.running { + return Ok(AutocompleteStartResult { started: false }); + } + state.running = true; + state.phase = "idle".to_string(); + state.debounce_ms = debounce_ms; + state.last_error = None; + + let engine = global_engine(); + state.task = Some(tokio::spawn(async move { + let mut last_refresh = Instant::now() - Duration::from_millis(debounce_ms); + loop { + let current_debounce_ms = { + let state = engine.inner.lock().await; + if !state.running { + break; + } + state.debounce_ms + }; + let _ = engine.try_reject_via_escape().await; + let _ = engine.try_accept_via_tab().await; + if last_refresh.elapsed() >= Duration::from_millis(current_debounce_ms) { + let refresh_result = + time::timeout(Duration::from_secs(15), engine.refresh(None)).await; + match refresh_result { + Ok(Err(err)) => { + let error_message = { + let mut state = engine.inner.lock().await; + state.phase = "error".to_string(); + state.last_error = Some(err); + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + state.last_error.clone() + }; + if let Some(error_message) = error_message { + let app_lower = engine + .inner + .lock() + .await + .app_name + .clone() + .unwrap_or_default() + .to_lowercase(); + if !app_lower.contains("openhuman") { + show_overflow_badge( + "error", + None, + Some(&error_message), + None, + None, + ); + } + } + } + Ok(Ok(())) => { + let mut state = engine.inner.lock().await; + if state.phase == "error" { + state.phase = "idle".to_string(); + } + state.last_error = None; + } + Err(_elapsed) => { + log::warn!("[autocomplete] refresh timed out after 15s, skipping"); + let mut state = engine.inner.lock().await; + state.phase = "idle".to_string(); + state.suggestion = None; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + } + } + last_refresh = Instant::now(); + } + time::sleep(Duration::from_millis(24)).await; + } + })); + + Ok(AutocompleteStartResult { started: true }) + } + + pub async fn stop(&self, _params: Option) -> AutocompleteStopResult { + let mut state = self.inner.lock().await; + state.running = false; + state.phase = "idle".to_string(); + state.last_escape_down = false; + state.last_overlay_signature = None; + state.last_error = None; + state.suggestion = None; + state.context = String::new(); + state.app_name = None; + if let Some(task) = state.task.take() { + task.abort(); + } + #[cfg(target_os = "macos")] + let _ = overlay_helper_quit(); + AutocompleteStopResult { stopped: true } + } + + pub async fn current( + &self, + params: Option, + ) -> Result { + let context_override = params + .and_then(|p| p.context) + .filter(|c| !c.trim().is_empty()); + self.refresh(context_override).await?; + let state = self.inner.lock().await; + Ok(AutocompleteCurrentResult { + app_name: state.app_name.clone(), + context: state.context.clone(), + suggestion: state.suggestion.clone(), + }) + } + + pub async fn debug_focus(&self) -> Result { + let focused = focused_text_context_verbose()?; + Ok(AutocompleteDebugFocusResult { + app_name: focused.app_name, + role: focused.role, + context: focused.text, + selected_text: focused.selected_text, + raw_error: focused.raw_error, + }) + } + + pub async fn accept( + &self, + params: AutocompleteAcceptParams, + ) -> Result { + let value = if let Some(value) = params.suggestion { + value + } else { + let state = self.inner.lock().await; + state + .suggestion + .as_ref() + .map(|s| s.value.clone()) + .unwrap_or_default() + }; + + let cleaned = sanitize_suggestion(&value); + if cleaned.is_empty() { + return Ok(AutocompleteAcceptResult { + accepted: false, + applied: false, + value: None, + reason: Some("no suggestion available".to_string()), + }); + } + + let should_apply = !params.skip_apply.unwrap_or(false); + + { + let mut state = self.inner.lock().await; + state.phase = "accepting".to_string(); + } + if should_apply { + // Validate the focused element still matches before inserting. + let (expected_app, expected_role) = { + let state = self.inner.lock().await; + (state.app_name.clone(), state.target_role.clone()) + }; + let apply_result = (|| -> Result<(), String> { + #[cfg(target_os = "macos")] + validate_focused_target(expected_app.as_deref(), expected_role.as_deref())?; + apply_text_to_focused_field(&cleaned)?; + Ok(()) + })(); + if let Err(e) = apply_result { + let mut state = self.inner.lock().await; + state.suggestion = None; + state.phase = "idle".to_string(); + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + state.last_overlay_signature = None; + return Err(e); + } + } + { + let mut state = self.inner.lock().await; + state.suggestion = None; + state.phase = "idle".to_string(); + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + state.last_overlay_signature = None; + } + if should_apply { + show_overflow_badge("accepted", Some(&cleaned), None, None, None); + } + + // Persist acceptance for personalisation (fire-and-forget). + // Dual-write: KV (UI list) + local docs (semantic search). + { + let (ctx, app) = { + let s = self.inner.lock().await; + (s.context.clone(), s.app_name.clone()) + }; + let sug = cleaned.clone(); + tokio::spawn(async move { + crate::openhuman::autocomplete::history::save_accepted_completion( + &ctx, + &sug, + app.as_deref(), + ) + .await; + crate::openhuman::autocomplete::history::save_completion_to_local_docs( + &ctx, + &sug, + app.as_deref(), + ) + .await; + }); + } + + Ok(AutocompleteAcceptResult { + accepted: true, + applied: should_apply, + value: Some(cleaned), + reason: None, + }) + } + + pub async fn set_style( + &self, + params: AutocompleteSetStyleParams, + ) -> Result { + let mut config = Config::load_or_init() + .await + .map_err(|e| format!("failed to load config: {e}"))?; + if let Some(enabled) = params.enabled { + config.autocomplete.enabled = enabled; + } + if let Some(debounce_ms) = params.debounce_ms { + config.autocomplete.debounce_ms = debounce_ms.clamp(50, 2000); + } + if let Some(max_chars) = params.max_chars { + config.autocomplete.max_chars = max_chars.clamp(64, 2048); + } + if let Some(style_preset) = params.style_preset { + config.autocomplete.style_preset = style_preset.trim().to_string(); + } + if let Some(style_instructions) = params.style_instructions { + config.autocomplete.style_instructions = if style_instructions.trim().is_empty() { + None + } else { + Some(style_instructions.trim().to_string()) + }; + } + if let Some(style_examples) = params.style_examples { + config.autocomplete.style_examples = style_examples + .into_iter() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .take(8) + .collect(); + } + if let Some(disabled_apps) = params.disabled_apps { + config.autocomplete.disabled_apps = disabled_apps + .into_iter() + .map(|s| s.trim().to_lowercase()) + .filter(|s| !s.is_empty()) + .collect(); + } + if let Some(accept_with_tab) = params.accept_with_tab { + config.autocomplete.accept_with_tab = accept_with_tab; + } + config.save().await.map_err(|e| e.to_string())?; + + let mut state = self.inner.lock().await; + state.debounce_ms = config.autocomplete.debounce_ms; + state.last_tab_down = false; + state.last_escape_down = false; + if !config.autocomplete.enabled { + state.running = false; + if let Some(task) = state.task.take() { + task.abort(); + } + state.suggestion = None; + state.last_overlay_signature = None; + #[cfg(target_os = "macos")] + let _ = overlay_helper_quit(); + } + + Ok(AutocompleteSetStyleResult { + config: config.autocomplete, + }) + } + + async fn refresh(&self, context_override: Option) -> Result<(), String> { + let is_in_app = context_override.is_some(); + let config = Config::load_or_init() + .await + .map_err(|e| format!("failed to load config: {e}"))?; + if !config.autocomplete.enabled { + let mut state = self.inner.lock().await; + state.suggestion = None; + state.phase = "disabled".to_string(); + return Ok(()); + } + { + let mut state = self.inner.lock().await; + state.phase = "capturing_context".to_string(); + } + + let focused = if let Some(context) = context_override { + FocusedTextContext { + app_name: Some("OpenHuman".to_string()), + role: None, + text: context, + selected_text: None, + raw_error: None, + bounds: None, + } + } else { + let focused = focused_text_context_verbose()?; + if let Some(err) = focused.raw_error.as_deref() { + if is_no_text_candidate_error(err) || err.contains("ERROR:-1728") { + let mut state = self.inner.lock().await; + state.app_name = focused.app_name; + state.context = String::new(); + state.suggestion = None; + state.phase = "idle".to_string(); + state.last_error = None; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + return Ok(()); + } + return Err(format!( + "focused text unavailable via accessibility api: {err}" + )); + } + focused + }; + + let app_lower = focused.app_name.clone().unwrap_or_default().to_lowercase(); + + // When OpenHuman itself is focused AND this is the background engine loop, + // skip AX-based refresh — the in-app React polling handles suggestions. + // When is_in_app (context_override provided), we still want inference to run. + if !is_in_app && app_lower.contains("openhuman") { + let mut state = self.inner.lock().await; + state.app_name = focused.app_name; + state.phase = "idle".to_string(); + state.last_error = None; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + return Ok(()); + } + + let is_terminalish = is_terminal_app(focused.app_name.as_deref()) + || looks_like_terminal_buffer(&focused.text); + let focused_text = if is_terminalish { + extract_terminal_input_context(&focused.text) + } else { + focused.text.clone() + }; + if config + .autocomplete + .disabled_apps + .iter() + .any(|needle| !needle.trim().is_empty() && app_lower.contains(needle)) + { + let mut state = self.inner.lock().await; + state.app_name = focused.app_name; + state.context = truncate_tail(&focused_text, config.autocomplete.max_chars); + state.suggestion = None; + state.phase = "blocked_app".to_string(); + state.last_error = None; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + return Ok(()); + } + + let context = truncate_tail(&focused_text, config.autocomplete.max_chars); + if context.trim().is_empty() { + let mut state = self.inner.lock().await; + state.app_name = focused.app_name; + state.context = context; + state.suggestion = None; + state.phase = "idle".to_string(); + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + return Ok(()); + } + + // Short-circuit: if context, frontmost app, AND role unchanged and we already have a suggestion, skip inference. + { + let mut state = self.inner.lock().await; + if state.context == context + && state.app_name == focused.app_name + && state.target_role == focused.role + && state.suggestion.is_some() + { + log::debug!("[autocomplete] context unchanged, returning cached suggestion"); + return Ok(()); + } + // Refresh metadata so try_accept_via_tab() sees current values + state.app_name = focused.app_name.clone(); + state.target_role = focused.role.clone(); + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + } + + { + let mut state = self.inner.lock().await; + state.phase = "generating".to_string(); + } + let service = local_ai::global(&config); + + // Build personalised style examples from three sources: + // 1. Semantically relevant past completions (local doc query) + // 2. Most recent past completions (KV recency signal / fallback) + // 3. Static user-configured examples + // Deduplicated and capped at 8 total. + let relevant_examples = + crate::openhuman::autocomplete::history::query_relevant_examples(&context, 4).await; + let recent_examples = + crate::openhuman::autocomplete::history::load_recent_examples(4).await; + let static_examples = config.autocomplete.style_examples.clone(); + + let merged_examples: Vec = { + let mut seen = std::collections::HashSet::new(); + let mut v = Vec::new(); + for ex in relevant_examples + .into_iter() + .chain(recent_examples) + .chain(static_examples) + { + if seen.insert(ex.clone()) { + v.push(ex); + } + if v.len() >= 8 { + break; + } + } + v + }; + + let generated = service + .inline_complete( + &config, + &context, + &config.autocomplete.style_preset, + config.autocomplete.style_instructions.as_deref(), + &merged_examples, + Some(36), + ) + .await?; + + let suggestion = sanitize_suggestion(&generated); + let app_name = focused.app_name.clone(); + let target_role = focused.role.clone(); + let mut state = self.inner.lock().await; + state.app_name = app_name.clone(); + state.target_role = target_role; + state.context = context; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + if suggestion.is_empty() { + state.suggestion = None; + state.phase = "idle".to_string(); + state.last_error = None; + state.last_overlay_signature = None; + return Ok(()); + } + state.suggestion = Some(AutocompleteSuggestion { + value: suggestion.clone(), + confidence: 0.72, + }); + state.phase = "ready".to_string(); + state.last_error = None; + let ready_signature = format!( + "ready:{}:{}", + app_name.as_deref().unwrap_or_default(), + suggestion + ); + if !is_in_app && state.last_overlay_signature.as_deref() != Some(ready_signature.as_str()) { + state.last_overlay_signature = Some(ready_signature); + drop(state); + show_overflow_badge( + "ready", + Some(&suggestion), + None, + app_name.as_deref(), + focused.bounds.as_ref(), + ); + return Ok(()); + } + Ok(()) + } + + async fn try_accept_via_tab(&self) -> Result<(), String> { + let accept_with_tab = Config::load_or_init() + .await + .map(|cfg| cfg.autocomplete.accept_with_tab) + .unwrap_or(true); + if !accept_with_tab { + let mut state = self.inner.lock().await; + state.last_tab_down = false; + return Ok(()); + } + + // Skip AX-based Tab accept when OpenHuman itself is focused — + // the in-app React handler manages insertion directly. + { + let state = self.inner.lock().await; + let app = state.app_name.as_deref().unwrap_or_default().to_lowercase(); + if app.contains("openhuman") { + return Ok(()); + } + } + + let is_down = is_tab_key_down(); + let pending = { + let mut state = self.inner.lock().await; + let edge = is_down && !state.last_tab_down; + state.last_tab_down = is_down; + if !edge { + None + } else { + state.suggestion.as_ref().map(|s| s.value.clone()) + } + }; + + if let Some(suggestion) = pending { + let cleaned = sanitize_suggestion(&suggestion); + if !cleaned.is_empty() { + // Validate the focused element still matches before inserting. + let (expected_app, expected_role) = { + let state = self.inner.lock().await; + (state.app_name.clone(), state.target_role.clone()) + }; + #[cfg(target_os = "macos")] + if let Err(e) = + validate_focused_target(expected_app.as_deref(), expected_role.as_deref()) + { + log::warn!("[autocomplete] tab-accept aborted: {e}"); + let mut state = self.inner.lock().await; + state.suggestion = None; + state.phase = "idle".to_string(); + state.last_overlay_signature = None; + return Ok(()); + } + { + let mut state = self.inner.lock().await; + state.phase = "accepting".to_string(); + } + apply_text_to_focused_field(&cleaned)?; + { + let mut state = self.inner.lock().await; + state.suggestion = None; + state.phase = "idle".to_string(); + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + state.last_overlay_signature = None; + } + { + let app_lower = self + .inner + .lock() + .await + .app_name + .clone() + .unwrap_or_default() + .to_lowercase(); + if !app_lower.contains("openhuman") { + show_overflow_badge("accepted", Some(&cleaned), None, None, None); + } + } + + // Persist acceptance for personalisation (fire-and-forget). + // Dual-write: KV (UI list) + local docs (semantic search). + { + let (ctx, app) = { + let s = self.inner.lock().await; + (s.context.clone(), s.app_name.clone()) + }; + let sug = cleaned.clone(); + tokio::spawn(async move { + crate::openhuman::autocomplete::history::save_accepted_completion( + &ctx, + &sug, + app.as_deref(), + ) + .await; + crate::openhuman::autocomplete::history::save_completion_to_local_docs( + &ctx, + &sug, + app.as_deref(), + ) + .await; + }); + } + } + } + + Ok(()) + } + + async fn try_reject_via_escape(&self) -> Result<(), String> { + let is_down = is_escape_key_down(); + let rejected = { + let mut state = self.inner.lock().await; + let edge = is_down && !state.last_escape_down; + state.last_escape_down = is_down; + if !edge || state.suggestion.is_none() { + None + } else { + let value = state.suggestion.as_ref().map(|s| s.value.clone()); + state.suggestion = None; + state.phase = "idle".to_string(); + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + state.last_overlay_signature = None; + value + } + }; + if let Some(value) = rejected { + let app_lower = self + .inner + .lock() + .await + .app_name + .clone() + .unwrap_or_default() + .to_lowercase(); + if !app_lower.contains("openhuman") { + show_overflow_badge("rejected", Some(&value), None, None, None); + } + } + Ok(()) + } +} + +pub static AUTOCOMPLETE_ENGINE: Lazy> = + Lazy::new(|| Arc::new(AutocompleteEngine::new())); + +pub fn global_engine() -> Arc { + AUTOCOMPLETE_ENGINE.clone() +} diff --git a/src/openhuman/autocomplete/core/focus.rs b/src/openhuman/autocomplete/core/focus.rs new file mode 100644 index 000000000..c0c75f924 --- /dev/null +++ b/src/openhuman/autocomplete/core/focus.rs @@ -0,0 +1,9 @@ +//! Accessibility focus, clipboard/paste insertion, and key state probes. +//! +//! Delegates to the shared `accessibility` middleware module. + +pub(super) use crate::openhuman::accessibility::apply_text_to_focused_field; +pub(super) use crate::openhuman::accessibility::focused_text_context_verbose; +pub(super) use crate::openhuman::accessibility::is_escape_key_down; +pub(super) use crate::openhuman::accessibility::is_tab_key_down; +pub(super) use crate::openhuman::accessibility::validate_focused_target; diff --git a/src/openhuman/autocomplete/core/mod.rs b/src/openhuman/autocomplete/core/mod.rs new file mode 100644 index 000000000..c48c06af9 --- /dev/null +++ b/src/openhuman/autocomplete/core/mod.rs @@ -0,0 +1,16 @@ +//! Autocomplete engine: macOS AX capture, local inline completion, overlay UI. + +mod engine; +mod focus; +mod overlay; +mod terminal; +mod text; +mod types; + +pub use engine::{global_engine, AutocompleteEngine, AUTOCOMPLETE_ENGINE}; +pub use types::{ + AutocompleteAcceptParams, AutocompleteAcceptResult, AutocompleteCurrentParams, + AutocompleteCurrentResult, AutocompleteDebugFocusResult, AutocompleteSetStyleParams, + AutocompleteSetStyleResult, AutocompleteStartParams, AutocompleteStartResult, + AutocompleteStatus, AutocompleteStopParams, AutocompleteStopResult, AutocompleteSuggestion, +}; diff --git a/src/openhuman/autocomplete/core/overlay.rs b/src/openhuman/autocomplete/core/overlay.rs new file mode 100644 index 000000000..434252757 --- /dev/null +++ b/src/openhuman/autocomplete/core/overlay.rs @@ -0,0 +1,135 @@ +//! Overflow badge, overlay display, and macOS notifications. +//! +//! Overlay rendering is delegated to the shared `accessibility` middleware module. + +#[cfg(target_os = "macos")] +use chrono::Utc; +#[cfg(target_os = "macos")] +use once_cell::sync::Lazy; +#[cfg(target_os = "macos")] +use std::sync::Mutex as StdMutex; + +use super::text::truncate_tail; +use crate::openhuman::accessibility::{self, ElementBounds}; + +#[cfg(target_os = "macos")] +static LAST_OVERFLOW_BADGE: Lazy>> = + Lazy::new(|| StdMutex::new(None)); + +#[cfg_attr(not(target_os = "macos"), allow(unused_variables))] +pub(super) fn show_overflow_badge( + kind: &str, + suggestion: Option<&str>, + error: Option<&str>, + app_name: Option<&str>, + anchor_bounds: Option<&ElementBounds>, +) { + #[cfg(target_os = "macos")] + { + const READY_THROTTLE_MS: i64 = 1_200; + let now_ms = Utc::now().timestamp_millis(); + let signature = format!( + "{}:{}:{}:{}", + kind, + app_name.unwrap_or_default(), + suggestion.unwrap_or_default(), + error.unwrap_or_default() + ); + + if let Ok(mut guard) = LAST_OVERFLOW_BADGE.lock() { + if let Some((last_signature, last_ms)) = guard.as_ref() { + if *last_signature == signature { + return; + } + if kind == "ready" && (now_ms - *last_ms) < READY_THROTTLE_MS { + return; + } + } + *guard = Some((signature, now_ms)); + } + + if kind == "ready" { + if let Some(suggestion_text) = suggestion { + // Use anchor bounds if available, otherwise pass zero bounds + // (the unified helper will fall back to mouse cursor position). + let fallback_bounds = ElementBounds { + x: 0, + y: 0, + width: 0, + height: 0, + }; + let bounds = if anchor_bounds.is_some() { + anchor_bounds.unwrap() + } else { + log::debug!( + "[autocomplete] overlay: no anchor bounds, falling back to zero bounds (mouse cursor); suggestion={:?}", + truncate_tail(suggestion_text, 40) + ); + &fallback_bounds + }; + if accessibility::show_overlay(bounds, suggestion_text, 1100).is_ok() { + return; + } + } + } else { + let _ = accessibility::hide_overlay(); + } + + // Notification fallback when overlay helper fails + let title = match kind { + "ready" => "OpenHuman suggestion", + "accepted" => "OpenHuman applied", + "rejected" => "OpenHuman dismissed", + "error" => "OpenHuman autocomplete error", + _ => "OpenHuman autocomplete", + }; + + let mut body = match kind { + "ready" => suggestion.unwrap_or_default().to_string(), + "accepted" => format!("Inserted: {}", suggestion.unwrap_or_default()), + "rejected" => "Suggestion dismissed.".to_string(), + "error" => error.unwrap_or("Autocomplete failed").to_string(), + _ => suggestion.unwrap_or_default().to_string(), + }; + if body.trim().is_empty() { + body = "No suggestion".to_string(); + } + body = truncate_tail(&body, 140); + + let subtitle = app_name.unwrap_or_default().trim().to_string(); + let escaped_title = escape_osascript_text(title); + let escaped_body = escape_osascript_text(&body); + let escaped_subtitle = escape_osascript_text(&subtitle); + + let script = if subtitle.is_empty() { + format!( + r#"display notification "{}" with title "{}""#, + escaped_body, escaped_title + ) + } else { + format!( + r#"display notification "{}" with title "{}" subtitle "{}""#, + escaped_body, escaped_title, escaped_subtitle + ) + }; + + std::thread::spawn(move || { + let _ = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output(); + }); + } +} + +#[cfg(target_os = "macos")] +fn escape_osascript_text(raw: &str) -> String { + raw.replace('\\', "\\\\") + .replace('\"', "\\\"") + .replace(['\n', '\r'], " ") +} + +/// Quit the overlay helper process. +pub(super) fn overlay_helper_quit() -> Result<(), String> { + accessibility::quit_overlay() +} diff --git a/src/openhuman/autocomplete/core/terminal.rs b/src/openhuman/autocomplete/core/terminal.rs new file mode 100644 index 000000000..9716a1b28 --- /dev/null +++ b/src/openhuman/autocomplete/core/terminal.rs @@ -0,0 +1,7 @@ +//! Terminal app detection and context extraction. +//! +//! Delegates to the shared `accessibility` middleware module. + +pub(super) use crate::openhuman::accessibility::extract_terminal_input_context; +pub(super) use crate::openhuman::accessibility::is_terminal_app; +pub(super) use crate::openhuman::accessibility::looks_like_terminal_buffer; diff --git a/src/openhuman/autocomplete/core/text.rs b/src/openhuman/autocomplete/core/text.rs new file mode 100644 index 000000000..563729668 --- /dev/null +++ b/src/openhuman/autocomplete/core/text.rs @@ -0,0 +1,28 @@ +//! Text utilities for autocomplete suggestions. + +use super::types::MAX_SUGGESTION_CHARS; + +pub(super) use crate::openhuman::accessibility::truncate_tail; + +/// Truncate to the first `max_chars` characters (preserves the start of the string). +pub(super) fn truncate_head(text: &str, max_chars: usize) -> String { + text.chars().take(max_chars).collect() +} + +pub(super) fn sanitize_suggestion(text: &str) -> String { + let first_line = text.lines().next().unwrap_or_default().trim(); + let cleaned = first_line + .trim_matches('"') + .replace('\t', " ") + .replace('\r', "") + .trim() + .to_string(); + if cleaned.is_empty() { + return String::new(); + } + truncate_head(&cleaned, MAX_SUGGESTION_CHARS) +} + +pub(super) fn is_no_text_candidate_error(err: &str) -> bool { + err.contains("ERROR:no_text_candidate_found") +} diff --git a/src/openhuman/autocomplete/core/types.rs b/src/openhuman/autocomplete/core/types.rs new file mode 100644 index 000000000..c6f093c07 --- /dev/null +++ b/src/openhuman/autocomplete/core/types.rs @@ -0,0 +1,100 @@ +use crate::openhuman::config::AutocompleteConfig; +use serde::{Deserialize, Serialize}; + +// Re-export platform types from the accessibility middleware. +pub(crate) use crate::openhuman::accessibility::FocusedTextContext; + +pub(crate) const MAX_SUGGESTION_CHARS: usize = 64; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSuggestion { + pub value: String, + pub confidence: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStatus { + pub platform_supported: bool, + pub enabled: bool, + pub running: bool, + pub phase: String, + pub debounce_ms: u64, + pub model_id: String, + pub app_name: Option, + pub last_error: Option, + pub updated_at_ms: Option, + pub suggestion: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStartParams { + pub debounce_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStartResult { + pub started: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStopParams { + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStopResult { + pub stopped: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteCurrentParams { + pub context: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteCurrentResult { + pub app_name: Option, + pub context: String, + pub suggestion: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteDebugFocusResult { + pub app_name: Option, + pub role: Option, + pub context: String, + pub selected_text: Option, + pub raw_error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteAcceptParams { + pub suggestion: Option, + /// When true, skip applying text via accessibility (caller already inserted it). + pub skip_apply: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteAcceptResult { + pub accepted: bool, + pub applied: bool, + pub value: Option, + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSetStyleParams { + pub enabled: Option, + pub debounce_ms: Option, + pub max_chars: Option, + pub style_preset: Option, + pub style_instructions: Option, + pub style_examples: Option>, + pub disabled_apps: Option>, + pub accept_with_tab: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSetStyleResult { + pub config: AutocompleteConfig, +} diff --git a/src/openhuman/mod.rs b/src/openhuman/mod.rs index 150a14190..5297656f1 100644 --- a/src/openhuman/mod.rs +++ b/src/openhuman/mod.rs @@ -11,6 +11,7 @@ // Many types/functions are not yet consumed but are intentionally exported. #![allow(dead_code)] +pub mod accessibility; pub mod agent; pub mod approval; pub mod autocomplete; diff --git a/src/openhuman/screen_intelligence/capture.rs b/src/openhuman/screen_intelligence/capture.rs index c67bacd06..4783e032f 100644 --- a/src/openhuman/screen_intelligence/capture.rs +++ b/src/openhuman/screen_intelligence/capture.rs @@ -1,248 +1,7 @@ +//! Timestamp helper (`now_ms`) for the screen intelligence engine. + use chrono::Utc; -#[cfg(target_os = "macos")] -use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; -#[cfg(target_os = "macos")] -use uuid::Uuid; - -use super::context::{AppContext, WindowBounds}; -#[cfg(target_os = "macos")] -use super::limits::MAX_SCREENSHOT_BYTES; - pub(crate) fn now_ms() -> i64 { Utc::now().timestamp_millis() } - -/// Capture mode used for a screenshot. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum CaptureMode { - Windowed, - Fullscreen, -} - -impl std::fmt::Display for CaptureMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - CaptureMode::Windowed => write!(f, "windowed"), - CaptureMode::Fullscreen => write!(f, "fullscreen"), - } - } -} - -pub(crate) fn capture_screen_image_ref_for_context( - context: Option<&AppContext>, -) -> Result { - #[cfg(target_os = "macos")] - { - let tmp_path = std::env::temp_dir().join(format!( - "openhuman_screen_intelligence_{}.png", - Uuid::new_v4() - )); - - let bounds = context.and_then(|ctx| ctx.bounds.clone()); - - // Determine capture mode: windowed if we have valid bounds, fullscreen otherwise. - let capture_mode = match &bounds { - Some(b) if b.width > 0 && b.height > 0 => CaptureMode::Windowed, - Some(b) => { - tracing::debug!( - "[screen_intelligence] invalid bounds ({}x{}), falling back to fullscreen", - b.width, - b.height - ); - CaptureMode::Fullscreen - } - None => { - tracing::debug!( - "[screen_intelligence] no window bounds available, falling back to fullscreen" - ); - CaptureMode::Fullscreen - } - }; - - let mut cmd = std::process::Command::new("screencapture"); - cmd.arg("-x").arg("-t").arg("png"); - - if capture_mode == CaptureMode::Windowed { - let b = bounds.as_ref().unwrap(); - let rect = format!("{},{},{},{}", b.x, b.y, b.width, b.height); - tracing::debug!( - "[screen_intelligence] capture mode=windowed rect={rect} app={:?}", - context.and_then(|c| c.app_name.as_deref()) - ); - cmd.arg("-R").arg(&rect); - } else { - tracing::debug!("[screen_intelligence] capture mode=fullscreen (primary display)"); - } - - cmd.arg(&tmp_path); - - let status = cmd - .status() - .map_err(|e| format!("screencapture failed to start: {e}"))?; - if !status.success() { - tracing::debug!( - "[screen_intelligence] screencapture exited with status: {:?}", - status.code() - ); - return Err("screencapture returned non-zero status".to_string()); - } - - let bytes = - std::fs::read(&tmp_path).map_err(|e| format!("failed to read screenshot: {e}"))?; - tracing::debug!( - "[screen_intelligence] captured {} bytes (mode={})", - bytes.len(), - capture_mode - ); - - // If fullscreen capture is too large, try downscaling with sips. - if bytes.len() > MAX_SCREENSHOT_BYTES && capture_mode == CaptureMode::Fullscreen { - tracing::debug!( - "[screen_intelligence] fullscreen capture {} bytes exceeds limit, downscaling with sips", - bytes.len() - ); - let sips_status = std::process::Command::new("sips") - .arg("--resampleWidth") - .arg("1280") - .arg(&tmp_path) - .status(); - match sips_status { - Ok(s) if s.success() => { - let resized = std::fs::read(&tmp_path) - .map_err(|e| format!("failed to read resized screenshot: {e}"))?; - let _ = std::fs::remove_file(&tmp_path); - tracing::debug!("[screen_intelligence] resized to {} bytes", resized.len()); - if resized.len() > MAX_SCREENSHOT_BYTES { - return Err( - "captured screenshot exceeds size limit after downscale".to_string() - ); - } - let encoded = BASE64_STANDARD.encode(resized); - return Ok(format!("data:image/png;base64,{encoded}")); - } - Ok(s) => { - let _ = std::fs::remove_file(&tmp_path); - tracing::debug!( - "[screen_intelligence] sips failed with status: {:?}", - s.code() - ); - return Err( - "captured screenshot exceeds size limit and downscale failed".to_string(), - ); - } - Err(e) => { - let _ = std::fs::remove_file(&tmp_path); - tracing::debug!("[screen_intelligence] sips not available: {e}"); - return Err("captured screenshot exceeds size limit".to_string()); - } - } - } - - let _ = std::fs::remove_file(&tmp_path); - - if bytes.len() > MAX_SCREENSHOT_BYTES { - return Err("captured screenshot exceeds size limit".to_string()); - } - - let encoded = BASE64_STANDARD.encode(bytes); - Ok(format!("data:image/png;base64,{encoded}")) - } - - #[cfg(not(target_os = "macos"))] - { - let _ = context; - Err("screen capture is unsupported on this platform".to_string()) - } -} - -/// Parse the raw stdout from the AppleScript foreground-context query. -/// -/// Expected format: 6 lines — app_name, window_title, x, y, width, height. -/// This is a pure function, fully testable without macOS. -pub(crate) fn parse_foreground_output(stdout: &str) -> Option { - let mut lines = stdout.lines(); - let app = lines.next().map(|s| s.trim().to_string()); - let title = lines.next().map(|s| s.trim().to_string()); - let x = lines.next().and_then(|s| s.trim().parse::().ok()); - let y = lines.next().and_then(|s| s.trim().parse::().ok()); - let width = lines.next().and_then(|s| s.trim().parse::().ok()); - let height = lines.next().and_then(|s| s.trim().parse::().ok()); - - let bounds = match (x, y, width, height) { - (Some(x), Some(y), Some(width), Some(height)) if width > 0 && height > 0 => { - Some(WindowBounds { - x, - y, - width, - height, - }) - } - _ => None, - }; - - Some(AppContext { - app_name: app.filter(|s| !s.is_empty()), - window_title: title.filter(|s| !s.is_empty()), - bounds, - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn foreground_context() -> Option { - let script = r#" - tell application "System Events" - set frontApp to name of first application process whose frontmost is true - set frontWindow to "" - set windowX to "" - set windowY to "" - set windowW to "" - set windowH to "" - try - tell process frontApp - if (count of windows) > 0 then - set w to front window - set frontWindow to name of w - set p to position of w - set s to size of w - set windowX to item 1 of p as text - set windowY to item 2 of p as text - set windowW to item 1 of s as text - set windowH to item 2 of s as text - end if - end tell - end try - return frontApp & "\n" & frontWindow & "\n" & windowX & "\n" & windowY & "\n" & windowW & "\n" & windowH - end tell - "#; - - let output = std::process::Command::new("osascript") - .arg("-e") - .arg(script) - .output() - .ok()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - tracing::debug!( - "[screen_intelligence] osascript failed: status={:?} stderr={}", - output.status.code(), - stderr.trim() - ); - return None; - } - - let text = String::from_utf8_lossy(&output.stdout); - let result = parse_foreground_output(&text); - tracing::debug!( - "[screen_intelligence] foreground_context: app={:?} bounds_present={}", - result.as_ref().and_then(|c| c.app_name.as_deref()), - result.as_ref().map(|c| c.bounds.is_some()).unwrap_or(false) - ); - result -} - -#[cfg(not(target_os = "macos"))] -pub(crate) fn foreground_context() -> Option { - None -} diff --git a/src/openhuman/screen_intelligence/context.rs b/src/openhuman/screen_intelligence/context.rs deleted file mode 100644 index 9bb526c15..000000000 --- a/src/openhuman/screen_intelligence/context.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Foreground window / app context for capture and policy. - -#[derive(Debug, Clone)] -pub(crate) struct WindowBounds { - pub(crate) x: i32, - pub(crate) y: i32, - pub(crate) width: i32, - pub(crate) height: i32, -} - -#[derive(Debug, Clone)] -pub(crate) struct AppContext { - pub(crate) app_name: Option, - pub(crate) window_title: Option, - pub(crate) bounds: Option, -} - -impl AppContext { - pub(crate) fn same_as(&self, other: &AppContext) -> bool { - self.app_name == other.app_name - && self.window_title == other.window_title - && self.bounds.as_ref().map(|b| (b.x, b.y, b.width, b.height)) - == other.bounds.as_ref().map(|b| (b.x, b.y, b.width, b.height)) - } - - pub(crate) fn as_compound_text(&self) -> String { - format!( - "{} {}", - self.app_name.clone().unwrap_or_default(), - self.window_title.clone().unwrap_or_default() - ) - .to_lowercase() - } -} diff --git a/src/openhuman/screen_intelligence/engine.rs b/src/openhuman/screen_intelligence/engine.rs index 26383b4f3..d89e64eef 100644 --- a/src/openhuman/screen_intelligence/engine.rs +++ b/src/openhuman/screen_intelligence/engine.rs @@ -7,24 +7,26 @@ use tokio::sync::Mutex; use tokio::task::JoinHandle; use tokio::time::{self, Duration}; -use super::capture::{capture_screen_image_ref_for_context, foreground_context, now_ms}; -use super::context::AppContext; +use super::capture::now_ms; use super::helpers::{ generate_suggestions, parse_vision_summary_output, persist_vision_summary, push_ephemeral_frame, push_ephemeral_vision_summary, truncate_tail, validate_input_action, }; use super::limits::{MAX_CONTEXT_CHARS, MAX_SUGGESTION_CHARS}; -use super::permissions::{detect_permissions, permission_to_str}; -#[cfg(target_os = "macos")] -use super::permissions::{ - open_macos_privacy_pane, request_accessibility_access, request_screen_recording_access, -}; use super::types::{ AccessibilityFeatures, AccessibilityStatus, AppContextInfo, AutocompleteCommitParams, AutocompleteCommitResult, AutocompleteSuggestParams, AutocompleteSuggestResult, CaptureFrame, CaptureImageRefResult, CaptureNowResult, CaptureTestResult, InputActionParams, - InputActionResult, PermissionKind, PermissionState, PermissionStatus, SessionStatus, - StartSessionParams, VisionFlushResult, VisionRecentResult, VisionSummary, + InputActionResult, SessionStatus, StartSessionParams, VisionFlushResult, VisionRecentResult, + VisionSummary, +}; +use crate::openhuman::accessibility::{ + capture_screen_image_ref_for_context, detect_permissions, foreground_context, + permission_to_str, AppContext, PermissionKind, PermissionState, PermissionStatus, +}; +#[cfg(target_os = "macos")] +use crate::openhuman::accessibility::{ + open_macos_privacy_pane, request_accessibility_access, request_screen_recording_access, }; struct SessionRuntime { @@ -864,13 +866,33 @@ impl AccessibilityEngine { .image_ref .clone() .ok_or_else(|| "frame has no image payload".to_string())?; + + // ── Compress & resize before sending to the LLM ───────────────── + tracing::trace!( + "[screen_intelligence] compress_screenshot: input image_ref len={}", + image_ref.len() + ); + let compressed = super::image_processing::compress_screenshot(&image_ref, None, None) + .map_err(|e| format!("image compression failed: {e}"))?; + tracing::trace!( + "[screen_intelligence] compress_screenshot: {}x{} -> {}x{}, {} -> {} bytes; vision_image_ref len={}", + compressed.original_dimensions.0, + compressed.original_dimensions.1, + compressed.final_dimensions.0, + compressed.final_dimensions.1, + compressed.original_bytes, + compressed.compressed_bytes, + compressed.data_uri.len() + ); + let vision_image_ref = compressed.data_uri; + let config = Config::load_or_init() .await .map_err(|e| format!("failed to load config: {e}"))?; let service = local_ai::global(&config); let prompt = "Analyze this UI screenshot. Return strict JSON with keys: ui_state, key_text, actionable_notes, confidence (0..1). Keep actionable_notes concise."; let raw = service - .vision_prompt(&config, prompt, &[image_ref], Some(180)) + .vision_prompt(&config, prompt, &[vision_image_ref], Some(180)) .await?; Ok(parse_vision_summary_output(frame, &raw)) } diff --git a/src/openhuman/screen_intelligence/image_processing.rs b/src/openhuman/screen_intelligence/image_processing.rs new file mode 100644 index 000000000..75e841562 --- /dev/null +++ b/src/openhuman/screen_intelligence/image_processing.rs @@ -0,0 +1,370 @@ +//! Image compression and resizing for screenshot intelligence. +//! +//! Before sending screenshots to the vision LLM we shrink them: +//! 1. Decode the base64 PNG data-URI into pixels. +//! 2. Resize so the longest edge fits within `max_dimension` (default 1024 px). +//! 3. Re-encode as JPEG at a configurable quality (default 72). +//! 4. Return a `data:image/jpeg;base64,…` URI ready for the Ollama vision API. +//! +//! Smaller images mean fewer tokens, faster inference, and lower memory pressure. + +use base64::{engine::general_purpose::STANDARD as B64, Engine}; +use image::codecs::jpeg::JpegEncoder; +use image::imageops::FilterType; +use image::{DynamicImage, ImageReader}; +use std::io::Cursor; + +/// Default longest-edge cap (pixels). Vision models rarely benefit from +/// more than 1024 px on the long side for UI-screenshot analysis. +pub(crate) const DEFAULT_MAX_DIMENSION: u32 = 1024; + +/// Default JPEG quality (1-100). 72 is a good trade-off: visually acceptable +/// for UI text while cutting size by ~70-85 % compared to PNG. +pub(crate) const DEFAULT_JPEG_QUALITY: u8 = 72; + +/// Result of compressing a screenshot. +#[derive(Debug, Clone)] +pub(crate) struct CompressedImage { + /// `data:image/jpeg;base64,…` ready for the vision API. + pub data_uri: String, + /// Original decoded size in bytes (raw PNG payload). + pub original_bytes: usize, + /// Compressed JPEG size in bytes. + pub compressed_bytes: usize, + /// Original dimensions (width, height). + pub original_dimensions: (u32, u32), + /// Final dimensions after resize (width, height). + pub final_dimensions: (u32, u32), +} + +/// Compress and resize a base64 PNG data-URI for vision LLM consumption. +/// +/// Accepts a full `data:image/png;base64,…` URI **or** a raw base64 string. +/// Returns `Err` if the payload cannot be decoded as a valid image. +pub(crate) fn compress_screenshot( + image_ref: &str, + max_dimension: Option, + jpeg_quality: Option, +) -> Result { + let max_dim = max_dimension.unwrap_or(DEFAULT_MAX_DIMENSION).max(64); + let quality = jpeg_quality.unwrap_or(DEFAULT_JPEG_QUALITY).clamp(10, 100); + + // ── 1. Strip data-URI prefix and decode base64 ────────────────────── + let b64_payload = strip_data_uri_prefix(image_ref); + let raw_bytes = B64 + .decode(b64_payload) + .map_err(|e| format!("base64 decode failed: {e}"))?; + let original_bytes = raw_bytes.len(); + + // ── 2. Decode into pixels ─────────────────────────────────────────── + let img = ImageReader::new(Cursor::new(&raw_bytes)) + .with_guessed_format() + .map_err(|e| format!("image format detection failed: {e}"))? + .decode() + .map_err(|e| format!("image decode failed: {e}"))?; + + let original_dimensions = (img.width(), img.height()); + + // ── 3. Resize if needed ───────────────────────────────────────────── + let resized = resize_to_fit(img, max_dim); + let final_dimensions = (resized.width(), resized.height()); + + // ── 4. Encode as JPEG ─────────────────────────────────────────────── + let jpeg_bytes = encode_jpeg(&resized, quality)?; + let compressed_bytes = jpeg_bytes.len(); + + // ── 5. Build data-URI ─────────────────────────────────────────────── + let b64_out = B64.encode(&jpeg_bytes); + let data_uri = format!("data:image/jpeg;base64,{b64_out}"); + + tracing::debug!( + "[screen_intelligence] image compressed: {}x{} -> {}x{}, {} -> {} bytes ({:.0}% reduction)", + original_dimensions.0, + original_dimensions.1, + final_dimensions.0, + final_dimensions.1, + original_bytes, + compressed_bytes, + (1.0 - compressed_bytes as f64 / original_bytes as f64) * 100.0, + ); + + Ok(CompressedImage { + data_uri, + original_bytes, + compressed_bytes, + original_dimensions, + final_dimensions, + }) +} + +/// Strip common data-URI prefixes, returning the raw base64 payload. +fn strip_data_uri_prefix(input: &str) -> &str { + // Handle: data:image/png;base64,… | data:image/jpeg;base64,… | data:image/*;base64,… + if let Some(pos) = input.find(";base64,") { + &input[pos + 8..] + } else { + input + } +} + +/// Resize `img` so neither dimension exceeds `max_dim`, preserving aspect ratio. +/// If both dimensions are already within bounds the image is returned unchanged. +fn resize_to_fit(img: DynamicImage, max_dim: u32) -> DynamicImage { + let (w, h) = (img.width(), img.height()); + if w <= max_dim && h <= max_dim { + return img; + } + + let scale = max_dim as f64 / w.max(h) as f64; + let new_w = ((w as f64 * scale).round() as u32).max(1); + let new_h = ((h as f64 * scale).round() as u32).max(1); + + // Lanczos3 gives crisp text edges — important for UI screenshots. + img.resize_exact(new_w, new_h, FilterType::Lanczos3) +} + +/// Encode a `DynamicImage` as JPEG bytes at the given quality. +fn encode_jpeg(img: &DynamicImage, quality: u8) -> Result, String> { + let rgb = img.to_rgb8(); + let mut buf: Vec = Vec::new(); + let encoder = JpegEncoder::new_with_quality(&mut buf, quality); + rgb.write_with_encoder(encoder) + .map_err(|e| format!("JPEG encode failed: {e}"))?; + Ok(buf) +} + +#[cfg(test)] +mod tests { + use super::*; + use image::{ImageBuffer, Rgb, RgbImage}; + + /// Helper: create a solid-color PNG image of given dimensions and return + /// its `data:image/png;base64,…` URI. + fn make_test_png(width: u32, height: u32, color: [u8; 3]) -> String { + let img: RgbImage = ImageBuffer::from_fn(width, height, |_, _| Rgb(color)); + let mut png_bytes: Vec = Vec::new(); + let encoder = image::codecs::png::PngEncoder::new(&mut png_bytes); + img.write_with_encoder(encoder).expect("PNG encode"); + let b64 = B64.encode(&png_bytes); + format!("data:image/png;base64,{b64}") + } + + // ── Basic compression ─────────────────────────────────────────────── + + #[test] + fn compress_reduces_size_for_large_image() { + let uri = make_test_png(2048, 1536, [100, 150, 200]); + let result = compress_screenshot(&uri, None, None).unwrap(); + + assert!( + result.compressed_bytes < result.original_bytes, + "JPEG should be smaller than PNG for a large solid image" + ); + assert_eq!(result.original_dimensions, (2048, 1536)); + assert!( + result.final_dimensions.0 <= DEFAULT_MAX_DIMENSION, + "width should be capped" + ); + assert!( + result.final_dimensions.1 <= DEFAULT_MAX_DIMENSION, + "height should be capped" + ); + assert!(result.data_uri.starts_with("data:image/jpeg;base64,")); + } + + #[test] + fn compress_preserves_aspect_ratio() { + let uri = make_test_png(2000, 1000, [255, 0, 0]); + let result = compress_screenshot(&uri, Some(500), None).unwrap(); + + // 2000x1000 → 500x250 (long edge capped at 500) + assert_eq!(result.final_dimensions.0, 500); + assert_eq!(result.final_dimensions.1, 250); + } + + #[test] + fn compress_portrait_image() { + let uri = make_test_png(600, 1800, [0, 255, 0]); + let result = compress_screenshot(&uri, Some(900), None).unwrap(); + + // 600x1800 → 300x900 (height is long edge) + assert_eq!(result.final_dimensions.1, 900); + assert_eq!(result.final_dimensions.0, 300); + } + + // ── No-resize path ────────────────────────────────────────────────── + + #[test] + fn small_image_not_resized() { + let uri = make_test_png(200, 150, [50, 50, 50]); + let result = compress_screenshot(&uri, Some(1024), None).unwrap(); + + assert_eq!(result.original_dimensions, result.final_dimensions); + } + + #[test] + fn exact_max_dimension_not_resized() { + let uri = make_test_png(1024, 768, [80, 80, 80]); + let result = compress_screenshot(&uri, Some(1024), None).unwrap(); + + assert_eq!(result.final_dimensions, (1024, 768)); + } + + // ── Quality settings ──────────────────────────────────────────────── + + #[test] + fn lower_quality_produces_smaller_output() { + let uri = make_test_png(800, 600, [128, 64, 200]); + let high = compress_screenshot(&uri, None, Some(95)).unwrap(); + let low = compress_screenshot(&uri, None, Some(30)).unwrap(); + + assert!( + low.compressed_bytes < high.compressed_bytes, + "quality 30 should produce smaller JPEG than quality 95" + ); + } + + #[test] + fn quality_clamped_to_valid_range() { + let uri = make_test_png(100, 100, [0, 0, 0]); + // quality below 10 should be clamped to 10 + let result = compress_screenshot(&uri, None, Some(1)).unwrap(); + assert!(result.compressed_bytes > 0); + + // quality above 100 should be clamped to 100 + let result2 = compress_screenshot(&uri, None, Some(255)).unwrap(); + assert!(result2.compressed_bytes > 0); + } + + // ── Data-URI prefix handling ──────────────────────────────────────── + + #[test] + fn handles_raw_base64_without_prefix() { + // Build raw base64 without data URI prefix + let img: RgbImage = ImageBuffer::from_fn(64, 64, |_, _| Rgb([255, 255, 255])); + let mut png_bytes: Vec = Vec::new(); + let encoder = image::codecs::png::PngEncoder::new(&mut png_bytes); + img.write_with_encoder(encoder).expect("PNG encode"); + let raw_b64 = B64.encode(&png_bytes); + + let result = compress_screenshot(&raw_b64, None, None).unwrap(); + assert_eq!(result.original_dimensions, (64, 64)); + } + + #[test] + fn handles_jpeg_data_uri_prefix() { + // Even if input is labeled as JPEG, we decode by content not prefix + let img: RgbImage = ImageBuffer::from_fn(64, 64, |_, _| Rgb([100, 100, 100])); + let mut png_bytes: Vec = Vec::new(); + let encoder = image::codecs::png::PngEncoder::new(&mut png_bytes); + img.write_with_encoder(encoder).expect("PNG encode"); + let b64 = B64.encode(&png_bytes); + let uri = format!("data:image/jpeg;base64,{b64}"); + + let result = compress_screenshot(&uri, None, None).unwrap(); + assert_eq!(result.original_dimensions, (64, 64)); + } + + // ── Edge cases ────────────────────────────────────────────────────── + + #[test] + fn tiny_1x1_image() { + let uri = make_test_png(1, 1, [255, 0, 0]); + let result = compress_screenshot(&uri, None, None).unwrap(); + + assert_eq!(result.original_dimensions, (1, 1)); + assert_eq!(result.final_dimensions, (1, 1)); + } + + #[test] + fn very_wide_panoramic_image() { + let uri = make_test_png(4000, 100, [0, 0, 255]); + let result = compress_screenshot(&uri, Some(1024), None).unwrap(); + + assert_eq!(result.final_dimensions.0, 1024); + // 4000x100 → 1024x26 (proportional) + assert!(result.final_dimensions.1 > 0); + assert!(result.final_dimensions.1 <= 100); + } + + #[test] + fn square_image() { + let uri = make_test_png(2000, 2000, [128, 128, 128]); + let result = compress_screenshot(&uri, Some(512), None).unwrap(); + + assert_eq!(result.final_dimensions, (512, 512)); + } + + #[test] + fn invalid_base64_returns_error() { + let result = compress_screenshot("data:image/png;base64,!!!invalid!!!", None, None); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("base64 decode failed")); + } + + #[test] + fn valid_base64_but_not_image_returns_error() { + let b64 = B64.encode(b"this is not an image"); + let uri = format!("data:image/png;base64,{b64}"); + let result = compress_screenshot(&uri, None, None); + assert!(result.is_err()); + } + + #[test] + fn min_max_dimension_floor() { + // max_dimension below 64 should be clamped to 64 + let uri = make_test_png(200, 200, [0, 0, 0]); + let result = compress_screenshot(&uri, Some(10), None).unwrap(); + assert!( + result.final_dimensions.0 >= 64, + "max_dimension should be floored to 64" + ); + } + + // ── strip_data_uri_prefix ─────────────────────────────────────────── + + #[test] + fn strip_prefix_png() { + let input = "data:image/png;base64,ABCD1234"; + assert_eq!(strip_data_uri_prefix(input), "ABCD1234"); + } + + #[test] + fn strip_prefix_jpeg() { + let input = "data:image/jpeg;base64,XYZ"; + assert_eq!(strip_data_uri_prefix(input), "XYZ"); + } + + #[test] + fn strip_prefix_no_prefix() { + let input = "ABCD1234"; + assert_eq!(strip_data_uri_prefix(input), "ABCD1234"); + } + + // ── Multicolored image (more realistic compression ratio) ─────────── + + #[test] + fn multicolored_image_compresses_well() { + // Create a gradient image that's more representative of real screenshots + let width = 1920u32; + let height = 1080u32; + let img: RgbImage = ImageBuffer::from_fn(width, height, |x, y| { + Rgb([(x % 256) as u8, (y % 256) as u8, ((x + y) % 256) as u8]) + }); + let mut png_bytes: Vec = Vec::new(); + let encoder = image::codecs::png::PngEncoder::new(&mut png_bytes); + img.write_with_encoder(encoder).expect("PNG encode"); + let b64 = B64.encode(&png_bytes); + let uri = format!("data:image/png;base64,{b64}"); + + let result = compress_screenshot(&uri, Some(1024), Some(72)).unwrap(); + + assert!(result.final_dimensions.0 <= 1024); + assert!(result.final_dimensions.1 <= 1024); + // For a gradient image, combined resize+JPEG should give significant savings + assert!( + result.compressed_bytes < result.original_bytes / 2, + "should achieve at least 50% size reduction on a 1920x1080 gradient" + ); + } +} diff --git a/src/openhuman/screen_intelligence/mod.rs b/src/openhuman/screen_intelligence/mod.rs index 8dc2c3fa6..fdef524d3 100644 --- a/src/openhuman/screen_intelligence/mod.rs +++ b/src/openhuman/screen_intelligence/mod.rs @@ -4,9 +4,9 @@ pub mod ops; mod schemas; mod capture; -mod context; mod engine; mod helpers; +mod image_processing; mod limits; mod permissions; mod types; diff --git a/src/openhuman/screen_intelligence/permissions.rs b/src/openhuman/screen_intelligence/permissions.rs index 2a49d997c..ae5872588 100644 --- a/src/openhuman/screen_intelligence/permissions.rs +++ b/src/openhuman/screen_intelligence/permissions.rs @@ -1,146 +1,2 @@ -#[cfg(target_os = "macos")] -use std::ffi::c_void; - -use super::types::{PermissionKind, PermissionState, PermissionStatus}; - -#[cfg(target_os = "macos")] -type CFAllocatorRef = *const c_void; -#[cfg(target_os = "macos")] -type CFDictionaryRef = *const c_void; -#[cfg(target_os = "macos")] -type CFBooleanRef = *const c_void; -#[cfg(target_os = "macos")] -type CFStringRef = *const c_void; - -#[cfg(target_os = "macos")] -#[link(name = "ApplicationServices", kind = "framework")] -extern "C" { - fn AXIsProcessTrusted() -> bool; - fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> bool; - static kAXTrustedCheckOptionPrompt: CFStringRef; - fn CGPreflightScreenCaptureAccess() -> bool; - fn CGRequestScreenCaptureAccess() -> bool; -} - -#[cfg(target_os = "macos")] -#[link(name = "CoreFoundation", kind = "framework")] -extern "C" { - static kCFAllocatorDefault: CFAllocatorRef; - static kCFBooleanTrue: CFBooleanRef; - fn CFDictionaryCreate( - allocator: CFAllocatorRef, - keys: *const *const c_void, - values: *const *const c_void, - num_values: isize, - key_callbacks: *const c_void, - value_callbacks: *const c_void, - ) -> CFDictionaryRef; - fn CFRelease(cf: *const c_void); -} - -#[cfg(target_os = "macos")] -#[link(name = "IOKit", kind = "framework")] -extern "C" { - fn IOHIDCheckAccess(request_type: i32) -> isize; -} - -#[cfg(target_os = "macos")] -const IOHID_REQUEST_TYPE_LISTEN_EVENT: i32 = 1; -#[cfg(target_os = "macos")] -const IOHID_ACCESS_GRANTED: isize = 0; -#[cfg(target_os = "macos")] -const IOHID_ACCESS_DENIED: isize = 1; -#[cfg(target_os = "macos")] -const IOHID_ACCESS_UNKNOWN: isize = 2; - -pub(crate) fn permission_to_str(permission: PermissionKind) -> &'static str { - match permission { - PermissionKind::ScreenRecording => "screen_recording", - PermissionKind::Accessibility => "accessibility", - PermissionKind::InputMonitoring => "input_monitoring", - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn open_macos_privacy_pane(pane: &str) { - let url = format!("x-apple.systempreferences:com.apple.preference.security?{pane}"); - let _ = std::process::Command::new("open").arg(url).status(); -} - -#[cfg(target_os = "macos")] -pub(crate) fn request_accessibility_access() { - unsafe { - let keys = [kAXTrustedCheckOptionPrompt as *const c_void]; - let values = [kCFBooleanTrue as *const c_void]; - let options = CFDictionaryCreate( - kCFAllocatorDefault, - keys.as_ptr(), - values.as_ptr(), - 1, - std::ptr::null(), - std::ptr::null(), - ); - let _ = AXIsProcessTrustedWithOptions(options); - if !options.is_null() { - CFRelease(options); - } - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn request_screen_recording_access() { - unsafe { - let _ = CGRequestScreenCaptureAccess(); - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn detect_accessibility_permission() -> PermissionState { - unsafe { - if AXIsProcessTrusted() { - PermissionState::Granted - } else { - PermissionState::Denied - } - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn detect_screen_recording_permission() -> PermissionState { - unsafe { - if CGPreflightScreenCaptureAccess() { - PermissionState::Granted - } else { - PermissionState::Denied - } - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn detect_input_monitoring_permission() -> PermissionState { - let access = unsafe { IOHIDCheckAccess(IOHID_REQUEST_TYPE_LISTEN_EVENT) }; - match access { - IOHID_ACCESS_GRANTED => PermissionState::Granted, - IOHID_ACCESS_DENIED => PermissionState::Denied, - IOHID_ACCESS_UNKNOWN => PermissionState::Unknown, - _ => PermissionState::Unknown, - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn detect_permissions() -> PermissionStatus { - PermissionStatus { - screen_recording: detect_screen_recording_permission(), - accessibility: detect_accessibility_permission(), - input_monitoring: detect_input_monitoring_permission(), - } -} - -#[cfg(not(target_os = "macos"))] -pub(crate) fn detect_permissions() -> PermissionStatus { - PermissionStatus { - screen_recording: PermissionState::Unsupported, - accessibility: PermissionState::Unsupported, - input_monitoring: PermissionState::Unsupported, - } -} +//! Permission detection — now in accessibility middleware. +//! This module is retained for module-tree compatibility. diff --git a/src/openhuman/screen_intelligence/tests.rs b/src/openhuman/screen_intelligence/tests.rs index be4d8691f..e3c4c87ff 100644 --- a/src/openhuman/screen_intelligence/tests.rs +++ b/src/openhuman/screen_intelligence/tests.rs @@ -2,13 +2,12 @@ use std::sync::Arc; use tokio::sync::Mutex; use tokio::time::{self, Duration}; -use super::capture::parse_foreground_output; -use super::context::AppContext; use super::engine::{global_engine, AccessibilityEngine, EngineState}; use super::helpers::{ generate_suggestions, parse_vision_summary_output, truncate_tail, validate_input_action, }; use super::types::{CaptureFrame, InputActionParams, StartSessionParams}; +use crate::openhuman::accessibility::{parse_foreground_output, AppContext}; use crate::openhuman::config::ScreenIntelligenceConfig; // ── parse_foreground_output ───────────────────────────────────────────── diff --git a/src/openhuman/screen_intelligence/types.rs b/src/openhuman/screen_intelligence/types.rs index cd806b6bf..b7b32e986 100644 --- a/src/openhuman/screen_intelligence/types.rs +++ b/src/openhuman/screen_intelligence/types.rs @@ -1,29 +1,8 @@ use crate::openhuman::config::ScreenIntelligenceConfig; use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum PermissionState { - Granted, - Denied, - Unknown, - Unsupported, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PermissionStatus { - pub screen_recording: PermissionState, - pub accessibility: PermissionState, - pub input_monitoring: PermissionState, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum PermissionKind { - ScreenRecording, - Accessibility, - InputMonitoring, -} +// Permission types are defined in the accessibility middleware; re-export for compatibility. +pub use crate::openhuman::accessibility::{PermissionKind, PermissionState, PermissionStatus}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AccessibilityFeatures { diff --git a/tests/screen_intelligence_vision_e2e.rs b/tests/screen_intelligence_vision_e2e.rs new file mode 100644 index 000000000..0c7a4ef25 --- /dev/null +++ b/tests/screen_intelligence_vision_e2e.rs @@ -0,0 +1,443 @@ +//! E2E tests for the screen-intelligence vision pipeline. +//! +//! Validates the full flow: generate image -> compress/resize -> parse vision +//! output -> persist to memory, all against real local storage in a temp workspace. +//! +//! Run with: `cargo test --test screen_intelligence_vision_e2e` + +use std::path::Path; +use std::sync::{Arc, Mutex, OnceLock}; + +use base64::{engine::general_purpose::STANDARD as B64, Engine}; +use image::codecs::jpeg::JpegEncoder; +use image::codecs::png::PngEncoder; +use image::imageops::FilterType; +use image::{ImageBuffer, Rgb, RgbImage}; +use tempfile::tempdir; + +use openhuman_core::openhuman::memory::embeddings::NoopEmbedding; +use openhuman_core::openhuman::memory::store::types::NamespaceDocumentInput; +use openhuman_core::openhuman::memory::store::UnifiedMemory; +use openhuman_core::openhuman::screen_intelligence::CaptureFrame; + +// ── Env isolation ──────────────────────────────────────────────────── + +struct EnvVarGuard { + key: &'static str, + old: Option, +} + +impl EnvVarGuard { + fn set_to_path(key: &'static str, path: &Path) -> Self { + let old = std::env::var(key).ok(); + std::env::set_var(key, path.as_os_str()); + Self { key, old } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + match &self.old { + Some(v) => std::env::set_var(self.key, v), + None => std::env::remove_var(self.key), + } + } +} + +static ENV_LOCK: OnceLock> = OnceLock::new(); + +fn env_lock() -> std::sync::MutexGuard<'static, ()> { + ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock poisoned") +} + +// ── Helpers ────────────────────────────────────────────────────────── + +/// Create a synthetic PNG data-URI simulating a desktop screenshot. +fn make_test_png_uri(width: u32, height: u32) -> String { + let img: RgbImage = ImageBuffer::from_fn(width, height, |x, y| { + Rgb([ + (x % 256) as u8, + (y % 256) as u8, + ((x * 3 + y * 7) % 256) as u8, + ]) + }); + let mut png_bytes: Vec = Vec::new(); + let encoder = PngEncoder::new(&mut png_bytes); + img.write_with_encoder(encoder).expect("PNG encode"); + let b64 = B64.encode(&png_bytes); + format!("data:image/png;base64,{b64}") +} + +fn make_capture_frame(image_ref: Option) -> CaptureFrame { + CaptureFrame { + captured_at_ms: chrono::Utc::now().timestamp_millis(), + reason: "e2e_test".to_string(), + app_name: Some("TestApp".to_string()), + window_title: Some("E2E Test Window".to_string()), + image_ref, + } +} + +/// Open a UnifiedMemory backed by NoopEmbedding in a temp dir. +fn open_test_memory(dir: &Path) -> UnifiedMemory { + let embedder: Arc = + Arc::new(NoopEmbedding); + UnifiedMemory::new(dir, embedder, Some(5)).expect("UnifiedMemory::new") +} + +/// Simulate what `parse_vision_summary_output` does, but from public types. +fn mock_vision_summary(frame: &CaptureFrame, raw_llm: &str) -> serde_json::Value { + let value: serde_json::Value = serde_json::from_str(raw_llm).unwrap_or_else(|_| { + serde_json::json!({ + "ui_state": "UI state unavailable", + "key_text": "", + "actionable_notes": raw_llm.trim(), + "confidence": 0.66, + }) + }); + serde_json::json!({ + "id": format!("vision-{}-e2e", frame.captured_at_ms), + "captured_at_ms": frame.captured_at_ms, + "app_name": frame.app_name, + "window_title": frame.window_title, + "ui_state": value.get("ui_state").and_then(|v| v.as_str()).unwrap_or("UI state unavailable"), + "key_text": value.get("key_text").and_then(|v| v.as_str()).unwrap_or(""), + "actionable_notes": value.get("actionable_notes").and_then(|v| v.as_str()).unwrap_or(raw_llm.trim()), + "confidence": value.get("confidence").and_then(|v| v.as_f64()).unwrap_or(0.66), + }) +} + +// ── Tests ──────────────────────────────────────────────────────────── + +/// Full pipeline: compress screenshot -> simulate LLM response -> persist to memory -> query back. +#[tokio::test] +async fn vision_pipeline_compress_parse_persist() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + + // ── Step 1: Generate a 1920x1080 screenshot ───────────────────── + let image_ref = make_test_png_uri(1920, 1080); + let original_b64_len = image_ref.len(); + assert!( + original_b64_len > 10_000, + "test image should be non-trivial" + ); + + // ── Step 2: Compress (same logic as image_processing module) ───── + let b64_payload = image_ref + .find(";base64,") + .map(|pos| &image_ref[pos + 8..]) + .unwrap_or(&image_ref); + let raw_bytes = B64.decode(b64_payload).expect("decode original"); + let original_size = raw_bytes.len(); + + let img = image::load_from_memory(&raw_bytes).expect("load image"); + assert_eq!(img.width(), 1920); + assert_eq!(img.height(), 1080); + + // Resize to 1024 on long edge + let max_dim = 1024u32; + let scale = max_dim as f64 / img.width().max(img.height()) as f64; + let new_w = (img.width() as f64 * scale).round() as u32; + let new_h = (img.height() as f64 * scale).round() as u32; + let resized = img.resize_exact(new_w, new_h, FilterType::Lanczos3); + assert!(resized.width() <= max_dim); + assert!(resized.height() <= max_dim); + + // JPEG encode + let rgb = resized.to_rgb8(); + let mut jpeg_buf: Vec = Vec::new(); + let encoder = JpegEncoder::new_with_quality(&mut jpeg_buf, 72); + rgb.write_with_encoder(encoder).expect("JPEG encode"); + let compressed_size = jpeg_buf.len(); + assert!( + compressed_size < original_size, + "compressed ({compressed_size}) should be smaller than original ({original_size})" + ); + + let compressed_uri = format!("data:image/jpeg;base64,{}", B64.encode(&jpeg_buf)); + assert!(compressed_uri.len() < original_b64_len); + + // ── Step 3: Simulate LLM vision response ──────────────────────── + let frame = make_capture_frame(Some(image_ref)); + let mock_llm_response = r#"{"ui_state": "code editor with terminal", "key_text": "fn main() { println!(\"hello\"); }", "actionable_notes": "User is editing Rust code in a split-pane layout", "confidence": 0.91}"#; + let summary = mock_vision_summary(&frame, mock_llm_response); + + assert_eq!( + summary["ui_state"].as_str().unwrap(), + "code editor with terminal" + ); + assert!((summary["confidence"].as_f64().unwrap() - 0.91).abs() < 0.01); + + // ── Step 4: Persist to memory ─────────────────────────────────── + let mem = open_test_memory(tmp.path()); + let content = serde_json::to_string(&summary).expect("serialize summary"); + let key = format!("screen_intelligence_{}", summary["id"].as_str().unwrap()); + mem.upsert_document(NamespaceDocumentInput { + namespace: "background".to_string(), + key: key.clone(), + title: key.clone(), + content: content.clone(), + source_type: "screenshot".to_string(), + priority: "medium".to_string(), + tags: vec!["screen_intelligence".to_string()], + metadata: serde_json::json!({}), + category: "screen_intelligence".to_string(), + session_id: None, + document_id: None, + }) + .await + .expect("upsert_document"); + + // ── Step 5: Query back from memory ────────────────────────────── + let result_json = mem + .list_documents(Some("background")) + .await + .expect("list_documents"); + let docs = result_json["documents"] + .as_array() + .expect("documents array"); + assert!(!docs.is_empty(), "should find the persisted vision summary"); + let found = docs.iter().any(|d| d["key"].as_str() == Some(&key)); + assert!(found, "should find document by key: {key}"); +} + +/// Multiple screenshots persisted and queryable. +#[tokio::test] +async fn multiple_vision_summaries_persist_and_query() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + + let mem = open_test_memory(tmp.path()); + + let scenarios = vec![ + ( + "Safari", + "GitHub PR Review", + 0.88, + "User reviewing pull request diffs", + ), + ( + "VSCode", + "main.rs - editor", + 0.92, + "Rust code editing with LSP diagnostics", + ), + ( + "Terminal", + "cargo test output", + 0.85, + "Test results showing 19 passed", + ), + ]; + + for (i, (app, window, confidence, notes)) in scenarios.iter().enumerate() { + let ts = chrono::Utc::now().timestamp_millis() + i as i64; + let summary = serde_json::json!({ + "id": format!("vision-{ts}-{app}"), + "captured_at_ms": ts, + "app_name": app, + "window_title": window, + "ui_state": "active", + "key_text": "", + "actionable_notes": notes, + "confidence": confidence, + }); + + let content = serde_json::to_string(&summary).expect("serialize"); + let key = format!("screen_intelligence_{}", summary["id"].as_str().unwrap()); + mem.upsert_document(NamespaceDocumentInput { + namespace: "background".to_string(), + key, + title: format!("{app} - {window}"), + content, + source_type: "screenshot".to_string(), + priority: "medium".to_string(), + tags: vec!["screen_intelligence".to_string()], + metadata: serde_json::json!({}), + category: "screen_intelligence".to_string(), + session_id: None, + document_id: None, + }) + .await + .expect("upsert"); + } + + let result_json = mem + .list_documents(Some("background")) + .await + .expect("list_documents"); + let docs = result_json["documents"] + .as_array() + .expect("documents array"); + assert_eq!( + docs.len(), + 3, + "should have 3 persisted summaries, got {}", + docs.len() + ); +} + +/// Malformed LLM response still produces a usable summary (fallback path). +#[test] +fn malformed_llm_response_handled_gracefully() { + let frame = make_capture_frame(None); + let broken = "Sorry, I cannot analyze this image due to unclear content."; + let summary = mock_vision_summary(&frame, broken); + + assert_eq!( + summary["ui_state"].as_str().unwrap(), + "UI state unavailable" + ); + assert!(summary["actionable_notes"] + .as_str() + .unwrap() + .contains("Sorry")); + assert!((summary["confidence"].as_f64().unwrap() - 0.66).abs() < 0.01); +} + +/// Compression pipeline handles various image sizes without panicking. +#[test] +fn compression_handles_various_sizes() { + let sizes = vec![ + (64, 64), // tiny + (800, 600), // small desktop + (1920, 1080), // full HD + (3840, 2160), // 4K + (100, 2000), // tall narrow + (3000, 50), // wide short + ]; + + let max_dim = 1024u32; + for (w, h) in sizes { + let uri = make_test_png_uri(w, h); + let b64_payload = uri + .find(";base64,") + .map(|pos| &uri[pos + 8..]) + .unwrap_or(&uri); + let raw = B64.decode(b64_payload).expect("decode"); + let img = image::load_from_memory(&raw).expect("load"); + assert_eq!(img.width(), w, "width mismatch for {w}x{h}"); + assert_eq!(img.height(), h, "height mismatch for {w}x{h}"); + + if w > max_dim || h > max_dim { + let scale = max_dim as f64 / w.max(h) as f64; + let nw = (w as f64 * scale).round() as u32; + let nh = (h as f64 * scale).round() as u32; + let resized = img.resize_exact(nw, nh, FilterType::Lanczos3); + assert!( + resized.width() <= max_dim, + "resized width exceeds max for {w}x{h}" + ); + assert!( + resized.height() <= max_dim, + "resized height exceeds max for {w}x{h}" + ); + + let rgb = resized.to_rgb8(); + let mut buf: Vec = Vec::new(); + let enc = JpegEncoder::new_with_quality(&mut buf, 72); + rgb.write_with_encoder(enc) + .unwrap_or_else(|e| panic!("JPEG encode failed for {w}x{h}: {e}")); + assert!( + !buf.is_empty(), + "JPEG output should not be empty for {w}x{h}" + ); + } + } +} + +/// Vision summary upsert is idempotent (same key overwrites, not duplicates). +#[tokio::test] +async fn vision_summary_upsert_is_idempotent() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + + let mem = open_test_memory(tmp.path()); + let key = "screen_intelligence_vision-12345-upsert-test".to_string(); + + // First insert + mem.upsert_document(NamespaceDocumentInput { + namespace: "background".to_string(), + key: key.clone(), + title: key.clone(), + content: r#"{"version": 1}"#.to_string(), + source_type: "screenshot".to_string(), + priority: "medium".to_string(), + tags: vec!["screen_intelligence".to_string()], + metadata: serde_json::json!({}), + category: "screen_intelligence".to_string(), + session_id: None, + document_id: None, + }) + .await + .expect("first upsert"); + + // Second insert with same key, different content + mem.upsert_document(NamespaceDocumentInput { + namespace: "background".to_string(), + key: key.clone(), + title: key.clone(), + content: r#"{"version": 2}"#.to_string(), + source_type: "screenshot".to_string(), + priority: "medium".to_string(), + tags: vec!["screen_intelligence".to_string()], + metadata: serde_json::json!({}), + category: "screen_intelligence".to_string(), + session_id: None, + document_id: None, + }) + .await + .expect("second upsert"); + + let result_json = mem + .list_documents(Some("background")) + .await + .expect("list_documents"); + let docs = result_json["documents"] + .as_array() + .expect("documents array"); + let matching: Vec<_> = docs + .iter() + .filter(|d| d["key"].as_str() == Some(&key)) + .collect(); + assert_eq!( + matching.len(), + 1, + "upsert should overwrite, not duplicate: found {} docs", + matching.len() + ); +} + +/// Verify that compression produces significant savings on realistic images. +#[test] +fn compression_savings_on_realistic_screenshot() { + let uri = make_test_png_uri(2560, 1440); // QHD resolution + let b64_payload = uri.find(";base64,").map(|pos| &uri[pos + 8..]).unwrap(); + let raw = B64.decode(b64_payload).expect("decode"); + let original_size = raw.len(); + + let img = image::load_from_memory(&raw).expect("load"); + let scale = 1024.0 / img.width().max(img.height()) as f64; + let nw = (img.width() as f64 * scale).round() as u32; + let nh = (img.height() as f64 * scale).round() as u32; + let resized = img.resize_exact(nw, nh, FilterType::Lanczos3); + + let rgb = resized.to_rgb8(); + let mut jpeg_buf: Vec = Vec::new(); + let enc = JpegEncoder::new_with_quality(&mut jpeg_buf, 72); + rgb.write_with_encoder(enc).expect("JPEG encode"); + + let ratio = jpeg_buf.len() as f64 / original_size as f64; + assert!( + ratio < 0.5, + "compression ratio should be under 50%, got {:.1}%", + ratio * 100.0 + ); +}