From c03b1dc2d57ce02f9f7399d1fa476ba8fbdb76a7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Sat, 28 Mar 2026 13:38:24 -0700 Subject: [PATCH] Add standalone core autocomplete with Gemma inline completion (#51) * refactor(core_server): improve code formatting and readability - Reformatted code in `core_server.rs` for better readability, including consistent indentation and line breaks. - Enhanced the clarity of function calls and JSON handling by spreading parameters across multiple lines. - Improved overall structure and maintainability of the dispatch and settings view response functions. * feat(accessibility): refactor AccessibilityAutomationConfig to ScreenIntelligenceConfig - Renamed AccessibilityAutomationConfig to ScreenIntelligenceConfig across the codebase for improved clarity. - Updated related structures and functions to utilize the new configuration, including default values and additional fields for enhanced functionality. - Ensured backward compatibility by adjusting references in the AccessibilityEngine and related modules. - Enhanced the AppContext structure to include window bounds, improving state management for UI components. * feat(screen-intelligence): introduce Screen Intelligence settings and functionality - Added a new Screen Intelligence module to manage window capture policies, vision summaries, and memory ingestion. - Implemented settings for enabling/disabling features, configuring capture policies, and managing allowlists/deny lists. - Updated the accessibility panel to include Screen Intelligence options and integrated permission requests for screen recording and accessibility. - Enhanced the core server to handle updates to Screen Intelligence settings and ensure proper state management across the application. - Refactored related components and Redux state management to support the new features, improving user experience and functionality. * chore: apply rust fmt and fix blocking eslint warning * feat(autocomplete): implement inline autocomplete functionality - Introduced a new Autocomplete module with commands for managing autocomplete features, including status, start, stop, current suggestions, accept, and style settings. - Added corresponding data structures and parameters for each command to facilitate interaction with the autocomplete engine. - Enhanced the core server to handle new autocomplete commands and integrated them into the CLI for user accessibility. - Updated configuration schema to include autocomplete settings, allowing for customization of debounce timing, style presets, and application-specific behavior. - Implemented a new inline completion method in the local AI service to support real-time text suggestions based on user input. * Improve core autocomplete focus traversal and debug visibility --- rust-core/src/core_server.rs | 204 ++++- rust-core/src/openhuman/autocomplete/mod.rs | 716 ++++++++++++++++++ rust-core/src/openhuman/config/mod.rs | 23 +- .../openhuman/config/schema/autocomplete.rs | 57 ++ rust-core/src/openhuman/config/schema/mod.rs | 6 + rust-core/src/openhuman/local_ai/mod.rs | 74 ++ rust-core/src/openhuman/mod.rs | 1 + 7 files changed, 1058 insertions(+), 23 deletions(-) create mode 100644 rust-core/src/openhuman/autocomplete/mod.rs create mode 100644 rust-core/src/openhuman/config/schema/autocomplete.rs diff --git a/rust-core/src/core_server.rs b/rust-core/src/core_server.rs index cd66ccbea..d14610740 100644 --- a/rust-core/src/core_server.rs +++ b/rust-core/src/core_server.rs @@ -14,6 +14,7 @@ use std::io; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::openhuman::autocomplete; use crate::openhuman::config::Config; use crate::openhuman::cron; use crate::openhuman::health; @@ -28,6 +29,12 @@ use crate::openhuman::{ }; use chrono::Utc; +pub use crate::openhuman::autocomplete::{ + AutocompleteAcceptParams, AutocompleteAcceptResult, AutocompleteCurrentParams, + AutocompleteCurrentResult, AutocompleteDebugFocusResult, AutocompleteSetStyleParams, + AutocompleteSetStyleResult, AutocompleteStartParams, AutocompleteStartResult, + AutocompleteStatus, AutocompleteStopParams, AutocompleteStopResult, +}; pub use crate::openhuman::screen_intelligence::{ AccessibilityStatus, AutocompleteCommitParams, AutocompleteCommitResult, AutocompleteSuggestParams, AutocompleteSuggestResult, CaptureImageRefResult, CaptureNowResult, @@ -1157,25 +1164,77 @@ async fn dispatch( )) } - "openhuman.accessibility_autocomplete_suggest" => { - let payload: AutocompleteSuggestParams = parse_params(params)?; - let result = screen_intelligence::global_engine() - .autocomplete_suggest(payload) - .await?; + "openhuman.autocomplete_status" => { + let result: AutocompleteStatus = autocomplete::global_engine().status().await; to_json_value(command_response( result, - vec!["screen intelligence autocomplete suggestions generated".to_string()], + vec!["autocomplete status fetched".to_string()], )) } - "openhuman.accessibility_autocomplete_commit" => { - let payload: AutocompleteCommitParams = parse_params(params)?; - let result = screen_intelligence::global_engine() - .autocomplete_commit(payload) - .await?; + "openhuman.autocomplete_start" => { + let payload: AutocompleteStartParams = parse_params(params)?; + let result: AutocompleteStartResult = + autocomplete::global_engine().start(payload).await?; to_json_value(command_response( result, - vec!["screen intelligence autocomplete suggestion committed".to_string()], + vec!["autocomplete started".to_string()], + )) + } + + "openhuman.autocomplete_stop" => { + let payload: Option = if params.is_null() { + None + } else { + Some(parse_params(params)?) + }; + let result: AutocompleteStopResult = autocomplete::global_engine().stop(payload).await; + to_json_value(command_response( + result, + vec!["autocomplete stopped".to_string()], + )) + } + + "openhuman.autocomplete_current" => { + let payload: Option = if params.is_null() { + None + } else { + Some(parse_params(params)?) + }; + let result: AutocompleteCurrentResult = + autocomplete::global_engine().current(payload).await?; + to_json_value(command_response( + result, + vec!["autocomplete suggestion fetched".to_string()], + )) + } + + "openhuman.autocomplete_debug_focus" => { + let result: AutocompleteDebugFocusResult = + autocomplete::global_engine().debug_focus().await?; + to_json_value(command_response( + result, + vec!["autocomplete focus debug fetched".to_string()], + )) + } + + "openhuman.autocomplete_accept" => { + let payload: AutocompleteAcceptParams = parse_params(params)?; + let result: AutocompleteAcceptResult = + autocomplete::global_engine().accept(payload).await?; + to_json_value(command_response( + result, + vec!["autocomplete suggestion accepted".to_string()], + )) + } + + "openhuman.autocomplete_set_style" => { + let payload: AutocompleteSetStyleParams = parse_params(params)?; + let result: AutocompleteSetStyleResult = + autocomplete::global_engine().set_style(payload).await?; + to_json_value(command_response( + result, + vec!["autocomplete style settings updated".to_string()], )) } @@ -1342,6 +1401,11 @@ enum CoreCommand { #[command(subcommand)] command: AccessibilityCommand, }, + /// Standalone inline autocomplete commands + Autocomplete { + #[command(subcommand)] + command: AutocompleteCommand, + }, /// Tool wrappers for local CLI testing Tools { #[command(subcommand)] @@ -1497,6 +1561,16 @@ enum AccessibilityCommand { VisionFlush, } +#[derive(Debug, Subcommand)] +enum AutocompleteCommand { + Status, + Start(AutocompleteStartCliArgs), + Stop(AutocompleteStopCliArgs), + Current(AutocompleteCurrentCliArgs), + Accept(AutocompleteAcceptCliArgs), + SetStyle(AutocompleteSetStyleCliArgs), +} + #[derive(Debug, Args)] struct RequestPermissionArgs { /// One of: screen_recording, accessibility, input_monitoring @@ -1535,6 +1609,50 @@ struct VisionRecentCliArgs { limit: Option, } +#[derive(Debug, Args)] +struct AutocompleteStartCliArgs { + #[arg(long)] + debounce_ms: Option, +} + +#[derive(Debug, Args)] +struct AutocompleteStopCliArgs { + #[arg(long)] + reason: Option, +} + +#[derive(Debug, Args)] +struct AutocompleteCurrentCliArgs { + #[arg(long)] + context: Option, +} + +#[derive(Debug, Args)] +struct AutocompleteAcceptCliArgs { + #[arg(long)] + suggestion: Option, +} + +#[derive(Debug, Args)] +struct AutocompleteSetStyleCliArgs { + #[arg(long)] + enabled: Option, + #[arg(long)] + debounce_ms: Option, + #[arg(long)] + max_chars: Option, + #[arg(long)] + style_preset: Option, + #[arg(long)] + style_instructions: Option, + #[arg(long)] + style_example: Vec, + #[arg(long)] + disabled_app: Vec, + #[arg(long)] + accept_with_tab: Option, +} + #[derive(Debug, Args)] struct BrowserSetArgs { #[arg(long)] @@ -2109,6 +2227,57 @@ async fn execute_core_cli(cli: CoreCli) -> Result { call_method("openhuman.accessibility_vision_flush", json!({})).await } }, + CoreCommand::Autocomplete { command } => match command { + AutocompleteCommand::Status => { + call_method("openhuman.autocomplete_status", json!({})).await + } + AutocompleteCommand::Start(args) => { + call_method( + "openhuman.autocomplete_start", + json!({ "debounce_ms": args.debounce_ms }), + ) + .await + } + AutocompleteCommand::Stop(args) => { + call_method( + "openhuman.autocomplete_stop", + json!({ "reason": args.reason }), + ) + .await + } + AutocompleteCommand::Current(args) => { + call_method( + "openhuman.autocomplete_current", + json!({ "context": args.context }), + ) + .await + } + AutocompleteCommand::Accept(args) => { + call_method( + "openhuman.autocomplete_accept", + json!({ "suggestion": args.suggestion }), + ) + .await + } + AutocompleteCommand::SetStyle(args) => { + let style_examples = (!args.style_example.is_empty()).then_some(args.style_example); + let disabled_apps = (!args.disabled_app.is_empty()).then_some(args.disabled_app); + call_method( + "openhuman.autocomplete_set_style", + json!({ + "enabled": args.enabled, + "debounce_ms": args.debounce_ms, + "max_chars": args.max_chars, + "style_preset": args.style_preset, + "style_instructions": args.style_instructions, + "style_examples": style_examples, + "disabled_apps": disabled_apps, + "accept_with_tab": args.accept_with_tab, + }), + ) + .await + } + }, CoreCommand::Tools { command } => match command { ToolsCommand::List => Ok(json!({ "result": { @@ -2268,4 +2437,15 @@ mod tests { assert!(err.contains("invalid params")); } + + #[tokio::test] + async fn autocomplete_status_rpc_returns_valid_schema() { + let raw = call_method("openhuman.autocomplete_status", json!({})) + .await + .expect("autocomplete status rpc should return"); + let payload: CommandResponse = + serde_json::from_value(raw).expect("autocomplete status payload should decode"); + + assert!(!payload.logs.is_empty()); + } } diff --git a/rust-core/src/openhuman/autocomplete/mod.rs b/rust-core/src/openhuman/autocomplete/mod.rs new file mode 100644 index 000000000..70ff232e0 --- /dev/null +++ b/rust-core/src/openhuman/autocomplete/mod.rs @@ -0,0 +1,716 @@ +use crate::openhuman::config::{AutocompleteConfig, Config}; +use crate::openhuman::local_ai; +use chrono::Utc; +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use tokio::time::{self, Duration, Instant}; + +const MAX_SUGGESTION_CHARS: usize = 64; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSuggestion { + pub value: String, + pub confidence: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStatus { + pub platform_supported: bool, + pub enabled: bool, + pub running: bool, + pub debounce_ms: u64, + pub model_id: String, + pub app_name: Option, + pub last_error: Option, + pub updated_at_ms: Option, + pub suggestion: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStartParams { + pub debounce_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStartResult { + pub started: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStopParams { + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteStopResult { + pub stopped: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteCurrentParams { + pub context: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteCurrentResult { + pub app_name: Option, + pub context: String, + pub suggestion: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteDebugFocusResult { + pub app_name: Option, + pub role: Option, + pub context: String, + pub selected_text: Option, + pub raw_error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteAcceptParams { + pub suggestion: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteAcceptResult { + pub accepted: bool, + pub applied: bool, + pub value: Option, + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSetStyleParams { + pub enabled: Option, + pub debounce_ms: Option, + pub max_chars: Option, + pub style_preset: Option, + pub style_instructions: Option, + pub style_examples: Option>, + pub disabled_apps: Option>, + pub accept_with_tab: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutocompleteSetStyleResult { + pub config: AutocompleteConfig, +} + +#[derive(Debug, Clone)] +struct FocusedTextContext { + app_name: Option, + role: Option, + text: String, + selected_text: Option, + raw_error: Option, +} + +fn is_text_role(role: Option<&str>) -> bool { + matches!( + role.unwrap_or_default(), + "AXTextArea" | "AXTextField" | "AXSearchField" | "AXComboBox" | "AXEditableText" + ) +} + +struct EngineState { + running: bool, + debounce_ms: u64, + app_name: Option, + context: String, + suggestion: Option, + last_error: Option, + updated_at_ms: Option, + last_tab_down: bool, + task: Option>, +} + +impl Default for EngineState { + fn default() -> Self { + Self { + running: false, + debounce_ms: 120, + app_name: None, + context: String::new(), + suggestion: None, + last_error: None, + updated_at_ms: None, + last_tab_down: false, + task: None, + } + } +} + +pub struct AutocompleteEngine { + inner: Mutex, +} + +impl AutocompleteEngine { + pub fn new() -> Self { + Self { + inner: Mutex::new(EngineState::default()), + } + } + + pub async fn status(&self) -> AutocompleteStatus { + let config = Config::load_or_init() + .await + .unwrap_or_else(|_| Config::default()); + let state = self.inner.lock().await; + + AutocompleteStatus { + platform_supported: cfg!(target_os = "macos"), + enabled: config.autocomplete.enabled, + running: state.running, + debounce_ms: state.debounce_ms, + model_id: config.local_ai.chat_model_id, + app_name: state.app_name.clone(), + last_error: state.last_error.clone(), + updated_at_ms: state.updated_at_ms, + suggestion: state.suggestion.clone(), + } + } + + pub async fn start( + &self, + params: AutocompleteStartParams, + ) -> Result { + if !cfg!(target_os = "macos") { + return Err("autocomplete is only supported on macOS".to_string()); + } + + let config = Config::load_or_init() + .await + .map_err(|e| format!("failed to load config: {e}"))?; + if !config.autocomplete.enabled { + return Ok(AutocompleteStartResult { started: false }); + } + + let debounce_ms = params + .debounce_ms + .unwrap_or(config.autocomplete.debounce_ms) + .clamp(50, 2000); + + let mut state = self.inner.lock().await; + if state.running { + return Ok(AutocompleteStartResult { started: false }); + } + state.running = true; + state.debounce_ms = debounce_ms; + state.last_error = None; + + let engine = global_engine(); + state.task = Some(tokio::spawn(async move { + let mut last_refresh = Instant::now() - Duration::from_millis(debounce_ms); + loop { + { + let state = engine.inner.lock().await; + if !state.running { + break; + } + } + let _ = engine.try_accept_via_tab().await; + if last_refresh.elapsed() >= Duration::from_millis(debounce_ms) { + if let Err(err) = engine.refresh(None).await { + let mut state = engine.inner.lock().await; + state.last_error = Some(err); + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + } else { + let mut state = engine.inner.lock().await; + state.last_error = None; + } + last_refresh = Instant::now(); + } + time::sleep(Duration::from_millis(24)).await; + } + })); + + Ok(AutocompleteStartResult { started: true }) + } + + pub async fn stop(&self, _params: Option) -> AutocompleteStopResult { + let mut state = self.inner.lock().await; + state.running = false; + if let Some(task) = state.task.take() { + task.abort(); + } + AutocompleteStopResult { stopped: true } + } + + pub async fn current( + &self, + params: Option, + ) -> Result { + let context_override = params + .and_then(|p| p.context) + .filter(|c| !c.trim().is_empty()); + self.refresh(context_override).await?; + let state = self.inner.lock().await; + Ok(AutocompleteCurrentResult { + app_name: state.app_name.clone(), + context: state.context.clone(), + suggestion: state.suggestion.clone(), + }) + } + + pub async fn debug_focus(&self) -> Result { + let focused = focused_text_context_verbose()?; + Ok(AutocompleteDebugFocusResult { + app_name: focused.app_name, + role: focused.role, + context: focused.text, + selected_text: focused.selected_text, + raw_error: focused.raw_error, + }) + } + + pub async fn accept( + &self, + params: AutocompleteAcceptParams, + ) -> Result { + let value = if let Some(value) = params.suggestion { + value + } else { + let state = self.inner.lock().await; + state + .suggestion + .as_ref() + .map(|s| s.value.clone()) + .unwrap_or_default() + }; + + let cleaned = sanitize_suggestion(&value); + if cleaned.is_empty() { + return Ok(AutocompleteAcceptResult { + accepted: false, + applied: false, + value: None, + reason: Some("no suggestion available".to_string()), + }); + } + + apply_text_to_focused_field(&cleaned)?; + let mut state = self.inner.lock().await; + state.suggestion = None; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + + Ok(AutocompleteAcceptResult { + accepted: true, + applied: true, + value: Some(cleaned), + reason: None, + }) + } + + pub async fn set_style( + &self, + params: AutocompleteSetStyleParams, + ) -> Result { + let mut config = Config::load_or_init() + .await + .map_err(|e| format!("failed to load config: {e}"))?; + if let Some(enabled) = params.enabled { + config.autocomplete.enabled = enabled; + } + if let Some(debounce_ms) = params.debounce_ms { + config.autocomplete.debounce_ms = debounce_ms.clamp(50, 2000); + } + if let Some(max_chars) = params.max_chars { + config.autocomplete.max_chars = max_chars.clamp(64, 2048); + } + if let Some(style_preset) = params.style_preset { + config.autocomplete.style_preset = style_preset.trim().to_string(); + } + if let Some(style_instructions) = params.style_instructions { + config.autocomplete.style_instructions = if style_instructions.trim().is_empty() { + None + } else { + Some(style_instructions.trim().to_string()) + }; + } + if let Some(style_examples) = params.style_examples { + config.autocomplete.style_examples = style_examples + .into_iter() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .take(8) + .collect(); + } + if let Some(disabled_apps) = params.disabled_apps { + config.autocomplete.disabled_apps = disabled_apps + .into_iter() + .map(|s| s.trim().to_lowercase()) + .filter(|s| !s.is_empty()) + .collect(); + } + if let Some(accept_with_tab) = params.accept_with_tab { + config.autocomplete.accept_with_tab = accept_with_tab; + } + config.save().await.map_err(|e| e.to_string())?; + + let mut state = self.inner.lock().await; + state.debounce_ms = config.autocomplete.debounce_ms; + state.last_tab_down = false; + if !config.autocomplete.enabled { + state.running = false; + if let Some(task) = state.task.take() { + task.abort(); + } + state.suggestion = None; + } + + Ok(AutocompleteSetStyleResult { + config: config.autocomplete, + }) + } + + async fn refresh(&self, context_override: Option) -> Result<(), String> { + let config = Config::load_or_init() + .await + .map_err(|e| format!("failed to load config: {e}"))?; + if !config.autocomplete.enabled { + let mut state = self.inner.lock().await; + state.suggestion = None; + return Ok(()); + } + + let focused = if let Some(context) = context_override { + FocusedTextContext { + app_name: None, + role: None, + text: context, + selected_text: None, + raw_error: None, + } + } else { + focused_text_context()? + }; + + let app_lower = focused.app_name.clone().unwrap_or_default().to_lowercase(); + if config + .autocomplete + .disabled_apps + .iter() + .any(|needle| !needle.trim().is_empty() && app_lower.contains(needle)) + { + let mut state = self.inner.lock().await; + state.app_name = focused.app_name; + state.context = truncate_tail(&focused.text, config.autocomplete.max_chars); + state.suggestion = None; + state.last_error = None; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + return Ok(()); + } + + let context = truncate_tail(&focused.text, config.autocomplete.max_chars); + if context.trim().is_empty() { + let mut state = self.inner.lock().await; + state.app_name = focused.app_name; + state.context = context; + state.suggestion = None; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + return Ok(()); + } + + let service = local_ai::global(&config); + let generated = service + .inline_complete( + &config, + &context, + &config.autocomplete.style_preset, + config.autocomplete.style_instructions.as_deref(), + &config.autocomplete.style_examples, + Some(36), + ) + .await + .unwrap_or_default(); + + let suggestion = sanitize_suggestion(&generated); + let mut state = self.inner.lock().await; + state.app_name = focused.app_name; + state.context = context; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + if suggestion.is_empty() { + state.suggestion = None; + state.last_error = None; + return Ok(()); + } + state.suggestion = Some(AutocompleteSuggestion { + value: suggestion, + confidence: 0.72, + }); + state.last_error = None; + Ok(()) + } + + async fn try_accept_via_tab(&self) -> Result<(), String> { + let accept_with_tab = Config::load_or_init() + .await + .map(|cfg| cfg.autocomplete.accept_with_tab) + .unwrap_or(true); + if !accept_with_tab { + let mut state = self.inner.lock().await; + state.last_tab_down = false; + return Ok(()); + } + + let is_down = is_tab_key_down(); + let pending = { + let mut state = self.inner.lock().await; + let edge = is_down && !state.last_tab_down; + state.last_tab_down = is_down; + if !edge { + None + } else { + state.suggestion.as_ref().map(|s| s.value.clone()) + } + }; + + if let Some(suggestion) = pending { + let cleaned = sanitize_suggestion(&suggestion); + if !cleaned.is_empty() { + apply_text_to_focused_field(&cleaned)?; + let mut state = self.inner.lock().await; + state.suggestion = None; + state.updated_at_ms = Some(Utc::now().timestamp_millis()); + } + } + + Ok(()) + } +} + +pub static AUTOCOMPLETE_ENGINE: Lazy> = + Lazy::new(|| Arc::new(AutocompleteEngine::new())); + +pub fn global_engine() -> Arc { + AUTOCOMPLETE_ENGINE.clone() +} + +fn truncate_tail(text: &str, max_chars: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= max_chars { + return text.to_string(); + } + chars[chars.len() - max_chars..].iter().collect() +} + +fn sanitize_suggestion(text: &str) -> String { + let first_line = text.lines().next().unwrap_or_default().trim(); + let cleaned = first_line + .trim_matches('"') + .replace('\t', " ") + .replace('\r', "") + .trim() + .to_string(); + if cleaned.is_empty() { + return String::new(); + } + truncate_tail(&cleaned, MAX_SUGGESTION_CHARS) +} + +#[cfg(target_os = "macos")] +fn focused_text_context() -> Result { + let ctx = focused_text_context_verbose()?; + if let Some(err) = ctx.raw_error.as_ref() { + return Err(format!( + "focused text unavailable via accessibility api: {err}" + )); + } + Ok(ctx) +} + +#[cfg(target_os = "macos")] +fn focused_text_context_verbose() -> Result { + let script = r#" + tell application "System Events" + set frontApp to first application process whose frontmost is true + set appName to name of frontApp + set roleValue to "unknown" + set textValue to "" + set selectedValue to "" + set errValue to "" + set targetRoles to {"AXTextArea", "AXTextField", "AXSearchField", "AXComboBox", "AXEditableText"} + + try + set focusedElement to value of attribute "AXFocusedUIElement" of frontApp + try + set roleValue to value of attribute "AXRole" of focusedElement as text + end try + try + set textValue to value of attribute "AXValue" of focusedElement as text + end try + if textValue is "missing value" then set textValue to "" + if textValue is "" then + try + set selectedValue to value of attribute "AXSelectedText" of focusedElement as text + end try + if selectedValue is "missing value" then set selectedValue to "" + if selectedValue is not "" then set textValue to selectedValue + end if + if textValue is "" then + try + set textValue to value of attribute "AXTitle" of focusedElement as text + end try + if textValue is "missing value" then set textValue to "" + end if + on error errMsg number errNum + set errValue to "ERROR:" & errNum & ":" & errMsg + end try + + if textValue is "" then + try + set focusedWindow to value of attribute "AXFocusedWindow" of frontApp + set childElems to value of attribute "AXChildren" of focusedWindow + repeat with childElem in childElems + set childRole to "" + set childValue to "" + try + set childRole to value of attribute "AXRole" of childElem as text + end try + if childRole is in targetRoles then + try + set childValue to value of attribute "AXValue" of childElem as text + end try + if childValue is "missing value" then set childValue to "" + if childValue is not "" then + set roleValue to childRole + set textValue to childValue + exit repeat + end if + end if + end repeat + on error errMsg2 number errNum2 + if errValue is "" then set errValue to "ERROR:" & errNum2 & ":" & errMsg2 + end try + end if + + if textValue is "" and errValue is "" then + set errValue to "ERROR:no_text_candidate_found" + end if + + return appName & "\n" & roleValue & "\n" & textValue & "\n" & selectedValue & "\n" & errValue + end tell + "#; + + let output = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .map_err(|e| format!("failed to run osascript: {e}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.is_empty() { + return Err("unable to query focused text context".to_string()); + } + return Err(format!("unable to query focused text context: {stderr}")); + } + + let text = String::from_utf8_lossy(&output.stdout); + let mut lines = text.lines(); + let app_name = lines + .next() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + let role = lines + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + let mut value = lines + .next() + .map(|s| normalize_ax_value(s.trim())) + .unwrap_or_default(); + let mut selected_text = lines + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + let mut raw_error = lines + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + + if !is_text_role(role.as_deref()) { + value.clear(); + selected_text = None; + if raw_error.is_none() { + raw_error = Some("ERROR:no_text_candidate_found".to_string()); + } + } + + Ok(FocusedTextContext { + app_name, + role, + text: value, + selected_text, + raw_error, + }) +} + +#[cfg(not(target_os = "macos"))] +fn focused_text_context() -> Result { + Err("autocomplete is only supported on macOS".to_string()) +} + +#[cfg(not(target_os = "macos"))] +fn focused_text_context_verbose() -> Result { + Err("autocomplete is only supported on macOS".to_string()) +} + +fn normalize_ax_value(raw: &str) -> String { + let v = raw.trim(); + if v.eq_ignore_ascii_case("missing value") { + String::new() + } else { + v.to_string() + } +} + +#[cfg(target_os = "macos")] +fn apply_text_to_focused_field(text: &str) -> Result<(), String> { + let escaped = text + .replace('\\', "\\\\") + .replace('\"', "\\\"") + .replace('\n', " "); + let script = format!( + r#"tell application "System Events" to keystroke "{}""#, + escaped + ); + let output = std::process::Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .map_err(|e| format!("failed to run osascript: {e}"))?; + if !output.status.success() { + return Err("failed to apply suggestion to focused text field".to_string()); + } + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +fn apply_text_to_focused_field(_text: &str) -> Result<(), String> { + Err("autocomplete is only supported on macOS".to_string()) +} + +#[cfg(target_os = "macos")] +fn is_tab_key_down() -> bool { + unsafe { CGEventSourceKeyState(KCG_EVENT_SOURCE_STATE_COMBINED_SESSION_STATE, KVK_TAB) } +} + +#[cfg(not(target_os = "macos"))] +fn is_tab_key_down() -> bool { + false +} + +#[cfg(target_os = "macos")] +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + fn CGEventSourceKeyState(state_id: i32, key: u16) -> bool; +} + +#[cfg(target_os = "macos")] +const KCG_EVENT_SOURCE_STATE_COMBINED_SESSION_STATE: i32 = 0; +#[cfg(target_os = "macos")] +const KVK_TAB: u16 = 48; diff --git a/rust-core/src/openhuman/config/mod.rs b/rust-core/src/openhuman/config/mod.rs index 0748f54a9..516db73d1 100644 --- a/rust-core/src/openhuman/config/mod.rs +++ b/rust-core/src/openhuman/config/mod.rs @@ -8,17 +8,18 @@ pub use daemon::DaemonConfig; pub use schema::{ apply_runtime_proxy_to_builder, build_runtime_proxy_client, build_runtime_proxy_client_with_timeouts, runtime_proxy_config, set_runtime_proxy_config, - AgentConfig, AuditConfig, AutonomyConfig, BrowserComputerUseConfig, BrowserConfig, - ChannelsConfig, ClassificationRule, CloudflareTunnelConfig, ComposioConfig, Config, CostConfig, - CronConfig, CustomTunnelConfig, DelegateAgentConfig, DiscordConfig, DockerRuntimeConfig, - EmbeddingRouteConfig, GatewayConfig, HardwareConfig, HardwareTransport, HeartbeatConfig, - HttpRequestConfig, IMessageConfig, IdentityConfig, LarkConfig, LocalAiConfig, MatrixConfig, - MemoryConfig, ModelRouteConfig, MultimodalConfig, NgrokTunnelConfig, ObservabilityConfig, - PeripheralBoardConfig, PeripheralsConfig, ProxyConfig, ProxyScope, QueryClassificationConfig, - ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, - SchedulerConfig, ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, SlackConfig, - StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode, - TailscaleTunnelConfig, TelegramConfig, TunnelConfig, WebSearchConfig, WebhookConfig, + AgentConfig, AuditConfig, AutocompleteConfig, AutonomyConfig, BrowserComputerUseConfig, + BrowserConfig, ChannelsConfig, ClassificationRule, CloudflareTunnelConfig, ComposioConfig, + Config, CostConfig, CronConfig, CustomTunnelConfig, DelegateAgentConfig, DiscordConfig, + DockerRuntimeConfig, EmbeddingRouteConfig, GatewayConfig, HardwareConfig, HardwareTransport, + HeartbeatConfig, HttpRequestConfig, IMessageConfig, IdentityConfig, LarkConfig, LocalAiConfig, + MatrixConfig, MemoryConfig, ModelRouteConfig, MultimodalConfig, NgrokTunnelConfig, + ObservabilityConfig, PeripheralBoardConfig, PeripheralsConfig, ProxyConfig, ProxyScope, + QueryClassificationConfig, ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, + SandboxBackend, SandboxConfig, SchedulerConfig, ScreenIntelligenceConfig, SecretsConfig, + SecurityConfig, SlackConfig, StorageConfig, StorageProviderConfig, StorageProviderSection, + StreamMode, TailscaleTunnelConfig, TelegramConfig, TunnelConfig, WebSearchConfig, + WebhookConfig, }; #[cfg(test)] diff --git a/rust-core/src/openhuman/config/schema/autocomplete.rs b/rust-core/src/openhuman/config/schema/autocomplete.rs new file mode 100644 index 000000000..97b2f0797 --- /dev/null +++ b/rust-core/src/openhuman/config/schema/autocomplete.rs @@ -0,0 +1,57 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct AutocompleteConfig { + #[serde(default = "default_enabled")] + pub enabled: bool, + #[serde(default = "default_debounce_ms")] + pub debounce_ms: u64, + #[serde(default = "default_max_chars")] + pub max_chars: usize, + #[serde(default = "default_style_preset")] + pub style_preset: String, + #[serde(default)] + pub style_instructions: Option, + #[serde(default)] + pub style_examples: Vec, + #[serde(default)] + pub disabled_apps: Vec, + #[serde(default = "default_accept_with_tab")] + pub accept_with_tab: bool, +} + +fn default_enabled() -> bool { + true +} + +fn default_debounce_ms() -> u64 { + 120 +} + +fn default_max_chars() -> usize { + 384 +} + +fn default_style_preset() -> String { + "balanced".to_string() +} + +fn default_accept_with_tab() -> bool { + true +} + +impl Default for AutocompleteConfig { + fn default() -> Self { + Self { + enabled: default_enabled(), + debounce_ms: default_debounce_ms(), + max_chars: default_max_chars(), + style_preset: default_style_preset(), + style_instructions: None, + style_examples: Vec::new(), + disabled_apps: vec!["terminal".to_string(), "code".to_string()], + accept_with_tab: default_accept_with_tab(), + } + } +} diff --git a/rust-core/src/openhuman/config/schema/mod.rs b/rust-core/src/openhuman/config/schema/mod.rs index 95ba61cef..dd10bb760 100644 --- a/rust-core/src/openhuman/config/schema/mod.rs +++ b/rust-core/src/openhuman/config/schema/mod.rs @@ -4,6 +4,7 @@ mod accessibility; mod agent; +mod autocomplete; mod autonomy; mod channels; mod defaults; @@ -23,6 +24,7 @@ mod tunnel; pub use accessibility::ScreenIntelligenceConfig; pub use agent::{AgentConfig, DelegateAgentConfig}; +pub use autocomplete::AutocompleteConfig; pub use autonomy::AutonomyConfig; pub use channels::{ AuditConfig, ChannelsConfig, DingTalkConfig, DiscordConfig, IMessageConfig, IrcConfig, @@ -90,6 +92,9 @@ pub struct Config { #[serde(default)] pub screen_intelligence: ScreenIntelligenceConfig, + #[serde(default)] + pub autocomplete: AutocompleteConfig, + #[serde(default)] pub reliability: ReliabilityConfig, @@ -187,6 +192,7 @@ impl Default for Config { autonomy: AutonomyConfig::default(), runtime: RuntimeConfig::default(), screen_intelligence: ScreenIntelligenceConfig::default(), + autocomplete: AutocompleteConfig::default(), reliability: ReliabilityConfig::default(), scheduler: SchedulerConfig::default(), agent: AgentConfig::default(), diff --git a/rust-core/src/openhuman/local_ai/mod.rs b/rust-core/src/openhuman/local_ai/mod.rs index a67992e8f..97e91c368 100644 --- a/rust-core/src/openhuman/local_ai/mod.rs +++ b/rust-core/src/openhuman/local_ai/mod.rs @@ -412,6 +412,57 @@ impl LocalAiService { )) } + pub async fn inline_complete( + &self, + config: &Config, + context: &str, + style_preset: &str, + style_instructions: Option<&str>, + style_examples: &[String], + max_tokens: Option, + ) -> Result { + if !config.local_ai.enabled { + return Ok(String::new()); + } + + let mut prompt = String::from( + "Complete the user's text with the most likely next words.\n\ + Return only the continuation suffix, no explanations.\n\ + Do not repeat text already written by the user.\n\ + Keep it natural and concise.\n\n", + ); + prompt.push_str(&format!("Style preset: {}\n", style_preset.trim())); + if let Some(instructions) = style_instructions { + if !instructions.trim().is_empty() { + prompt.push_str(&format!("Style instructions: {}\n", instructions.trim())); + } + } + if !style_examples.is_empty() { + prompt.push_str("Style examples:\n"); + for example in style_examples.iter().take(8) { + let trimmed = example.trim(); + if !trimmed.is_empty() { + prompt.push_str("- "); + prompt.push_str(trimmed); + prompt.push('\n'); + } + } + } + prompt.push_str("\nUser text:\n"); + prompt.push_str(context.trim()); + + let raw = self + .inference( + config, + "You are a low-latency inline text completion assistant.", + &prompt, + max_tokens.or(Some(36)), + true, + ) + .await?; + Ok(sanitize_inline_completion(&raw)) + } + pub async fn vision_prompt( &self, config: &Config, @@ -1664,3 +1715,26 @@ fn parse_suggestions(raw: &str, limit: usize) -> Vec { }) .collect() } + +fn sanitize_inline_completion(raw: &str) -> String { + let line = raw.lines().next().unwrap_or_default().trim(); + if line.is_empty() { + return String::new(); + } + + let mut cleaned = line + .trim_matches('"') + .trim_start_matches(|c: char| matches!(c, '-' | '*' | '>' | '1'..='9' | '.' | ')')) + .trim() + .replace('\t', " "); + + if cleaned.eq_ignore_ascii_case("none") || cleaned.eq_ignore_ascii_case("n/a") { + return String::new(); + } + + if cleaned.chars().count() > 128 { + cleaned = cleaned.chars().take(128).collect(); + } + + cleaned +} diff --git a/rust-core/src/openhuman/mod.rs b/rust-core/src/openhuman/mod.rs index d9562325a..e88bdeeb3 100644 --- a/rust-core/src/openhuman/mod.rs +++ b/rust-core/src/openhuman/mod.rs @@ -14,6 +14,7 @@ pub mod accessibility; pub mod agent; pub mod approval; +pub mod autocomplete; pub mod channels; pub mod config; pub mod cost;