From e3ebaca8fd6c688d0c2444998de2d170f3fe6e44 Mon Sep 17 00:00:00 2001 From: Mega Mind <146339422+M3gA-Mind@users.noreply.github.com> Date: Thu, 4 Jun 2026 16:50:11 +0530 Subject: [PATCH] feat(computer): main-thread synthetic-input executor + CEF crash fix (1/7 of #3307) (#3340) --- app/src-tauri/src/lib.rs | 33 +++ .../developing/architecture/tauri-shell.md | 22 ++ .../tools/impl/browser/screenshot.rs | 223 +++++++++++++----- src/openhuman/tools/impl/computer/keyboard.rs | 146 ++++++------ .../tools/impl/computer/main_thread.rs | 74 ++++++ src/openhuman/tools/impl/computer/mod.rs | 2 + src/openhuman/tools/impl/computer/mouse.rs | 217 +++++++++-------- 7 files changed, 473 insertions(+), 244 deletions(-) create mode 100644 src/openhuman/tools/impl/computer/main_thread.rs diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 41c01934a..7df029b44 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -2800,6 +2800,39 @@ pub fn run() { // let _ = window.show(); // } + // Synthetic-input main-thread executor. enigo's macOS keyboard-layout + // lookup (TSMGetInputSourceProperty) MUST run on the app main thread + // or it traps (`_dispatch_assert_queue_fail`/EXC_BREAKPOINT) and + // crashes the CEF host (Change 1.15, confirmed via crash report). The + // keyboard/mouse tools run on tokio workers, so they dispatch their + // enigo ops here via the native registry; we run each on the real + // main thread through `run_on_main_thread`. + { + use openhuman_core::core::event_bus::register_native_global; + use openhuman_core::openhuman::tools::{ + MainThreadInputOp, INPUT_ON_MAIN_THREAD_METHOD, + }; + let input_app = app.handle().clone(); + register_native_global::, _, _>( + INPUT_ON_MAIN_THREAD_METHOD, + move |req| { + let input_app = input_app.clone(); + async move { + let (tx, rx) = tokio::sync::oneshot::channel(); + let run = req.run; + input_app + .run_on_main_thread(move || { + let _ = tx.send((run)()); + }) + .map_err(|e| format!("run_on_main_thread dispatch failed: {e}"))?; + rx.await + .map_err(|_| "main-thread input op was cancelled".to_string()) + } + }, + ); + log::info!("[computer] registered main-thread synthetic-input executor"); + } + // Tray icon setup moved to RunEvent::Ready (see below) — GTK is only // initialized after the event loop starts, so we must delay tray creation // until the Ready event fires. Creating the tray here would panic on diff --git a/gitbooks/developing/architecture/tauri-shell.md b/gitbooks/developing/architecture/tauri-shell.md index 5929c79ff..b4f3d0a75 100644 --- a/gitbooks/developing/architecture/tauri-shell.md +++ b/gitbooks/developing/architecture/tauri-shell.md @@ -158,6 +158,28 @@ From **`workspace_paths.rs`** (closes `#1402`). These commands accept workspace- | `reveal_workspace_path` | Reveal an existing workspace file or directory in the OS file manager. | | `preview_workspace_text` | Read a capped UTF-8 text preview from an existing workspace file. | +### Synthetic input main-thread executor (native registry, not `invoke`) + +Registered in **`lib.rs`** at startup under the event-bus native-request method +`computer.input_on_main_thread` (`INPUT_ON_MAIN_THREAD_METHOD`, defined in +`openhuman_core::openhuman::tools::computer::main_thread`). This is **not** a +`@tauri-apps/api` `invoke` command — it is an in-process native request the +**core** dispatches to the **shell** so synthetic input runs on the real app +main thread. + +Why: enigo's macOS keyboard-layout lookup (`TSMGetInputSourceProperty`) traps +(`_dispatch_assert_queue_fail` / `EXC_BREAKPOINT`) and crashes the CEF host when +called off the main thread. The `mouse` / `keyboard` tools therefore never call +enigo on their tokio worker; they build a closure and dispatch it here, where +the shell runs it via `AppHandle::run_on_main_thread`. + +| Field | Shape | +| ------------ | -------------------------------------------------------------------------------------------------- | +| Method | `computer.input_on_main_thread` | +| Request | `MainThreadInputOp { run: Box Result + Send> }` (passed by value) | +| Response | `Result` — `Ok(message)` on success, `Err(reason)` on failure | +| Availability | Desktop only. Headless / CLI builds register no executor; the core call then returns a clean `Err`. | + ### Removed / not present The following **do not** exist in the current `generate_handler!` list: `exchange_token`, `get_auth_state`, `socket_connect`, `start_telegram_login`. Authentication and sockets are handled in the **React** app and **core** process, not via these IPC names. diff --git a/src/openhuman/tools/impl/browser/screenshot.rs b/src/openhuman/tools/impl/browser/screenshot.rs index 1a247830e..4a723ac42 100644 --- a/src/openhuman/tools/impl/browser/screenshot.rs +++ b/src/openhuman/tools/impl/browser/screenshot.rs @@ -9,8 +9,6 @@ use std::time::Duration; /// Maximum time to wait for a screenshot command to complete. const SCREENSHOT_TIMEOUT_SECS: u64 = 15; -/// Maximum base64 payload size to return (2 MB of base64 ≈ 1.5 MB image). -const MAX_BASE64_BYTES: usize = 2_097_152; /// Tool for capturing screenshots using platform-native commands. /// @@ -132,59 +130,102 @@ impl ScreenshotTool { } } - /// Read the screenshot file and return base64-encoded result. + /// Read the screenshot file and return a base64 data-URL the model can see. + /// + /// Full-screen Retina captures are multi-MB PNGs that blow the inline + /// budget. Rather than dropping the image (which leaves vision-driven + /// control blind), downscale oversized captures to a JPEG that fits — the + /// model can then actually see the screen. Reports the *shown* dimensions so + /// callers know the coordinate space they're reading. async fn read_and_encode(output_path: &std::path::Path) -> anyhow::Result { - // Check file size before reading to prevent OOM on large screenshots - const MAX_RAW_BYTES: u64 = 1_572_864; // ~1.5 MB (base64 expands ~33%) - if let Ok(meta) = tokio::fs::metadata(output_path).await { - if meta.len() > MAX_RAW_BYTES { - return Ok(ToolResult::success(format!( - "Screenshot saved to: {}\nSize: {} bytes (too large to base64-encode inline)", - output_path.display(), - meta.len(), - ))); + // ~1.5 MB raw → ~2 MB base64, a safe inline payload size. + const MAX_RAW_BYTES: usize = 1_572_864; + + let bytes = match tokio::fs::read(output_path).await { + Ok(b) => b, + Err(e) => { + return Ok(ToolResult::error(format!( + "Failed to read screenshot file: {e}" + ))) } + }; + let ext = output_path + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("png") + .to_lowercase(); + + // Fits as-is → return verbatim. + if bytes.len() <= MAX_RAW_BYTES { + let mime = match ext.as_str() { + "jpg" | "jpeg" => "image/jpeg", + "bmp" => "image/bmp", + "gif" => "image/gif", + "webp" => "image/webp", + _ => "image/png", + }; + return Ok(Self::data_url_result(output_path, &bytes, mime, None)); } - match tokio::fs::read(output_path).await { - Ok(bytes) => { - use base64::Engine; - let size = bytes.len(); - let mut encoded = base64::engine::general_purpose::STANDARD.encode(&bytes); - let truncated = if encoded.len() > MAX_BASE64_BYTES { - encoded.truncate(crate::openhuman::util::floor_char_boundary( - &encoded, - MAX_BASE64_BYTES, - )); - true - } else { - false - }; - - let mut output_msg = format!( - "Screenshot saved to: {}\nSize: {size} bytes\nBase64 length: {}", - output_path.display(), - encoded.len(), - ); - if truncated { - output_msg.push_str(" (truncated)"); - } - let mime = match output_path.extension().and_then(|e| e.to_str()) { - Some("jpg" | "jpeg") => "image/jpeg", - Some("bmp") => "image/bmp", - Some("gif") => "image/gif", - Some("webp") => "image/webp", - _ => "image/png", - }; - let _ = write!(output_msg, "\ndata:{mime};base64,{encoded}"); - - Ok(ToolResult::success(output_msg)) - } - Err(e) => Ok(ToolResult::error(format!( - "Failed to read screenshot file: {e}" + // Too large → downscale to a JPEG that fits (CPU work off the runtime). + match tokio::task::spawn_blocking(move || downscale_to_jpeg(&bytes, MAX_RAW_BYTES)).await { + Ok(Ok((jpeg, w, h))) => Ok(Self::data_url_result( + output_path, + &jpeg, + "image/jpeg", + Some((w, h)), + )), + Ok(Err(e)) => Ok(ToolResult::success(format!( + "Screenshot saved to: {} (could not downscale for inline view: {e})", + output_path.display() ))), + Err(e) => Ok(ToolResult::error(format!("downscale task failed: {e}"))), } } + + /// Build a success result carrying a base64 data-URL of `data`. + fn data_url_result( + output_path: &std::path::Path, + data: &[u8], + mime: &str, + shown_dims: Option<(u32, u32)>, + ) -> ToolResult { + use base64::Engine; + let encoded = base64::engine::general_purpose::STANDARD.encode(data); + let mut msg = format!("Screenshot saved to: {}\n", output_path.display()); + if let Some((w, h)) = shown_dims { + let _ = write!( + msg, + "Downscaled to {w}x{h}px for inline view (coordinates you read are in this {w}x{h} space).\n" + ); + } + let _ = write!(msg, "data:{mime};base64,{encoded}"); + ToolResult::success(msg) + } +} + +/// Decode image bytes, downscale (preserving aspect ratio), and JPEG-encode so +/// the result is ≤ `max_bytes`. Returns `(jpeg_bytes, width, height)`. +fn downscale_to_jpeg(bytes: &[u8], max_bytes: usize) -> Result<(Vec, u32, u32), String> { + let img = image::load_from_memory(bytes).map_err(|e| format!("decode: {e}"))?; + let mut last: Option<(Vec, u32, u32)> = None; + for max_dim in [1568u32, 1280, 1024, 768, 600] { + // Drop alpha before JPEG-encoding: JPEG has no alpha channel, so an + // RGBA capture (PNG screenshots often carry one) would otherwise fail + // to encode and leave vision-driven control blind. + let thumb = img.thumbnail(max_dim, max_dim).to_rgb8(); // fits within max_dim², keeps aspect + let (w, h) = (thumb.width(), thumb.height()); + let mut buf = std::io::Cursor::new(Vec::new()); + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 72) + .encode_image(&thumb) + .map_err(|e| format!("jpeg encode: {e}"))?; + let out = buf.into_inner(); + if out.len() <= max_bytes { + return Ok((out, w, h)); + } + last = Some((out, w, h)); + } + last.ok_or_else(|| "could not produce a fitting JPEG".to_string()) } #[async_trait] @@ -228,6 +269,56 @@ mod tests { use super::*; use crate::openhuman::security::{AutonomyLevel, SecurityPolicy}; + #[test] + fn downscale_to_jpeg_shrinks_oversized_capture() { + // A 1600x1200 PNG of noise is well over a tight budget; downscaling must + // produce a smaller JPEG that still decodes, so the model can see it. + let mut img = image::RgbImage::new(1600, 1200); + for (i, px) in img.pixels_mut().enumerate() { + *px = image::Rgb([(i % 251) as u8, (i % 253) as u8, (i % 247) as u8]); + } + let mut png = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgb8(img) + .write_to(&mut png, image::ImageFormat::Png) + .expect("encode png"); + let png = png.into_inner(); + + let max = 400_000usize; + let (jpeg, w, h) = downscale_to_jpeg(&png, max).expect("downscale"); + assert!(jpeg.len() <= max, "jpeg {} should be <= {max}", jpeg.len()); + assert!( + w <= 1568 && h <= 1568, + "dims {w}x{h} should be capped to 1568" + ); + assert!( + jpeg.len() < png.len(), + "jpeg should be smaller than source png" + ); + // Result must be a valid, decodable image at the reported dims. + let decoded = image::load_from_memory(&jpeg).expect("jpeg decodes"); + assert_eq!((decoded.width(), decoded.height()), (w, h)); + } + + #[test] + fn downscale_to_jpeg_handles_rgba_input() { + // PNG screenshots frequently carry an alpha channel. JPEG has none, so + // the encoder must run on RGB — otherwise an RGBA capture fails to + // encode and leaves vision-driven control blind. + let mut img = image::RgbaImage::new(1600, 1200); + for (i, px) in img.pixels_mut().enumerate() { + *px = image::Rgba([(i % 251) as u8, (i % 253) as u8, (i % 247) as u8, 128]); + } + let mut png = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(img) + .write_to(&mut png, image::ImageFormat::Png) + .expect("encode png"); + let png = png.into_inner(); + + let (jpeg, w, h) = downscale_to_jpeg(&png, 400_000).expect("rgba downscales"); + let decoded = image::load_from_memory(&jpeg).expect("jpeg decodes"); + assert_eq!((decoded.width(), decoded.height()), (w, h)); + } + fn test_security() -> Arc { Arc::new(SecurityPolicy { autonomy: AutonomyLevel::Full, @@ -439,24 +530,38 @@ mod tests { // ── read_and_encode: large file returns saved-path-only message ─────────── #[tokio::test] - async fn read_and_encode_large_file_skips_base64() { - use tokio::io::AsyncWriteExt; + async fn read_and_encode_large_file_downscales_to_viewable_jpeg() { + // A large *real* PNG (over MAX_RAW_BYTES) must be downscaled to an inline + // JPEG data-URL the model can see — not dropped (the old behavior left + // vision-driven control blind). let dir = tempfile::TempDir::new().unwrap(); let path = dir.path().join("big.png"); - let mut f = tokio::fs::File::create(&path).await.unwrap(); - // Write ~1.6 MB to exceed the MAX_RAW_BYTES threshold (1.5 MB) - let chunk = vec![0u8; 1024]; - for _ in 0..1600 { - f.write_all(&chunk).await.unwrap(); + let mut img = image::RgbImage::new(2200, 1500); + for (i, px) in img.pixels_mut().enumerate() { + *px = image::Rgb([(i % 251) as u8, (i % 253) as u8, (i % 247) as u8]); } - drop(f); + image::DynamicImage::ImageRgb8(img) + .save_with_format(&path, image::ImageFormat::Png) + .unwrap(); + assert!( + tokio::fs::metadata(&path).await.unwrap().len() > 1_572_864, + "test PNG should exceed the inline budget" + ); let result = ScreenshotTool::read_and_encode(&path).await.unwrap(); - assert!(!result.is_error, "large file should not be an error result"); assert!( - result.output().contains("too large to base64-encode"), - "large file should skip base64, got: {}", + !result.is_error, + "should not error, got: {}", result.output() ); + let out = result.output(); + assert!( + out.contains("data:image/jpeg;base64,"), + "should inline a jpeg: {out}" + ); + assert!( + out.contains("Downscaled to"), + "should report downscale: {out}" + ); } } diff --git a/src/openhuman/tools/impl/computer/keyboard.rs b/src/openhuman/tools/impl/computer/keyboard.rs index 9bbfecb70..9e9a9a629 100644 --- a/src/openhuman/tools/impl/computer/keyboard.rs +++ b/src/openhuman/tools/impl/computer/keyboard.rs @@ -4,6 +4,7 @@ //! via platform-native APIs (Core Graphics on macOS, SendInput on Windows, //! X11/libxdo on Linux). +use super::main_thread::run_input_on_main; use crate::openhuman::security::SecurityPolicy; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; use async_trait::async_trait; @@ -186,21 +187,18 @@ impl Tool for KeyboardTool { } let len = text.len(); - tokio::task::spawn_blocking(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - enigo - .text(&text) - .map_err(|e| anyhow::anyhow!("text typing failed: {e}"))?; - info!( - tool = "keyboard", - action = "type", - chars = len, - "[computer] typed text" - ); - Ok(ToolResult::success(format!("Typed {len} characters"))) - }) - .await? + into_result( + "type", + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + enigo + .text(&text) + .map_err(|e| format!("text typing failed: {e}"))?; + Ok(format!("Typed {len} characters")) + }) + .await, + ) } "press" => { @@ -214,21 +212,18 @@ impl Tool for KeyboardTool { anyhow::anyhow!("Unknown key '{key_name}'. Use names like Enter, Tab, Escape, F1-F12, a-z, 0-9, Space, etc.") })?; - tokio::task::spawn_blocking(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - enigo - .key(key, Direction::Click) - .map_err(|e| anyhow::anyhow!("key press failed: {e}"))?; - info!( - tool = "keyboard", - action = "press", - key = key_name.as_str(), - "[computer] pressed key" - ); - Ok(ToolResult::success(format!("Pressed key '{key_name}'"))) - }) - .await? + into_result( + "press", + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + enigo + .key(key, Direction::Click) + .map_err(|e| format!("key press failed: {e}"))?; + Ok(format!("Pressed key '{key_name}'")) + }) + .await, + ) } "hotkey" => { @@ -288,51 +283,42 @@ impl Tool for KeyboardTool { } let combo_desc = key_names.join("+"); - tokio::task::spawn_blocking(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; + into_result( + "hotkey", + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; - // Press keys in order, tracking which were successfully - // pressed so we can release them on error. - let mut pressed_keys: Vec = Vec::with_capacity(keys.len()); - let press_result: Result<(), anyhow::Error> = (|| { - for key in &keys { - enigo.key(*key, Direction::Press).map_err(|e| { - anyhow::anyhow!("key press failed for {key:?}: {e}") - })?; - pressed_keys.push(*key); - std::thread::sleep(HOTKEY_INTER_KEY_DELAY); + // Press keys in order, tracking which were pressed so we + // can release them on error. + let mut pressed_keys: Vec = Vec::with_capacity(keys.len()); + let press_result: Result<(), String> = (|| { + for key in &keys { + enigo + .key(*key, Direction::Press) + .map_err(|e| format!("key press failed for {key:?}: {e}"))?; + pressed_keys.push(*key); + std::thread::sleep(HOTKEY_INTER_KEY_DELAY); + } + Ok(()) + })(); + + // Always release pressed keys in reverse, even on error. + for key in pressed_keys.iter().rev() { + if let Err(e) = enigo.key(*key, Direction::Release) { + tracing::warn!( + tool = "keyboard", + key = ?key, + error = %e, + "[computer] best-effort key release failed during cleanup" + ); + } } - Ok(()) - })(); - - // Always release all successfully pressed keys in reverse - // order, even if a press failed partway through. - for key in pressed_keys.iter().rev() { - if let Err(e) = enigo.key(*key, Direction::Release) { - tracing::warn!( - tool = "keyboard", - key = ?key, - error = %e, - "[computer] best-effort key release failed during cleanup" - ); - } - } - - // Now propagate any press error. - press_result?; - - info!( - tool = "keyboard", - action = "hotkey", - combo = combo_desc.as_str(), - "[computer] hotkey executed" - ); - Ok(ToolResult::success(format!( - "Executed hotkey: {combo_desc}" - ))) - }) - .await? + press_result?; + Ok(format!("Executed hotkey: {combo_desc}")) + }) + .await, + ) } other => Ok(ToolResult::error(format!( @@ -342,6 +328,20 @@ impl Tool for KeyboardTool { } } +/// Map a main-thread input op result to a `ToolResult`, logging the outcome. +fn into_result(action: &str, r: Result) -> anyhow::Result { + match r { + Ok(msg) => { + info!(tool = "keyboard", action, "[computer] {msg}"); + Ok(ToolResult::success(msg)) + } + Err(e) => { + tracing::warn!(tool = "keyboard", action, "[computer] failed: {e}"); + Ok(ToolResult::error(e)) + } + } +} + #[cfg(test)] #[path = "keyboard_tests.rs"] mod tests; diff --git a/src/openhuman/tools/impl/computer/main_thread.rs b/src/openhuman/tools/impl/computer/main_thread.rs new file mode 100644 index 000000000..fe1825dd2 --- /dev/null +++ b/src/openhuman/tools/impl/computer/main_thread.rs @@ -0,0 +1,74 @@ +//! Main-thread bridge for synthetic input (mouse/keyboard). +//! +//! macOS's Text Input Source APIs (`TSMGetInputSourceProperty`), which enigo +//! calls during keyboard-layout lookup, **must run on the app's main thread**. +//! Running them on a tokio worker (or `spawn_blocking`) traps with +//! `_dispatch_assert_queue_fail` / `EXC_BREAKPOINT` and crashes the CEF host +//! (tracker §1.8 / Change 1.15 — confirmed via crash report). +//! +//! So the keyboard/mouse tools never call enigo on their own thread. They build +//! a closure and hand it to [`run_input_on_main`], which dispatches it — over +//! the native request registry — to a handler the Tauri shell registers at +//! startup, which runs it on the real main thread via +//! `AppHandle::run_on_main_thread`. + +use crate::core::event_bus::request_native_global; + +/// Native-registry method the Tauri shell handles to run an input op on the +/// main thread. The shell registers a handler under this key at startup. +pub const INPUT_ON_MAIN_THREAD_METHOD: &str = "computer.input_on_main_thread"; + +/// A synthetic-input operation to run on the app's main thread. `run` performs +/// the enigo calls and returns a human-readable success message (`Ok`) or an +/// error string (`Err`). Carried by value through the native registry (no +/// serialization — the boxed `FnOnce` passes through unchanged). +pub struct MainThreadInputOp { + pub run: Box Result + Send>, +} + +/// Dispatch `op` to the app main thread and await its result. +/// +/// Returns an error when no main-thread executor is registered (headless / CLI +/// builds have no Tauri main thread — synthetic input is a desktop capability). +pub async fn run_input_on_main(op: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, +{ + let req = MainThreadInputOp { run: Box::new(op) }; + match request_native_global::>( + INPUT_ON_MAIN_THREAD_METHOD, + req, + ) + .await + { + Ok(inner) => inner, + Err(e) => Err(format!( + "synthetic input requires the desktop app's main-thread executor (unavailable: {e})" + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_bus::register_native_global; + + /// With an executor registered (as the desktop shell does at startup), the + /// op's result passes straight back through the native registry — both the + /// success and the error variant. Registering here also exercises + /// `MainThreadInputOp` construction and the `Ok(inner) => inner` arm. + #[tokio::test] + async fn dispatches_op_result_through_registered_executor() { + // Stand-in for the Tauri main-thread handler: just run the op inline. + register_native_global::, _, _>( + INPUT_ON_MAIN_THREAD_METHOD, + |req| async move { Ok((req.run)()) }, + ); + + let ok = run_input_on_main(|| Ok("clicked".to_string())).await; + assert_eq!(ok, Ok("clicked".to_string())); + + let err = run_input_on_main(|| Err("enigo failed".to_string())).await; + assert_eq!(err, Err("enigo failed".to_string())); + } +} diff --git a/src/openhuman/tools/impl/computer/mod.rs b/src/openhuman/tools/impl/computer/mod.rs index ec8363c0f..6603105d9 100644 --- a/src/openhuman/tools/impl/computer/mod.rs +++ b/src/openhuman/tools/impl/computer/mod.rs @@ -1,8 +1,10 @@ mod ax_interact; mod human_path; mod keyboard; +mod main_thread; mod mouse; pub use ax_interact::AxInteractTool; pub use keyboard::KeyboardTool; +pub use main_thread::{run_input_on_main, MainThreadInputOp, INPUT_ON_MAIN_THREAD_METHOD}; pub use mouse::MouseTool; diff --git a/src/openhuman/tools/impl/computer/mouse.rs b/src/openhuman/tools/impl/computer/mouse.rs index bcaf554e7..40f016e5b 100644 --- a/src/openhuman/tools/impl/computer/mouse.rs +++ b/src/openhuman/tools/impl/computer/mouse.rs @@ -5,6 +5,7 @@ //! SendInput on Windows, X11/libxdo on Linux). use super::human_path::{human_path, HumanPathOptions}; +use super::main_thread::run_input_on_main; use crate::openhuman::security::SecurityPolicy; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; use async_trait::async_trait; @@ -226,69 +227,57 @@ impl Tool for MouseTool { "move" => { let (x, y) = require_xy(&args)?; let human_like = human_like_enabled(&args)?; - tokio::task::spawn_blocking(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - humanized_move(&mut enigo, x, y, human_like)?; - info!( - tool = "mouse", - action = "move", - x = x, - y = y, - "[computer] cursor moved" - ); - Ok(ToolResult::success(format!("Moved cursor to ({x}, {y})"))) - }) - .await? + into_result( + "move", + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + humanized_move(&mut enigo, x, y, human_like).map_err(|e| e.to_string())?; + Ok(format!("Moved cursor to ({x}, {y})")) + }) + .await, + ) } "click" => { let (x, y) = require_xy(&args)?; let button = parse_button(&args)?; let human_like = human_like_enabled(&args)?; - tokio::task::spawn_blocking(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - humanized_move(&mut enigo, x, y, human_like)?; - enigo - .button(button, Direction::Click) - .map_err(|e| anyhow::anyhow!("button click failed: {e}"))?; - info!( - tool = "mouse", action = "click", - x = x, y = y, button = ?button, - "[computer] clicked" - ); - Ok(ToolResult::success(format!( - "Clicked {button:?} at ({x}, {y})" - ))) - }) - .await? + into_result( + "click", + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + humanized_move(&mut enigo, x, y, human_like).map_err(|e| e.to_string())?; + enigo + .button(button, Direction::Click) + .map_err(|e| format!("button click failed: {e}"))?; + Ok(format!("Clicked {button:?} at ({x}, {y})")) + }) + .await, + ) } "double_click" => { let (x, y) = require_xy(&args)?; let button = parse_button(&args)?; let human_like = human_like_enabled(&args)?; - tokio::task::spawn_blocking(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - humanized_move(&mut enigo, x, y, human_like)?; - enigo - .button(button, Direction::Click) - .map_err(|e| anyhow::anyhow!("button click failed: {e}"))?; - enigo - .button(button, Direction::Click) - .map_err(|e| anyhow::anyhow!("button click failed: {e}"))?; - info!( - tool = "mouse", action = "double_click", - x = x, y = y, button = ?button, - "[computer] double-clicked" - ); - Ok(ToolResult::success(format!( - "Double-clicked {button:?} at ({x}, {y})" - ))) - }) - .await? + into_result( + "double_click", + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + humanized_move(&mut enigo, x, y, human_like).map_err(|e| e.to_string())?; + enigo + .button(button, Direction::Click) + .map_err(|e| format!("button click failed: {e}"))?; + enigo + .button(button, Direction::Click) + .map_err(|e| format!("button click failed: {e}"))?; + Ok(format!("Double-clicked {button:?} at ({x}, {y})")) + }) + .await, + ) } "drag" => { @@ -308,44 +297,40 @@ impl Tool for MouseTool { let sx = start_x as i32; let sy = start_y as i32; - tokio::task::spawn_blocking(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - humanized_move(&mut enigo, sx, sy, human_like)?; - enigo - .button(button, Direction::Press) - .map_err(|e| anyhow::anyhow!("button press failed: {e}"))?; + into_result( + "drag", + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + humanized_move(&mut enigo, sx, sy, human_like) + .map_err(|e| e.to_string())?; + enigo + .button(button, Direction::Press) + .map_err(|e| format!("button press failed: {e}"))?; - // After press succeeds, guarantee release even on error. - let drag_result: Result<(), anyhow::Error> = (|| { - humanized_move(&mut enigo, end_x, end_y, human_like)?; - Ok(()) - })(); + // After press succeeds, guarantee release even on error. + let drag_result: Result<(), String> = (|| { + humanized_move(&mut enigo, end_x, end_y, human_like) + .map_err(|e| e.to_string())?; + Ok(()) + })(); - // Always release — best-effort cleanup. - if let Err(e) = enigo.button(button, Direction::Release) { - warn!( - tool = "mouse", - button = ?button, - error = %e, - "[computer] best-effort button release failed during drag cleanup" - ); - } - - // Propagate the drag error if the move failed. - drag_result?; - - info!( - tool = "mouse", action = "drag", - start_x = sx, start_y = sy, - end_x = end_x, end_y = end_y, button = ?button, - "[computer] dragged" - ); - Ok(ToolResult::success(format!( - "Dragged {button:?} from ({sx}, {sy}) to ({end_x}, {end_y})" - ))) - }) - .await? + // Always release — best-effort cleanup. + if let Err(e) = enigo.button(button, Direction::Release) { + warn!( + tool = "mouse", + button = ?button, + error = %e, + "[computer] best-effort button release failed during drag cleanup" + ); + } + drag_result?; + Ok(format!( + "Dragged {button:?} from ({sx}, {sy}) to ({end_x}, {end_y})" + )) + }) + .await, + ) } "scroll" => { @@ -373,31 +358,25 @@ impl Tool for MouseTool { )); } - tokio::task::spawn_blocking(move || { - let mut enigo = Enigo::new(&Settings::default()) - .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - if scroll_y != 0 { - enigo - .scroll(scroll_y, enigo::Axis::Vertical) - .map_err(|e| anyhow::anyhow!("vertical scroll failed: {e}"))?; - } - if scroll_x != 0 { - enigo - .scroll(scroll_x, enigo::Axis::Horizontal) - .map_err(|e| anyhow::anyhow!("horizontal scroll failed: {e}"))?; - } - info!( - tool = "mouse", - action = "scroll", - scroll_x = scroll_x, - scroll_y = scroll_y, - "[computer] scrolled" - ); - Ok(ToolResult::success(format!( - "Scrolled (x={scroll_x}, y={scroll_y})" - ))) - }) - .await? + into_result( + "scroll", + run_input_on_main(move || { + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("Failed to create enigo instance: {e}"))?; + if scroll_y != 0 { + enigo + .scroll(scroll_y, enigo::Axis::Vertical) + .map_err(|e| format!("vertical scroll failed: {e}"))?; + } + if scroll_x != 0 { + enigo + .scroll(scroll_x, enigo::Axis::Horizontal) + .map_err(|e| format!("horizontal scroll failed: {e}"))?; + } + Ok(format!("Scrolled (x={scroll_x}, y={scroll_y})")) + }) + .await, + ) } other => Ok(ToolResult::error(format!( @@ -407,6 +386,20 @@ impl Tool for MouseTool { } } +/// Map a main-thread input op result to a `ToolResult`, logging the outcome. +fn into_result(action: &str, r: Result) -> anyhow::Result { + match r { + Ok(msg) => { + info!(tool = "mouse", action, "[computer] {msg}"); + Ok(ToolResult::success(msg)) + } + Err(e) => { + warn!(tool = "mouse", action, "[computer] failed: {e}"); + Ok(ToolResult::error(e)) + } + } +} + #[cfg(test)] #[path = "mouse_tests.rs"] mod tests;