diff --git a/examples/mouse_smoke.rs b/examples/mouse_smoke.rs new file mode 100644 index 000000000..e03665025 --- /dev/null +++ b/examples/mouse_smoke.rs @@ -0,0 +1,89 @@ +//! Manual smoke test for humanized MouseTool (#682). +//! +//! Run with: `cargo run --example mouse_smoke --release` +//! +//! Watch your cursor — it should curve, not teleport. Keep your hand +//! off the mouse during the run. + +use openhuman_core::openhuman::security::SecurityPolicy; +use openhuman_core::openhuman::tools::{MouseTool, Tool}; +use serde_json::json; +use std::sync::Arc; +use std::time::Instant; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "debug".into()), + ) + .init(); + + let policy = Arc::new(SecurityPolicy { + max_actions_per_hour: 1000, + ..SecurityPolicy::default() + }); + let tool = MouseTool::new(policy); + + println!("\n=== smoke 1: humanized move (default) ==="); + let t0 = Instant::now(); + let res = tool + .execute(json!({ "action": "move", "x": 800, "y": 500 })) + .await?; + println!("elapsed = {:?}", t0.elapsed()); + println!("result = {res:?}"); + assert!(!res.is_error, "humanized move should succeed"); + + tokio::time::sleep(std::time::Duration::from_millis(800)).await; + + println!("\n=== smoke 2: instant teleport (human_like=false) ==="); + let t0 = Instant::now(); + let res = tool + .execute(json!({ "action": "move", "x": 200, "y": 200, "human_like": false })) + .await?; + println!("elapsed = {:?}", t0.elapsed()); + println!("result = {res:?}"); + assert!(!res.is_error, "teleport move should succeed"); + + tokio::time::sleep(std::time::Duration::from_millis(800)).await; + + println!("\n=== smoke 3: humanized move long distance ==="); + let t0 = Instant::now(); + let res = tool + .execute(json!({ "action": "move", "x": 1400, "y": 800 })) + .await?; + println!("elapsed = {:?}", t0.elapsed()); + println!("result = {res:?}"); + assert!(!res.is_error); + + tokio::time::sleep(std::time::Duration::from_millis(800)).await; + + println!("\n=== smoke 4: humanized drag ==="); + let t0 = Instant::now(); + let res = tool + .execute(json!({ + "action": "drag", + "start_x": 600, "start_y": 400, + "x": 1000, "y": 600, + })) + .await?; + println!("elapsed = {:?}", t0.elapsed()); + println!("result = {res:?}"); + assert!(!res.is_error, "drag should succeed"); + + tokio::time::sleep(std::time::Duration::from_millis(800)).await; + + println!("\n=== smoke 5: humanized click ==="); + // Click in dead screen area to avoid collateral. + let t0 = Instant::now(); + let res = tool + .execute(json!({ "action": "click", "x": 50, "y": 50 })) + .await?; + println!("elapsed = {:?}", t0.elapsed()); + println!("result = {res:?}"); + assert!(!res.is_error); + + println!("\n✓ smoke complete — verify visually that motion was curved + paced for human-like runs and instant for the teleport."); + Ok(()) +} diff --git a/src/openhuman/tools/impl/computer/human_path.rs b/src/openhuman/tools/impl/computer/human_path.rs new file mode 100644 index 000000000..97db71fdf --- /dev/null +++ b/src/openhuman/tools/impl/computer/human_path.rs @@ -0,0 +1,136 @@ +use rand::Rng; +use std::f64::consts::TAU; + +#[derive(Debug, Clone, Copy)] +pub struct HumanPathOptions { + /// Total number of interpolation steps. Default 25. + pub steps: usize, + /// Mean dwell time between steps in milliseconds. Default 12 ms. + pub mean_step_ms: f64, + /// Std-dev of dwell time. Default 4 ms. + pub stddev_step_ms: f64, + /// Bezier control-point lateral deviation factor. Default 0.3. + pub curvature: f64, +} + +impl Default for HumanPathOptions { + fn default() -> Self { + Self { + steps: 25, + mean_step_ms: 12.0, + stddev_step_ms: 4.0, + curvature: 0.3, + } + } +} + +/// Returns `(x, y, dwell_ms)` steps for a humanized cursor path. +pub fn human_path( + start: (i32, i32), + end: (i32, i32), + opts: &HumanPathOptions, + rng: &mut R, +) -> Vec<(i32, i32, u64)> { + if start == end || opts.steps == 0 { + return vec![(end.0, end.1, 0)]; + } + + let start_f = (start.0 as f64, start.1 as f64); + let end_f = (end.0 as f64, end.1 as f64); + let dx = end_f.0 - start_f.0; + let dy = end_f.1 - start_f.1; + let dist = dx.hypot(dy); + let steps = if dist < 5.0 { + opts.steps.min(3) + } else { + opts.steps + }; + if steps == 0 { + return vec![(end.0, end.1, 0)]; + } + + let perp = (-dy / dist, dx / dist); + let curvature = opts.curvature.max(0.0); + let deviation = curvature * dist; + let p1_offset = sample_normal(0.0, deviation, rng); + let p2_offset = sample_normal(0.0, deviation, rng); + let p1 = offset_perp(lerp(start_f, end_f, 0.33), perp, p1_offset); + let p2 = offset_perp(lerp(start_f, end_f, 0.66), perp, p2_offset); + + (0..=steps) + .map(|step| { + let t = step as f64 / steps as f64; + let (x, y) = cubic_bezier(start_f, p1, p2, end_f, t); + (x.round() as i32, y.round() as i32, dwell_ms(opts, rng)) + }) + .collect() +} + +fn lerp(start: (f64, f64), end: (f64, f64), t: f64) -> (f64, f64) { + ( + start.0 + (end.0 - start.0) * t, + start.1 + (end.1 - start.1) * t, + ) +} + +fn offset_perp(point: (f64, f64), perp: (f64, f64), offset: f64) -> (f64, f64) { + (point.0 + perp.0 * offset, point.1 + perp.1 * offset) +} + +fn cubic_bezier( + p0: (f64, f64), + p1: (f64, f64), + p2: (f64, f64), + p3: (f64, f64), + t: f64, +) -> (f64, f64) { + let one_minus = 1.0 - t; + let a = one_minus.powi(3); + let b = 3.0 * one_minus.powi(2) * t; + let c = 3.0 * one_minus * t.powi(2); + let d = t.powi(3); + ( + a * p0.0 + b * p1.0 + c * p2.0 + d * p3.0, + a * p0.1 + b * p1.1 + c * p2.1 + d * p3.1, + ) +} + +fn dwell_ms(opts: &HumanPathOptions, rng: &mut R) -> u64 { + let mean = finite_or_default(opts.mean_step_ms, HumanPathOptions::default().mean_step_ms); + let stddev = finite_or_default( + opts.stddev_step_ms, + HumanPathOptions::default().stddev_step_ms, + ) + .max(0.0); + let sample = if stddev == 0.0 { + mean + } else { + let raw = sample_normal(mean, stddev, rng); + raw.clamp(mean - 3.0 * stddev, mean + 3.0 * stddev) + }; + sample.max(1.0).round() as u64 +} + +fn finite_or_default(value: f64, default: f64) -> f64 { + if value.is_finite() { + value + } else { + default + } +} + +fn sample_normal(mean: f64, stddev: f64, rng: &mut R) -> f64 { + if stddev <= 0.0 { + return mean; + } + let u1 = rng + .random::() + .clamp(f64::MIN_POSITIVE, 1.0 - f64::EPSILON); + let u2 = rng.random::(); + let z0 = (-2.0 * u1.ln()).sqrt() * (TAU * u2).cos(); + mean + z0 * stddev +} + +#[cfg(test)] +#[path = "human_path_tests.rs"] +mod tests; diff --git a/src/openhuman/tools/impl/computer/human_path_tests.rs b/src/openhuman/tools/impl/computer/human_path_tests.rs new file mode 100644 index 000000000..fe62b96f0 --- /dev/null +++ b/src/openhuman/tools/impl/computer/human_path_tests.rs @@ -0,0 +1,93 @@ +use super::*; +use rand::{rngs::StdRng, SeedableRng}; + +fn seeded() -> StdRng { + StdRng::seed_from_u64(42) +} + +#[test] +fn start_equals_end_returns_single_point() { + let mut rng = seeded(); + let path = human_path((10, 20), (10, 20), &HumanPathOptions::default(), &mut rng); + assert_eq!(path, vec![(10, 20, 0)]); +} + +#[test] +fn steps_zero_returns_single_point() { + let mut rng = seeded(); + let opts = HumanPathOptions { + steps: 0, + ..HumanPathOptions::default() + }; + let path = human_path((10, 20), (30, 40), &opts, &mut rng); + assert_eq!(path, vec![(30, 40, 0)]); +} + +#[test] +fn path_starts_at_start_and_ends_at_end() { + let mut rng = seeded(); + let path = human_path((10, 20), (210, 120), &HumanPathOptions::default(), &mut rng); + assert_eq!((path.first().unwrap().0, path.first().unwrap().1), (10, 20)); + assert_eq!((path.last().unwrap().0, path.last().unwrap().1), (210, 120)); +} + +#[test] +fn path_has_expected_step_count() { + let mut rng = seeded(); + let opts = HumanPathOptions { + steps: 8, + ..HumanPathOptions::default() + }; + let path = human_path((0, 0), (100, 0), &opts, &mut rng); + assert_eq!(path.len(), 9); +} + +#[test] +fn tiny_move_caps_step_count() { + let mut rng = seeded(); + let opts = HumanPathOptions { + steps: 25, + ..HumanPathOptions::default() + }; + let path = human_path((0, 0), (4, 0), &opts, &mut rng); + assert_eq!(path.len(), 4); +} + +#[test] +fn dwell_times_within_3_sigma() { + let mut rng = seeded(); + let opts = HumanPathOptions { + steps: 40, + mean_step_ms: 12.0, + stddev_step_ms: 4.0, + ..HumanPathOptions::default() + }; + let path = human_path((0, 0), (100, 0), &opts, &mut rng); + assert!(path.iter().all(|(_, _, dwell)| (0..=24).contains(dwell))); +} + +#[test] +fn path_curves_off_straight_line() { + let mut rng = seeded(); + let opts = HumanPathOptions { + steps: 25, + curvature: 0.8, + ..HumanPathOptions::default() + }; + let path = human_path((0, 0), (100, 0), &opts, &mut rng); + assert!(path + .iter() + .skip(1) + .take(path.len() - 2) + .any(|(_, y, _)| y.abs() > 1)); +} + +#[test] +fn deterministic_with_seeded_rng() { + let opts = HumanPathOptions::default(); + let mut first_rng = StdRng::seed_from_u64(7); + let mut second_rng = StdRng::seed_from_u64(7); + let first = human_path((5, 9), (150, 90), &opts, &mut first_rng); + let second = human_path((5, 9), (150, 90), &opts, &mut second_rng); + assert_eq!(first, second); +} diff --git a/src/openhuman/tools/impl/computer/mod.rs b/src/openhuman/tools/impl/computer/mod.rs index d9136c81f..ce9fde911 100644 --- a/src/openhuman/tools/impl/computer/mod.rs +++ b/src/openhuman/tools/impl/computer/mod.rs @@ -1,3 +1,4 @@ +mod human_path; mod keyboard; mod mouse; diff --git a/src/openhuman/tools/impl/computer/mouse.rs b/src/openhuman/tools/impl/computer/mouse.rs index c91c30e04..8e7dc4738 100644 --- a/src/openhuman/tools/impl/computer/mouse.rs +++ b/src/openhuman/tools/impl/computer/mouse.rs @@ -4,13 +4,14 @@ //! dragging, and scrolling via platform-native APIs (Core Graphics on macOS, //! SendInput on Windows, X11/libxdo on Linux). +use super::human_path::{human_path, HumanPathOptions}; use crate::openhuman::security::SecurityPolicy; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; use async_trait::async_trait; use enigo::{Button, Coordinate, Direction, Enigo, Mouse, Settings}; use serde_json::{json, Value}; -use std::sync::Arc; -use tracing::{debug, info, warn}; +use std::{sync::Arc, thread, time::Duration}; +use tracing::{debug, info, trace, warn}; /// Coordinate safety bound — reject values outside this range. const MAX_COORD: i64 = 32768; @@ -54,6 +55,15 @@ fn require_xy(args: &Value) -> anyhow::Result<(i32, i32)> { Ok((x as i32, y as i32)) } +fn human_like_enabled(args: &Value) -> anyhow::Result { + match args.get("human_like") { + None => Ok(true), + Some(v) => v + .as_bool() + .ok_or_else(|| anyhow::anyhow!("'human_like' must be a boolean, got {v}")), + } +} + fn validate_coord(name: &str, value: i64) -> anyhow::Result<()> { if !(0..=MAX_COORD).contains(&value) { anyhow::bail!("'{name}' coordinate {value} is out of range (0..{MAX_COORD})"); @@ -61,6 +71,68 @@ fn validate_coord(name: &str, value: i64) -> anyhow::Result<()> { Ok(()) } +/// Clamp a sampled bezier waypoint into the same screen-coord band that +/// `validate_coord` enforces on caller-supplied endpoints. Bezier control +/// points are zero-centered Gaussians, so perpendicular offsets can push +/// intermediate `(x, y)` outside `0..=MAX_COORD` even when the start and +/// end are valid — clamp before handing to `enigo.move_mouse`. +fn clamp_waypoint(value: i32) -> i32 { + value.clamp(0, MAX_COORD as i32) +} + +fn planned_mouse_path( + start: (i32, i32), + end: (i32, i32), + human_like: bool, + opts: &HumanPathOptions, + rng: &mut R, +) -> Vec<(i32, i32, u64)> { + if human_like { + human_path(start, end, opts, rng) + } else { + vec![(end.0, end.1, 0)] + } +} + +fn humanized_move( + enigo: &mut Enigo, + end_x: i32, + end_y: i32, + human_like: bool, +) -> anyhow::Result<()> { + if !human_like { + enigo + .move_mouse(end_x, end_y, Coordinate::Abs) + .map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?; + return Ok(()); + } + + let start = enigo + .location() + .map_err(|e| anyhow::anyhow!("location failed: {e}"))?; + let opts = HumanPathOptions::default(); + let mut rng = rand::rng(); + let path = planned_mouse_path(start, (end_x, end_y), true, &opts, &mut rng); + debug!( + start = ?start, + end = ?(end_x, end_y), + steps = path.len(), + "[mouse][humanized] generated path" + ); + for (raw_x, raw_y, dwell) in path { + let x = clamp_waypoint(raw_x); + let y = clamp_waypoint(raw_y); + trace!(x, y, dwell_ms = dwell, "[mouse][humanized] step"); + enigo + .move_mouse(x, y, Coordinate::Abs) + .map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?; + if dwell > 0 { + thread::sleep(Duration::from_millis(dwell)); + } + } + Ok(()) +} + #[async_trait] impl Tool for MouseTool { fn name(&self) -> &str { @@ -116,6 +188,11 @@ impl Tool for MouseTool { "scroll_y": { "type": "integer", "description": "Vertical scroll amount (positive = down, negative = up). For scroll action." + }, + "human_like": { + "type": "boolean", + "description": "Default true. Set false for instant teleport (faster, but easier to fingerprint).", + "default": true } }, "required": ["action"] @@ -146,12 +223,11 @@ impl Tool for MouseTool { match action { "move" => { let (x, y) = require_xy(&args)?; + let human_like = human_like_enabled(&args)?; tokio::task::spawn_blocking(move || { let mut enigo = Enigo::new(&Settings::default()) .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - enigo - .move_mouse(x, y, Coordinate::Abs) - .map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?; + humanized_move(&mut enigo, x, y, human_like)?; info!( tool = "mouse", action = "move", @@ -167,12 +243,11 @@ impl Tool for MouseTool { "click" => { let (x, y) = require_xy(&args)?; let button = parse_button(&args)?; + let human_like = human_like_enabled(&args)?; tokio::task::spawn_blocking(move || { let mut enigo = Enigo::new(&Settings::default()) .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - enigo - .move_mouse(x, y, Coordinate::Abs) - .map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?; + humanized_move(&mut enigo, x, y, human_like)?; enigo .button(button, Direction::Click) .map_err(|e| anyhow::anyhow!("button click failed: {e}"))?; @@ -191,12 +266,11 @@ impl Tool for MouseTool { "double_click" => { let (x, y) = require_xy(&args)?; let button = parse_button(&args)?; + let human_like = human_like_enabled(&args)?; tokio::task::spawn_blocking(move || { let mut enigo = Enigo::new(&Settings::default()) .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - enigo - .move_mouse(x, y, Coordinate::Abs) - .map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?; + humanized_move(&mut enigo, x, y, human_like)?; enigo .button(button, Direction::Click) .map_err(|e| anyhow::anyhow!("button click failed: {e}"))?; @@ -228,24 +302,21 @@ impl Tool for MouseTool { validate_coord("start_y", start_y)?; let (end_x, end_y) = require_xy(&args)?; let button = parse_button(&args)?; + let human_like = human_like_enabled(&args)?; let sx = start_x as i32; let sy = start_y as i32; tokio::task::spawn_blocking(move || { let mut enigo = Enigo::new(&Settings::default()) .map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?; - enigo - .move_mouse(sx, sy, Coordinate::Abs) - .map_err(|e| anyhow::anyhow!("move_mouse (start) failed: {e}"))?; + humanized_move(&mut enigo, sx, sy, human_like)?; enigo .button(button, Direction::Press) .map_err(|e| anyhow::anyhow!("button press failed: {e}"))?; // After press succeeds, guarantee release even on error. let drag_result: Result<(), anyhow::Error> = (|| { - enigo - .move_mouse(end_x, end_y, Coordinate::Abs) - .map_err(|e| anyhow::anyhow!("move_mouse (end) failed: {e}"))?; + humanized_move(&mut enigo, end_x, end_y, human_like)?; Ok(()) })(); diff --git a/src/openhuman/tools/impl/computer/mouse_tests.rs b/src/openhuman/tools/impl/computer/mouse_tests.rs index 06a02f8b8..cb1932c3b 100644 --- a/src/openhuman/tools/impl/computer/mouse_tests.rs +++ b/src/openhuman/tools/impl/computer/mouse_tests.rs @@ -1,4 +1,5 @@ use super::*; +use rand::SeedableRng; fn make_tool() -> MouseTool { MouseTool::new(Arc::new(SecurityPolicy::default())) @@ -24,6 +25,15 @@ fn schema_enumerates_actions() { assert!(names.contains(&"scroll")); } +#[test] +fn schema_includes_human_like_default_true() { + let tool = make_tool(); + let schema = tool.parameters_schema(); + let human_like = &schema["properties"]["human_like"]; + assert_eq!(human_like["type"], json!("boolean")); + assert_eq!(human_like["default"], json!(true)); +} + #[test] fn permission_is_dangerous() { let tool = make_tool(); @@ -40,6 +50,48 @@ fn coord_validation_rejects_negative() { assert!(validate_coord("x", -1).is_err()); } +#[test] +fn clamp_waypoint_floors_at_zero() { + assert_eq!(clamp_waypoint(-1), 0); + assert_eq!(clamp_waypoint(-9999), 0); +} + +#[test] +fn clamp_waypoint_caps_at_max() { + assert_eq!(clamp_waypoint(MAX_COORD as i32), MAX_COORD as i32); + assert_eq!(clamp_waypoint(MAX_COORD as i32 + 100), MAX_COORD as i32); +} + +#[test] +fn clamp_waypoint_passes_through_in_range() { + assert_eq!(clamp_waypoint(0), 0); + assert_eq!(clamp_waypoint(500), 500); + assert_eq!(clamp_waypoint(MAX_COORD as i32 - 1), MAX_COORD as i32 - 1); +} + +#[test] +fn humanized_path_clamped_for_edge_endpoints() { + // Bezier control points are zero-centered Gaussians scaled by + // distance, so perpendicular offsets can push waypoints negative + // or beyond MAX_COORD even when start/end are valid edge coords. + // Verify the clamp covers every waypoint regardless of seed. + let opts = HumanPathOptions { + steps: 25, + curvature: 0.8, + ..HumanPathOptions::default() + }; + for seed in 0u64..32 { + let mut rng = rand::rngs::StdRng::seed_from_u64(seed); + let path = planned_mouse_path((0, 0), (200, 0), true, &opts, &mut rng); + for (x, y, _) in &path { + let cx = clamp_waypoint(*x); + let cy = clamp_waypoint(*y); + assert!((0..=MAX_COORD as i32).contains(&cx), "seed={seed} cx={cx}"); + assert!((0..=MAX_COORD as i32).contains(&cy), "seed={seed} cy={cy}"); + } + } +} + #[test] fn coord_validation_rejects_overflow() { assert!(validate_coord("x", MAX_COORD + 1).is_err()); @@ -167,6 +219,42 @@ fn require_xy_valid_returns_tuple() { assert_eq!(y, 200); } +#[test] +fn human_like_defaults_true() { + assert!(human_like_enabled(&json!({})).unwrap()); +} + +#[test] +fn human_like_false_is_accepted() { + assert!(!human_like_enabled(&json!({"human_like": false})).unwrap()); +} + +#[test] +fn human_like_non_bool_returns_error() { + assert!(human_like_enabled(&json!({"human_like": "false"})).is_err()); +} + +#[test] +fn humanized_move_visits_intermediate_points() { + let opts = HumanPathOptions { + steps: 5, + ..HumanPathOptions::default() + }; + let mut rng = rand::rngs::StdRng::seed_from_u64(3); + let path = planned_mouse_path((0, 0), (100, 0), true, &opts, &mut rng); + assert!(path.len() > 1); + assert_eq!((path.first().unwrap().0, path.first().unwrap().1), (0, 0)); + assert_eq!((path.last().unwrap().0, path.last().unwrap().1), (100, 0)); +} + +#[test] +fn human_like_false_skips_humanization() { + let opts = HumanPathOptions::default(); + let mut rng = rand::rngs::StdRng::seed_from_u64(3); + let path = planned_mouse_path((0, 0), (100, 0), false, &opts, &mut rng); + assert_eq!(path, vec![(100, 0, 0)]); +} + // ── security: read-only autonomy blocks all actions ─────────────────────── #[tokio::test]