feat(tools/computer): humanize MouseTool cursor motion (#682) (#1309)

This commit is contained in:
oxoxDev
2026-05-07 12:05:56 -07:00
committed by GitHub
parent 3bbf4b5d9f
commit 7b0eff2d14
6 changed files with 495 additions and 17 deletions
+89
View File
@@ -0,0 +1,89 @@
//! Manual smoke test for humanized MouseTool (#682).
//!
//! Run with: `cargo run --example mouse_smoke --release`
//!
//! Watch your cursor — it should curve, not teleport. Keep your hand
//! off the mouse during the run.
use openhuman_core::openhuman::security::SecurityPolicy;
use openhuman_core::openhuman::tools::{MouseTool, Tool};
use serde_json::json;
use std::sync::Arc;
use std::time::Instant;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "debug".into()),
)
.init();
let policy = Arc::new(SecurityPolicy {
max_actions_per_hour: 1000,
..SecurityPolicy::default()
});
let tool = MouseTool::new(policy);
println!("\n=== smoke 1: humanized move (default) ===");
let t0 = Instant::now();
let res = tool
.execute(json!({ "action": "move", "x": 800, "y": 500 }))
.await?;
println!("elapsed = {:?}", t0.elapsed());
println!("result = {res:?}");
assert!(!res.is_error, "humanized move should succeed");
tokio::time::sleep(std::time::Duration::from_millis(800)).await;
println!("\n=== smoke 2: instant teleport (human_like=false) ===");
let t0 = Instant::now();
let res = tool
.execute(json!({ "action": "move", "x": 200, "y": 200, "human_like": false }))
.await?;
println!("elapsed = {:?}", t0.elapsed());
println!("result = {res:?}");
assert!(!res.is_error, "teleport move should succeed");
tokio::time::sleep(std::time::Duration::from_millis(800)).await;
println!("\n=== smoke 3: humanized move long distance ===");
let t0 = Instant::now();
let res = tool
.execute(json!({ "action": "move", "x": 1400, "y": 800 }))
.await?;
println!("elapsed = {:?}", t0.elapsed());
println!("result = {res:?}");
assert!(!res.is_error);
tokio::time::sleep(std::time::Duration::from_millis(800)).await;
println!("\n=== smoke 4: humanized drag ===");
let t0 = Instant::now();
let res = tool
.execute(json!({
"action": "drag",
"start_x": 600, "start_y": 400,
"x": 1000, "y": 600,
}))
.await?;
println!("elapsed = {:?}", t0.elapsed());
println!("result = {res:?}");
assert!(!res.is_error, "drag should succeed");
tokio::time::sleep(std::time::Duration::from_millis(800)).await;
println!("\n=== smoke 5: humanized click ===");
// Click in dead screen area to avoid collateral.
let t0 = Instant::now();
let res = tool
.execute(json!({ "action": "click", "x": 50, "y": 50 }))
.await?;
println!("elapsed = {:?}", t0.elapsed());
println!("result = {res:?}");
assert!(!res.is_error);
println!("\n✓ smoke complete — verify visually that motion was curved + paced for human-like runs and instant for the teleport.");
Ok(())
}
@@ -0,0 +1,136 @@
use rand::Rng;
use std::f64::consts::TAU;
#[derive(Debug, Clone, Copy)]
pub struct HumanPathOptions {
/// Total number of interpolation steps. Default 25.
pub steps: usize,
/// Mean dwell time between steps in milliseconds. Default 12 ms.
pub mean_step_ms: f64,
/// Std-dev of dwell time. Default 4 ms.
pub stddev_step_ms: f64,
/// Bezier control-point lateral deviation factor. Default 0.3.
pub curvature: f64,
}
impl Default for HumanPathOptions {
fn default() -> Self {
Self {
steps: 25,
mean_step_ms: 12.0,
stddev_step_ms: 4.0,
curvature: 0.3,
}
}
}
/// Returns `(x, y, dwell_ms)` steps for a humanized cursor path.
pub fn human_path<R: Rng>(
start: (i32, i32),
end: (i32, i32),
opts: &HumanPathOptions,
rng: &mut R,
) -> Vec<(i32, i32, u64)> {
if start == end || opts.steps == 0 {
return vec![(end.0, end.1, 0)];
}
let start_f = (start.0 as f64, start.1 as f64);
let end_f = (end.0 as f64, end.1 as f64);
let dx = end_f.0 - start_f.0;
let dy = end_f.1 - start_f.1;
let dist = dx.hypot(dy);
let steps = if dist < 5.0 {
opts.steps.min(3)
} else {
opts.steps
};
if steps == 0 {
return vec![(end.0, end.1, 0)];
}
let perp = (-dy / dist, dx / dist);
let curvature = opts.curvature.max(0.0);
let deviation = curvature * dist;
let p1_offset = sample_normal(0.0, deviation, rng);
let p2_offset = sample_normal(0.0, deviation, rng);
let p1 = offset_perp(lerp(start_f, end_f, 0.33), perp, p1_offset);
let p2 = offset_perp(lerp(start_f, end_f, 0.66), perp, p2_offset);
(0..=steps)
.map(|step| {
let t = step as f64 / steps as f64;
let (x, y) = cubic_bezier(start_f, p1, p2, end_f, t);
(x.round() as i32, y.round() as i32, dwell_ms(opts, rng))
})
.collect()
}
fn lerp(start: (f64, f64), end: (f64, f64), t: f64) -> (f64, f64) {
(
start.0 + (end.0 - start.0) * t,
start.1 + (end.1 - start.1) * t,
)
}
fn offset_perp(point: (f64, f64), perp: (f64, f64), offset: f64) -> (f64, f64) {
(point.0 + perp.0 * offset, point.1 + perp.1 * offset)
}
fn cubic_bezier(
p0: (f64, f64),
p1: (f64, f64),
p2: (f64, f64),
p3: (f64, f64),
t: f64,
) -> (f64, f64) {
let one_minus = 1.0 - t;
let a = one_minus.powi(3);
let b = 3.0 * one_minus.powi(2) * t;
let c = 3.0 * one_minus * t.powi(2);
let d = t.powi(3);
(
a * p0.0 + b * p1.0 + c * p2.0 + d * p3.0,
a * p0.1 + b * p1.1 + c * p2.1 + d * p3.1,
)
}
fn dwell_ms<R: Rng>(opts: &HumanPathOptions, rng: &mut R) -> u64 {
let mean = finite_or_default(opts.mean_step_ms, HumanPathOptions::default().mean_step_ms);
let stddev = finite_or_default(
opts.stddev_step_ms,
HumanPathOptions::default().stddev_step_ms,
)
.max(0.0);
let sample = if stddev == 0.0 {
mean
} else {
let raw = sample_normal(mean, stddev, rng);
raw.clamp(mean - 3.0 * stddev, mean + 3.0 * stddev)
};
sample.max(1.0).round() as u64
}
fn finite_or_default(value: f64, default: f64) -> f64 {
if value.is_finite() {
value
} else {
default
}
}
fn sample_normal<R: Rng>(mean: f64, stddev: f64, rng: &mut R) -> f64 {
if stddev <= 0.0 {
return mean;
}
let u1 = rng
.random::<f64>()
.clamp(f64::MIN_POSITIVE, 1.0 - f64::EPSILON);
let u2 = rng.random::<f64>();
let z0 = (-2.0 * u1.ln()).sqrt() * (TAU * u2).cos();
mean + z0 * stddev
}
#[cfg(test)]
#[path = "human_path_tests.rs"]
mod tests;
@@ -0,0 +1,93 @@
use super::*;
use rand::{rngs::StdRng, SeedableRng};
fn seeded() -> StdRng {
StdRng::seed_from_u64(42)
}
#[test]
fn start_equals_end_returns_single_point() {
let mut rng = seeded();
let path = human_path((10, 20), (10, 20), &HumanPathOptions::default(), &mut rng);
assert_eq!(path, vec![(10, 20, 0)]);
}
#[test]
fn steps_zero_returns_single_point() {
let mut rng = seeded();
let opts = HumanPathOptions {
steps: 0,
..HumanPathOptions::default()
};
let path = human_path((10, 20), (30, 40), &opts, &mut rng);
assert_eq!(path, vec![(30, 40, 0)]);
}
#[test]
fn path_starts_at_start_and_ends_at_end() {
let mut rng = seeded();
let path = human_path((10, 20), (210, 120), &HumanPathOptions::default(), &mut rng);
assert_eq!((path.first().unwrap().0, path.first().unwrap().1), (10, 20));
assert_eq!((path.last().unwrap().0, path.last().unwrap().1), (210, 120));
}
#[test]
fn path_has_expected_step_count() {
let mut rng = seeded();
let opts = HumanPathOptions {
steps: 8,
..HumanPathOptions::default()
};
let path = human_path((0, 0), (100, 0), &opts, &mut rng);
assert_eq!(path.len(), 9);
}
#[test]
fn tiny_move_caps_step_count() {
let mut rng = seeded();
let opts = HumanPathOptions {
steps: 25,
..HumanPathOptions::default()
};
let path = human_path((0, 0), (4, 0), &opts, &mut rng);
assert_eq!(path.len(), 4);
}
#[test]
fn dwell_times_within_3_sigma() {
let mut rng = seeded();
let opts = HumanPathOptions {
steps: 40,
mean_step_ms: 12.0,
stddev_step_ms: 4.0,
..HumanPathOptions::default()
};
let path = human_path((0, 0), (100, 0), &opts, &mut rng);
assert!(path.iter().all(|(_, _, dwell)| (0..=24).contains(dwell)));
}
#[test]
fn path_curves_off_straight_line() {
let mut rng = seeded();
let opts = HumanPathOptions {
steps: 25,
curvature: 0.8,
..HumanPathOptions::default()
};
let path = human_path((0, 0), (100, 0), &opts, &mut rng);
assert!(path
.iter()
.skip(1)
.take(path.len() - 2)
.any(|(_, y, _)| y.abs() > 1));
}
#[test]
fn deterministic_with_seeded_rng() {
let opts = HumanPathOptions::default();
let mut first_rng = StdRng::seed_from_u64(7);
let mut second_rng = StdRng::seed_from_u64(7);
let first = human_path((5, 9), (150, 90), &opts, &mut first_rng);
let second = human_path((5, 9), (150, 90), &opts, &mut second_rng);
assert_eq!(first, second);
}
+1
View File
@@ -1,3 +1,4 @@
mod human_path;
mod keyboard;
mod mouse;
+88 -17
View File
@@ -4,13 +4,14 @@
//! dragging, and scrolling via platform-native APIs (Core Graphics on macOS,
//! SendInput on Windows, X11/libxdo on Linux).
use super::human_path::{human_path, HumanPathOptions};
use crate::openhuman::security::SecurityPolicy;
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult};
use async_trait::async_trait;
use enigo::{Button, Coordinate, Direction, Enigo, Mouse, Settings};
use serde_json::{json, Value};
use std::sync::Arc;
use tracing::{debug, info, warn};
use std::{sync::Arc, thread, time::Duration};
use tracing::{debug, info, trace, warn};
/// Coordinate safety bound — reject values outside this range.
const MAX_COORD: i64 = 32768;
@@ -54,6 +55,15 @@ fn require_xy(args: &Value) -> anyhow::Result<(i32, i32)> {
Ok((x as i32, y as i32))
}
fn human_like_enabled(args: &Value) -> anyhow::Result<bool> {
match args.get("human_like") {
None => Ok(true),
Some(v) => v
.as_bool()
.ok_or_else(|| anyhow::anyhow!("'human_like' must be a boolean, got {v}")),
}
}
fn validate_coord(name: &str, value: i64) -> anyhow::Result<()> {
if !(0..=MAX_COORD).contains(&value) {
anyhow::bail!("'{name}' coordinate {value} is out of range (0..{MAX_COORD})");
@@ -61,6 +71,68 @@ fn validate_coord(name: &str, value: i64) -> anyhow::Result<()> {
Ok(())
}
/// Clamp a sampled bezier waypoint into the same screen-coord band that
/// `validate_coord` enforces on caller-supplied endpoints. Bezier control
/// points are zero-centered Gaussians, so perpendicular offsets can push
/// intermediate `(x, y)` outside `0..=MAX_COORD` even when the start and
/// end are valid — clamp before handing to `enigo.move_mouse`.
fn clamp_waypoint(value: i32) -> i32 {
value.clamp(0, MAX_COORD as i32)
}
fn planned_mouse_path<R: rand::Rng>(
start: (i32, i32),
end: (i32, i32),
human_like: bool,
opts: &HumanPathOptions,
rng: &mut R,
) -> Vec<(i32, i32, u64)> {
if human_like {
human_path(start, end, opts, rng)
} else {
vec![(end.0, end.1, 0)]
}
}
fn humanized_move(
enigo: &mut Enigo,
end_x: i32,
end_y: i32,
human_like: bool,
) -> anyhow::Result<()> {
if !human_like {
enigo
.move_mouse(end_x, end_y, Coordinate::Abs)
.map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?;
return Ok(());
}
let start = enigo
.location()
.map_err(|e| anyhow::anyhow!("location failed: {e}"))?;
let opts = HumanPathOptions::default();
let mut rng = rand::rng();
let path = planned_mouse_path(start, (end_x, end_y), true, &opts, &mut rng);
debug!(
start = ?start,
end = ?(end_x, end_y),
steps = path.len(),
"[mouse][humanized] generated path"
);
for (raw_x, raw_y, dwell) in path {
let x = clamp_waypoint(raw_x);
let y = clamp_waypoint(raw_y);
trace!(x, y, dwell_ms = dwell, "[mouse][humanized] step");
enigo
.move_mouse(x, y, Coordinate::Abs)
.map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?;
if dwell > 0 {
thread::sleep(Duration::from_millis(dwell));
}
}
Ok(())
}
#[async_trait]
impl Tool for MouseTool {
fn name(&self) -> &str {
@@ -116,6 +188,11 @@ impl Tool for MouseTool {
"scroll_y": {
"type": "integer",
"description": "Vertical scroll amount (positive = down, negative = up). For scroll action."
},
"human_like": {
"type": "boolean",
"description": "Default true. Set false for instant teleport (faster, but easier to fingerprint).",
"default": true
}
},
"required": ["action"]
@@ -146,12 +223,11 @@ impl Tool for MouseTool {
match action {
"move" => {
let (x, y) = require_xy(&args)?;
let human_like = human_like_enabled(&args)?;
tokio::task::spawn_blocking(move || {
let mut enigo = Enigo::new(&Settings::default())
.map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?;
enigo
.move_mouse(x, y, Coordinate::Abs)
.map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?;
humanized_move(&mut enigo, x, y, human_like)?;
info!(
tool = "mouse",
action = "move",
@@ -167,12 +243,11 @@ impl Tool for MouseTool {
"click" => {
let (x, y) = require_xy(&args)?;
let button = parse_button(&args)?;
let human_like = human_like_enabled(&args)?;
tokio::task::spawn_blocking(move || {
let mut enigo = Enigo::new(&Settings::default())
.map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?;
enigo
.move_mouse(x, y, Coordinate::Abs)
.map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?;
humanized_move(&mut enigo, x, y, human_like)?;
enigo
.button(button, Direction::Click)
.map_err(|e| anyhow::anyhow!("button click failed: {e}"))?;
@@ -191,12 +266,11 @@ impl Tool for MouseTool {
"double_click" => {
let (x, y) = require_xy(&args)?;
let button = parse_button(&args)?;
let human_like = human_like_enabled(&args)?;
tokio::task::spawn_blocking(move || {
let mut enigo = Enigo::new(&Settings::default())
.map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?;
enigo
.move_mouse(x, y, Coordinate::Abs)
.map_err(|e| anyhow::anyhow!("move_mouse failed: {e}"))?;
humanized_move(&mut enigo, x, y, human_like)?;
enigo
.button(button, Direction::Click)
.map_err(|e| anyhow::anyhow!("button click failed: {e}"))?;
@@ -228,24 +302,21 @@ impl Tool for MouseTool {
validate_coord("start_y", start_y)?;
let (end_x, end_y) = require_xy(&args)?;
let button = parse_button(&args)?;
let human_like = human_like_enabled(&args)?;
let sx = start_x as i32;
let sy = start_y as i32;
tokio::task::spawn_blocking(move || {
let mut enigo = Enigo::new(&Settings::default())
.map_err(|e| anyhow::anyhow!("Failed to create enigo instance: {e}"))?;
enigo
.move_mouse(sx, sy, Coordinate::Abs)
.map_err(|e| anyhow::anyhow!("move_mouse (start) failed: {e}"))?;
humanized_move(&mut enigo, sx, sy, human_like)?;
enigo
.button(button, Direction::Press)
.map_err(|e| anyhow::anyhow!("button press failed: {e}"))?;
// After press succeeds, guarantee release even on error.
let drag_result: Result<(), anyhow::Error> = (|| {
enigo
.move_mouse(end_x, end_y, Coordinate::Abs)
.map_err(|e| anyhow::anyhow!("move_mouse (end) failed: {e}"))?;
humanized_move(&mut enigo, end_x, end_y, human_like)?;
Ok(())
})();
@@ -1,4 +1,5 @@
use super::*;
use rand::SeedableRng;
fn make_tool() -> MouseTool {
MouseTool::new(Arc::new(SecurityPolicy::default()))
@@ -24,6 +25,15 @@ fn schema_enumerates_actions() {
assert!(names.contains(&"scroll"));
}
#[test]
fn schema_includes_human_like_default_true() {
let tool = make_tool();
let schema = tool.parameters_schema();
let human_like = &schema["properties"]["human_like"];
assert_eq!(human_like["type"], json!("boolean"));
assert_eq!(human_like["default"], json!(true));
}
#[test]
fn permission_is_dangerous() {
let tool = make_tool();
@@ -40,6 +50,48 @@ fn coord_validation_rejects_negative() {
assert!(validate_coord("x", -1).is_err());
}
#[test]
fn clamp_waypoint_floors_at_zero() {
assert_eq!(clamp_waypoint(-1), 0);
assert_eq!(clamp_waypoint(-9999), 0);
}
#[test]
fn clamp_waypoint_caps_at_max() {
assert_eq!(clamp_waypoint(MAX_COORD as i32), MAX_COORD as i32);
assert_eq!(clamp_waypoint(MAX_COORD as i32 + 100), MAX_COORD as i32);
}
#[test]
fn clamp_waypoint_passes_through_in_range() {
assert_eq!(clamp_waypoint(0), 0);
assert_eq!(clamp_waypoint(500), 500);
assert_eq!(clamp_waypoint(MAX_COORD as i32 - 1), MAX_COORD as i32 - 1);
}
#[test]
fn humanized_path_clamped_for_edge_endpoints() {
// Bezier control points are zero-centered Gaussians scaled by
// distance, so perpendicular offsets can push waypoints negative
// or beyond MAX_COORD even when start/end are valid edge coords.
// Verify the clamp covers every waypoint regardless of seed.
let opts = HumanPathOptions {
steps: 25,
curvature: 0.8,
..HumanPathOptions::default()
};
for seed in 0u64..32 {
let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
let path = planned_mouse_path((0, 0), (200, 0), true, &opts, &mut rng);
for (x, y, _) in &path {
let cx = clamp_waypoint(*x);
let cy = clamp_waypoint(*y);
assert!((0..=MAX_COORD as i32).contains(&cx), "seed={seed} cx={cx}");
assert!((0..=MAX_COORD as i32).contains(&cy), "seed={seed} cy={cy}");
}
}
}
#[test]
fn coord_validation_rejects_overflow() {
assert!(validate_coord("x", MAX_COORD + 1).is_err());
@@ -167,6 +219,42 @@ fn require_xy_valid_returns_tuple() {
assert_eq!(y, 200);
}
#[test]
fn human_like_defaults_true() {
assert!(human_like_enabled(&json!({})).unwrap());
}
#[test]
fn human_like_false_is_accepted() {
assert!(!human_like_enabled(&json!({"human_like": false})).unwrap());
}
#[test]
fn human_like_non_bool_returns_error() {
assert!(human_like_enabled(&json!({"human_like": "false"})).is_err());
}
#[test]
fn humanized_move_visits_intermediate_points() {
let opts = HumanPathOptions {
steps: 5,
..HumanPathOptions::default()
};
let mut rng = rand::rngs::StdRng::seed_from_u64(3);
let path = planned_mouse_path((0, 0), (100, 0), true, &opts, &mut rng);
assert!(path.len() > 1);
assert_eq!((path.first().unwrap().0, path.first().unwrap().1), (0, 0));
assert_eq!((path.last().unwrap().0, path.last().unwrap().1), (100, 0));
}
#[test]
fn human_like_false_skips_humanization() {
let opts = HumanPathOptions::default();
let mut rng = rand::rngs::StdRng::seed_from_u64(3);
let path = planned_mouse_path((0, 0), (100, 0), false, &opts, &mut rng);
assert_eq!(path, vec![(100, 0, 0)]);
}
// ── security: read-only autonomy blocks all actions ───────────────────────
#[tokio::test]