diff --git a/src/openhuman/agent/harness/session/builder/factory.rs b/src/openhuman/agent/harness/session/builder/factory.rs index 916be0001..6609309ee 100644 --- a/src/openhuman/agent/harness/session/builder/factory.rs +++ b/src/openhuman/agent/harness/session/builder/factory.rs @@ -350,41 +350,67 @@ impl Agent { let profile_mcp_allowlist: Option> = profile.and_then(|p| p.allowed_mcp_servers.clone()); - let mut tools = tools::all_tools_with_runtime( - Arc::new(config.clone()), - &security, - runtime, - audit, - memory.clone(), - &config.browser, - &config.http_request, - &config.action_dir, - &config.agents, - config, - profile_skill_allowlist.as_ref(), - profile_mcp_allowlist.as_deref(), - ); - - // Filter tools by user preference stored in app state. - { + // Load the user's persisted tool preferences once. They drive two + // things below: granting the App UI Control / App Automation mutation + // opt-in (#3762) and filtering the tool set to the enabled snapshot. + let enabled_tools: Vec = { use crate::openhuman::app_state::load_stored_app_state; match load_stored_app_state(config) { - Ok(stored) => { - if let Some(ref tasks) = stored.onboarding_tasks { - if !tasks.enabled_tools.is_empty() { - crate::openhuman::tools::filter_tools_by_user_preference( - &mut tools, - &tasks.enabled_tools, - ); - } - } - } + Ok(stored) => stored + .onboarding_tasks + .map(|tasks| tasks.enabled_tools) + .unwrap_or_default(), Err(e) => { log::warn!( "[session-builder] failed to load app state for tool filtering: {e}" ); + Vec::new() } } + }; + + // Enabling the "App UI Control" (`ax_interact`) or "App Automation" + // (`automate`) tool in Settings → Features grants the mutating + // click/type actions its description promises — not just the read-only + // `list`. Previously those actions required the UI-less + // `computer_control.ax_interact_mutations` flag or Full autonomy, so the + // toggle silently did nothing on the default (Supervised) autonomy + // (#3762). The actions stay approval-gated and bound by the + // sensitive-app denylist; Full autonomy continues to grant this + // independently via `app_control_enabled`. + let adjusted_config: Config; + let tool_config: &Config = if !config.computer_control.ax_interact_mutations + && tools::enables_app_ui_control_mutations(&enabled_tools) + { + let mut c = config.clone(); + c.computer_control.ax_interact_mutations = true; + log::debug!( + "[session-builder] action=grant_app_ui_control_mutations source=features_toggle" + ); + adjusted_config = c; + &adjusted_config + } else { + config + }; + + let mut tools = tools::all_tools_with_runtime( + Arc::new(tool_config.clone()), + &security, + runtime, + audit, + memory.clone(), + &tool_config.browser, + &tool_config.http_request, + &tool_config.action_dir, + &tool_config.agents, + tool_config, + profile_skill_allowlist.as_ref(), + profile_mcp_allowlist.as_deref(), + ); + + // Filter tools by the user preference loaded above. + if !enabled_tools.is_empty() { + crate::openhuman::tools::filter_tools_by_user_preference(&mut tools, &enabled_tools); } if read_only_tools_only { diff --git a/src/openhuman/tools/mod.rs b/src/openhuman/tools/mod.rs index f1476727d..0229106e9 100644 --- a/src/openhuman/tools/mod.rs +++ b/src/openhuman/tools/mod.rs @@ -64,4 +64,4 @@ pub use traits::{ PermissionLevel, Tool, ToolCallOptions, ToolCategory, ToolContent, ToolResult, ToolScope, ToolSpec, }; -pub(crate) use user_filter::filter_tools_by_user_preference; +pub(crate) use user_filter::{enables_app_ui_control_mutations, filter_tools_by_user_preference}; diff --git a/src/openhuman/tools/user_filter.rs b/src/openhuman/tools/user_filter.rs index 6c29f92dc..7071e75cc 100644 --- a/src/openhuman/tools/user_filter.rs +++ b/src/openhuman/tools/user_filter.rs @@ -354,6 +354,30 @@ fn expand_enabled_tool_names(enabled_tool_names: &[String]) -> HashSet { expanded } +/// True when the persisted tool-preference snapshot opts into the mutating +/// app-control actions — i.e. the user enabled "App UI Control" (`ax_interact`) +/// or "App Automation" (`automate`) in Settings → Features → Tools. +/// +/// Those tools' descriptions promise clicking buttons and typing into fields, +/// but enabling the toggle alone only made the read-only `list` action +/// available — the mutating `press` / `set_value` actions required a separate, +/// UI-less `computer_control.ax_interact_mutations` flag or Full autonomy +/// (#3762). The session builder uses this to treat enabling the tool as the +/// user-facing opt-in for those mutations, equivalent to setting that flag. The +/// actions stay approval-gated and bound by the sensitive-app denylist. +/// +/// Reuses [`expand_enabled_tool_names`] so it is robust to both persisted +/// formats (UI toggle ids or Rust tool names). An empty snapshot ("not yet +/// configured" / all-enabled) is treated as no explicit opt-in, preserving the +/// safe default. +pub(crate) fn enables_app_ui_control_mutations(enabled_tool_names: &[String]) -> bool { + if enabled_tool_names.is_empty() { + return false; + } + let expanded = expand_enabled_tool_names(enabled_tool_names); + expanded.contains("ax_interact") || expanded.contains("automate") +} + /// Given the list of enabled tools from app state, retain only tools that are /// either infrastructure (not filterable), explicitly enabled, or a default-ON /// capability the snapshot never explicitly opted out of. @@ -442,7 +466,10 @@ pub(crate) fn filter_tools_by_user_preference( #[cfg(test)] mod tests { - use super::{expand_enabled_tool_names, filter_tools_by_user_preference}; + use super::{ + enables_app_ui_control_mutations, expand_enabled_tool_names, + filter_tools_by_user_preference, + }; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; @@ -602,4 +629,32 @@ mod tests { filter_tools_by_user_preference(&mut t, &["totally_unknown".to_string()]); assert_eq!(names(&t).len(), 2); } + + // #3762: enabling the App UI Control / App Automation tool is the opt-in + // for the mutating click/type actions. + #[test] + fn app_ui_control_opt_in_detects_ax_interact_and_automate() { + assert!(enables_app_ui_control_mutations(&[ + "ax_interact".to_string() + ])); + assert!(enables_app_ui_control_mutations(&["automate".to_string()])); + // Present alongside other enabled tools. + assert!(enables_app_ui_control_mutations(&[ + "cron_add".to_string(), + "automate".to_string(), + ])); + } + + #[test] + fn app_ui_control_opt_in_false_when_absent_or_empty() { + assert!(!enables_app_ui_control_mutations(&[])); + assert!(!enables_app_ui_control_mutations(&[ + "cron_add".to_string(), + "service_start".to_string(), + ])); + // Unknown entries never opt in. + assert!(!enables_app_ui_control_mutations(&[ + "totally_unknown".to_string() + ])); + } }