fix(tools): App UI Control toggle grants click/type mutations (#3762) (#3985)

src/openhuman/agent/harness/session/builder/factory.rs

@@ -350,41 +350,67 @@ impl Agent {
         let profile_mcp_allowlist: Option<Vec<String>> =
             profile.and_then(|p| p.allowed_mcp_servers.clone());
 
-        let mut tools = tools::all_tools_with_runtime(
-            Arc::new(config.clone()),
-            &security,
-            runtime,
-            audit,
-            memory.clone(),
-            &config.browser,
-            &config.http_request,
-            &config.action_dir,
-            &config.agents,
-            config,
-            profile_skill_allowlist.as_ref(),
-            profile_mcp_allowlist.as_deref(),
-        );
-
-        // Filter tools by user preference stored in app state.
-        {
+        // Load the user's persisted tool preferences once. They drive two
+        // things below: granting the App UI Control / App Automation mutation
+        // opt-in (#3762) and filtering the tool set to the enabled snapshot.
+        let enabled_tools: Vec<String> = {
             use crate::openhuman::app_state::load_stored_app_state;
             match load_stored_app_state(config) {
-                Ok(stored) => {
-                    if let Some(ref tasks) = stored.onboarding_tasks {
-                        if !tasks.enabled_tools.is_empty() {
-                            crate::openhuman::tools::filter_tools_by_user_preference(
-                                &mut tools,
-                                &tasks.enabled_tools,
-                            );
-                        }
-                    }
-                }
+                Ok(stored) => stored
+                    .onboarding_tasks
+                    .map(|tasks| tasks.enabled_tools)
+                    .unwrap_or_default(),
                 Err(e) => {
                     log::warn!(
                         "[session-builder] failed to load app state for tool filtering: {e}"
                     );
+                    Vec::new()
                 }
             }
+        };
+
+        // Enabling the "App UI Control" (`ax_interact`) or "App Automation"
+        // (`automate`) tool in Settings → Features grants the mutating
+        // click/type actions its description promises — not just the read-only
+        // `list`. Previously those actions required the UI-less
+        // `computer_control.ax_interact_mutations` flag or Full autonomy, so the
+        // toggle silently did nothing on the default (Supervised) autonomy
+        // (#3762). The actions stay approval-gated and bound by the
+        // sensitive-app denylist; Full autonomy continues to grant this
+        // independently via `app_control_enabled`.
+        let adjusted_config: Config;
+        let tool_config: &Config = if !config.computer_control.ax_interact_mutations
+            && tools::enables_app_ui_control_mutations(&enabled_tools)
+        {
+            let mut c = config.clone();
+            c.computer_control.ax_interact_mutations = true;
+            log::debug!(
+                "[session-builder] action=grant_app_ui_control_mutations source=features_toggle"
+            );
+            adjusted_config = c;
+            &adjusted_config
+        } else {
+            config
+        };
+
+        let mut tools = tools::all_tools_with_runtime(
+            Arc::new(tool_config.clone()),
+            &security,
+            runtime,
+            audit,
+            memory.clone(),
+            &tool_config.browser,
+            &tool_config.http_request,
+            &tool_config.action_dir,
+            &tool_config.agents,
+            tool_config,
+            profile_skill_allowlist.as_ref(),
+            profile_mcp_allowlist.as_deref(),
+        );
+
+        // Filter tools by the user preference loaded above.
+        if !enabled_tools.is_empty() {
+            crate::openhuman::tools::filter_tools_by_user_preference(&mut tools, &enabled_tools);
         }
 
         if read_only_tools_only {

src/openhuman/tools/mod.rs

@@ -64,4 +64,4 @@ pub use traits::{
     PermissionLevel, Tool, ToolCallOptions, ToolCategory, ToolContent, ToolResult, ToolScope,
     ToolSpec,
 };
-pub(crate) use user_filter::filter_tools_by_user_preference;
+pub(crate) use user_filter::{enables_app_ui_control_mutations, filter_tools_by_user_preference};

src/openhuman/tools/user_filter.rs

@@ -354,6 +354,30 @@ fn expand_enabled_tool_names(enabled_tool_names: &[String]) -> HashSet<String> {
     expanded
 }
 
+/// True when the persisted tool-preference snapshot opts into the mutating
+/// app-control actions — i.e. the user enabled "App UI Control" (`ax_interact`)
+/// or "App Automation" (`automate`) in Settings → Features → Tools.
+///
+/// Those tools' descriptions promise clicking buttons and typing into fields,
+/// but enabling the toggle alone only made the read-only `list` action
+/// available — the mutating `press` / `set_value` actions required a separate,
+/// UI-less `computer_control.ax_interact_mutations` flag or Full autonomy
+/// (#3762). The session builder uses this to treat enabling the tool as the
+/// user-facing opt-in for those mutations, equivalent to setting that flag. The
+/// actions stay approval-gated and bound by the sensitive-app denylist.
+///
+/// Reuses [`expand_enabled_tool_names`] so it is robust to both persisted
+/// formats (UI toggle ids or Rust tool names). An empty snapshot ("not yet
+/// configured" / all-enabled) is treated as no explicit opt-in, preserving the
+/// safe default.
+pub(crate) fn enables_app_ui_control_mutations(enabled_tool_names: &[String]) -> bool {
+    if enabled_tool_names.is_empty() {
+        return false;
+    }
+    let expanded = expand_enabled_tool_names(enabled_tool_names);
+    expanded.contains("ax_interact") || expanded.contains("automate")
+}
+
 /// Given the list of enabled tools from app state, retain only tools that are
 /// either infrastructure (not filterable), explicitly enabled, or a default-ON
 /// capability the snapshot never explicitly opted out of.
@@ -442,7 +466,10 @@ pub(crate) fn filter_tools_by_user_preference(
 
 #[cfg(test)]
 mod tests {
-    use super::{expand_enabled_tool_names, filter_tools_by_user_preference};
+    use super::{
+        enables_app_ui_control_mutations, expand_enabled_tool_names,
+        filter_tools_by_user_preference,
+    };
     use crate::openhuman::tools::traits::{Tool, ToolResult};
     use async_trait::async_trait;
 
@@ -602,4 +629,32 @@ mod tests {
         filter_tools_by_user_preference(&mut t, &["totally_unknown".to_string()]);
         assert_eq!(names(&t).len(), 2);
     }
+
+    // #3762: enabling the App UI Control / App Automation tool is the opt-in
+    // for the mutating click/type actions.
+    #[test]
+    fn app_ui_control_opt_in_detects_ax_interact_and_automate() {
+        assert!(enables_app_ui_control_mutations(&[
+            "ax_interact".to_string()
+        ]));
+        assert!(enables_app_ui_control_mutations(&["automate".to_string()]));
+        // Present alongside other enabled tools.
+        assert!(enables_app_ui_control_mutations(&[
+            "cron_add".to_string(),
+            "automate".to_string(),
+        ]));
+    }
+
+    #[test]
+    fn app_ui_control_opt_in_false_when_absent_or_empty() {
+        assert!(!enables_app_ui_control_mutations(&[]));
+        assert!(!enables_app_ui_control_mutations(&[
+            "cron_add".to_string(),
+            "service_start".to_string(),
+        ]));
+        // Unknown entries never opt in.
+        assert!(!enables_app_ui_control_mutations(&[
+            "totally_unknown".to_string()
+        ]));
+    }
 }