From bb1f2c3c8d0ce6587fb4e17252807396ddb29c33 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Sun, 5 Apr 2026 22:17:10 -0700 Subject: [PATCH] feat: screen intelligence pipeline + CLI + keep_screenshots config (#339) * feat: add keep_screenshots functionality to Screen Intelligence settings - Introduced a new `keep_screenshots` option in the Screen Intelligence settings, allowing users to save captured screenshots to the workspace instead of deleting them after processing. - Updated relevant components and tests to reflect this new feature, ensuring consistent behavior across the application. - Enhanced the user interface to include a checkbox for the `keep_screenshots` setting, improving user experience and configurability. * feat: add `openhuman screen-intelligence` CLI for standalone testing Adds a dedicated CLI (mirroring `openhuman skills`) to run the screen intelligence capture + vision pipeline without the full desktop app. Subcommands: run, status, capture, start, stop. Makes save_screenshot_to_disk public so the CLI capture --keep flag works. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: add keep_screenshots to ScreenIntelligencePanel test assertion Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .../ScreenIntelligenceDebugPanel.test.tsx | 1 + .../panels/ScreenIntelligencePanel.tsx | 15 + .../__tests__/AccessibilityPanel.test.tsx | 1 + .../ScreenIntelligencePanel.test.tsx | 2 + .../services/__tests__/coreRpcClient.test.ts | 1 + .../__tests__/accessibilitySlice.test.ts | 1 + app/src/utils/tauriCommands.ts | 2 + src/core/cli.rs | 3 + src/core/mod.rs | 1 + src/core/screen_intelligence_cli.rs | 511 ++++++++++++++++++ src/openhuman/config/ops.rs | 4 + src/openhuman/config/schema/accessibility.rs | 5 + src/openhuman/config/schemas.rs | 3 + src/openhuman/screen_intelligence/engine.rs | 87 +++ 14 files changed, 637 insertions(+) create mode 100644 src/core/screen_intelligence_cli.rs diff --git a/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx b/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx index fbe61d6ac..9d1c82c6a 100644 --- a/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx +++ b/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx @@ -74,6 +74,7 @@ const sampleStatus: AccessibilityStatus = { session_ttl_secs: 300, panic_stop_hotkey: 'Cmd+Shift+.', autocomplete_enabled: true, + keep_screenshots: false, allowlist: [], denylist: [], }, diff --git a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx index 686751c03..6d54619bb 100644 --- a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx +++ b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx @@ -71,6 +71,7 @@ const ScreenIntelligencePanel = () => { 'all_except_blacklist' ); const [baselineFps, setBaselineFps] = useState('1'); + const [keepScreenshots, setKeepScreenshots] = useState(false); const [allowlistText, setAllowlistText] = useState(''); const [denylistText, setDenylistText] = useState(''); const [isSavingConfig, setIsSavingConfig] = useState(false); @@ -104,6 +105,7 @@ const ScreenIntelligencePanel = () => { status.config.policy_mode === 'whitelist_only' ? 'whitelist_only' : 'all_except_blacklist' ); setBaselineFps(String(status.config.baseline_fps ?? 1)); + setKeepScreenshots(status.config.keep_screenshots ?? false); setAllowlistText((status.config.allowlist ?? []).join('\n')); setDenylistText((status.config.denylist ?? []).join('\n')); }, [status?.config]); @@ -143,6 +145,7 @@ const ScreenIntelligencePanel = () => { enabled, policy_mode: policyMode, baseline_fps: Number.isFinite(fps) && fps > 0 ? fps : 1, + keep_screenshots: keepScreenshots, allowlist: allowlistText .split('\n') .map(v => v.trim()) @@ -279,6 +282,18 @@ const ScreenIntelligencePanel = () => { /> + +
Allowlist (one rule per line)