feat: screen intelligence pipeline + CLI + keep_screenshots config (#339)

* feat: add keep_screenshots functionality to Screen Intelligence settings

- Introduced a new `keep_screenshots` option in the Screen Intelligence settings, allowing users to save captured screenshots to the workspace instead of deleting them after processing.
- Updated relevant components and tests to reflect this new feature, ensuring consistent behavior across the application.
- Enhanced the user interface to include a checkbox for the `keep_screenshots` setting, improving user experience and configurability.

* feat: add `openhuman screen-intelligence` CLI for standalone testing

Adds a dedicated CLI (mirroring `openhuman skills`) to run the screen
intelligence capture + vision pipeline without the full desktop app.

Subcommands: run, status, capture, start, stop.
Makes save_screenshot_to_disk public so the CLI capture --keep flag works.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: add keep_screenshots to ScreenIntelligencePanel test assertion

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Steven Enamakel
2026-04-05 22:17:10 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 8757bb9f15
commit bb1f2c3c8d
14 changed files with 637 additions and 0 deletions
@@ -74,6 +74,7 @@ const sampleStatus: AccessibilityStatus = {
session_ttl_secs: 300,
panic_stop_hotkey: 'Cmd+Shift+.',
autocomplete_enabled: true,
keep_screenshots: false,
allowlist: [],
denylist: [],
},
@@ -71,6 +71,7 @@ const ScreenIntelligencePanel = () => {
'all_except_blacklist'
);
const [baselineFps, setBaselineFps] = useState<string>('1');
const [keepScreenshots, setKeepScreenshots] = useState<boolean>(false);
const [allowlistText, setAllowlistText] = useState('');
const [denylistText, setDenylistText] = useState('');
const [isSavingConfig, setIsSavingConfig] = useState(false);
@@ -104,6 +105,7 @@ const ScreenIntelligencePanel = () => {
status.config.policy_mode === 'whitelist_only' ? 'whitelist_only' : 'all_except_blacklist'
);
setBaselineFps(String(status.config.baseline_fps ?? 1));
setKeepScreenshots(status.config.keep_screenshots ?? false);
setAllowlistText((status.config.allowlist ?? []).join('\n'));
setDenylistText((status.config.denylist ?? []).join('\n'));
}, [status?.config]);
@@ -143,6 +145,7 @@ const ScreenIntelligencePanel = () => {
enabled,
policy_mode: policyMode,
baseline_fps: Number.isFinite(fps) && fps > 0 ? fps : 1,
keep_screenshots: keepScreenshots,
allowlist: allowlistText
.split('\n')
.map(v => v.trim())
@@ -279,6 +282,18 @@ const ScreenIntelligencePanel = () => {
/>
</label>
<label className="flex items-center justify-between rounded-xl border border-stone-200 bg-stone-50 px-3 py-2">
<div>
<span className="text-sm text-stone-700">Keep Screenshots</span>
<p className="text-xs text-stone-400">Save captured screenshots to the workspace instead of deleting after processing</p>
</div>
<input
type="checkbox"
checked={keepScreenshots}
onChange={event => setKeepScreenshots(event.target.checked)}
/>
</label>
<div className="space-y-1">
<div className="text-xs text-stone-600">Allowlist (one rule per line)</div>
<textarea
@@ -46,6 +46,7 @@ const status: AccessibilityStatus = {
session_ttl_secs: 300,
panic_stop_hotkey: 'Cmd+Shift+.',
autocomplete_enabled: true,
keep_screenshots: false,
allowlist: [],
denylist: ['wallet'],
},
@@ -78,6 +78,7 @@ const baseStatus: AccessibilityStatus = {
session_ttl_secs: 300,
panic_stop_hotkey: 'Cmd+Shift+.',
autocomplete_enabled: true,
keep_screenshots: false,
allowlist: ['Code'],
denylist: ['1Password'],
},
@@ -188,6 +189,7 @@ describe('ScreenIntelligencePanel', () => {
enabled: true,
policy_mode: 'all_except_blacklist',
baseline_fps: 1,
keep_screenshots: false,
allowlist: ['Code'],
denylist: ['1Password'],
});
@@ -41,6 +41,7 @@ function sampleAccessibilityStatus(
session_ttl_secs: 300,
panic_stop_hotkey: 'Cmd+Shift+.',
autocomplete_enabled: true,
keep_screenshots: false,
allowlist: [],
denylist: [],
},
@@ -45,6 +45,7 @@ const sampleStatus: AccessibilityStatus = {
session_ttl_secs: 300,
panic_stop_hotkey: 'Cmd+Shift+.',
autocomplete_enabled: true,
keep_screenshots: false,
allowlist: [],
denylist: ['wallet'],
},
+2
View File
@@ -793,6 +793,7 @@ export interface AccessibilityConfig {
session_ttl_secs: number;
panic_stop_hotkey: string;
autocomplete_enabled: boolean;
keep_screenshots: boolean;
allowlist: string[];
denylist: string[];
}
@@ -1039,6 +1040,7 @@ export interface ScreenIntelligenceSettingsUpdate {
baseline_fps?: number | null;
vision_enabled?: boolean | null;
autocomplete_enabled?: boolean | null;
keep_screenshots?: boolean | null;
allowlist?: string[] | null;
denylist?: string[] | null;
}