mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
feat: screen intelligence pipeline + CLI + keep_screenshots config (#339)
* feat: add keep_screenshots functionality to Screen Intelligence settings - Introduced a new `keep_screenshots` option in the Screen Intelligence settings, allowing users to save captured screenshots to the workspace instead of deleting them after processing. - Updated relevant components and tests to reflect this new feature, ensuring consistent behavior across the application. - Enhanced the user interface to include a checkbox for the `keep_screenshots` setting, improving user experience and configurability. * feat: add `openhuman screen-intelligence` CLI for standalone testing Adds a dedicated CLI (mirroring `openhuman skills`) to run the screen intelligence capture + vision pipeline without the full desktop app. Subcommands: run, status, capture, start, stop. Makes save_screenshot_to_disk public so the CLI capture --keep flag works. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: add keep_screenshots to ScreenIntelligencePanel test assertion Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Steven Enamakel
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
8757bb9f15
commit
bb1f2c3c8d
@@ -74,6 +74,7 @@ const sampleStatus: AccessibilityStatus = {
|
||||
session_ttl_secs: 300,
|
||||
panic_stop_hotkey: 'Cmd+Shift+.',
|
||||
autocomplete_enabled: true,
|
||||
keep_screenshots: false,
|
||||
allowlist: [],
|
||||
denylist: [],
|
||||
},
|
||||
|
||||
@@ -71,6 +71,7 @@ const ScreenIntelligencePanel = () => {
|
||||
'all_except_blacklist'
|
||||
);
|
||||
const [baselineFps, setBaselineFps] = useState<string>('1');
|
||||
const [keepScreenshots, setKeepScreenshots] = useState<boolean>(false);
|
||||
const [allowlistText, setAllowlistText] = useState('');
|
||||
const [denylistText, setDenylistText] = useState('');
|
||||
const [isSavingConfig, setIsSavingConfig] = useState(false);
|
||||
@@ -104,6 +105,7 @@ const ScreenIntelligencePanel = () => {
|
||||
status.config.policy_mode === 'whitelist_only' ? 'whitelist_only' : 'all_except_blacklist'
|
||||
);
|
||||
setBaselineFps(String(status.config.baseline_fps ?? 1));
|
||||
setKeepScreenshots(status.config.keep_screenshots ?? false);
|
||||
setAllowlistText((status.config.allowlist ?? []).join('\n'));
|
||||
setDenylistText((status.config.denylist ?? []).join('\n'));
|
||||
}, [status?.config]);
|
||||
@@ -143,6 +145,7 @@ const ScreenIntelligencePanel = () => {
|
||||
enabled,
|
||||
policy_mode: policyMode,
|
||||
baseline_fps: Number.isFinite(fps) && fps > 0 ? fps : 1,
|
||||
keep_screenshots: keepScreenshots,
|
||||
allowlist: allowlistText
|
||||
.split('\n')
|
||||
.map(v => v.trim())
|
||||
@@ -279,6 +282,18 @@ const ScreenIntelligencePanel = () => {
|
||||
/>
|
||||
</label>
|
||||
|
||||
<label className="flex items-center justify-between rounded-xl border border-stone-200 bg-stone-50 px-3 py-2">
|
||||
<div>
|
||||
<span className="text-sm text-stone-700">Keep Screenshots</span>
|
||||
<p className="text-xs text-stone-400">Save captured screenshots to the workspace instead of deleting after processing</p>
|
||||
</div>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={keepScreenshots}
|
||||
onChange={event => setKeepScreenshots(event.target.checked)}
|
||||
/>
|
||||
</label>
|
||||
|
||||
<div className="space-y-1">
|
||||
<div className="text-xs text-stone-600">Allowlist (one rule per line)</div>
|
||||
<textarea
|
||||
|
||||
@@ -46,6 +46,7 @@ const status: AccessibilityStatus = {
|
||||
session_ttl_secs: 300,
|
||||
panic_stop_hotkey: 'Cmd+Shift+.',
|
||||
autocomplete_enabled: true,
|
||||
keep_screenshots: false,
|
||||
allowlist: [],
|
||||
denylist: ['wallet'],
|
||||
},
|
||||
|
||||
@@ -78,6 +78,7 @@ const baseStatus: AccessibilityStatus = {
|
||||
session_ttl_secs: 300,
|
||||
panic_stop_hotkey: 'Cmd+Shift+.',
|
||||
autocomplete_enabled: true,
|
||||
keep_screenshots: false,
|
||||
allowlist: ['Code'],
|
||||
denylist: ['1Password'],
|
||||
},
|
||||
@@ -188,6 +189,7 @@ describe('ScreenIntelligencePanel', () => {
|
||||
enabled: true,
|
||||
policy_mode: 'all_except_blacklist',
|
||||
baseline_fps: 1,
|
||||
keep_screenshots: false,
|
||||
allowlist: ['Code'],
|
||||
denylist: ['1Password'],
|
||||
});
|
||||
|
||||
@@ -41,6 +41,7 @@ function sampleAccessibilityStatus(
|
||||
session_ttl_secs: 300,
|
||||
panic_stop_hotkey: 'Cmd+Shift+.',
|
||||
autocomplete_enabled: true,
|
||||
keep_screenshots: false,
|
||||
allowlist: [],
|
||||
denylist: [],
|
||||
},
|
||||
|
||||
@@ -45,6 +45,7 @@ const sampleStatus: AccessibilityStatus = {
|
||||
session_ttl_secs: 300,
|
||||
panic_stop_hotkey: 'Cmd+Shift+.',
|
||||
autocomplete_enabled: true,
|
||||
keep_screenshots: false,
|
||||
allowlist: [],
|
||||
denylist: ['wallet'],
|
||||
},
|
||||
|
||||
@@ -793,6 +793,7 @@ export interface AccessibilityConfig {
|
||||
session_ttl_secs: number;
|
||||
panic_stop_hotkey: string;
|
||||
autocomplete_enabled: boolean;
|
||||
keep_screenshots: boolean;
|
||||
allowlist: string[];
|
||||
denylist: string[];
|
||||
}
|
||||
@@ -1039,6 +1040,7 @@ export interface ScreenIntelligenceSettingsUpdate {
|
||||
baseline_fps?: number | null;
|
||||
vision_enabled?: boolean | null;
|
||||
autocomplete_enabled?: boolean | null;
|
||||
keep_screenshots?: boolean | null;
|
||||
allowlist?: string[] | null;
|
||||
denylist?: string[] | null;
|
||||
}
|
||||
|
||||
@@ -55,6 +55,9 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> {
|
||||
"call" => run_call_command(&args[1..]),
|
||||
"repl" | "shell" => crate::core::repl::run_repl(&args[1..]),
|
||||
"skills" => crate::core::skills_cli::run_skills_command(&args[1..]),
|
||||
"screen-intelligence" => {
|
||||
crate::core::screen_intelligence_cli::run_screen_intelligence_command(&args[1..])
|
||||
}
|
||||
namespace => run_namespace_command(namespace, &args[1..], &grouped),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ pub mod jsonrpc;
|
||||
pub mod logging;
|
||||
pub mod repl;
|
||||
pub mod rpc_log;
|
||||
pub mod screen_intelligence_cli;
|
||||
pub mod skills_cli;
|
||||
pub mod socketio;
|
||||
pub mod types;
|
||||
|
||||
@@ -0,0 +1,511 @@
|
||||
//! `openhuman screen-intelligence` — standalone CLI for the screen intelligence loop.
|
||||
//!
|
||||
//! Boots **only** the screen intelligence engine (accessibility capture + local-AI
|
||||
//! vision) without the full desktop app, Socket.IO, or skills runtime. Useful for
|
||||
//! testing the capture → save → vision-analysis pipeline from a terminal.
|
||||
//!
|
||||
//! Usage:
|
||||
//! openhuman screen-intelligence run [--port <u16>] [-v]
|
||||
//! openhuman screen-intelligence status
|
||||
//! openhuman screen-intelligence capture [--keep]
|
||||
//! openhuman screen-intelligence start [--ttl <secs>] [-v]
|
||||
//! openhuman screen-intelligence stop
|
||||
|
||||
use anyhow::Result;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Entry point for `openhuman screen-intelligence <subcommand>`.
|
||||
pub fn run_screen_intelligence_command(args: &[String]) -> Result<()> {
|
||||
if args.is_empty() || is_help(&args[0]) {
|
||||
print_help();
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match args[0].as_str() {
|
||||
"run" => run_server(&args[1..]),
|
||||
"status" => run_status(&args[1..]),
|
||||
"capture" => run_capture(&args[1..]),
|
||||
"start" => run_start_session(&args[1..]),
|
||||
"stop" => run_stop_session(&args[1..]),
|
||||
other => Err(anyhow::anyhow!(
|
||||
"unknown screen-intelligence subcommand '{other}'. Run `openhuman screen-intelligence --help`."
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct CliOpts {
|
||||
port: u16,
|
||||
verbose: bool,
|
||||
ttl_secs: u64,
|
||||
keep: bool,
|
||||
}
|
||||
|
||||
fn parse_opts(args: &[String]) -> Result<(CliOpts, Vec<String>)> {
|
||||
let mut port: u16 = 7797;
|
||||
let mut verbose = false;
|
||||
let mut ttl_secs: u64 = 300;
|
||||
let mut keep = false;
|
||||
let mut rest = Vec::new();
|
||||
let mut i = 0;
|
||||
|
||||
while i < args.len() {
|
||||
match args[i].as_str() {
|
||||
"--port" => {
|
||||
let val = args
|
||||
.get(i + 1)
|
||||
.ok_or_else(|| anyhow::anyhow!("missing value for --port"))?;
|
||||
port = val
|
||||
.parse()
|
||||
.map_err(|e| anyhow::anyhow!("invalid --port: {e}"))?;
|
||||
i += 2;
|
||||
}
|
||||
"--ttl" => {
|
||||
let val = args
|
||||
.get(i + 1)
|
||||
.ok_or_else(|| anyhow::anyhow!("missing value for --ttl"))?;
|
||||
ttl_secs = val
|
||||
.parse()
|
||||
.map_err(|e| anyhow::anyhow!("invalid --ttl: {e}"))?;
|
||||
i += 2;
|
||||
}
|
||||
"--keep" => {
|
||||
keep = true;
|
||||
i += 1;
|
||||
}
|
||||
"-v" | "--verbose" => {
|
||||
verbose = true;
|
||||
i += 1;
|
||||
}
|
||||
"-h" | "--help" => {
|
||||
rest.push(args[i].clone());
|
||||
i += 1;
|
||||
}
|
||||
_ => {
|
||||
rest.push(args[i].clone());
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((
|
||||
CliOpts {
|
||||
port,
|
||||
verbose,
|
||||
ttl_secs,
|
||||
keep,
|
||||
},
|
||||
rest,
|
||||
))
|
||||
}
|
||||
|
||||
/// Bootstrap the screen intelligence engine with config.
|
||||
async fn bootstrap_engine(
|
||||
verbose: bool,
|
||||
) -> Result<Arc<crate::openhuman::screen_intelligence::AccessibilityEngine>> {
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::screen_intelligence::global_engine;
|
||||
|
||||
let config = Config::load_or_init()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("config load failed: {e}"))?;
|
||||
|
||||
let engine = global_engine();
|
||||
let _ = engine
|
||||
.apply_config(config.screen_intelligence.clone())
|
||||
.await;
|
||||
|
||||
if verbose {
|
||||
log::info!(
|
||||
"[screen-intelligence-cli] engine initialized, enabled={}, vision={}, keep_screenshots={}, workspace={}",
|
||||
config.screen_intelligence.enabled,
|
||||
config.screen_intelligence.vision_enabled,
|
||||
config.screen_intelligence.keep_screenshots,
|
||||
config.workspace_dir.display(),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(engine)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subcommands
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `openhuman screen-intelligence run` — start a minimal JSON-RPC server with the
|
||||
/// screen intelligence engine, useful for integration testing.
|
||||
fn run_server(args: &[String]) -> Result<()> {
|
||||
let (opts, rest) = parse_opts(args)?;
|
||||
|
||||
if rest.iter().any(|a| is_help(a)) {
|
||||
println!("Usage: openhuman screen-intelligence run [--port <u16>] [-v]");
|
||||
println!();
|
||||
println!("Start a lightweight JSON-RPC server exposing screen intelligence RPC methods.");
|
||||
println!();
|
||||
println!(" --port <u16> Listen port (default: 7797)");
|
||||
println!(" -v, --verbose Enable debug logging");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
crate::core::logging::init_for_cli_run(opts.verbose);
|
||||
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
rt.block_on(async {
|
||||
let _engine = bootstrap_engine(opts.verbose).await?;
|
||||
|
||||
let app = build_router();
|
||||
|
||||
let bind_addr = format!("127.0.0.1:{}", opts.port);
|
||||
let listener = tokio::net::TcpListener::bind(&bind_addr).await?;
|
||||
|
||||
log::info!(
|
||||
"[screen-intelligence-cli] ready — http://{bind_addr}/rpc (JSON-RPC 2.0)"
|
||||
);
|
||||
|
||||
eprintln!();
|
||||
eprintln!(" Screen intelligence dev server listening on http://{bind_addr}");
|
||||
eprintln!(" JSON-RPC endpoint: POST http://{bind_addr}/rpc");
|
||||
eprintln!(" Health check: GET http://{bind_addr}/health");
|
||||
eprintln!(" Press Ctrl+C to stop.");
|
||||
eprintln!();
|
||||
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// `openhuman screen-intelligence status` — print current engine status as JSON.
|
||||
fn run_status(args: &[String]) -> Result<()> {
|
||||
if args.iter().any(|a| is_help(a)) {
|
||||
println!("Usage: openhuman screen-intelligence status [-v]");
|
||||
println!();
|
||||
println!("Print current screen intelligence engine status (permissions, session, config).");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (opts, _) = parse_opts(args)?;
|
||||
init_quiet_logging(opts.verbose);
|
||||
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
rt.block_on(async {
|
||||
let engine = bootstrap_engine(opts.verbose).await?;
|
||||
let status = engine.status().await;
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&status).unwrap_or_else(|_| format!("{:?}", status))
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// `openhuman screen-intelligence capture` — take a single screenshot and print info.
|
||||
fn run_capture(args: &[String]) -> Result<()> {
|
||||
if args.iter().any(|a| is_help(a)) {
|
||||
println!("Usage: openhuman screen-intelligence capture [--keep] [-v]");
|
||||
println!();
|
||||
println!("Take a single screenshot, optionally save to workspace, and print diagnostics.");
|
||||
println!();
|
||||
println!(" --keep Save the screenshot to {{workspace}}/screenshots/");
|
||||
println!(" -v, --verbose Enable debug logging");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (opts, _) = parse_opts(args)?;
|
||||
init_quiet_logging(opts.verbose);
|
||||
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
rt.block_on(async {
|
||||
let engine = bootstrap_engine(opts.verbose).await?;
|
||||
let result = engine.capture_test().await;
|
||||
|
||||
if result.ok {
|
||||
eprintln!(" Capture: OK");
|
||||
eprintln!(" Mode: {}", result.capture_mode);
|
||||
eprintln!(" Timing: {}ms", result.timing_ms);
|
||||
if let Some(bytes) = result.bytes_estimate {
|
||||
eprintln!(" Size: {} bytes", bytes);
|
||||
}
|
||||
if let Some(ctx) = &result.context {
|
||||
eprintln!(
|
||||
" App: {}",
|
||||
ctx.app_name.as_deref().unwrap_or("unknown")
|
||||
);
|
||||
eprintln!(
|
||||
" Window: {}",
|
||||
ctx.window_title.as_deref().unwrap_or("unknown")
|
||||
);
|
||||
}
|
||||
|
||||
// Save to disk if --keep
|
||||
if opts.keep {
|
||||
if let Some(image_ref) = &result.image_ref {
|
||||
let config = crate::openhuman::config::Config::load_or_init()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("config load failed: {e}"))?;
|
||||
|
||||
let frame = crate::openhuman::screen_intelligence::CaptureFrame {
|
||||
captured_at_ms: chrono::Utc::now().timestamp_millis(),
|
||||
reason: "cli_capture".to_string(),
|
||||
app_name: result
|
||||
.context
|
||||
.as_ref()
|
||||
.and_then(|c| c.app_name.clone()),
|
||||
window_title: result
|
||||
.context
|
||||
.as_ref()
|
||||
.and_then(|c| c.window_title.clone()),
|
||||
image_ref: Some(image_ref.clone()),
|
||||
};
|
||||
|
||||
match crate::openhuman::screen_intelligence::AccessibilityEngine::save_screenshot_to_disk(
|
||||
&config.workspace_dir,
|
||||
&frame,
|
||||
) {
|
||||
Ok(path) => {
|
||||
eprintln!(" Saved: {}", path.display());
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" Save failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
eprintln!(" Capture: FAILED");
|
||||
if let Some(err) = &result.error {
|
||||
eprintln!(" Error: {err}");
|
||||
}
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// Also print as JSON for machine-readable output.
|
||||
let mut json_result = serde_json::to_value(&result).unwrap_or_default();
|
||||
// Strip image_ref from JSON output (too large for terminal).
|
||||
if let Some(obj) = json_result.as_object_mut() {
|
||||
obj.remove("image_ref");
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json_result).unwrap_or_default()
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// `openhuman screen-intelligence start` — start a capture + vision session.
|
||||
fn run_start_session(args: &[String]) -> Result<()> {
|
||||
if args.iter().any(|a| is_help(a)) {
|
||||
println!("Usage: openhuman screen-intelligence start [--ttl <secs>] [-v]");
|
||||
println!();
|
||||
println!("Start a screen intelligence capture session with vision analysis.");
|
||||
println!("The session runs until TTL expires or Ctrl+C is pressed.");
|
||||
println!();
|
||||
println!(" --ttl <secs> Session duration (default: 300, max: 3600)");
|
||||
println!(" -v, --verbose Enable debug logging");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (opts, _) = parse_opts(args)?;
|
||||
crate::core::logging::init_for_cli_run(opts.verbose);
|
||||
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
rt.block_on(async {
|
||||
let engine = bootstrap_engine(opts.verbose).await?;
|
||||
|
||||
let params = crate::openhuman::screen_intelligence::StartSessionParams {
|
||||
consent: true,
|
||||
ttl_secs: Some(opts.ttl_secs),
|
||||
screen_monitoring: Some(true),
|
||||
device_control: Some(false),
|
||||
predictive_input: Some(false),
|
||||
};
|
||||
|
||||
match engine.start_session(params).await {
|
||||
Ok(session) => {
|
||||
eprintln!(" Session started!");
|
||||
eprintln!(" TTL: {}s", session.ttl_secs);
|
||||
eprintln!(" Vision: {}", session.vision_enabled);
|
||||
eprintln!(" Panic hotkey: {}", session.panic_hotkey);
|
||||
eprintln!();
|
||||
eprintln!(" Capturing screenshots and running vision analysis...");
|
||||
eprintln!(" Press Ctrl+C to stop.");
|
||||
eprintln!();
|
||||
|
||||
// Print periodic status updates until the session ends.
|
||||
let mut tick = tokio::time::interval(std::time::Duration::from_secs(5));
|
||||
loop {
|
||||
tick.tick().await;
|
||||
let status = engine.status().await;
|
||||
if !status.session.active {
|
||||
eprintln!(
|
||||
"\n Session ended: {}",
|
||||
status.session.stop_reason.unwrap_or_else(|| "unknown".into())
|
||||
);
|
||||
break;
|
||||
}
|
||||
eprintln!(
|
||||
" [{}] captures={} vision={} queue={} last_app={:?}",
|
||||
chrono::Utc::now().format("%H:%M:%S"),
|
||||
status.session.capture_count,
|
||||
status.session.vision_state,
|
||||
status.session.vision_queue_depth,
|
||||
status.session.last_context.as_deref().unwrap_or("-"),
|
||||
);
|
||||
if let Some(summary) = &status.session.last_vision_summary {
|
||||
let truncated = if summary.len() > 100 {
|
||||
format!("{}…", &summary[..100])
|
||||
} else {
|
||||
summary.clone()
|
||||
};
|
||||
eprintln!(" notes: {truncated}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" Failed to start session: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// `openhuman screen-intelligence stop` — stop an active session.
|
||||
fn run_stop_session(args: &[String]) -> Result<()> {
|
||||
if args.iter().any(|a| is_help(a)) {
|
||||
println!("Usage: openhuman screen-intelligence stop [-v]");
|
||||
println!();
|
||||
println!("Stop the active screen intelligence session.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (opts, _) = parse_opts(args)?;
|
||||
init_quiet_logging(opts.verbose);
|
||||
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
rt.block_on(async {
|
||||
let engine = bootstrap_engine(opts.verbose).await?;
|
||||
let session = engine.stop_session(Some("cli_stop".to_string())).await;
|
||||
eprintln!(
|
||||
" Session stopped: {}",
|
||||
session
|
||||
.stop_reason
|
||||
.unwrap_or_else(|| "no active session".into())
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Minimal HTTP router
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn build_router() -> axum::Router {
|
||||
use axum::routing::{get, post};
|
||||
|
||||
axum::Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/rpc", post(rpc))
|
||||
.route("/status", get(status_endpoint))
|
||||
}
|
||||
|
||||
async fn health() -> impl axum::response::IntoResponse {
|
||||
axum::Json(serde_json::json!({ "ok": true, "mode": "screen-intelligence-dev" }))
|
||||
}
|
||||
|
||||
async fn rpc(
|
||||
axum::Json(req): axum::Json<crate::core::types::RpcRequest>,
|
||||
) -> axum::response::Response {
|
||||
use crate::core::types::{RpcError, RpcFailure, RpcSuccess};
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
let id = req.id.clone();
|
||||
let state = crate::core::jsonrpc::default_state();
|
||||
|
||||
match crate::core::jsonrpc::invoke_method(state, req.method.as_str(), req.params).await {
|
||||
Ok(value) => (
|
||||
axum::http::StatusCode::OK,
|
||||
axum::Json(RpcSuccess {
|
||||
jsonrpc: "2.0",
|
||||
id,
|
||||
result: value,
|
||||
}),
|
||||
)
|
||||
.into_response(),
|
||||
Err(message) => (
|
||||
axum::http::StatusCode::OK,
|
||||
axum::Json(RpcFailure {
|
||||
jsonrpc: "2.0",
|
||||
id,
|
||||
error: RpcError {
|
||||
code: -32000,
|
||||
message,
|
||||
data: None,
|
||||
},
|
||||
}),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn status_endpoint() -> impl axum::response::IntoResponse {
|
||||
let engine = crate::openhuman::screen_intelligence::global_engine();
|
||||
let status = engine.status().await;
|
||||
axum::Json(serde_json::to_value(&status).unwrap_or_default())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn init_quiet_logging(verbose: bool) {
|
||||
if !verbose && std::env::var_os("RUST_LOG").is_none() {
|
||||
std::env::set_var("RUST_LOG", "warn");
|
||||
}
|
||||
crate::core::logging::init_for_cli_run(verbose);
|
||||
}
|
||||
|
||||
fn is_help(value: &str) -> bool {
|
||||
matches!(value, "-h" | "--help" | "help")
|
||||
}
|
||||
|
||||
fn print_help() {
|
||||
println!("openhuman screen-intelligence — screen intelligence runtime\n");
|
||||
println!("Usage:");
|
||||
println!(" openhuman screen-intelligence run [--port <u16>] [-v]");
|
||||
println!(" openhuman screen-intelligence status [-v]");
|
||||
println!(" openhuman screen-intelligence capture [--keep] [-v]");
|
||||
println!(" openhuman screen-intelligence start [--ttl <secs>] [-v]");
|
||||
println!(" openhuman screen-intelligence stop [-v]");
|
||||
println!();
|
||||
println!("Subcommands:");
|
||||
println!(" run Start a lightweight JSON-RPC server with screen intelligence");
|
||||
println!(" status Print current engine status (permissions, session, config)");
|
||||
println!(" capture Take a single screenshot and print diagnostics");
|
||||
println!(" start Start a capture + vision session (runs until TTL or Ctrl+C)");
|
||||
println!(" stop Stop the active session");
|
||||
println!();
|
||||
println!("Common options:");
|
||||
println!(" --port <u16> Server port for 'run' (default: 7797)");
|
||||
println!(" --ttl <secs> Session TTL for 'start' (default: 300)");
|
||||
println!(" --keep Save screenshot to disk (for 'capture')");
|
||||
println!(" -v, --verbose Enable debug logging");
|
||||
}
|
||||
@@ -164,6 +164,7 @@ pub struct ScreenIntelligenceSettingsPatch {
|
||||
pub baseline_fps: Option<f32>,
|
||||
pub vision_enabled: Option<bool>,
|
||||
pub autocomplete_enabled: Option<bool>,
|
||||
pub keep_screenshots: Option<bool>,
|
||||
pub allowlist: Option<Vec<String>>,
|
||||
pub denylist: Option<Vec<String>>,
|
||||
}
|
||||
@@ -281,6 +282,9 @@ pub async fn apply_screen_intelligence_settings(
|
||||
if let Some(autocomplete_enabled) = update.autocomplete_enabled {
|
||||
config.screen_intelligence.autocomplete_enabled = autocomplete_enabled;
|
||||
}
|
||||
if let Some(keep_screenshots) = update.keep_screenshots {
|
||||
config.screen_intelligence.keep_screenshots = keep_screenshots;
|
||||
}
|
||||
if let Some(allowlist) = update.allowlist {
|
||||
config.screen_intelligence.allowlist = allowlist;
|
||||
}
|
||||
|
||||
@@ -19,6 +19,10 @@ pub struct ScreenIntelligenceConfig {
|
||||
pub panic_stop_hotkey: String,
|
||||
#[serde(default = "default_autocomplete_enabled")]
|
||||
pub autocomplete_enabled: bool,
|
||||
/// When `true`, captured screenshots are saved to `{workspace_dir}/screenshots/`
|
||||
/// instead of being discarded after vision processing. Default: `false`.
|
||||
#[serde(default)]
|
||||
pub keep_screenshots: bool,
|
||||
#[serde(default)]
|
||||
pub allowlist: Vec<String>,
|
||||
#[serde(default)]
|
||||
@@ -68,6 +72,7 @@ impl Default for ScreenIntelligenceConfig {
|
||||
session_ttl_secs: default_session_ttl_secs(),
|
||||
panic_stop_hotkey: default_panic_stop_hotkey(),
|
||||
autocomplete_enabled: default_autocomplete_enabled(),
|
||||
keep_screenshots: false,
|
||||
allowlist: vec![],
|
||||
denylist: vec![
|
||||
"1password".to_string(),
|
||||
|
||||
@@ -45,6 +45,7 @@ struct ScreenIntelligenceSettingsUpdate {
|
||||
baseline_fps: Option<f32>,
|
||||
vision_enabled: Option<bool>,
|
||||
autocomplete_enabled: Option<bool>,
|
||||
keep_screenshots: Option<bool>,
|
||||
allowlist: Option<Vec<String>>,
|
||||
denylist: Option<Vec<String>>,
|
||||
}
|
||||
@@ -240,6 +241,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
||||
},
|
||||
optional_bool("vision_enabled", "Enable vision analysis."),
|
||||
optional_bool("autocomplete_enabled", "Enable autocomplete integration."),
|
||||
optional_bool("keep_screenshots", "Keep screenshots on disk after vision processing."),
|
||||
FieldSchema {
|
||||
name: "allowlist",
|
||||
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(
|
||||
@@ -481,6 +483,7 @@ fn handle_update_screen_intelligence_settings(params: Map<String, Value>) -> Con
|
||||
baseline_fps: update.baseline_fps,
|
||||
vision_enabled: update.vision_enabled,
|
||||
autocomplete_enabled: update.autocomplete_enabled,
|
||||
keep_screenshots: update.keep_screenshots,
|
||||
allowlist: update.allowlist,
|
||||
denylist: update.denylist,
|
||||
};
|
||||
|
||||
@@ -2,6 +2,7 @@ use crate::openhuman::config::{Config, ScreenIntelligenceConfig};
|
||||
use crate::openhuman::local_ai;
|
||||
use once_cell::sync::Lazy;
|
||||
use std::collections::VecDeque;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::task::JoinHandle;
|
||||
@@ -832,6 +833,54 @@ impl AccessibilityEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Save a screenshot PNG to `{workspace_dir}/screenshots/{timestamp}_{app}.png`.
|
||||
/// Returns the file path on success.
|
||||
pub fn save_screenshot_to_disk(
|
||||
workspace_dir: &std::path::Path,
|
||||
frame: &CaptureFrame,
|
||||
) -> Result<PathBuf, String> {
|
||||
use base64::{engine::general_purpose::STANDARD as B64, Engine};
|
||||
|
||||
let image_ref = frame
|
||||
.image_ref
|
||||
.as_deref()
|
||||
.ok_or_else(|| "frame has no image payload".to_string())?;
|
||||
|
||||
let b64_payload = if let Some(pos) = image_ref.find(";base64,") {
|
||||
&image_ref[pos + 8..]
|
||||
} else {
|
||||
image_ref
|
||||
};
|
||||
|
||||
let raw_bytes = B64
|
||||
.decode(b64_payload)
|
||||
.map_err(|e| format!("base64 decode for screenshot save failed: {e}"))?;
|
||||
|
||||
let screenshots_dir = workspace_dir.join("screenshots");
|
||||
std::fs::create_dir_all(&screenshots_dir)
|
||||
.map_err(|e| format!("failed to create screenshots dir: {e}"))?;
|
||||
|
||||
let app_slug = frame
|
||||
.app_name
|
||||
.as_deref()
|
||||
.unwrap_or("unknown")
|
||||
.chars()
|
||||
.map(|c| if c.is_alphanumeric() || c == '-' { c } else { '_' })
|
||||
.collect::<String>();
|
||||
let filename = format!("{}_{}.png", frame.captured_at_ms, app_slug);
|
||||
let file_path = screenshots_dir.join(&filename);
|
||||
|
||||
std::fs::write(&file_path, &raw_bytes)
|
||||
.map_err(|e| format!("failed to write screenshot {filename}: {e}"))?;
|
||||
|
||||
tracing::debug!(
|
||||
"[screen_intelligence] screenshot saved: {} ({} bytes)",
|
||||
file_path.display(),
|
||||
raw_bytes.len()
|
||||
);
|
||||
Ok(file_path)
|
||||
}
|
||||
|
||||
async fn run_vision_worker(
|
||||
self: Arc<Self>,
|
||||
mut rx: tokio::sync::mpsc::UnboundedReceiver<CaptureFrame>,
|
||||
@@ -844,6 +893,32 @@ impl AccessibilityEngine {
|
||||
frame.app_name,
|
||||
frame.reason
|
||||
);
|
||||
|
||||
// Read config for keep_screenshots and workspace_dir.
|
||||
let (keep_screenshots, workspace_dir) = match Config::load_or_init().await {
|
||||
Ok(cfg) => (
|
||||
cfg.screen_intelligence.keep_screenshots,
|
||||
cfg.workspace_dir.clone(),
|
||||
),
|
||||
Err(_) => (false, PathBuf::from(".")),
|
||||
};
|
||||
|
||||
// Save screenshot to disk (always — we need it on disk for the workspace).
|
||||
// If keep_screenshots is false, we delete after vision processing.
|
||||
let saved_path = if frame.image_ref.is_some() {
|
||||
match Self::save_screenshot_to_disk(&workspace_dir, &frame) {
|
||||
Ok(path) => Some(path),
|
||||
Err(err) => {
|
||||
tracing::debug!(
|
||||
"[screen_intelligence] screenshot save failed (non-fatal): {err}"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
{
|
||||
let mut state = self.inner.lock().await;
|
||||
if let Some(session) = state.session.as_mut() {
|
||||
@@ -856,6 +931,18 @@ impl AccessibilityEngine {
|
||||
|
||||
let result = self.analyze_frame_with_vision(frame).await;
|
||||
|
||||
// Clean up screenshot file if keep_screenshots is false.
|
||||
if !keep_screenshots {
|
||||
if let Some(path) = saved_path {
|
||||
if let Err(err) = std::fs::remove_file(&path) {
|
||||
tracing::trace!(
|
||||
"[screen_intelligence] failed to remove temp screenshot {}: {err}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut state = self.inner.lock().await;
|
||||
let Some(session) = state.session.as_mut() else {
|
||||
break;
|
||||
|
||||
Reference in New Issue
Block a user