diff --git a/src/openhuman/screen_intelligence/engine.rs b/src/openhuman/screen_intelligence/engine.rs index 11eef260a..ad9bb7861 100644 --- a/src/openhuman/screen_intelligence/engine.rs +++ b/src/openhuman/screen_intelligence/engine.rs @@ -46,6 +46,9 @@ struct SessionRuntime { vision_queue_depth: usize, last_vision_at_ms: Option, last_vision_summary: Option, + vision_persist_count: u64, + last_vision_persisted_key: Option, + last_vision_persist_error: Option, vision_summaries: VecDeque, vision_task: Option>, vision_tx: Option>, @@ -154,6 +157,9 @@ impl AccessibilityEngine { vision_queue_depth: 0, last_vision_at_ms: None, last_vision_summary: None, + vision_persist_count: 0, + last_vision_persisted_key: None, + last_vision_persist_error: None, vision_summaries: VecDeque::new(), vision_task: None, vision_tx: None, @@ -241,6 +247,9 @@ impl AccessibilityEngine { vision_queue_depth: session.vision_queue_depth, last_vision_at_ms: session.last_vision_at_ms, last_vision_summary: session.last_vision_summary.clone(), + vision_persist_count: session.vision_persist_count, + last_vision_persisted_key: session.last_vision_persisted_key.clone(), + last_vision_persist_error: session.last_vision_persist_error.clone(), }, None => SessionStatus { active: false, @@ -260,6 +269,9 @@ impl AccessibilityEngine { vision_queue_depth: 0, last_vision_at_ms: None, last_vision_summary: None, + vision_persist_count: 0, + last_vision_persisted_key: None, + last_vision_persist_error: None, }, }; @@ -373,9 +385,16 @@ impl AccessibilityEngine { return Err("accessibility permission is not granted".to_string()); } + let screen_monitoring_requested = params.screen_monitoring.unwrap_or(true); + if screen_monitoring_requested + && state.permissions.screen_recording != PermissionState::Granted + { + return Err("screen recording permission is not granted".to_string()); + } + let now = now_ms(); let expires_at_ms = now + (ttl_secs as i64 * 1000); - state.features.screen_monitoring = params.screen_monitoring.unwrap_or(true); + state.features.screen_monitoring = screen_monitoring_requested; state.features.device_control = params.device_control.unwrap_or(true); state.features.predictive_input = params .predictive_input @@ -392,11 +411,14 @@ impl AccessibilityEngine { frames: VecDeque::new(), last_context: None, task: None, - vision_enabled: true, + vision_enabled: state.config.vision_enabled, vision_state: "idle".to_string(), vision_queue_depth: 0, last_vision_at_ms: None, last_vision_summary: None, + vision_persist_count: 0, + last_vision_persisted_key: None, + last_vision_persist_error: None, vision_summaries: VecDeque::new(), vision_task: None, vision_tx: None, @@ -664,16 +686,75 @@ impl AccessibilityEngine { }); }; - let summary = self - .analyze_frame_with_vision(frame) + let summary = match self.analyze_frame_with_vision(frame).await { + Ok(summary) => summary, + Err(err) => { + let mut state = self.inner.lock().await; + if let Some(session) = state.session.as_mut() { + session.vision_queue_depth = session.vision_queue_depth.saturating_sub(1); + session.vision_state = "error".to_string(); + } + state.last_error = Some(format!("vision_flush_analysis_failed: {err}")); + return Err(format!("vision flush failed: {err}")); + } + }; + + let persist = persist_vision_summary(summary.clone()) .await - .map_err(|e| format!("vision flush failed: {e}"))?; + .map_err(|err| format!("vision summary persistence failed: {err}")); + + { + let mut state = self.inner.lock().await; + if let Some(session) = state.session.as_mut() { + session.vision_queue_depth = session.vision_queue_depth.saturating_sub(1); + push_ephemeral_vision_summary(&mut session.vision_summaries, summary.clone()); + session.last_vision_at_ms = Some(summary.captured_at_ms); + session.last_vision_summary = Some(summary.actionable_notes.clone()); + match &persist { + Ok(result) => { + session.vision_state = "ready".to_string(); + session.vision_persist_count = + session.vision_persist_count.saturating_add(1); + session.last_vision_persisted_key = Some(result.key.clone()); + session.last_vision_persist_error = None; + } + Err(err) => { + session.vision_state = "error".to_string(); + session.last_vision_persist_error = Some(err.clone()); + state.last_error = Some(format!("vision_flush_persist_failed: {err}")); + } + } + } + } + + if let Err(err) = persist { + return Err(format!("vision flush failed: {err}")); + } + Ok(VisionFlushResult { accepted: true, summary: Some(summary), }) } + /// Deterministic pipeline hook used by tests and diagnostics: + /// analyze one frame with the local vision model and persist the summary to memory. + pub async fn analyze_and_persist_frame( + &self, + frame: CaptureFrame, + ) -> Result { + let summary = self.analyze_frame_with_vision(frame).await?; + let persisted = persist_vision_summary(summary.clone()) + .await + .map_err(|err| format!("vision summary persistence failed: {err}"))?; + tracing::debug!( + "[screen_intelligence] analyze_and_persist_frame completed (namespace={} key={})", + persisted.namespace, + persisted.key + ); + Ok(summary) + } + /// Standalone capture test — works without an active session. /// Returns rich diagnostics for the debug UI. pub async fn capture_test(&self) -> CaptureTestResult { @@ -949,31 +1030,59 @@ impl AccessibilityEngine { } } - let mut state = self.inner.lock().await; - let Some(session) = state.session.as_mut() else { - break; - }; - session.vision_queue_depth = session.vision_queue_depth.saturating_sub(1); - match result { - Ok(summary) => { - tracing::debug!( - "[screen_intelligence] vision analysis complete: confidence={:.2} notes={}", - summary.confidence, - &summary.actionable_notes[..summary.actionable_notes.len().min(80)] - ); - push_ephemeral_vision_summary(&mut session.vision_summaries, summary.clone()); - session.last_vision_at_ms = Some(summary.captured_at_ms); - session.last_vision_summary = Some(summary.actionable_notes.clone()); - session.vision_state = "ready".to_string(); - let summary_for_store = summary.clone(); - tokio::spawn(async move { - persist_vision_summary(summary_for_store).await; - }); + let mut summary_to_persist: Option = None; + { + let mut state = self.inner.lock().await; + let Some(session) = state.session.as_mut() else { + break; + }; + session.vision_queue_depth = session.vision_queue_depth.saturating_sub(1); + match result { + Ok(summary) => { + tracing::debug!( + "[screen_intelligence] vision analysis complete (summary_id={} confidence={:.2})", + summary.id, + summary.confidence + ); + push_ephemeral_vision_summary( + &mut session.vision_summaries, + summary.clone(), + ); + session.last_vision_at_ms = Some(summary.captured_at_ms); + session.last_vision_summary = Some(summary.actionable_notes.clone()); + session.vision_state = "ready".to_string(); + summary_to_persist = Some(summary); + } + Err(err) => { + tracing::debug!("[screen_intelligence] vision analysis failed: {err}"); + session.vision_state = "error".to_string(); + state.last_error = Some(err); + } } - Err(err) => { - tracing::debug!("[screen_intelligence] vision analysis failed: {err}"); - session.vision_state = "error".to_string(); - state.last_error = Some(err); + } + + if let Some(summary_for_store) = summary_to_persist { + match persist_vision_summary(summary_for_store).await { + Ok(persisted) => { + let mut state = self.inner.lock().await; + if let Some(session) = state.session.as_mut() { + session.vision_persist_count = + session.vision_persist_count.saturating_add(1); + session.last_vision_persisted_key = Some(persisted.key.clone()); + session.last_vision_persist_error = None; + } + } + Err(err) => { + tracing::debug!( + "[screen_intelligence] vision summary persistence failed: {err}" + ); + let mut state = self.inner.lock().await; + if let Some(session) = state.session.as_mut() { + session.vision_state = "error".to_string(); + session.last_vision_persist_error = Some(err.clone()); + } + state.last_error = Some(format!("vision_summary_persist_failed: {err}")); + } } } } @@ -1010,6 +1119,33 @@ impl AccessibilityEngine { let config = Config::load_or_init() .await .map_err(|e| format!("failed to load config: {e}"))?; + if !config.local_ai.enabled { + return Err( + "screen intelligence vision requires local_ai.enabled=true in config".to_string(), + ); + } + let provider = config.local_ai.provider.trim().to_ascii_lowercase(); + if provider != "ollama" { + return Err(format!( + "screen intelligence vision requires local provider 'ollama' (found '{}')", + config.local_ai.provider + )); + } + if let Ok(mock_raw) = std::env::var("OPENHUMAN_SCREEN_INTELLIGENCE_MOCK_VISION_JSON") { + if !mock_raw.trim().is_empty() { + tracing::debug!( + "[screen_intelligence] using mocked vision output from OPENHUMAN_SCREEN_INTELLIGENCE_MOCK_VISION_JSON" + ); + return Ok(parse_vision_summary_output(frame, &mock_raw)); + } + } + + tracing::debug!( + "[screen_intelligence] running local vision inference (provider={} model={} compressed_bytes={})", + provider, + config.local_ai.vision_model_id, + compressed.compressed_bytes + ); let service = local_ai::global(&config); let prompt = "Analyze this UI screenshot. Return strict JSON with keys: ui_state, key_text, actionable_notes, confidence (0..1). Keep actionable_notes concise."; let raw = service @@ -1091,6 +1227,9 @@ mod tests { vision_queue_depth: 0, last_vision_at_ms: None, last_vision_summary: None, + vision_persist_count: 0, + last_vision_persisted_key: None, + last_vision_persist_error: None, vision_summaries: VecDeque::new(), vision_task: None, vision_tx: None, diff --git a/src/openhuman/screen_intelligence/helpers.rs b/src/openhuman/screen_intelligence/helpers.rs index 08fbc6bf0..a10e9a7f3 100644 --- a/src/openhuman/screen_intelligence/helpers.rs +++ b/src/openhuman/screen_intelligence/helpers.rs @@ -7,6 +7,17 @@ use uuid::Uuid; use super::limits::{MAX_CONTEXT_CHARS, MAX_EPHEMERAL_FRAMES, MAX_EPHEMERAL_VISION_SUMMARIES}; use super::types::{AutocompleteSuggestion, CaptureFrame, InputActionParams, VisionSummary}; +pub(crate) const VISION_MEMORY_NAMESPACE: &str = "background"; +pub(crate) const VISION_MEMORY_SOURCE_TYPE: &str = "screenshot"; +pub(crate) const VISION_MEMORY_CATEGORY: &str = "screen_intelligence"; +pub(crate) const VISION_MEMORY_TAG: &str = "screen_intelligence"; + +#[derive(Debug, Clone)] +pub(crate) struct PersistVisionSummaryResult { + pub namespace: String, + pub key: String, +} + pub(crate) fn validate_input_action(action: &InputActionParams) -> Result<(), String> { match action.action.as_str() { "mouse_move" | "mouse_click" | "mouse_drag" => { @@ -106,12 +117,18 @@ pub(crate) fn parse_vision_summary_output(frame: CaptureFrame, raw: &str) -> Vis } } -pub(crate) async fn persist_vision_summary(summary: VisionSummary) { +pub(crate) async fn persist_vision_summary( + summary: VisionSummary, +) -> Result { let config = match Config::load_or_init().await { Ok(cfg) => cfg, Err(err) => { - tracing::debug!("vision summary persistence skipped: config load failed: {err}"); - return; + let message = format!("config load failed: {err}"); + tracing::debug!( + "[screen_intelligence] vision summary persistence skipped: {}", + message + ); + return Err(message); } }; @@ -128,35 +145,56 @@ pub(crate) async fn persist_vision_summary(summary: VisionSummary) { ) { Ok(mem) => mem, Err(err) => { - tracing::debug!("vision summary persistence skipped: memory init failed: {err}"); - return; + let message = format!("memory init failed: {err}"); + tracing::debug!( + "[screen_intelligence] vision summary persistence skipped: {}", + message + ); + return Err(message); } }; let content = match serde_json::to_string(&summary) { Ok(content) => content, Err(err) => { - tracing::debug!("vision summary persistence skipped: serialization failed: {err}"); - return; + let message = format!("serialization failed: {err}"); + tracing::debug!( + "[screen_intelligence] vision summary persistence skipped: {}", + message + ); + return Err(message); } }; let key = format!("screen_intelligence_{}", summary.id); - let _ = mem - .upsert_document(NamespaceDocumentInput { - namespace: "background".to_string(), - key: key.clone(), - title: key, - content, - source_type: "screenshot".to_string(), - priority: "medium".to_string(), - tags: vec!["screen_intelligence".to_string()], - metadata: serde_json::json!({}), - category: "screen_intelligence".to_string(), - session_id: None, - document_id: None, - }) - .await; + mem.upsert_document(NamespaceDocumentInput { + namespace: VISION_MEMORY_NAMESPACE.to_string(), + key: key.clone(), + title: key.clone(), + content, + source_type: VISION_MEMORY_SOURCE_TYPE.to_string(), + priority: "medium".to_string(), + tags: vec![VISION_MEMORY_TAG.to_string()], + metadata: serde_json::json!({}), + category: VISION_MEMORY_CATEGORY.to_string(), + session_id: None, + document_id: None, + }) + .await + .map_err(|err| format!("memory upsert failed: {err}"))?; + + tracing::debug!( + "[screen_intelligence] persisted vision summary into unified memory (namespace={} key={} app={:?} captured_at_ms={})", + VISION_MEMORY_NAMESPACE, + key, + summary.app_name, + summary.captured_at_ms + ); + + Ok(PersistVisionSummaryResult { + namespace: VISION_MEMORY_NAMESPACE.to_string(), + key, + }) } pub(crate) fn truncate_tail(text: &str, max_chars: usize) -> String { diff --git a/src/openhuman/screen_intelligence/tests.rs b/src/openhuman/screen_intelligence/tests.rs index a7cbb496b..2f1da3459 100644 --- a/src/openhuman/screen_intelligence/tests.rs +++ b/src/openhuman/screen_intelligence/tests.rs @@ -1,14 +1,101 @@ -use std::sync::Arc; +use std::path::Path; +use std::sync::{Arc, OnceLock}; use tokio::sync::Mutex; use tokio::time::{self, Duration}; +use base64::{engine::general_purpose::STANDARD as B64, Engine}; +use image::codecs::png::PngEncoder; +use image::{ImageBuffer, Rgb, RgbImage}; +use tempfile::tempdir; + use super::engine::{AccessibilityEngine, EngineState}; use super::helpers::{ generate_suggestions, parse_vision_summary_output, truncate_tail, validate_input_action, }; use super::types::{CaptureFrame, InputActionParams, StartSessionParams}; use crate::openhuman::accessibility::{parse_foreground_output, AppContext}; -use crate::openhuman::config::ScreenIntelligenceConfig; +use crate::openhuman::config::{Config, ScreenIntelligenceConfig}; +use crate::openhuman::memory::embeddings::NoopEmbedding; +use crate::openhuman::memory::store::UnifiedMemory; + +struct EnvVarGuard { + key: &'static str, + old: Option, +} + +impl EnvVarGuard { + fn set_to_path(key: &'static str, path: &Path) -> Self { + let old = std::env::var(key).ok(); + std::env::set_var(key, path.as_os_str()); + Self { key, old } + } + + fn set(key: &'static str, value: &str) -> Self { + let old = std::env::var(key).ok(); + std::env::set_var(key, value); + Self { key, old } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + match &self.old { + Some(value) => std::env::set_var(self.key, value), + None => std::env::remove_var(self.key), + } + } +} + +static SCREEN_INTELLIGENCE_ENV_LOCK: OnceLock> = OnceLock::new(); + +fn screen_intelligence_env_lock() -> std::sync::MutexGuard<'static, ()> { + match SCREEN_INTELLIGENCE_ENV_LOCK + .get_or_init(|| std::sync::Mutex::new(())) + .lock() + { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +fn write_screen_intelligence_test_config( + workspace_root: &Path, + local_ai_enabled: bool, + local_ai_provider: &str, +) { + let cfg = format!( + r#"default_temperature = 0.7 + +[memory] +backend = "sqlite" +auto_save = true +embedding_provider = "none" +embedding_model = "none" +embedding_dimensions = 0 + +[local_ai] +enabled = {local_ai_enabled} +provider = "{local_ai_provider}" + +[secrets] +encrypt = false +"# + ); + std::fs::create_dir_all(workspace_root).expect("mkdir test workspace root"); + let config_path = workspace_root.join("config.toml"); + std::fs::write(&config_path, &cfg).expect("write test config"); + let _: Config = toml::from_str(&cfg).expect("test config should deserialize"); +} + +fn make_test_png_uri(width: u32, height: u32) -> String { + let img: RgbImage = ImageBuffer::from_fn(width, height, |x, y| { + Rgb([(x % 255) as u8, (y % 255) as u8, ((x + y) % 255) as u8]) + }); + let mut png_bytes: Vec = Vec::new(); + img.write_with_encoder(PngEncoder::new(&mut png_bytes)) + .expect("PNG encode"); + format!("data:image/png;base64,{}", B64.encode(&png_bytes)) +} // ── parse_foreground_output ───────────────────────────────────────────── @@ -548,3 +635,187 @@ async fn capture_now_without_session_is_rejected_without_hanging() { "capture_now should not produce a frame without a session" ); } + +// ── save_screenshot_to_disk ───────────────────────────────────────────── + +#[test] +fn save_screenshot_to_disk_writes_png_to_workspace() { + use base64::{engine::general_purpose::STANDARD as B64, Engine}; + use image::codecs::png::PngEncoder; + use image::{ImageBuffer, Rgb, RgbImage}; + use tempfile::tempdir; + + let tmp = tempdir().expect("tempdir"); + + // Build a tiny 4x4 solid-colour PNG as data URI + let img: RgbImage = ImageBuffer::from_fn(4, 4, |_, _| Rgb([100u8, 149u8, 237u8])); + let mut png_bytes: Vec = Vec::new(); + img.write_with_encoder(PngEncoder::new(&mut png_bytes)) + .expect("PNG encode"); + let image_ref = format!("data:image/png;base64,{}", B64.encode(&png_bytes)); + + let frame = CaptureFrame { + captured_at_ms: 1700000000200, + reason: "unit_test_save".to_string(), + app_name: Some("UnitTestApp".to_string()), + window_title: Some("Test Window".to_string()), + image_ref: Some(image_ref), + }; + + let result = AccessibilityEngine::save_screenshot_to_disk(tmp.path(), &frame); + assert!( + result.is_ok(), + "save_screenshot_to_disk should succeed: {:?}", + result + ); + + let path = result.unwrap(); + assert!( + path.exists(), + "[screen_intelligence] saved PNG file should exist at {}", + path.display() + ); + assert_eq!( + path.extension().and_then(|e| e.to_str()), + Some("png"), + "saved file should have .png extension" + ); + let metadata = std::fs::metadata(&path).expect("file metadata"); + assert!(metadata.len() > 0, "saved PNG should not be empty"); + assert!( + path.to_string_lossy().contains("1700000000200"), + "filename should include capture timestamp" + ); +} + +#[test] +fn save_screenshot_to_disk_rejects_frame_without_image_ref() { + use tempfile::tempdir; + + let tmp = tempdir().expect("tempdir"); + + let frame = CaptureFrame { + captured_at_ms: 1700000000201, + reason: "unit_test_no_image".to_string(), + app_name: Some("TestApp".to_string()), + window_title: None, + image_ref: None, // no image payload + }; + + let result = AccessibilityEngine::save_screenshot_to_disk(tmp.path(), &frame); + assert!( + result.is_err(), + "[screen_intelligence] save_screenshot_to_disk should return Err when frame has no image_ref" + ); + let err = result.unwrap_err(); + assert!(!err.is_empty(), "error message should not be empty"); +} + +// ── deterministic vision pipeline (mocked local output) ──────────────────── + +#[tokio::test] +async fn analyze_and_persist_frame_writes_unified_memory_document() { + let _env_lock = screen_intelligence_env_lock(); + let tmp = tempdir().expect("tempdir"); + let _workspace = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _mock = EnvVarGuard::set( + "OPENHUMAN_SCREEN_INTELLIGENCE_MOCK_VISION_JSON", + r#"{"ui_state":"editor","key_text":"fn main() {}","actionable_notes":"Rust source is open","confidence":0.93}"#, + ); + write_screen_intelligence_test_config(tmp.path(), true, "ollama"); + + let engine = Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(ScreenIntelligenceConfig::default())), + }); + let frame = CaptureFrame { + captured_at_ms: 1700000000300, + reason: "pipeline_test".to_string(), + app_name: Some("PipelineApp".to_string()), + window_title: Some("Main.rs".to_string()), + image_ref: Some(make_test_png_uri(320, 200)), + }; + let summary = engine + .analyze_and_persist_frame(frame) + .await + .expect("analyze_and_persist_frame should succeed with mocked vision output"); + assert_eq!(summary.ui_state, "editor"); + assert_eq!(summary.actionable_notes, "Rust source is open"); + + let config = Config::load_or_init().await.expect("load config"); + let mem = UnifiedMemory::new( + &config.workspace_dir, + Arc::new(NoopEmbedding), + config.memory.sqlite_open_timeout_secs, + ) + .expect("memory init"); + let list = mem + .list_documents(Some("background")) + .await + .expect("list documents"); + let documents = list["documents"] + .as_array() + .expect("documents array should exist"); + let key = format!("screen_intelligence_{}", summary.id); + assert!( + documents + .iter() + .any(|doc| doc["key"].as_str() == Some(&key)), + "expected persisted vision summary key in background namespace: {key}" + ); +} + +#[tokio::test] +async fn analyze_and_persist_frame_rejects_non_local_provider() { + let _env_lock = screen_intelligence_env_lock(); + let tmp = tempdir().expect("tempdir"); + let _workspace = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", tmp.path()); + write_screen_intelligence_test_config(tmp.path(), true, "openai"); + + let engine = Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(ScreenIntelligenceConfig::default())), + }); + let frame = CaptureFrame { + captured_at_ms: 1700000000301, + reason: "provider_guard_test".to_string(), + app_name: Some("PipelineApp".to_string()), + window_title: Some("Guard".to_string()), + image_ref: Some(make_test_png_uri(160, 120)), + }; + + let err = engine + .analyze_and_persist_frame(frame) + .await + .expect_err("non-local providers should be rejected"); + assert!( + err.contains("provider 'ollama'"), + "unexpected error for non-local provider: {err}" + ); +} + +#[tokio::test] +async fn analyze_and_persist_frame_rejects_disabled_local_ai() { + let _env_lock = screen_intelligence_env_lock(); + let tmp = tempdir().expect("tempdir"); + let _workspace = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", tmp.path()); + write_screen_intelligence_test_config(tmp.path(), false, "ollama"); + + let engine = Arc::new(AccessibilityEngine { + inner: Mutex::new(EngineState::new(ScreenIntelligenceConfig::default())), + }); + let frame = CaptureFrame { + captured_at_ms: 1700000000302, + reason: "local_ai_disabled_test".to_string(), + app_name: Some("PipelineApp".to_string()), + window_title: Some("Guard".to_string()), + image_ref: Some(make_test_png_uri(160, 120)), + }; + + let err = engine + .analyze_and_persist_frame(frame) + .await + .expect_err("disabled local ai should be rejected"); + assert!( + err.contains("local_ai.enabled=true"), + "unexpected error when local ai is disabled: {err}" + ); +} diff --git a/src/openhuman/screen_intelligence/types.rs b/src/openhuman/screen_intelligence/types.rs index 580b090c5..d775f768b 100644 --- a/src/openhuman/screen_intelligence/types.rs +++ b/src/openhuman/screen_intelligence/types.rs @@ -31,6 +31,9 @@ pub struct SessionStatus { pub vision_queue_depth: usize, pub last_vision_at_ms: Option, pub last_vision_summary: Option, + pub vision_persist_count: u64, + pub last_vision_persisted_key: Option, + pub last_vision_persist_error: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/tests/json_rpc_e2e.rs b/tests/json_rpc_e2e.rs index a3a348954..062f52fee 100644 --- a/tests/json_rpc_e2e.rs +++ b/tests/json_rpc_e2e.rs @@ -789,6 +789,144 @@ async fn json_rpc_screen_intelligence_capture_test_returns_stable_shape() { rpc_join.abort(); } +#[tokio::test] +async fn json_rpc_screen_intelligence_status_returns_stable_shape() { + let _env_lock = json_rpc_e2e_env_lock(); + let tmp = tempdir().expect("tempdir"); + let home = tmp.path(); + let openhuman_home = home.join(".openhuman"); + + let _home_guard = EnvVarGuard::set_to_path("HOME", home); + let _workspace_guard = EnvVarGuard::unset("OPENHUMAN_WORKSPACE"); + let _backend_url_guard = EnvVarGuard::unset("BACKEND_URL"); + let _vite_backend_guard = EnvVarGuard::unset("VITE_BACKEND_URL"); + + let (mock_addr, mock_join) = serve_on_ephemeral(mock_upstream_router()).await; + let mock_origin = format!("http://{}", mock_addr); + write_min_config(&openhuman_home, &mock_origin); + + let (rpc_addr, rpc_join) = serve_on_ephemeral(build_core_http_router(false)).await; + let rpc_base = format!("http://{}", rpc_addr); + tokio::time::sleep(Duration::from_millis(100)).await; + + let status = post_json_rpc( + &rpc_base, + 1003, + "openhuman.screen_intelligence_status", + json!({}), + ) + .await; + let result = assert_no_jsonrpc_error(&status, "screen_intelligence_status"); + let status_result = result.get("result").unwrap_or(result); + + // Required top-level fields + assert!( + status_result + .get("platform_supported") + .and_then(Value::as_bool) + .is_some(), + "expected bool platform_supported: {status_result}" + ); + assert!( + status_result + .get("is_context_blocked") + .and_then(Value::as_bool) + .is_some(), + "expected bool is_context_blocked: {status_result}" + ); + + // session block + let session = status_result + .get("session") + .expect("expected session object"); + assert!( + session.get("active").and_then(Value::as_bool).is_some(), + "expected bool session.active: {status_result}" + ); + assert_eq!( + session.get("active").and_then(Value::as_bool), + Some(false), + "session should not be active without start_session: {status_result}" + ); + assert!( + session + .get("capture_count") + .and_then(Value::as_u64) + .is_some(), + "expected u64 session.capture_count: {status_result}" + ); + assert!( + session + .get("vision_persist_count") + .and_then(Value::as_u64) + .is_some(), + "expected u64 session.vision_persist_count: {status_result}" + ); + assert!( + session.get("last_vision_persist_error").is_some(), + "expected nullable session.last_vision_persist_error: {status_result}" + ); + + // permissions block + let perms = status_result + .get("permissions") + .expect("expected permissions object"); + assert!( + perms + .get("screen_recording") + .and_then(Value::as_str) + .is_some(), + "expected string permissions.screen_recording: {status_result}" + ); + + mock_join.abort(); + rpc_join.abort(); +} + +#[tokio::test] +async fn json_rpc_screen_intelligence_vision_recent_returns_empty_without_session() { + let _env_lock = json_rpc_e2e_env_lock(); + let tmp = tempdir().expect("tempdir"); + let home = tmp.path(); + let openhuman_home = home.join(".openhuman"); + + let _home_guard = EnvVarGuard::set_to_path("HOME", home); + let _workspace_guard = EnvVarGuard::unset("OPENHUMAN_WORKSPACE"); + let _backend_url_guard = EnvVarGuard::unset("BACKEND_URL"); + let _vite_backend_guard = EnvVarGuard::unset("VITE_BACKEND_URL"); + + let (mock_addr, mock_join) = serve_on_ephemeral(mock_upstream_router()).await; + let mock_origin = format!("http://{}", mock_addr); + write_min_config(&openhuman_home, &mock_origin); + + let (rpc_addr, rpc_join) = serve_on_ephemeral(build_core_http_router(false)).await; + let rpc_base = format!("http://{}", rpc_addr); + tokio::time::sleep(Duration::from_millis(100)).await; + + let recent = post_json_rpc( + &rpc_base, + 1004, + "openhuman.screen_intelligence_vision_recent", + json!({ "limit": 10 }), + ) + .await; + let result = assert_no_jsonrpc_error(&recent, "screen_intelligence_vision_recent"); + let recent_result = result.get("result").unwrap_or(result); + + let summaries = recent_result + .get("summaries") + .and_then(Value::as_array) + .expect("expected summaries array: {recent_result}"); + assert!( + summaries.is_empty(), + "vision_recent should return empty list without an active session, got {} items", + summaries.len() + ); + + mock_join.abort(); + rpc_join.abort(); +} + #[cfg(target_os = "macos")] #[tokio::test] async fn json_rpc_autocomplete_runtime_settings_and_logs_flow() { diff --git a/tests/screen_intelligence_vision_e2e.rs b/tests/screen_intelligence_vision_e2e.rs index 0c7a4ef25..6a5d05efc 100644 --- a/tests/screen_intelligence_vision_e2e.rs +++ b/tests/screen_intelligence_vision_e2e.rs @@ -1,9 +1,29 @@ //! E2E tests for the screen-intelligence vision pipeline. //! -//! Validates the full flow: generate image -> compress/resize -> parse vision -//! output -> persist to memory, all against real local storage in a temp workspace. +//! ## Platform support //! -//! Run with: `cargo test --test screen_intelligence_vision_e2e` +//! | Test group | Linux CI | macOS local | +//! |-------------------------------------|----------|-------------| +//! | Compression + image processing | ✅ | ✅ | +//! | Memory persistence (UnifiedMemory) | ✅ | ✅ | +//! | Screenshot save/cleanup (disk I/O) | ✅ | ✅ | +//! | Real screen capture (permission) | ❌ | ✅ (manual) | +//! | Local LLM vision analysis | ❌ | ✅ (manual) | +//! +//! ### Running +//! ``` +//! cargo test --test screen_intelligence_vision_e2e +//! ``` +//! Cross-platform CI tests use `OPENHUMAN_SCREEN_INTELLIGENCE_MOCK_VISION_JSON` to validate the +//! real engine pipeline without requiring macOS permissions or a running Ollama server. +//! +//! ### macOS E2E checklist (manual, requires Screen Recording permission) +//! 1. Grant Screen Recording to the `openhuman-core` binary in System Settings › Privacy & Security. +//! 2. Run: `cargo test --test screen_intelligence_vision_e2e -- --nocapture` +//! 3. Ensure Ollama is running with a vision-capable model (e.g. `ollama run minicpm-v`). +//! 4. Call `openhuman.screen_intelligence_capture_test` via `cargo test --test json_rpc_e2e json_rpc_screen_intelligence`. +//! 5. Run ignored real-capture test: +//! `cargo test --test screen_intelligence_vision_e2e macos_real_capture_cycle_persists_summary -- --ignored --nocapture` use std::path::Path; use std::sync::{Arc, Mutex, OnceLock}; @@ -19,6 +39,9 @@ use openhuman_core::openhuman::memory::embeddings::NoopEmbedding; use openhuman_core::openhuman::memory::store::types::NamespaceDocumentInput; use openhuman_core::openhuman::memory::store::UnifiedMemory; use openhuman_core::openhuman::screen_intelligence::CaptureFrame; +use openhuman_core::openhuman::screen_intelligence::{ + global_engine, AccessibilityEngine, VisionSummary, +}; // ── Env isolation ──────────────────────────────────────────────────── @@ -33,6 +56,18 @@ impl EnvVarGuard { std::env::set_var(key, path.as_os_str()); Self { key, old } } + + fn set(key: &'static str, value: &str) -> Self { + let old = std::env::var(key).ok(); + std::env::set_var(key, value); + Self { key, old } + } + + fn unset(key: &'static str) -> Self { + let old = std::env::var(key).ok(); + std::env::remove_var(key); + Self { key, old } + } } impl Drop for EnvVarGuard { @@ -47,10 +82,10 @@ impl Drop for EnvVarGuard { static ENV_LOCK: OnceLock> = OnceLock::new(); fn env_lock() -> std::sync::MutexGuard<'static, ()> { - ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env lock poisoned") + match ENV_LOCK.get_or_init(|| Mutex::new(())).lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } } // ── Helpers ────────────────────────────────────────────────────────── @@ -88,6 +123,38 @@ fn open_test_memory(dir: &Path) -> UnifiedMemory { UnifiedMemory::new(dir, embedder, Some(5)).expect("UnifiedMemory::new") } +fn write_screen_intelligence_test_config( + root: &Path, + local_ai_enabled: bool, + local_ai_provider: &str, +) { + let cfg = format!( + r#"default_temperature = 0.7 + +[memory] +backend = "sqlite" +auto_save = true +embedding_provider = "none" +embedding_model = "none" +embedding_dimensions = 0 + +[local_ai] +enabled = {local_ai_enabled} +provider = "{local_ai_provider}" + +[screen_intelligence] +keep_screenshots = false + +[secrets] +encrypt = false +"# + ); + std::fs::create_dir_all(root).expect("mkdir test root"); + std::fs::write(root.join("config.toml"), &cfg).expect("write config"); + let _: openhuman_core::openhuman::config::Config = + toml::from_str(&cfg).expect("test config should deserialize"); +} + /// Simulate what `parse_vision_summary_output` does, but from public types. fn mock_vision_summary(frame: &CaptureFrame, raw_llm: &str) -> serde_json::Value { let value: serde_json::Value = serde_json::from_str(raw_llm).unwrap_or_else(|_| { @@ -441,3 +508,268 @@ fn compression_savings_on_realistic_screenshot() { ratio * 100.0 ); } + +/// save_screenshot_to_disk writes a valid PNG file to the workspace directory. +#[test] +fn save_screenshot_to_disk_creates_png_file() { + let png_uri = make_test_png_uri(32, 32); + let frame = CaptureFrame { + captured_at_ms: 1700000000001, + reason: "e2e_disk_save_test".to_string(), + app_name: Some("DiskSaveApp".to_string()), + window_title: Some("E2E Save Test".to_string()), + image_ref: Some(png_uri), + }; + + let tmp = tempdir().expect("tempdir"); + let result = AccessibilityEngine::save_screenshot_to_disk(tmp.path(), &frame); + + assert!( + result.is_ok(), + "[screen_intelligence] save_screenshot_to_disk should succeed: {:?}", + result + ); + let saved_path = result.unwrap(); + assert!( + saved_path.exists(), + "[screen_intelligence] saved PNG file should exist at {}", + saved_path.display() + ); + assert_eq!( + saved_path.extension().and_then(|e| e.to_str()), + Some("png"), + "saved file should have .png extension" + ); + let metadata = std::fs::metadata(&saved_path).expect("file metadata"); + assert!(metadata.len() > 0, "saved PNG should not be empty"); +} + +/// Simulates the keep_screenshots=false cleanup path: save then immediately remove. +#[test] +fn save_screenshot_to_disk_cleanup_simulates_keep_screenshots_false() { + let png_uri = make_test_png_uri(32, 32); + let frame = CaptureFrame { + captured_at_ms: 1700000000002, + reason: "e2e_cleanup_test".to_string(), + app_name: Some("CleanupApp".to_string()), + window_title: Some("E2E Cleanup Test".to_string()), + image_ref: Some(png_uri), + }; + + let tmp = tempdir().expect("tempdir"); + let result = AccessibilityEngine::save_screenshot_to_disk(tmp.path(), &frame); + assert!( + result.is_ok(), + "[screen_intelligence] save should succeed before cleanup: {:?}", + result + ); + + let saved_path = result.unwrap(); + assert!(saved_path.exists(), "file should exist before cleanup"); + + // Simulate what the vision worker does when keep_screenshots=false + std::fs::remove_file(&saved_path).expect("remove_file should succeed"); + + assert!( + !saved_path.exists(), + "[screen_intelligence] file should no longer exist after cleanup: {}", + saved_path.display() + ); +} + +/// VisionSummary struct serializes and deserializes correctly, and is queryable after persistence. +/// +/// Tests two things independently: +/// 1. `VisionSummary` serde roundtrip in memory (proves struct attributes are correct). +/// 2. Persisting to UnifiedMemory and verifying the key is listed (proves `persist_vision_summary` +/// writes to the right namespace with the right key format). +#[tokio::test] +async fn vision_summary_struct_persist_and_deserialize_roundtrip() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + + let summary = VisionSummary { + id: "vision-1700000000100-roundtrip-test".to_string(), + captured_at_ms: 1700000000100, + app_name: Some("RoundtripApp".to_string()), + window_title: Some("Roundtrip Test Window".to_string()), + ui_state: "code editor with Rust file open".to_string(), + key_text: "fn main() {}".to_string(), + actionable_notes: "Developer is writing Rust code".to_string(), + confidence: 0.93, + }; + + // ── Step 1: serde roundtrip in memory (no DB) ────────────────────────── + // This proves VisionSummary has correct Serialize/Deserialize attributes and + // that the JSON format matches what persist_vision_summary stores. + let serialized = serde_json::to_string(&summary).expect("serialize VisionSummary"); + let deserialized: VisionSummary = + serde_json::from_str(&serialized).expect("deserialize VisionSummary"); + + assert_eq!(deserialized.id, summary.id, "id roundtrip"); + assert_eq!( + deserialized.ui_state, summary.ui_state, + "ui_state roundtrip" + ); + assert_eq!( + deserialized.key_text, summary.key_text, + "key_text roundtrip" + ); + assert_eq!( + deserialized.actionable_notes, summary.actionable_notes, + "actionable_notes roundtrip" + ); + assert_eq!( + deserialized.app_name, summary.app_name, + "app_name roundtrip" + ); + assert!( + (deserialized.confidence - summary.confidence).abs() < 0.01, + "confidence roundtrip: expected {}, got {}", + summary.confidence, + deserialized.confidence + ); + + // ── Step 2: persist to UnifiedMemory, verify queryable by key ───────── + // Matches exactly what persist_vision_summary() does (namespace, key format, tags). + let mem = open_test_memory(tmp.path()); + let key = format!("screen_intelligence_{}", summary.id); + mem.upsert_document(NamespaceDocumentInput { + namespace: "background".to_string(), + key: key.clone(), + title: key.clone(), + content: serialized, + source_type: "screenshot".to_string(), + priority: "medium".to_string(), + tags: vec!["screen_intelligence".to_string()], + metadata: serde_json::json!({}), + category: "screen_intelligence".to_string(), + session_id: None, + document_id: None, + }) + .await + .expect("upsert_document"); + + let result_json = mem + .list_documents(Some("background")) + .await + .expect("list_documents"); + let docs = result_json["documents"] + .as_array() + .expect("documents array"); + + assert!( + docs.iter().any(|d| d["key"].as_str() == Some(&key)), + "[screen_intelligence] persisted VisionSummary should be queryable by key: {key}" + ); +} + +/// Exercises the real engine pipeline (compress -> parse -> persist) with mocked local-vision +/// output so Linux CI can validate behavior without macOS permissions or Ollama runtime. +#[tokio::test] +async fn engine_pipeline_with_mocked_local_vision_persists_to_memory() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _workspace = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _mock = EnvVarGuard::set( + "OPENHUMAN_SCREEN_INTELLIGENCE_MOCK_VISION_JSON", + r#"{"ui_state":"browser with docs","key_text":"README.md","actionable_notes":"User is reading project docs","confidence":0.89}"#, + ); + write_screen_intelligence_test_config(tmp.path(), true, "ollama"); + + let frame = make_capture_frame(Some(make_test_png_uri(960, 540))); + let summary = global_engine() + .analyze_and_persist_frame(frame) + .await + .expect("mocked engine pipeline should succeed"); + assert_eq!(summary.ui_state, "browser with docs"); + + let config = openhuman_core::openhuman::config::Config::load_or_init() + .await + .expect("load config"); + let mem = open_test_memory(&config.workspace_dir); + let docs = mem + .list_documents(Some("background")) + .await + .expect("list documents")["documents"] + .as_array() + .cloned() + .expect("documents array"); + let key = format!("screen_intelligence_{}", summary.id); + assert!( + docs.iter().any(|doc| doc["key"].as_str() == Some(&key)), + "expected persisted summary key in memory: {key}" + ); +} + +/// Ensures screen-intelligence vision refuses non-local providers to avoid remote fallback. +#[tokio::test] +async fn engine_pipeline_rejects_non_local_provider() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _workspace = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", tmp.path()); + write_screen_intelligence_test_config(tmp.path(), true, "openai"); + + let frame = make_capture_frame(Some(make_test_png_uri(320, 240))); + let err = global_engine() + .analyze_and_persist_frame(frame) + .await + .expect_err("non-local providers should be rejected"); + assert!( + err.contains("provider 'ollama'"), + "unexpected provider guard error: {err}" + ); +} + +/// Manual macOS-only smoke test for the real capture -> local vision -> memory persistence chain. +/// Run manually with: +/// `cargo test --test screen_intelligence_vision_e2e macos_real_capture_cycle_persists_summary -- --ignored --nocapture` +#[cfg(target_os = "macos")] +#[tokio::test] +#[ignore = "requires Screen Recording permission + local Ollama vision model"] +async fn macos_real_capture_cycle_persists_summary() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _workspace = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _mock = EnvVarGuard::unset("OPENHUMAN_SCREEN_INTELLIGENCE_MOCK_VISION_JSON"); + write_screen_intelligence_test_config(tmp.path(), true, "ollama"); + + let capture = global_engine().capture_test().await; + assert!( + capture.ok, + "capture_test failed; ensure Screen Recording permission is granted: {:?}", + capture.error + ); + let image_ref = capture + .image_ref + .clone() + .expect("capture_test should return image_ref on success"); + let frame = make_capture_frame(Some(image_ref)); + + let summary = global_engine() + .analyze_and_persist_frame(frame) + .await + .expect("real local-vision inference should succeed"); + assert!( + !summary.actionable_notes.is_empty(), + "summary should include actionable notes" + ); + + let config = openhuman_core::openhuman::config::Config::load_or_init() + .await + .expect("load config"); + let mem = open_test_memory(&config.workspace_dir); + let docs = mem + .list_documents(Some("background")) + .await + .expect("list documents")["documents"] + .as_array() + .cloned() + .expect("documents array"); + let key = format!("screen_intelligence_{}", summary.id); + assert!( + docs.iter().any(|doc| doc["key"].as_str() == Some(&key)), + "expected persisted summary key after real capture cycle: {key}" + ); +}