diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 2a76c7e9c..1b7cc4e69 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -18,6 +18,7 @@ mod mascot_native_window; mod meet_audio; mod meet_call; mod meet_scanner; +mod meet_video; mod native_notifications; mod notification_settings; mod process_kill; diff --git a/app/src-tauri/src/meet_audio/inject.rs b/app/src-tauri/src/meet_audio/inject.rs index d0a951573..ff50ef086 100644 --- a/app/src-tauri/src/meet_audio/inject.rs +++ b/app/src-tauri/src/meet_audio/inject.rs @@ -108,6 +108,21 @@ pub async fn install_audio_bridge(meet_url: &str) -> Result<(CdpConn, String), S // logs and we still return Ok so the listen path keeps working. confirm_bridge_alive(&mut cdp, &session).await; + // Camera bridge is injected *after* the audio bridge has confirmed + // the post-reload page is alive. Pre-document registration of a + // 56 KB script (the inlined mascot SVGs) reliably crashed the CEF + // 146 renderer during reload — see `meet_video::inject` for the + // rationale. Meet's first getUserMedia call only fires after the + // user clicks "Ask to join" (multiple seconds), so a post-reload + // Runtime.evaluate lands well before it's needed. + if let Err(err) = + crate::meet_video::inject::install_camera_bridge_post_reload(&mut cdp, &session).await + { + log::warn!("[meet-audio] camera bridge install failed: {err} (falling back to static Y4M)"); + } else { + crate::meet_video::inject::confirm_bridge_alive(&mut cdp, &session).await; + } + Ok((cdp, session)) } diff --git a/app/src-tauri/src/meet_video/camera_bridge.js b/app/src-tauri/src/meet_video/camera_bridge.js new file mode 100644 index 000000000..8433fc3e4 --- /dev/null +++ b/app/src-tauri/src/meet_video/camera_bridge.js @@ -0,0 +1,211 @@ +// OpenHuman Meet camera bridge. +// +// Replaces the agent's outbound video stream with a programmatically +// drawn mascot. Runs at document-start in the Meet webview (installed +// via Page.addScriptToEvaluateOnNewDocument by `inject.rs`), so the +// monkey-patches on `navigator.mediaDevices.{getUserMedia, +// enumerateDevices}` are in place before Meet's app code requests the +// camera. +// +// The two `__OPENHUMAN_MASCOT_*_DATAURI__` placeholders are substituted +// from Rust at install time with `data:image/svg+xml;base64,...` URIs +// for the idle and thinking mascot SVGs, so the bridge is fully +// self-contained — no network fetch from inside meet.google.com's +// origin sandbox. +(function () { + if (window.__openhumanCameraBridge) return; + const TAG = '[openhuman-camera-bridge]'; + const W = 640; + const H = 480; + const FPS = 30; + const TOGGLE_INTERVAL_MS = 5000; + + const MASCOTS = { + idle: '__OPENHUMAN_MASCOT_IDLE_DATAURI__', + thinking: '__OPENHUMAN_MASCOT_THINKING_DATAURI__', + }; + + // Mood state. `currentMood` drives every frame; `setMood` is the + // public host-callable API and also the sink for the auto-toggle. + let currentMood = 'idle'; + const moodImgs = { idle: null, thinking: null }; + + function loadImage(src) { + return new Promise(function (resolve, reject) { + const img = new Image(); + img.onload = function () { resolve(img); }; + img.onerror = function (e) { + console.warn(TAG, 'image decode failed for src head=', (src || '').slice(0, 120)); + reject(new Error('img.onerror')); + }; + img.src = src; + }); + } + + // Build the canvas + decode mascots once. Ready promise gates the + // capture-stream construction so getUserMedia callers always get a + // canvas with valid frames in flight. + const canvas = document.createElement('canvas'); + canvas.width = W; + canvas.height = H; + const ctx = canvas.getContext('2d', { alpha: false }); + // Calm, off-white background — matches Meet's tile chrome. + ctx.fillStyle = '#F7F4EE'; + ctx.fillRect(0, 0, W, H); + + const ready = (async function () { + try { + moodImgs.idle = await loadImage(MASCOTS.idle); + moodImgs.thinking = await loadImage(MASCOTS.thinking); + console.log(TAG, 'mascots decoded', + 'idle=' + moodImgs.idle.naturalWidth + 'x' + moodImgs.idle.naturalHeight, + 'thinking=' + moodImgs.thinking.naturalWidth + 'x' + moodImgs.thinking.naturalHeight); + } catch (err) { + console.warn(TAG, 'mascot decode failed', err); + } + })(); + + // setInterval-driven render loop, NOT requestAnimationFrame: the + // meet window is frequently backgrounded behind the main openhuman + // window during the agent flow, and Chromium throttles rAF to ~0Hz + // in background tabs. setInterval keeps firing regardless of focus, + // which is what we need for the outbound camera to stay live. + // The small per-frame phase counter drives a gentle sine-wave bob + // so the camera reads as a live feed (Meet's outbound codec drops + // static frames, which can show up as a "frozen camera" banner). + let frame = 0; + function tick() { + frame++; + ctx.fillStyle = '#F7F4EE'; + ctx.fillRect(0, 0, W, H); + const img = moodImgs[currentMood]; + if (img) { + const margin = 0.12; + const tw = W * (1 - 2 * margin); + const th = H * (1 - 2 * margin); + const scale = Math.min(tw / img.naturalWidth, th / img.naturalHeight); + const bob = Math.sin(frame / (FPS * 2 / Math.PI)) * 6; + const dw = img.naturalWidth * scale; + const dh = img.naturalHeight * scale; + const dx = (W - dw) / 2; + const dy = (H - dh) / 2 + bob; + ctx.drawImage(img, dx, dy, dw, dh); + } + } + setInterval(tick, Math.round(1000 / FPS)); + + // Capture-stream once both mascots are decoded; before then the + // canvas just shows the background fill, which is fine — Meet won't + // ask for the camera until the user is past the lobby anyway. + const stream = canvas.captureStream(FPS); + const fakeVideoTrack = stream.getVideoTracks()[0]; + if (fakeVideoTrack) { + // Lie about device label so Meet's tile shows a friendly name. + try { + Object.defineProperty(fakeVideoTrack, 'label', { + value: 'OpenHuman Mascot', + configurable: true, + }); + } catch (_) {} + } + + // ---- monkey-patch ---------------------------------------------------- + // Important: the audio bridge (audio_bridge.js) installs its own + // getUserMedia override BEFORE we run, and it already handles every + // shape of constraint correctly — including audio+video, where it + // pulls the fake-camera Y4M video and splices in its own audio. We + // must NOT build a new MediaStream from cloned tracks: doing so + // creates duplicate audio senders against the same destination, + // which manifests at WebRTC negotiation as + // "BUNDLE group contains a codec collision between [111: audio/opus] + // and [111: audio/opus]" and breaks the Meet join flow. + // + // Correct shape: let the audio bridge produce the canonical stream, + // then swap *only* the video track in place. + const md = navigator.mediaDevices; + if (!md) { + console.warn(TAG, 'navigator.mediaDevices missing — cannot install bridge'); + return; + } + const origGetUserMedia = md.getUserMedia ? md.getUserMedia.bind(md) : null; + if (!origGetUserMedia) { + console.warn(TAG, 'navigator.mediaDevices.getUserMedia missing — cannot install bridge'); + return; + } + + function wantsVideo(constraints) { + if (!constraints) return false; + const v = constraints.video; + return v === true || (v && typeof v === 'object'); + } + + md.getUserMedia = async function (constraints) { + console.log(TAG, 'getUserMedia intercepted', JSON.stringify(constraints || {})); + if (!wantsVideo(constraints)) { + return origGetUserMedia(constraints); + } + await ready; + // Run the existing chain (audio bridge + Chromium) with the full + // original constraints so it returns a properly assembled stream. + const realStream = await origGetUserMedia(constraints); + // Drop whatever video track came back (the fake-camera Y4M) and + // splice our canvas track in. addTrack/removeTrack on a live + // MediaStream is the supported way to mutate a stream returned + // from getUserMedia without re-allocating it. + try { + realStream.getVideoTracks().forEach(function (t) { + realStream.removeTrack(t); + t.stop(); + }); + } catch (err) { + console.warn(TAG, 'failed to strip original video tracks', err); + } + const ours = stream.getVideoTracks()[0]; + if (ours) { + realStream.addTrack(ours.clone()); + } else { + console.warn(TAG, 'no canvas video track available — returning audio-only'); + } + return realStream; + }; + + // Note: we deliberately do NOT override enumerateDevices. The + // process-level --use-fake-device-for-media-stream flag already + // surfaces a "Fake Video Capture" device, which Meet picks by + // default. Returning custom plain objects from enumerateDevices + // can break Meet's device-picker code paths that expect real + // MediaDeviceInfo instances. + + // ---- host API -------------------------------------------------------- + window.__openhumanSetMood = function (mood) { + if (!Object.prototype.hasOwnProperty.call(MASCOTS, mood)) { + console.warn(TAG, 'unknown mood', mood); + return false; + } + if (currentMood !== mood) { + currentMood = mood; + console.log(TAG, 'mood ->', mood); + } + return true; + }; + window.__openhumanCameraBridgeInfo = function () { + return { + installed: true, + currentMood: currentMood, + hasIdle: !!moodImgs.idle, + hasThinking: !!moodImgs.thinking, + frame: frame, + }; + }; + + // Default driver: toggle every 5s. Once the agent state machine wires + // host-side `set_mood` calls, we can drop this fallback — but having + // it on by default keeps the visible behavior even if the host loop + // never starts. + setInterval(function () { + window.__openhumanSetMood(currentMood === 'idle' ? 'thinking' : 'idle'); + }, TOGGLE_INTERVAL_MS); + + window.__openhumanCameraBridge = true; + console.log(TAG, 'installed'); +})(); diff --git a/app/src-tauri/src/meet_video/inject.rs b/app/src-tauri/src/meet_video/inject.rs new file mode 100644 index 000000000..f4d6aba9b --- /dev/null +++ b/app/src-tauri/src/meet_video/inject.rs @@ -0,0 +1,131 @@ +//! Install the OpenHuman camera bridge into the Meet webview via CDP. +//! +//! ## Why post-reload `Runtime.evaluate`, not `addScriptToEvaluateOnNewDocument` +//! +//! The natural shape would be to mirror [`crate::meet_audio::inject`]: +//! register via `Page.addScriptToEvaluateOnNewDocument`, then ride the +//! audio bridge's `Page.reload` so all three scripts run at +//! document-start. We tried that. With CEF 146 + a 56 KB camera bridge +//! (the inlined mascot SVGs as data URIs are the bulk), registering a +//! third pre-document script consistently crashed the renderer during +//! the reload — `meet-scanner` would see +//! `cdp error: {"code":-32000,"message":"Target crashed"}` within ~1 s +//! of opening, the page was gone before either readiness probe could +//! answer, and the user saw a blank Meet window. +//! +//! The camera bridge only needs to be in place before Meet's first +//! `getUserMedia` call, which happens after the user (or +//! `meet_scanner`) clicks "Ask to join" — multiple seconds after the +//! navigation completes. Plenty of room to inject via +//! `Runtime.evaluate` once the post-reload page is up. +//! +//! Lifecycle: +//! 1. `meet_audio::inject::install_audio_bridge` registers + reloads +//! (unchanged). +//! 2. After the audio bridge's readiness probe confirms the new doc is +//! live, [`install_camera_bridge_post_reload`] evaluates the bridge +//! JS directly. No second reload, no pre-document script. + +use serde_json::{json, Value}; +use std::time::Duration; + +use crate::cdp::CdpConn; + +/// Inject the camera bridge into the Meet page's main world via +/// `Runtime.evaluate`. Called *after* the audio bridge's Page.reload +/// has settled, so we land on the live, post-reload document. +/// +/// Returns `Ok(())` if the evaluation didn't throw page-side. Errors +/// are non-fatal at the call site: the audio path keeps working and +/// Meet falls back to the static-Y4M outbound camera. +pub async fn install_camera_bridge_post_reload( + cdp: &mut CdpConn, + session: &str, +) -> Result<(), String> { + let js = super::build_camera_bridge_js(); + log::info!( + "[meet-camera] inject session={session} bridge_chars={}", + js.chars().count() + ); + let res = cdp + .call( + "Runtime.evaluate", + json!({ + "expression": js, + // returnByValue:false because the bridge IIFE returns + // undefined; we only care about exceptionDetails. + "awaitPromise": false, + }), + Some(session), + ) + .await + .map_err(|e| format!("Runtime.evaluate(camera bridge): {e}"))?; + if let Some(exception) = res.get("exceptionDetails") { + return Err(format!("page exception: {exception}")); + } + Ok(()) +} + +/// Best-effort readiness probe — logs the bridge's self-reported state +/// once it's live. Mirrors the audio bridge's `confirm_bridge_alive` +/// shape so a failure here is observable in the same place. +pub async fn confirm_bridge_alive(cdp: &mut CdpConn, session: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while tokio::time::Instant::now() < deadline { + let res = cdp + .call( + "Runtime.evaluate", + json!({ + "expression": "(typeof window.__openhumanCameraBridgeInfo === 'function') \ + ? JSON.stringify(window.__openhumanCameraBridgeInfo()) \ + : null", + "returnByValue": true, + }), + Some(session), + ) + .await; + if let Ok(v) = res { + let value = v + .get("result") + .and_then(|r| r.get("value")) + .cloned() + .unwrap_or(Value::Null); + if let Some(s) = value.as_str() { + log::info!("[meet-camera] bridge alive info={s}"); + return; + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + log::warn!("[meet-camera] bridge readiness probe timed out"); +} + +/// Host-side mood control. Future hookup: the meet-agent state machine +/// (`src/openhuman/meet_agent/session.rs`) calls this on phase +/// transitions so the camera reflects what the agent is actually doing +/// instead of running on the JS-side 5s auto-toggle. Until that's +/// wired, the bridge's own `setInterval` provides the visible toggle. +#[allow(dead_code)] +pub async fn set_mood(cdp: &mut CdpConn, session: &str, mood: &str) -> Result<(), String> { + // Mood is an internal enum — guard against accidental injection + // even though the call site is internal. + if !mood.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(format!("invalid mood: {mood}")); + } + let expression = format!( + "(typeof window.__openhumanSetMood === 'function') \ + ? window.__openhumanSetMood('{mood}') : false" + ); + let res = cdp + .call( + "Runtime.evaluate", + json!({ "expression": expression, "returnByValue": true }), + Some(session), + ) + .await + .map_err(|e| format!("Runtime.evaluate set_mood: {e}"))?; + if let Some(exception) = res.get("exceptionDetails") { + return Err(format!("page exception: {exception}")); + } + Ok(()) +} diff --git a/app/src-tauri/src/meet_video/mod.rs b/app/src-tauri/src/meet_video/mod.rs new file mode 100644 index 000000000..671c62249 --- /dev/null +++ b/app/src-tauri/src/meet_video/mod.rs @@ -0,0 +1,163 @@ +//! Meet camera bridge — overrides the agent webview's outbound video +//! track with a programmatically rendered mascot. +//! +//! ## Why JS injection (not a CEF patch) +//! +//! The `--use-file-for-fake-video-capture` flag we already pass at +//! browser startup (see [`crate::fake_camera`]) reads a single Y4M and +//! loops at EOF, which produces a static image. The flag is process- +//! level; we cannot rebind it per-call without rebuilding Chromium +//! from source. Until we own that build pipeline, the only way to +//! produce a *dynamic* outbound camera is to intercept +//! `navigator.mediaDevices.getUserMedia` at the JS layer and substitute +//! a `MediaStream` from a `` we own. +//! +//! This is a deliberate, scoped exception to the "no JS injection into +//! CEF child webviews" rule (see CLAUDE.md). Google Meet is already on +//! the grandfathered list (`audio_bridge.js`, `captions_bridge.js`), +//! and the camera bridge follows the same install path. +//! +//! ## Pieces +//! +//! - [`camera_bridge.js`] (sibling file, embedded via `include_str!`): +//! page-side bridge. Builds a hidden 640×480 canvas, decodes the +//! idle + thinking mascot SVGs, runs an rAF loop, exposes the +//! resulting `canvas.captureStream(30)` through monkey-patched +//! `getUserMedia` + `enumerateDevices`. Carries an unconditional 5s +//! mood toggle as the default driver; the host can also call +//! `window.__openhumanSetMood(name)` over CDP at any time. +//! +//! - [`inject`] — installs the bridge via CDP +//! `Page.addScriptToEvaluateOnNewDocument`. Wired into +//! [`crate::meet_audio::inject::install_audio_bridge`] so a single +//! `Page.reload` boots all three bridges (audio + captions + camera). +//! +//! - This file — embeds the two mascot SVGs at build time and templates +//! them into the bridge JS as `data:image/svg+xml;base64,...` URIs, +//! keeping the bridge fully self-contained inside the Meet origin. + +pub mod inject; + +// SVG data URIs use URL encoding rather than base64 because: +// 1. base64-encoded `data:image/svg+xml` has tripped on strict +// image-src CSPs in some Meet builds, manifesting as the bridge's +// "mascot decode failed Event" warning with no further detail. +// 2. The SVGs already minify well; url-encoding only inflates the +// reserved characters, while base64 inflates the whole payload by +// 33%. Net wire size is comparable. + +/// Idle mascot SVG (calm, eyes-forward). Rasterized into the canvas +/// during the bridge's `ready` promise. +const MASCOT_IDLE_SVG: &str = include_str!("../../../../remotion/public/idelMascot.svg"); + +/// Thinking mascot SVG (book-reading pose) — toggled in/out as the +/// agent's "thinking" state. Picked over `Cupholding`/`syicsmile` for +/// the most legible mood difference; revisit when phase 2 swaps the +/// static SVG for a live Remotion-driven OSR feed. +const MASCOT_THINKING_SVG: &str = include_str!("../../../../remotion/public/Bookreading.svg"); + +/// Bridge JS template. Two `__OPENHUMAN_MASCOT_*_DATAURI__` tokens are +/// substituted at install time with base64'd SVG data URIs. +const CAMERA_BRIDGE_TEMPLATE: &str = include_str!("camera_bridge.js"); + +/// Build the page-side camera bridge JS with the mascot SVGs inlined as +/// data URIs. Cheap to compute and stable per process; the inject path +/// can memoize via `OnceLock` if it ever grows hot. +pub fn build_camera_bridge_js() -> String { + let idle = svg_to_data_uri(MASCOT_IDLE_SVG); + let thinking = svg_to_data_uri(MASCOT_THINKING_SVG); + CAMERA_BRIDGE_TEMPLATE + .replace("__OPENHUMAN_MASCOT_IDLE_DATAURI__", &idle) + .replace("__OPENHUMAN_MASCOT_THINKING_DATAURI__", &thinking) +} + +/// URL-encode an SVG into a `data:image/svg+xml` URI suitable for +/// ``. Conservative whitelist of unreserved characters per +/// RFC 3986 plus a few path-safe extras; everything else is +/// percent-encoded byte-by-byte (UTF-8). Earlier passes that escaped +/// only the obvious breakers (`<`, `>`, `"`, `#`, `%`) left raw spaces +/// in attribute values like `viewBox="0 0 1000 1000"`, which Chromium +/// rejects in data URIs (manifests as the bridge's +/// "mascot decode failed Event" warning with no further detail). +fn svg_to_data_uri(svg: &str) -> String { + fn is_unreserved(b: u8) -> bool { + matches!(b, + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'-' | b'_' | b'.' | b'~' + // Sub-delims + path-safe that don't trip data-URI parsers + // and keep the SVG body itself parseable. Notably: '/' + // and ':' are fine inside path components per RFC 3986. + // + // Apostrophe ('\'') is deliberately NOT on this list: the + // resulting URI is interpolated into single-quoted JS + // string literals in `camera_bridge.js`, so a raw '\'' in + // the SVG would terminate the string and break the bridge + // load. It gets percent-encoded as %27. + | b'/' | b':' | b';' | b'=' | b',' | b'(' | b')' + | b'*' | b'!' + ) + } + + let mut out = String::with_capacity(svg.len() * 2 + 64); + out.push_str("data:image/svg+xml;charset=utf-8,"); + for byte in svg.bytes() { + if is_unreserved(byte) { + out.push(byte as char); + } else { + use std::fmt::Write as _; + let _ = write!(out, "%{byte:02X}"); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_substitutes_both_dataurus() { + let js = build_camera_bridge_js(); + assert!(!js.contains("__OPENHUMAN_MASCOT_IDLE_DATAURI__")); + assert!(!js.contains("__OPENHUMAN_MASCOT_THINKING_DATAURI__")); + let count = js.matches("data:image/svg+xml;charset=utf-8,").count(); + assert!(count >= 2, "expected at least 2 data URIs, got {count}"); + } + + #[test] + fn url_encoding_escapes_single_quote_for_js_string_context() { + // The data URI is interpolated into single-quoted JS literals + // in `camera_bridge.js` (e.g. `MASCOTS = { idle: '...' }`), so + // a raw apostrophe would terminate the string and break the + // bridge. Must come back as `%27`. + let uri = svg_to_data_uri(""); + let body = uri.trim_start_matches("data:image/svg+xml;charset=utf-8,"); + assert!(!body.contains('\'')); + assert!(body.contains("%27")); + } + + #[test] + fn url_encoding_escapes_reserved_chars() { + let uri = svg_to_data_uri("\n"); + assert!(uri.starts_with("data:image/svg+xml;charset=utf-8,")); + let body = uri.trim_start_matches("data:image/svg+xml;charset=utf-8,"); + // The breakers — '<', '>', '"', '\n' — must not appear unescaped. + assert!(!body.contains('<')); + assert!(!body.contains('>')); + assert!(!body.contains('"')); + assert!(!body.contains('\n')); + assert!(body.contains("%3C")); + assert!(body.contains("%3E")); + assert!(body.contains("%22")); + } + + #[test] + fn embedded_mascots_are_nonempty() { + assert!(MASCOT_IDLE_SVG.len() > 100); + assert!(MASCOT_THINKING_SVG.len() > 100); + assert!(MASCOT_IDLE_SVG.contains("