diff --git a/app/src-tauri/src/meet_video/camera_bridge.js b/app/src-tauri/src/meet_video/camera_bridge.js index 736e48da0..edd2631a2 100644 --- a/app/src-tauri/src/meet_video/camera_bridge.js +++ b/app/src-tauri/src/meet_video/camera_bridge.js @@ -9,7 +9,7 @@ // // Primary: connect to `ws://127.0.0.1:` (Rust-hosted, see // `frame_bus.rs`) and pump incoming binary JPEG frames straight onto -// our 640×480 capture canvas. This is what the user sees in Meet. +// our 1280x720 capture canvas. This is what the user sees in Meet. // // Fallback: if the WS hasn't delivered a frame in the last 500 ms (or // the port is 0 — meaning the producer never came up), draw the @@ -23,8 +23,8 @@ (function () { if (window.__openhumanCameraBridge) return; const TAG = '[openhuman-camera-bridge]'; - const W = 640; - const H = 480; + const W = 1280; + const H = 720; const FPS = 30; const FRAME_BUS_PORT = __OPENHUMAN_FRAME_BUS_PORT__; // The static-SVG path is **cold-start only**: we use it before the @@ -63,6 +63,12 @@ let nextRecvSeq = 0; let lastAcceptedSeq = -1; let wsState = 'init'; + let lastRemoteBitmapInfo = null; + let lastDrawSource = 'cold-start'; + let lastCanvasProbe = null; + let lastOutboundVideoStats = null; + let lastOutboundStatsAt = 0; + const peerConnections = new Set(); function loadImage(src) { return new Promise(function (resolve, reject) { @@ -145,6 +151,12 @@ } latestRemoteBitmap = bitmap; latestRemoteAt = Date.now(); + lastRemoteBitmapInfo = { + width: bitmap.width || null, + height: bitmap.height || null, + bytes: ev.data.byteLength || 0, + seq: mySeq, + }; lastAcceptedSeq = mySeq; remoteFrameCount++; } catch (err) { @@ -170,6 +182,45 @@ // setInterval keeps firing regardless of focus, which is what we need // for the outbound camera to stay live. let frame = 0; + function sampleCanvasPixels() { + try { + const cols = 7; + const rows = 5; + let sum = 0; + let min = 255; + let max = 0; + let count = 0; + let dark = 0; + let bright = 0; + for (let y = 0; y < rows; y++) { + for (let x = 0; x < cols; x++) { + const px = Math.max(0, Math.min(W - 1, Math.floor(((x + 0.5) * W) / cols))); + const py = Math.max(0, Math.min(H - 1, Math.floor(((y + 0.5) * H) / rows))); + const d = ctx.getImageData(px, py, 1, 1).data; + const luma = Math.round((d[0] * 0.299) + (d[1] * 0.587) + (d[2] * 0.114)); + sum += luma; + min = Math.min(min, luma); + max = Math.max(max, luma); + if (luma < 8) dark++; + if (luma > 32) bright++; + count++; + } + } + lastCanvasProbe = { + avgLuma: Math.round(sum / Math.max(1, count)), + minLuma: min, + maxLuma: max, + darkSamples: dark, + brightSamples: bright, + sampleCount: count, + source: lastDrawSource, + frame: frame, + }; + } catch (err) { + lastCanvasProbe = { error: String((err && err.message) || err), source: lastDrawSource }; + } + } + function tick() { frame++; if (latestRemoteBitmap) { @@ -189,6 +240,8 @@ const dx = (W - dw) / 2; const dy = (H - dh) / 2 + (Math.sin(frame / (FPS * 2 / Math.PI)) * 0.5); ctx.drawImage(latestRemoteBitmap, dx, dy, dw, dh); + lastDrawSource = 'remote'; + if (frame % FPS === 0) sampleCanvasPixels(); return; } // Cold-start fallback: static SVG with a gentle bob so the camera @@ -208,6 +261,8 @@ const dy = (H - dh) / 2 + bob; ctx.drawImage(img, dx, dy, dw, dh); } + lastDrawSource = 'fallback'; + if (frame % FPS === 0) sampleCanvasPixels(); } setInterval(tick, Math.round(1000 / FPS)); @@ -220,6 +275,9 @@ configurable: true, }); } catch (_) {} + try { + fakeVideoTrack.contentHint = 'motion'; + } catch (_) {} } // ---- monkey-patch ---------------------------------------------------- @@ -252,6 +310,149 @@ return v === true || (v && typeof v === 'object'); } + function makeMascotTrack() { + const ours = stream.getVideoTracks()[0]; + if (!ours) return null; + const clone = ours.clone(); + try { + Object.defineProperty(clone, 'label', { + value: 'OpenHuman Mascot', + configurable: true, + }); + } catch (_) {} + try { + clone.contentHint = 'motion'; + } catch (_) {} + return clone; + } + + function isVideoTrack(track) { + return !!track && track.kind === 'video'; + } + + function isVideoTransceiverInit(init) { + if (!init || typeof init !== 'object') return false; + if (Array.isArray(init.streams) && init.streams.some(function (s) { + return s && typeof s.getVideoTracks === 'function' && s.getVideoTracks().length > 0; + })) return true; + return false; + } + + function sanitizeVideoSenderInit(init) { + if (!init || typeof init !== 'object' || !Array.isArray(init.sendEncodings)) return init; + if (init.sendEncodings.length <= 1) return init; + const next = Object.assign({}, init); + const first = Object.assign({}, init.sendEncodings[0] || {}); + delete first.rid; + delete first.scalabilityMode; + first.scaleResolutionDownBy = 1; + next.sendEncodings = [first]; + console.log(TAG, 'collapsed video sendEncodings to one layer for mascot'); + return next; + } + + async function collectOutboundVideoStats() { + const now = Date.now(); + if (now - lastOutboundStatsAt < 2000) return; + lastOutboundStatsAt = now; + try { + for (const pc of peerConnections) { + if (!pc || typeof pc.getSenders !== 'function') continue; + const senders = pc.getSenders().filter(function (sender) { + return sender && sender.track && sender.track.kind === 'video'; + }); + for (const sender of senders) { + if (typeof sender.getStats !== 'function') continue; + const report = await sender.getStats(); + report.forEach(function (stat) { + if (stat && stat.type === 'outbound-rtp' && (stat.kind === 'video' || stat.mediaType === 'video')) { + lastOutboundVideoStats = { + framesEncoded: stat.framesEncoded ?? null, + framesSent: stat.framesSent ?? null, + bytesSent: stat.bytesSent ?? null, + frameWidth: stat.frameWidth ?? null, + frameHeight: stat.frameHeight ?? null, + qualityLimitationReason: stat.qualityLimitationReason ?? null, + timestamp: Math.round(stat.timestamp || 0), + }; + } + }); + } + } + } catch (err) { + lastOutboundVideoStats = { error: String((err && err.message) || err) }; + } + } + + function sampleVideoLuma(video) { + try { + if (!video || !video.videoWidth || !video.videoHeight || video.readyState < 2) return null; + const c = document.createElement('canvas'); + c.width = 16; + c.height = 9; + const cctx = c.getContext('2d', { alpha: false }); + if (!cctx) return null; + cctx.drawImage(video, 0, 0, c.width, c.height); + const data = cctx.getImageData(0, 0, c.width, c.height).data; + let sum = 0; + let min = 255; + let max = 0; + for (let i = 0; i < data.length; i += 4) { + const luma = Math.round((data[i] * 0.299) + (data[i + 1] * 0.587) + (data[i + 2] * 0.114)); + sum += luma; + min = Math.min(min, luma); + max = Math.max(max, luma); + } + const count = data.length / 4; + return { avgLuma: Math.round(sum / Math.max(1, count)), minLuma: min, maxLuma: max }; + } catch (err) { + return { error: String((err && err.message) || err) }; + } + } + + function probeVideoElements() { + try { + return Array.prototype.slice.call(document.querySelectorAll('video'), 0, 12).map(function (video, idx) { + const rect = video.getBoundingClientRect ? video.getBoundingClientRect() : null; + const tracks = video.srcObject && typeof video.srcObject.getVideoTracks === 'function' + ? video.srcObject.getVideoTracks().map(function (track) { + let settings = {}; + try { settings = track.getSettings ? track.getSettings() : {}; } catch (_) {} + return { + // track.label is intentionally omitted — on real devices it + // contains the camera/microphone device name, which is PII. + enabled: !!track.enabled, + muted: !!track.muted, + readyState: track.readyState || '', + width: settings.width || null, + height: settings.height || null, + frameRate: settings.frameRate || null, + }; + }) + : []; + return { + idx: idx, + videoWidth: video.videoWidth || 0, + videoHeight: video.videoHeight || 0, + readyState: video.readyState, + paused: !!video.paused, + currentTime: Math.round((video.currentTime || 0) * 1000) / 1000, + visible: !!rect && rect.width > 0 && rect.height > 0, + rect: rect ? { + width: Math.round(rect.width), + height: Math.round(rect.height), + x: Math.round(rect.x), + y: Math.round(rect.y), + } : null, + tracks: tracks, + luma: sampleVideoLuma(video), + }; + }); + } catch (err) { + return [{ error: String((err && err.message) || err) }]; + } + } + md.getUserMedia = async function (constraints) { console.log(TAG, 'getUserMedia intercepted', JSON.stringify(constraints || {})); if (!wantsVideo(constraints)) { @@ -269,13 +470,86 @@ } const ours = stream.getVideoTracks()[0]; if (ours) { - realStream.addTrack(ours.clone()); + realStream.addTrack(makeMascotTrack()); } else { console.warn(TAG, 'no canvas video track available — returning audio-only'); } return realStream; }; + const NativeRTCPeerConnection = window.RTCPeerConnection || window.webkitRTCPeerConnection; + if (NativeRTCPeerConnection && !NativeRTCPeerConnection.__openhumanCameraPatched) { + const origAddTrack = NativeRTCPeerConnection.prototype.addTrack; + const origAddTransceiver = NativeRTCPeerConnection.prototype.addTransceiver; + const origGetSenders = NativeRTCPeerConnection.prototype.getSenders; + + if (origAddTrack) { + NativeRTCPeerConnection.prototype.addTrack = function (track) { + peerConnections.add(this); + const args = Array.prototype.slice.call(arguments); + if (isVideoTrack(track)) { + const mascot = makeMascotTrack(); + if (mascot) { + args[0] = mascot; + console.log(TAG, 'RTCPeerConnection.addTrack video -> mascot'); + } + } + return origAddTrack.apply(this, args); + }; + } + + if (origAddTransceiver) { + NativeRTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { + peerConnections.add(this); + let nextTrackOrKind = trackOrKind; + let nextInit = init; + const direction = init && init.direction; + const willSend = !direction || direction === 'sendrecv' || direction === 'sendonly'; + if (willSend && (isVideoTrack(trackOrKind) || isVideoTransceiverInit(init))) { + const mascot = makeMascotTrack(); + if (mascot) { + nextTrackOrKind = mascot; + nextInit = sanitizeVideoSenderInit(init); + console.log(TAG, 'RTCPeerConnection.addTransceiver video -> mascot'); + } + } + return origAddTransceiver.call(this, nextTrackOrKind, nextInit); + }; + } + + if (origGetSenders) { + NativeRTCPeerConnection.prototype.getSenders = function () { + peerConnections.add(this); + return origGetSenders.apply(this, arguments); + }; + } + + NativeRTCPeerConnection.__openhumanCameraPatched = true; + } + + if (window.RTCRtpSender && window.RTCRtpSender.prototype && window.RTCRtpSender.prototype.replaceTrack) { + const origReplaceTrack = window.RTCRtpSender.prototype.replaceTrack; + if (!origReplaceTrack.__openhumanCameraPatched) { + const patchedReplaceTrack = function (track) { + const args = Array.prototype.slice.call(arguments); + if (isVideoTrack(track)) { + const mascot = makeMascotTrack(); + if (mascot) { + args[0] = mascot; + console.log(TAG, 'RTCRtpSender.replaceTrack video -> mascot'); + } + } + return origReplaceTrack.apply(this, args); + }; + patchedReplaceTrack.__openhumanCameraPatched = true; + window.RTCRtpSender.prototype.replaceTrack = patchedReplaceTrack; + } + } + + setInterval(function () { + void collectOutboundVideoStats(); + }, 2000); + // ---- host API -------------------------------------------------------- window.__openhumanSetMood = function (mood) { if (!Object.prototype.hasOwnProperty.call(MASCOTS, mood)) { @@ -300,6 +574,18 @@ remoteFrameCount: remoteFrameCount, droppedOutOfOrder: droppedOutOfOrder, remoteFreshMs: latestRemoteAt ? (Date.now() - latestRemoteAt) : null, + lastRemoteBitmapInfo: lastRemoteBitmapInfo, + lastDrawSource: lastDrawSource, + canvasProbe: lastCanvasProbe, + outboundVideoStats: lastOutboundVideoStats, + videoTrack: fakeVideoTrack ? { + label: fakeVideoTrack.label, + enabled: fakeVideoTrack.enabled, + muted: fakeVideoTrack.muted, + readyState: fakeVideoTrack.readyState, + settings: fakeVideoTrack.getSettings ? fakeVideoTrack.getSettings() : null, + } : null, + videoElements: probeVideoElements(), }; }; diff --git a/app/src-tauri/src/meet_video/inject.rs b/app/src-tauri/src/meet_video/inject.rs index dd498aa76..d4bc564e5 100644 --- a/app/src-tauri/src/meet_video/inject.rs +++ b/app/src-tauri/src/meet_video/inject.rs @@ -194,6 +194,60 @@ pub fn spawn_diagnostics_poller(meet_url: String) { .get("frameBusPort") .and_then(|x| x.as_u64()) .unwrap_or(0); + let source = parsed + .get("lastDrawSource") + .and_then(|x| x.as_str()) + .unwrap_or("?"); + let canvas_probe = parsed.get("canvasProbe").unwrap_or(&Value::Null); + let canvas_luma = canvas_probe.get("avgLuma").and_then(|x| x.as_i64()); + let canvas_min = canvas_probe.get("minLuma").and_then(|x| x.as_i64()); + let canvas_max = canvas_probe.get("maxLuma").and_then(|x| x.as_i64()); + let remote_bitmap = parsed.get("lastRemoteBitmapInfo").unwrap_or(&Value::Null); + let remote_w = remote_bitmap.get("width").and_then(|x| x.as_u64()); + let remote_h = remote_bitmap.get("height").and_then(|x| x.as_u64()); + let remote_bytes = remote_bitmap.get("bytes").and_then(|x| x.as_u64()); + let outbound = parsed.get("outboundVideoStats").unwrap_or(&Value::Null); + let frames_encoded = outbound.get("framesEncoded").and_then(|x| x.as_u64()); + let bytes_sent = outbound.get("bytesSent").and_then(|x| x.as_u64()); + let encoded_w = outbound.get("frameWidth").and_then(|x| x.as_u64()); + let encoded_h = outbound.get("frameHeight").and_then(|x| x.as_u64()); + let quality_reason = outbound + .get("qualityLimitationReason") + .and_then(|x| x.as_str()) + .unwrap_or("?"); + let video_elements = parsed + .get("videoElements") + .and_then(|x| x.as_array()) + .cloned() + .unwrap_or_default(); + let video_count = video_elements.len(); + let mascot_video = video_elements.iter().find(|video| { + video + .get("tracks") + .and_then(|x| x.as_array()) + .map(|tracks| { + tracks.iter().any(|track| { + track + .get("label") + .and_then(|x| x.as_str()) + .map(|label| label.contains("OpenHuman Mascot")) + .unwrap_or(false) + }) + }) + .unwrap_or(false) + }); + let mascot_video_summary = mascot_video + .map(|video| { + let vw = video.get("videoWidth").and_then(|x| x.as_u64()); + let vh = video.get("videoHeight").and_then(|x| x.as_u64()); + let ready = video.get("readyState").and_then(|x| x.as_u64()); + let luma = video + .get("luma") + .and_then(|x| x.get("avgLuma")) + .and_then(|x| x.as_i64()); + format!("w={vw:?} h={vh:?} ready={ready:?} luma={luma:?}") + }) + .unwrap_or_else(|| "none".to_string()); let delta_frames = frames.saturating_sub(last_frames); let delta_dropped = dropped.saturating_sub(last_dropped); let fps = (delta_frames as f32) / 2.0; @@ -201,7 +255,12 @@ pub fn spawn_diagnostics_poller(meet_url: String) { "[meet-camera-diag] tick={tick} ws={ws_state} port={port} \ frames_total={frames} fps_2s={fps:.1} \ dropped_total={dropped} new_dropped={delta_dropped} \ - fresh_ms={fresh_ms:?} bridge_frame={frame} mood={mood}" + fresh_ms={fresh_ms:?} bridge_frame={frame} mood={mood} source={source} \ + remote={remote_w:?}x{remote_h:?}/{remote_bytes:?}B \ + canvas_luma={canvas_luma:?}/{canvas_min:?}-{canvas_max:?} \ + outbound_frames={frames_encoded:?} outbound_bytes={bytes_sent:?} \ + outbound_size={encoded_w:?}x{encoded_h:?} quality={quality_reason} \ + videos={video_count} mascot_video={mascot_video_summary}" ); last_frames = frames; last_dropped = dropped; diff --git a/app/src/components/intelligence/PixiGraph.tsx b/app/src/components/intelligence/PixiGraph.tsx index c05c4cde6..3cf8351aa 100644 --- a/app/src/components/intelligence/PixiGraph.tsx +++ b/app/src/components/intelligence/PixiGraph.tsx @@ -93,7 +93,6 @@ export function PixiGraph({ mountedModeRef.current = null; void pending.then(handle => handle?.destroy()); }; - // eslint-disable-next-line react-hooks/exhaustive-deps }, [nodes, edges, mode]); useEffect(() => { diff --git a/app/src/components/skills/MeetingBotsCard.tsx b/app/src/components/skills/MeetingBotsCard.tsx index 94f379128..f1c735a9c 100644 --- a/app/src/components/skills/MeetingBotsCard.tsx +++ b/app/src/components/skills/MeetingBotsCard.tsx @@ -1,23 +1,33 @@ // Meeting bots entry point on the Skills "Integrations" section. // -// Surfaces as a compact, fun banner: clicking opens a modal that opens -// a dedicated CEF webview pointed at the Meet URL. The bot's outbound -// camera is the mascot canvas (`meet_video::camera_bridge`) and its -// outbound audio is the synthesized speech pump (`meet_audio`). Zoom -// and Teams are shown as "coming soon" — only Google Meet has the CEF -// bridge pipeline today. +// Surfaces as a compact banner: clicking opens a modal that asks the +// backend to send a Recall.ai-hosted mascot bot into the meeting. The +// backend streams replies, harness requests, and the final transcript +// back through the core Socket.IO bridge. -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { RiveMascot, type MascotFace } from '../../features/human/Mascot'; import { useT } from '../../lib/i18n/I18nContext'; import { - joinMeetCall, + joinMeetViaBackendBot, + leaveBackendMeetBot, listMeetCalls, type MascotMeetPlatform, type MeetCallRecord, } from '../../services/meetCallService'; -import { useAppSelector } from '../../store/hooks'; -import { selectPersonaDisplayName } from '../../store/personaSlice'; +import { useAppDispatch, useAppSelector } from '../../store/hooks'; +import { + type BackendMeetHarnessEvent, + type BackendMeetReplyEvent, + type BackendMeetStatus, + resetBackendMeet, + selectBackendMeetLastHarness, + selectBackendMeetLastReply, + selectBackendMeetStatus, + selectBackendMeetUrl, + setBackendMeetJoining, +} from '../../store/backendMeetSlice'; type Toast = { type: 'success' | 'error' | 'info'; title: string; message?: string }; @@ -42,27 +52,135 @@ const PLATFORMS: PlatformDef[] = [ platform: 'zoom', labelKey: 'skills.meetingBots.platforms.zoom', domainHintKey: 'skills.meetingBots.platformHints.zoom', - comingSoon: true, }, { platform: 'teams', labelKey: 'skills.meetingBots.platforms.teams', domainHintKey: 'skills.meetingBots.platformHints.teams', - comingSoon: true, }, ]; export default function MeetingBotsCard({ onToast }: Props) { const [open, setOpen] = useState(false); + const status = useAppSelector(selectBackendMeetStatus); + + const showActive = status === 'active' || status === 'joining'; return ( <> - setOpen(true)} /> + {showActive ? ( + + ) : ( + setOpen(true)} /> + )} {open && setOpen(false)} onToast={onToast} />} ); } +function faceFromMeetState( + status: BackendMeetStatus, + lastReply: BackendMeetReplyEvent | null, + lastHarness: BackendMeetHarnessEvent | null, +): MascotFace { + if (status === 'joining') return 'thinking'; + if (status === 'error') return 'concerned'; + if (status === 'ended') return 'happy'; + if (lastHarness) return 'thinking'; + if (lastReply) { + const e = (lastReply.emotion ?? '').toLowerCase(); + if (e.includes('happy') || e.includes('pleased') || e.includes('joy') || e.includes('excit')) return 'happy'; + if (e.includes('celebrat') || e.includes('proud')) return 'celebrating'; + if (e.includes('concern') || e.includes('worried') || e.includes('unsure')) return 'concerned'; + if (e.includes('curious') || e.includes('interest')) return 'curious'; + } + return 'idle'; +} + +function ActiveMeetingView({ onToast }: Props) { + const { t } = useT(); + const dispatch = useAppDispatch(); + const status = useAppSelector(selectBackendMeetStatus); + const meetUrl = useAppSelector(selectBackendMeetUrl); + const lastReply = useAppSelector(selectBackendMeetLastReply); + const lastHarness = useAppSelector(selectBackendMeetLastHarness); + const face = faceFromMeetState(status, lastReply, lastHarness); + const meetingCode = useMemo(() => { + if (!meetUrl) return ''; + try { + const tail = new URL(meetUrl).pathname.replace(/^\/+/, ''); + return tail || meetUrl; + } catch { return meetUrl; } + }, [meetUrl]); + + const [leaving, setLeaving] = useState(false); + + const handleLeave = async () => { + if (leaving) return; + setLeaving(true); + try { + await leaveBackendMeetBot('user-requested'); + } catch (err) { + onToast?.({ type: 'error', title: t('skills.meetingBots.couldNotStartTitle'), message: String(err) }); + } finally { + setLeaving(false); + } + }; + + const statusText = { + joining: t('skills.meetingBots.liveStatusJoining'), + active: t('skills.meetingBots.liveStatusActive'), + ended: t('skills.meetingBots.liveStatusEnded'), + error: t('skills.meetingBots.liveStatusError'), + idle: '', + }[status] ?? ''; + + const canLeave = status === 'active' || status === 'joining'; + const isDone = status === 'ended' || status === 'error'; + + return ( +
+
+ + + {canLeave && ( + + )} + {isDone && ( + + )} +
+
+
+ +
+
+
+ {t('skills.meetingBots.liveTitle')} +
+
{statusText}
+ {meetingCode && ( +
{meetingCode}
+ )} + {lastReply?.reply && ( +
+ “{lastReply.reply}” +
+ )} +
+
+
+ ); +} + function MeetingBotsBanner({ onClick }: { onClick: () => void }) { const { t } = useT(); return ( @@ -124,26 +242,12 @@ interface ModalProps { export function MeetingBotsModal({ onClose, onToast }: ModalProps) { const { t } = useT(); + const dispatch = useAppDispatch(); const [platform, setPlatform] = useState('gmeet'); const [meetUrl, setMeetUrl] = useState(''); const [displayName, setDisplayName] = useState('OpenHuman'); - // Privacy lock: the bot will only react to the wake word when this - // exact name is the speaker in Meet's captions. Anyone else who - // says "hey openhuman …" is silently ignored — preventing a - // remote participant from issuing tool calls in the owner's - // name. Empty fails closed; the submit handler will surface an - // explicit error before opening the CEF window. - // - // Effective value = Persona display name (Settings → Persona) until - // the user types into the field — the "name prompt" UX complaint in - // #2945, so repeat callers don't retype the same value every meeting. - // Once the user edits the field, the dirty flag latches and the - // input becomes fully controlled — Persona changes no longer - // overwrite their input, and clearing the field stays empty. - const personaDisplayName = useAppSelector(selectPersonaDisplayName); - const [ownerDisplayNameDraft, setOwnerDisplayNameDraft] = useState(''); - const [isOwnerNameEdited, setIsOwnerNameEdited] = useState(false); - const ownerDisplayName = isOwnerNameEdited ? ownerDisplayNameDraft : personaDisplayName; + const [respondToParticipant, setRespondToParticipant] = useState(''); + const [wakePhrase, setWakePhrase] = useState('Hey OpenHuman'); const [submitting, setSubmitting] = useState(false); const [error, setError] = useState(null); // Recent-calls history loaded from core when the modal opens. @@ -194,18 +298,21 @@ export function MeetingBotsModal({ onClose, onToast }: ModalProps) { } setSubmitting(true); try { - // Flow A: local CEF webview with mascot canvas + synthesized audio. - // joinMeetCall opens an off-screen CEF window per request_id, - // installs the audio/video bridges via CDP, then meet_scanner - // drives the join automatically. Returns once the window has - // been created — meet_audio + meet_scanner take it from there. - // - // ownerDisplayName is the privacy lock: the wake-word gate in - // the core only accepts captions whose speaker matches this - // value (case-insensitive, "(host)" / "(you)" suffix stripped). - // Anyone else in the room saying the wake phrase is dropped - // without dispatching a tool turn. - await joinMeetCall({ meetUrl, displayName, ownerDisplayName }); + // Optimistically update Redux state so the banner transitions to + // the ActiveMeetingView immediately, before the backend responds. + dispatch(setBackendMeetJoining({ meetUrl: meetUrl.trim() })); + // Backend Recall.ai bot: sends the mascot into the meeting via + // the backend's Recall.ai integration. The backend joins as a + // participant, renders the mascot as the bot's camera feed, and + // streams transcript events back over Socket.IO. + await joinMeetViaBackendBot({ + meetUrl, + displayName, + platform, + agentName: displayName, + respondToParticipant: respondToParticipant.trim() || undefined, + wakePhrase: wakePhrase.trim() || undefined, + }); onToast?.({ type: 'success', title: t('skills.meetingBots.joiningTitle'), @@ -312,28 +419,38 @@ export function MeetingBotsModal({ onClose, onToast }: ModalProps) { + + {error && ( @@ -354,7 +471,7 @@ export function MeetingBotsModal({ onClose, onToast }: ModalProps) {