From 796468c0344b25ba6360fc8e4bff5403b3eef112 Mon Sep 17 00:00:00 2001 From: YellowSnnowmann <167776381+YellowSnnowmann@users.noreply.github.com> Date: Wed, 8 Jul 2026 02:26:57 +0530 Subject: [PATCH] =?UTF-8?q?feat:=20dual=20mascots=20in=20meetings=20?= =?UTF-8?q?=E2=80=94=20per-mascot=20voice=20+=20speaker=20alternation=20(#?= =?UTF-8?q?4277)=20(#4585)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/src-tauri/src/lib.rs | 4 + app/src-tauri/src/meet_audio/mod.rs | 15 +- app/src-tauri/src/meet_audio/speak_pump.rs | 137 ++++- app/src-tauri/src/meet_call/mod.rs | 42 ++ app/src/components/meetings/MeetComposer.tsx | 34 ++ app/src/components/meetings/UpcomingTable.tsx | 54 +- .../meetings/__tests__/meetingUtils.test.ts | 132 ++++- app/src/components/meetings/meetingUtils.ts | 75 ++- .../settings/panels/MascotPanel.tsx | 95 +++ .../settings/panels/PerMascotVoiceRow.tsx | 245 ++++++++ .../panels/__tests__/MascotPanel.test.tsx | 88 +++ .../__tests__/PerMascotVoiceRow.test.tsx | 203 +++++++ app/src/features/meet/MascotFrameProducer.tsx | 358 +++++++++--- .../__tests__/MascotFrameProducer.test.tsx | 553 ++++++++++++++++-- .../__tests__/mascotFrameCompositor.test.ts | 214 +++++++ .../meet/__tests__/useMeetingMascots.test.ts | 136 +++++ .../features/meet/mascotFrameCompositor.ts | 129 ++++ app/src/features/meet/useMeetingMascots.ts | 150 +++++ app/src/lib/i18n/ar.ts | 7 + app/src/lib/i18n/bn.ts | 7 + app/src/lib/i18n/de.ts | 7 + app/src/lib/i18n/en.ts | 7 + app/src/lib/i18n/es.ts | 7 + app/src/lib/i18n/fr.ts | 7 + app/src/lib/i18n/hi.ts | 7 + app/src/lib/i18n/id.ts | 7 + app/src/lib/i18n/it.ts | 7 + app/src/lib/i18n/ko.ts | 7 + app/src/lib/i18n/pl.ts | 7 + app/src/lib/i18n/pt.ts | 7 + app/src/lib/i18n/ru.ts | 7 + app/src/lib/i18n/zh-CN.ts | 7 + .../__tests__/meetCallService.test.ts | 83 +++ app/src/services/meetCallService.ts | 84 ++- app/src/store/__tests__/mascotSlice.test.ts | 172 ++++++ app/src/store/mascotSlice.ts | 184 +++++- src/core/event_bus/events.rs | 4 + src/openhuman/agent_meetings/bus.rs | 4 + src/openhuman/agent_meetings/in_call.rs | 90 ++- src/openhuman/agent_meetings/ops.rs | 91 +++ src/openhuman/agent_meetings/schemas.rs | 10 + src/openhuman/agent_meetings/types.rs | 36 +- src/openhuman/meet_agent/brain/access.rs | 4 +- src/openhuman/meet_agent/brain/speech.rs | 9 +- src/openhuman/meet_agent/brain/turns.rs | 32 +- src/openhuman/meet_agent/rpc.rs | 50 +- src/openhuman/meet_agent/session.rs | 126 ++++ src/openhuman/meet_agent/types.rs | 13 + src/openhuman/socket/event_handlers.rs | 13 +- 49 files changed, 3581 insertions(+), 186 deletions(-) create mode 100644 app/src/components/settings/panels/PerMascotVoiceRow.tsx create mode 100644 app/src/components/settings/panels/__tests__/PerMascotVoiceRow.test.tsx create mode 100644 app/src/features/meet/__tests__/mascotFrameCompositor.test.ts create mode 100644 app/src/features/meet/__tests__/useMeetingMascots.test.ts create mode 100644 app/src/features/meet/mascotFrameCompositor.ts create mode 100644 app/src/features/meet/useMeetingMascots.ts diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 4aae5e711..1840d17fc 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -3678,6 +3678,10 @@ pub fn run() { // wake gate will fail-closed (no wakes fire) which // is the safe posture for an automated harness. owner_display_name: String::new(), + // No mascot config on the dev-auto path → single + // default voice (issue #4277). + primary_voice_id: None, + secondary_voice_id: None, }; match meet_call::meet_call_open_window(app_handle.clone(), state, args) .await diff --git a/app/src-tauri/src/meet_audio/mod.rs b/app/src-tauri/src/meet_audio/mod.rs index fbd0769d0..a60f582b9 100644 --- a/app/src-tauri/src/meet_audio/mod.rs +++ b/app/src-tauri/src/meet_audio/mod.rs @@ -89,13 +89,19 @@ pub async fn start( meet_url: String, owner_display_name: String, bot_display_name: String, + primary_voice_id: Option, + secondary_voice_id: Option, ) -> Result<(), String> { log::info!( "[meet-audio] start request_id={request_id} url_prefix={} \ - owner_chars={} bot_chars={}", + owner_chars={} bot_chars={} voices={}", truncate_for_log(&meet_url, 64), owner_display_name.chars().count(), - bot_display_name.chars().count() + bot_display_name.chars().count(), + [&primary_voice_id, &secondary_voice_id] + .into_iter() + .filter(|v| v.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false)) + .count() ); if let Some(state) = app.try_state::() { @@ -125,6 +131,11 @@ pub async fn start( // " in the history list. The URL the shell built // the CEF window with is the canonical value. "meet_url": meet_url, + // Per-mascot TTS voices for speaker alternation (issue #4277). + // Null for single-mascot calls → core keeps the backend + // default voice (unchanged behavior). + "primary_voice_id": primary_voice_id, + "secondary_voice_id": secondary_voice_id, }), ) .await?; diff --git a/app/src-tauri/src/meet_audio/speak_pump.rs b/app/src-tauri/src/meet_audio/speak_pump.rs index 4b83a15ce..2cda72a79 100644 --- a/app/src-tauri/src/meet_audio/speak_pump.rs +++ b/app/src-tauri/src/meet_audio/speak_pump.rs @@ -98,10 +98,10 @@ pub fn start( break; } _ = tick.tick() => { - let had_pcm = match poll_and_feed(&request_id_for_task, &mut cdp, &session_id).await { - Ok(had) => { + let (had_pcm, active_slot) = match poll_and_feed(&request_id_for_task, &mut cdp, &session_id).await { + Ok((had, slot)) => { feed_errors = 0; - had + (had, Some(slot)) } Err(err) => { feed_errors += 1; @@ -118,11 +118,12 @@ pub fn start( // A failed tick is *not* evidence the bot // stopped speaking — leave the hangover to // expire naturally so transient CDP errors - // don't flicker the mascot's mouth shut. - false + // don't flicker the mascot's mouth shut. No + // fresh slot data → keep the last-known slot. + (false, None) } }; - speaking_state.tick(had_pcm, &app, &request_id_for_task); + speaking_state.tick(had_pcm, active_slot, &app, &request_id_for_task); } } } @@ -148,6 +149,14 @@ struct SpeakingTracker { /// every tick that carries PCM; the state flips back to `false` /// only once `now > hangover_until` AND a tick with no PCM lands. hangover_until: Option, + /// Which mascot (0 = primary, 1 = secondary) is speaking the current + /// outbound audio, as reported by `meet_agent_poll_speech`. For + /// two-mascot calls the brain alternates this per reply; the frontend + /// lip-syncs this slot and reacts the other. Emitted on the + /// speaking-state edge AND whenever the slot changes mid-speech (so a + /// back-to-back reply from the other mascot still switches lip-sync + /// even if the hangover bridged the gap). 0 for single-mascot calls. + active_slot: u8, } impl SpeakingTracker { @@ -155,6 +164,7 @@ impl SpeakingTracker { Self { reported: false, hangover_until: None, + active_slot: 0, } } @@ -162,12 +172,21 @@ impl SpeakingTracker { /// is whether `poll_and_feed` saw a non-empty `pcm_base64` for /// this tick. Emits the Tauri event only when the reported /// state actually flips. - fn tick(&mut self, had_pcm: bool, app: &AppHandle, request_id: &str) { + /// `active_slot` is the speaking mascot slot for this tick, or `None` + /// when the tick had no fresh poll data (e.g. a CDP error) — in that + /// case the last-known slot is retained. + fn tick( + &mut self, + had_pcm: bool, + active_slot: Option, + app: &AppHandle, + request_id: &str, + ) { if had_pcm { // Extend the hangover. If we were idle, flip up to // speaking — the user hears audio starting now. self.hangover_until = Some(Instant::now() + SPEAKING_HANGOVER); - self.set_reported(true, app, request_id); + self.update(true, active_slot, app, request_id); return; } // No PCM this tick. If the hangover hasn't expired, stay in @@ -182,7 +201,7 @@ impl SpeakingTracker { self.hangover_until = None; } // Hangover expired or never armed → bot is genuinely idle. - self.set_reported(false, app, request_id); + self.update(false, active_slot, app, request_id); } /// Force the reported state to `false` and emit an event if that's @@ -190,30 +209,71 @@ impl SpeakingTracker { /// can't get stuck mid-talk. fn force_off(&mut self, app: &AppHandle, request_id: &str) { self.hangover_until = None; - self.set_reported(false, app, request_id); + self.update(false, None, app, request_id); } - fn set_reported(&mut self, next: bool, app: &AppHandle, request_id: &str) { - if self.reported == next { + /// Update reported speaking state + active slot, emitting the Tauri + /// event when either the speaking flag flips OR (while speaking) the + /// active mascot slot changes. `slot = None` keeps the last-known + /// slot (used when a tick carried no fresh poll data). + fn update( + &mut self, + next: bool, + slot: Option, + app: &AppHandle, + request_id: &str, + ) { + let (should_emit, resolved_slot) = + next_speaking_state(self.reported, self.active_slot, next, slot); + self.reported = next; + self.active_slot = resolved_slot; + if !should_emit { return; } - self.reported = next; let payload = serde_json::json!({ "requestId": request_id, "speaking": next, + // Which mascot is speaking this audio (0 = primary, 1 = + // secondary). Frontend lip-syncs this slot, reacts the other. + "activeMascotSlot": self.active_slot, }); if let Err(err) = app.emit(SPEAKING_STATE_EVENT, payload) { // Best-effort: a missing renderer (closed window mid-tick) // is the common case and not worth raising the log level. log::debug!( - "[meet-audio] speaking-state emit failed request_id={request_id} speaking={next} err={err}" + "[meet-audio] speaking-state emit failed request_id={request_id} speaking={next} slot={} err={err}", + self.active_slot ); } else { - log::debug!("[meet-audio] speaking-state -> {next} request_id={request_id}"); + log::debug!( + "[meet-audio] speaking-state -> {next} slot={} request_id={request_id}", + self.active_slot + ); } } } +/// Pure decision for [`SpeakingTracker::update`]: given the previously +/// reported speaking flag + slot and the incoming `next`/`slot`, compute +/// whether the Tauri event should be emitted and which slot to resolve to. +/// +/// Extracted so the emit logic can be unit-tested without a live +/// `AppHandle`. `slot = None` retains the previous slot (a tick with no +/// fresh poll data). We emit when the speaking flag flips OR (while +/// speaking) the active slot changes — the latter switches lip-sync on a +/// back-to-back reply from the other mascot even if the hangover bridged +/// the gap. +fn next_speaking_state( + prev_reported: bool, + prev_slot: u8, + next: bool, + slot: Option, +) -> (bool, u8) { + let resolved = slot.unwrap_or(prev_slot); + let should_emit = prev_reported != next || (next && prev_slot != resolved); + (should_emit, resolved) +} + /// No-op pump used when bridge install failed at session start. Keeps /// the rest of the session lifecycle uniform — `MeetAudioSession` can /// still hold a `SpeakPump` regardless of speak-path readiness. @@ -231,7 +291,7 @@ async fn poll_and_feed( request_id: &str, cdp: &mut CdpConn, session_id: &str, -) -> Result { +) -> Result<(bool, u8), String> { let v = super::rpc_call( "openhuman.meet_agent_poll_speech", serde_json::json!({ "request_id": request_id }), @@ -249,6 +309,12 @@ async fn poll_and_feed( .get("flush_pending") .and_then(|x| x.as_bool()) .unwrap_or(false); + // Which mascot slot is speaking this audio (0 = primary, 1 = + // secondary). Absent on older cores / single-mascot → 0. + let active_slot = v + .get("active_mascot_slot") + .and_then(|x| x.as_u64()) + .unwrap_or(0) as u8; // Barge-in: brain set flush_pending when it cancelled the previous // outbound. Stop in-flight playback inside the JS bridge BEFORE we @@ -278,10 +344,45 @@ async fn poll_and_feed( bytes.len() ); inject::feed_pcm_chunk(cdp, session_id, pcm_b64).await?; - return Ok(true); + return Ok((true, active_slot)); } if utterance_done { log::info!("[meet-audio] speak pump utterance complete request_id={request_id}"); } - Ok(false) + Ok((false, active_slot)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn speaking_edge_and_slot_change_emit_logic() { + // idle -> speaking: a real edge, emits. + let (emit, slot) = next_speaking_state(false, 0, true, Some(0)); + assert!(emit, "idle -> speaking must emit"); + assert_eq!(slot, 0); + + // speaking -> speaking, same slot: no edge, no slot change, no emit. + let (emit, slot) = next_speaking_state(true, 0, true, Some(0)); + assert!(!emit, "speaking -> speaking same slot must not emit"); + assert_eq!(slot, 0); + + // speaking(slot 0) -> speaking(slot 1): mid-speech mascot switch emits + // and resolves to the new slot so lip-sync moves to the other mascot. + let (emit, slot) = next_speaking_state(true, 0, true, Some(1)); + assert!(emit, "speaking slot change must emit"); + assert_eq!(slot, 1, "resolved slot must be the new slot 1"); + + // slot = None retains the previous slot; with the same reported flag + // that is neither an edge nor a slot change, so no emit. + let (emit, slot) = next_speaking_state(true, 1, true, None); + assert!(!emit, "no fresh slot + same reported must not emit"); + assert_eq!(slot, 1, "None retains the previous slot"); + + // speaking -> idle: a real edge, emits. + let (emit, slot) = next_speaking_state(true, 1, false, None); + assert!(emit, "speaking -> idle must emit"); + assert_eq!(slot, 1, "None retains the previous slot on the way down"); + } } diff --git a/app/src-tauri/src/meet_call/mod.rs b/app/src-tauri/src/meet_call/mod.rs index eb9e24aa4..592aa2789 100644 --- a/app/src-tauri/src/meet_call/mod.rs +++ b/app/src-tauri/src/meet_call/mod.rs @@ -93,6 +93,15 @@ pub struct OpenWindowArgs { /// fails closed in core (no wakes fire). #[serde(default)] pub owner_display_name: String, + /// ElevenLabs voice for the primary mascot (issue #4277). When the + /// user has two mascots enabled the core alternates the speaking + /// voice per reply. Absent/empty → core keeps its default voice. + #[serde(default)] + pub primary_voice_id: Option, + /// ElevenLabs voice for the secondary mascot. Absent when only one + /// mascot is enabled. + #[serde(default)] + pub secondary_voice_id: Option, } /// Open a dedicated top-level CEF webview window pointed at the Meet URL. @@ -262,6 +271,8 @@ pub async fn meet_call_open_window( let url_for_audio = parsed.to_string(); let bot_for_audio = args.display_name.clone(); let owner_for_audio = args.owner_display_name.clone(); + let primary_voice_for_audio = args.primary_voice_id.clone(); + let secondary_voice_for_audio = args.secondary_voice_id.clone(); tauri::async_runtime::spawn(async move { if let Err(err) = crate::meet_audio::start( app_for_audio, @@ -269,6 +280,8 @@ pub async fn meet_call_open_window( url_for_audio, owner_for_audio, bot_for_audio, + primary_voice_for_audio, + secondary_voice_for_audio, ) .await { @@ -551,6 +564,35 @@ mod tests { ); } + #[test] + fn open_window_args_deserializes_voice_ids() { + use serde_json::json; + + // Both voice ids present → parse into Some(...). Proves the + // #4277 per-mascot voice fields ride through the command payload. + let args: OpenWindowArgs = serde_json::from_value(json!({ + "request_id": "r", + "meet_url": "u", + "display_name": "d", + "primary_voice_id": "va", + "secondary_voice_id": "vb", + })) + .unwrap(); + assert_eq!(args.primary_voice_id.as_deref(), Some("va")); + assert_eq!(args.secondary_voice_id.as_deref(), Some("vb")); + + // Both omitted → #[serde(default)] yields None (single-mascot + // callers and older shells that don't forward voices still parse). + let args: OpenWindowArgs = serde_json::from_value(json!({ + "request_id": "r", + "meet_url": "u", + "display_name": "d", + })) + .unwrap(); + assert_eq!(args.primary_voice_id, None); + assert_eq!(args.secondary_voice_id, None); + } + #[test] fn meet_call_state_default_is_empty() { let state = MeetCallState::default(); diff --git a/app/src/components/meetings/MeetComposer.tsx b/app/src/components/meetings/MeetComposer.tsx index 5b69b6361..a64af50b4 100644 --- a/app/src/components/meetings/MeetComposer.tsx +++ b/app/src/components/meetings/MeetComposer.tsx @@ -9,6 +9,7 @@ import debug from 'debug'; import { type RefObject, useEffect, useRef, useState } from 'react'; +import { useMascotManifest } from '../../features/human/Mascot/manifest/useMascotManifest'; import { useComposioIntegrations } from '../../lib/composio/hooks'; import { useT } from '../../lib/i18n/I18nContext'; import { @@ -25,12 +26,15 @@ import { useAppDispatch, useAppSelector } from '../../store/hooks'; import { selectCustomPrimaryColor, selectCustomSecondaryColor, + selectDualMascotEnabled, selectMascotColor, + selectMeetingMascotVoicePair, selectSelectedMascotId, } from '../../store/mascotSlice'; import { selectPersonaDescription, selectPersonaDisplayName } from '../../store/personaSlice'; import Button from '../ui/Button'; import { + buildMeetingMascots, platformLabel, platformUrlPlaceholder, resolveMeetingBotMascotId, @@ -76,6 +80,15 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) { const mascotColor = useAppSelector(selectMascotColor); const customPrimaryColor = useAppSelector(selectCustomPrimaryColor); const customSecondaryColor = useAppSelector(selectCustomSecondaryColor); + // Dual-mascot config (issue #4277): when a distinct second mascot is enabled + // we send both slots (each with its own voice) so the backend bot renders + // two mascots and alternates who speaks. Single-mascot keeps the legacy + // `mascotId` path below untouched. + const dualMascotEnabled = useAppSelector(selectDualMascotEnabled); + const mascotVoicePair = useAppSelector(selectMeetingMascotVoicePair); + // Manifest drives name-addressed routing (#4277 follow-up): each dual slot is + // tagged with its display name so "Hey Toshi …" routes to that mascot. + const { manifest } = useMascotManifest(); // ── Meet slice ─────────────────────────────────────────────────────────── const meetStatus = useAppSelector(selectBackendMeetStatus); @@ -129,6 +142,17 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) { mascotColor === 'custom' ? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor } : undefined; + // Two-mascot slots for the backend bot (issue #4277) — built via the shared + // helper so this live-join path and the UpcomingTable scheduled-join path stay + // behaviorally identical. + const mascots = buildMeetingMascots({ + dualMascotEnabled, + mascotVoicePair, + manifest, + mascotId, + riveColors, + agentName, + }); const wakePhrase = listenOnly ? undefined : `Hey ${agentName}`; const handleSubmit = async (event: React.FormEvent) => { @@ -143,6 +167,14 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) { !listenOnly, meetingId ); + // Name-addressing (#4277 follow-up) trace: the exact mascot ids + names sent + // to the backend. If `mascots` is undefined or a slot's `name` is empty, + // name addressing ("Hey Toshi") can't work — the backend needs both names. + log( + '[composer] join mascots=%o wakePhrase=%s', + mascots?.map(m => ({ mascotId: m.mascotId, name: m.name })), + wakePhrase + ); try { // Await the RPC BEFORE dispatching setBackendMeetJoining so that a // synchronous rejection (bad URL, auth failure) can be shown inline @@ -159,6 +191,8 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) { systemPrompt, mascotId, riveColors, + // Dual-mascot slots (issue #4277); undefined for single-mascot calls. + mascots, correlationId: meetingId, respondToParticipant: displayedRespondTo.trim() || undefined, wakePhrase, diff --git a/app/src/components/meetings/UpcomingTable.tsx b/app/src/components/meetings/UpcomingTable.tsx index 971f70400..7ff6bcc6d 100644 --- a/app/src/components/meetings/UpcomingTable.tsx +++ b/app/src/components/meetings/UpcomingTable.tsx @@ -14,6 +14,7 @@ import debug from 'debug'; import { useState } from 'react'; +import { useMascotManifest } from '../../features/human/Mascot/manifest/useMascotManifest'; import { useT } from '../../lib/i18n/I18nContext'; import { joinMeetViaBackendBot, @@ -26,13 +27,21 @@ import { useAppSelector } from '../../store/hooks'; import { selectCustomPrimaryColor, selectCustomSecondaryColor, + selectDualMascotEnabled, selectMascotColor, + selectMeetingMascotVoicePair, selectSelectedMascotId, } from '../../store/mascotSlice'; import { selectPersonaDescription, selectPersonaDisplayName } from '../../store/personaSlice'; import Button from '../ui/Button'; import { type JoinPolicy, JoinPolicyToggle } from './JoinPolicyToggle'; -import { inferPlatformFromUrl, platformLabel, platformLogoUrl } from './meetingUtils'; +import { + buildMeetingMascots, + inferPlatformFromUrl, + platformLabel, + platformLogoUrl, + resolveMeetingBotMascotId, +} from './meetingUtils'; import { useUpcomingMeetings } from './useUpcomingMeetings'; const log = debug('meetings:upcoming-table'); @@ -359,6 +368,14 @@ export function UpcomingTable({ const mascotColor = useAppSelector(selectMascotColor); const customPrimaryColor = useAppSelector(selectCustomPrimaryColor); const customSecondaryColor = useAppSelector(selectCustomSecondaryColor); + // Dual-mascot config (issue #4277) — see MeetComposer for the rationale; + // both join sites resolve the two-mascot slots the same way so a "Join now" + // and a manual join behave identically. + const dualMascotEnabled = useAppSelector(selectDualMascotEnabled); + const mascotVoicePair = useAppSelector(selectMeetingMascotVoicePair); + // Manifest drives name-addressed routing (#4277 follow-up): tag each dual slot + // with its display name so "Hey Toshi …" routes to that mascot. + const { manifest } = useMascotManifest(); // Live in-call state — lets a row detect that its meeting is already joined // and suppress the "Join now" button. correlationId is a fresh per-join UUID @@ -371,12 +388,29 @@ export function UpcomingTable({ return Boolean(backendMeetUrl && m.meet_url && backendMeetUrl === m.meet_url); }; - // Resolve bot join params the same way MeetComposer does. - const mascotId = selectedMascotId ?? (mascotColor === 'custom' ? undefined : mascotColor); + // Resolve bot join params the same way MeetComposer does — via + // resolveMeetingBotMascotId, so a manifest-only id the backend bot doesn't + // recognize is dropped here too (a raw `selectedMascotId` fallback would let + // it through and diverge from the MeetComposer join). + const mascotId = resolveMeetingBotMascotId(selectedMascotId, mascotColor); const riveColors = mascotColor === 'custom' ? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor } : undefined; + // Bot join name — hoisted to component-body scope because both the `mascots` + // array here and the wake phrase inside handleJoin need it. + const agentName = personaDisplayName.trim() || 'Tiny'; + // Two-mascot slots (issue #4277) — built via the shared helper so this + // scheduled-join path and the MeetComposer live-join path stay behaviorally + // identical. + const mascots = buildMeetingMascots({ + dualMascotEnabled, + mascotVoicePair, + manifest, + mascotId, + riveColors, + agentName, + }); const handleJoin = async (meeting: UpcomingMeeting) => { if (!meeting.meet_url) return; @@ -388,9 +422,8 @@ export function UpcomingTable({ const anchor = replyDisplayName.trim(); // Reply mode gates the bot behind a wake phrase so it only reacts when // addressed ("Hey Alex, …"), never to every caption from the anchor — - // mirroring MeetComposer. The bot joins as `agentName`, so the phrase must - // match it. Listen-only joins (no anchor) send no wake phrase. - const agentName = personaDisplayName.trim() || 'Tiny'; + // mirroring MeetComposer. The bot joins as `agentName` (hoisted above), so + // the phrase must match it. Listen-only joins (no anchor) send no wake phrase. const wakePhrase = anchor ? `Hey ${agentName}` : undefined; // Mint a fresh correlation id per join. It becomes the call record's // `request_id` (recent-calls list key + per-call detail filename), so it @@ -408,6 +441,13 @@ export function UpcomingTable({ correlationId ); setJoiningId(meeting.calendar_event_id); + // Name-addressing (#4277 follow-up) trace: mascot ids + names sent to the + // backend. Empty `name` on a slot ⇒ name addressing can't route to it. + log( + '[upcoming] join mascots=%o wakePhrase=%s', + mascots?.map(m => ({ mascotId: m.mascotId, name: m.name })), + wakePhrase + ); try { await joinMeetViaBackendBot({ meetUrl: meeting.meet_url, @@ -415,6 +455,8 @@ export function UpcomingTable({ agentName, systemPrompt: personaDescription || undefined, mascotId: mascotId || undefined, + // Dual-mascot slots (issue #4277); undefined for single-mascot calls. + mascots, respondToParticipant: anchor || undefined, wakePhrase, listenOnly: !anchor, diff --git a/app/src/components/meetings/__tests__/meetingUtils.test.ts b/app/src/components/meetings/__tests__/meetingUtils.test.ts index 4bb4d224a..ec3c2094a 100644 --- a/app/src/components/meetings/__tests__/meetingUtils.test.ts +++ b/app/src/components/meetings/__tests__/meetingUtils.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest'; +import type { MascotManifest } from '../../../features/human/Mascot/manifest/types'; import type { ComposioConnection } from '../../../lib/composio/types'; import { + buildMeetingMascots, deriveDisplayNameFromEmail, inferPlatformFromUrl, MEETING_PLATFORMS, @@ -257,8 +259,16 @@ describe('resolveMeetingBotMascotId', () => { expect(resolveMeetingBotMascotId('navy', 'yellow')).toBe('navy'); }); - it('falls back to the legacy mascot color for a manifest-only mascot id', () => { - expect(resolveMeetingBotMascotId('river-guide', 'yellow')).toBe('yellow'); + it('keeps the "toshi" manifest mascot id the backend now ships as an asset', () => { + expect(resolveMeetingBotMascotId('toshi', 'yellow')).toBe('toshi'); + }); + + it('keeps the "tiny-mascot" manifest mascot id the backend now ships as an asset', () => { + expect(resolveMeetingBotMascotId('tiny-mascot', 'navy')).toBe('tiny-mascot'); + }); + + it('falls back to the legacy mascot color for a manifest-only mascot id the backend still lacks', () => { + expect(resolveMeetingBotMascotId('jarvis', 'yellow')).toBe('yellow'); }); it('uses the mascot color when no mascot id is selected', () => { @@ -269,3 +279,121 @@ describe('resolveMeetingBotMascotId', () => { expect(resolveMeetingBotMascotId('river-guide', 'custom')).toBeUndefined(); }); }); + +// --------------------------------------------------------------------------- +// buildMeetingMascots +// --------------------------------------------------------------------------- + +describe('buildMeetingMascots', () => { + // findMascot only reads `.id`/`.name`, so a partial manifest is enough. + const manifest = { + mascots: [ + { id: 'toshi', name: 'Toshi' }, + { id: 'tiny-mascot', name: 'Tiny' }, + ], + } as unknown as MascotManifest; + + const pair = { + primary: { mascotId: 'toshi', voiceId: 'voice-a' }, + secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' }, + }; + + it('returns undefined when dual mode is off', () => { + expect( + buildMeetingMascots({ + dualMascotEnabled: false, + mascotVoicePair: pair, + manifest, + mascotId: 'toshi', + agentName: 'Tiny', + }) + ).toBeUndefined(); + }); + + it('returns undefined when no secondary is configured', () => { + expect( + buildMeetingMascots({ + dualMascotEnabled: true, + mascotVoicePair: { primary: { mascotId: 'toshi', voiceId: 'voice-a' } }, + manifest, + mascotId: 'toshi', + agentName: 'Tiny', + }) + ).toBeUndefined(); + }); + + it('returns undefined when the primary slot id cannot be resolved', () => { + // primary has no explicit mascot AND the resolved single-path id is undefined. + expect( + buildMeetingMascots({ + dualMascotEnabled: true, + mascotVoicePair: { + primary: { voiceId: 'voice-a' }, + secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' }, + }, + manifest, + mascotId: undefined, + agentName: 'Tiny', + }) + ).toBeUndefined(); + }); + + it('falls back to the resolved single-path mascotId for the primary slot', () => { + const mascots = buildMeetingMascots({ + dualMascotEnabled: true, + mascotVoicePair: { + primary: { voiceId: 'voice-a' }, + secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' }, + }, + manifest, + mascotId: 'toshi', + agentName: 'Tiny', + }); + expect(mascots?.[0].mascotId).toBe('toshi'); + }); + + it('builds two slots with manifest names and per-slot voices', () => { + const riveColors = { primaryColor: '#111', secondaryColor: '#222' }; + const mascots = buildMeetingMascots({ + dualMascotEnabled: true, + mascotVoicePair: pair, + manifest, + mascotId: 'toshi', + riveColors, + agentName: 'Tiny', + }); + expect(mascots).toEqual([ + { mascotId: 'toshi', name: 'Toshi', voiceId: 'voice-a', riveColors }, + { mascotId: 'tiny-mascot', name: 'Tiny', voiceId: 'voice-b', riveColors }, + ]); + }); + + it('resolves names by slot so either mascot order works', () => { + const mascots = buildMeetingMascots({ + dualMascotEnabled: true, + mascotVoicePair: { + primary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' }, + secondary: { mascotId: 'toshi', voiceId: 'voice-a' }, + }, + manifest, + mascotId: 'tiny-mascot', + agentName: 'Tiny', + }); + expect(mascots?.map(m => m.name)).toEqual(['Tiny', 'Toshi']); + }); + + it('falls back to agentName for the primary when its manifest entry is missing', () => { + const mascots = buildMeetingMascots({ + dualMascotEnabled: true, + mascotVoicePair: { + primary: { mascotId: 'river-guide', voiceId: 'voice-a' }, + secondary: { mascotId: 'toshi', voiceId: 'voice-a' }, + }, + manifest, + mascotId: 'river-guide', + agentName: 'Persona', + }); + expect(mascots?.[0].name).toBe('Persona'); + expect(mascots?.[1].name).toBe('Toshi'); + }); +}); diff --git a/app/src/components/meetings/meetingUtils.ts b/app/src/components/meetings/meetingUtils.ts index 441d274d6..86f158d00 100644 --- a/app/src/components/meetings/meetingUtils.ts +++ b/app/src/components/meetings/meetingUtils.ts @@ -5,17 +5,31 @@ * previously embedded inside MeetingBotsCard so they can be unit-tested in * isolation and shared across the split composer components. */ +import { findMascot } from '../../features/human/Mascot/manifest/manifestService'; +import type { MascotManifest } from '../../features/human/Mascot/manifest/types'; import type { MascotColor } from '../../features/human/Mascot/mascotPalette'; import type { ComposioConnection } from '../../lib/composio/types'; -import type { MeetingPlatform } from '../../services/meetCallService'; +import type { BackendMeetJoinInput, MeetingPlatform } from '../../services/meetCallService'; import { composioLogoUrl } from '../composio/toolkitMeta'; /** * Mascot ids the meeting-bot backend recognizes. Newer manifest-only mascot * ids (e.g. "river-guide") aren't supported there, so the bot falls back to the * legacy mascot color for them. + * + * "toshi" and "tiny-mascot" are the tinyhumansai/mascots manifest ids that the + * Recall meeting-bot backend now ships as real mascot assets, so they pass + * through unchanged instead of being collapsed to a color. */ -const MEETING_BOT_MASCOT_IDS = new Set(['yellow', 'blue', 'burgundy', 'black', 'navy']); +const MEETING_BOT_MASCOT_IDS = new Set([ + 'yellow', + 'blue', + 'burgundy', + 'black', + 'navy', + 'toshi', + 'tiny-mascot', +]); /** * Resolve the mascot id to send to the meeting bot: the selected mascot id when @@ -31,6 +45,63 @@ export function resolveMeetingBotMascotId( return undefined; } +/** Voice/mascot pair the composer selects for the two speaking slots. */ +type MeetingMascotVoicePair = { + primary: { mascotId?: string | null; voiceId?: string }; + secondary?: { mascotId?: string | null; voiceId?: string } | null; +}; + +/** + * Build the backend bot's dual-mascot `mascots` array (issue #4277), shared by + * both join sites — MeetComposer (live join) and UpcomingTable (scheduled join) + * — so the two paths stay behaviorally identical. They had already begun to + * drift, which is exactly the class of bug centralising this prevents. + * + * Slot 0 (primary) reuses the resolved single-path `mascotId` when the voice + * pair's primary carries no explicit mascot; slot 1 is the secondary. Slot names + * come from the manifest so name-addressed routing works in either order ("Hey + * Toshi" → whichever slot Toshi occupies); the primary falls back to `agentName` + * only when its manifest entry is unavailable. Per-mascot colors are out of + * scope — both slots reuse the single `riveColors`. + * + * Returns `undefined` (→ the single-mascot `mascotId` join) unless dual mode is + * on, a secondary is configured, AND both slot ids resolve to a concrete value: + * a blank slot-0 id would make the backend drop it and render the secondary + * alone, mismatching the on-camera primary. + */ +export function buildMeetingMascots(input: { + dualMascotEnabled: boolean; + mascotVoicePair: MeetingMascotVoicePair; + manifest: MascotManifest | null; + mascotId: string | undefined; + riveColors?: { primaryColor?: string; secondaryColor?: string }; + agentName: string; +}): BackendMeetJoinInput['mascots'] { + const { dualMascotEnabled, mascotVoicePair, manifest, mascotId, riveColors, agentName } = input; + const primarySlotId = mascotVoicePair.primary.mascotId ?? mascotId; + const secondarySlotId = mascotVoicePair.secondary?.mascotId ?? undefined; + if (!dualMascotEnabled || !mascotVoicePair.secondary || !primarySlotId || !secondarySlotId) { + return undefined; + } + const primaryName = + (manifest ? findMascot(manifest, primarySlotId)?.name : undefined) ?? agentName; + const secondaryName = manifest ? findMascot(manifest, secondarySlotId)?.name : undefined; + return [ + { + mascotId: primarySlotId, + name: primaryName, + voiceId: mascotVoicePair.primary.voiceId, + riveColors, + }, + { + mascotId: secondarySlotId, + name: secondaryName, + voiceId: mascotVoicePair.secondary.voiceId, + riveColors, + }, + ]; +} + // --------------------------------------------------------------------------- // Platform registry // --------------------------------------------------------------------------- diff --git a/app/src/components/settings/panels/MascotPanel.tsx b/app/src/components/settings/panels/MascotPanel.tsx index a97be584c..cc2b42381 100644 --- a/app/src/components/settings/panels/MascotPanel.tsx +++ b/app/src/components/settings/panels/MascotPanel.tsx @@ -22,6 +22,7 @@ import { selectMascotVoiceGender, selectMascotVoiceId, selectMascotVoiceUseLocaleDefault, + selectSecondaryMascotId, selectSelectedMascotId, setCustomMascotGifUrl, setCustomPrimaryColor, @@ -30,6 +31,7 @@ import { setMascotVoiceGender, setMascotVoiceId, setMascotVoiceUseLocaleDefault, + setSecondaryMascotId, setSelectedMascotId, SUPPORTED_MASCOT_COLORS, } from '../../../store/mascotSlice'; @@ -41,6 +43,7 @@ import { ELEVENLABS_VOICE_PRESETS, isCuratedVoicePreset, } from './elevenlabsVoicePresets'; +import PerMascotVoiceRow from './PerMascotVoiceRow'; interface ColorOption { id: MascotColor; @@ -70,6 +73,7 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => { const customPrimary = useAppSelector(selectCustomPrimaryColor); const customSecondary = useAppSelector(selectCustomSecondaryColor); const selectedMascotId = useAppSelector(selectSelectedMascotId); + const secondaryMascotId = useAppSelector(selectSecondaryMascotId); const customMascotGifUrl = useAppSelector(selectCustomMascotGifUrl); const storedVoiceId = useAppSelector(selectMascotVoiceId); const voiceGender = useAppSelector(selectMascotVoiceGender); @@ -126,6 +130,21 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => { // null ("default") case has to clear it here so the stage falls back to // the default manifest mascot rather than the GIF. if (id == null) dispatch(setCustomMascotGifUrl(null)); + // A newly-picked primary that collides with the current secondary would + // leave both slots pointing at the same mascot; clear the secondary so + // the duo never duplicates. The reducer's `selectDualMascotEnabled` + // guard already treats a collision as single-mascot, but clearing here + // keeps the picker's rendered state honest. + if (id != null && id === secondaryMascotId) dispatch(setSecondaryMascotId(null)); + }; + + // ── Second-mascot picker (issue #4277) ─────────────────────────── + // Enable / clear the meeting duo's second mascot. `null` (the "None" + // option) drops back to single-mascot. Picking the primary's id is + // disabled in the dropdown, so this only ever dispatches a distinct id + // or null. + const handleSelectSecondaryMascot = (id: string | null) => { + dispatch(setSecondaryMascotId(id)); }; const onSaveCustomGif = () => { @@ -683,6 +702,82 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => { {t('settings.mascot.characterDesc')}

+ + {/* ── Meeting duo: second mascot + per-mascot voices (issue #4277) ─ + Only meaningful for manifest mascots — a custom GIF avatar is a + single-figure path, so the whole block hides while one is set. */} + {manifest && manifest.mascots.length > 0 && !customMascotGifUrl && ( +
+

+ {t('settings.mascot.secondaryHeading')} +

+ + {/* Second-mascot picker — bespoke label + select combo mirroring + the voice preset dropdown. The primary's id is disabled so the + duo can never duplicate a single mascot. */} +
+ +
+

+ {t('settings.mascot.secondaryDesc')} +

+ + {/* Per-mascot voices — a row per mascot whose voice is actually + addressable in `mascotVoices` (keyed by a concrete manifest + id). The join path (`selectMeetingMascotVoicePair`) resolves + the primary slot's voice from `mascotVoices[selectedMascotId]`, + so the primary row only appears once a specific primary mascot + is pinned; on the default mascot the effective single voice + (governed by the Voice section above) is what plays. Each row + writes its own `mascotVoices` entry and owns a guarded preview. */} + {secondaryMascotId != null && secondaryMascotId !== selectedMascotId && ( +
+

+ {t('settings.mascot.perMascotVoiceHeading')} +

+ {selectedMascotId != null && ( + + )} + +
+ )} +
+ )} ); diff --git a/app/src/components/settings/panels/PerMascotVoiceRow.tsx b/app/src/components/settings/panels/PerMascotVoiceRow.tsx new file mode 100644 index 000000000..c5ddca6c2 --- /dev/null +++ b/app/src/components/settings/panels/PerMascotVoiceRow.tsx @@ -0,0 +1,245 @@ +import { useEffect, useRef, useState } from 'react'; + +import { synthesizeSpeech } from '../../../features/human/voice/ttsClient'; +import { useT } from '../../../lib/i18n/I18nContext'; +import { useAppDispatch, useAppSelector } from '../../../store/hooks'; +import { + selectEffectiveMascotVoiceId, + selectMascotVoiceFor, + selectMascotVoiceGender, + setMascotVoice, +} from '../../../store/mascotSlice'; +import Button from '../../ui/Button'; +import { SettingsSelect, SettingsTextField } from '../controls'; +import { ELEVENLABS_VOICE_PRESETS, isCuratedVoicePreset } from './elevenlabsVoicePresets'; + +interface PerMascotVoiceRowProps { + /** Manifest mascot id this row controls the voice for. Writes land in + * `mascotVoices[mascotId]` via `setMascotVoice`. */ + mascotId: string; + /** Human-readable heading for the row (e.g. the mascot's name), already + * localized by the caller. */ + label: string; + /** Stable test hook so both the primary and secondary rows expose + * distinct `data-testid`s (`mascot-voice-{primary,secondary}-*`). */ + testIdPrefix: string; +} + +/** + * Per-mascot reply-voice control (issue #4277). A trimmed sibling of the + * primary voice section in `MascotPanel`: it reuses the same preset + * dropdown + custom-paste + guarded `synthesizeSpeech` preview, but writes + * to `mascotVoices[mascotId]` via `setMascotVoice` instead of the single + * `voiceId`. The current value is the per-mascot override + * (`selectMascotVoiceFor`) falling back to the effective single voice + * (`selectEffectiveMascotVoiceId`), so a mascot with no override sounds + * exactly like the single-voice behaviour until the user picks one. + * + * Extracted from `MascotPanel` to keep that file within the ~500-line + * budget while the per-mascot map (primary + secondary) doubles the voice + * UI. Each instance owns its own preview-abort guard (`previewRequestIdRef`) + * so the two rows never share an in-flight preview. + */ +const PerMascotVoiceRow = ({ mascotId, label, testIdPrefix }: PerMascotVoiceRowProps) => { + const { t } = useT(); + const dispatch = useAppDispatch(); + // Per-mascot override, or the effective single voice when unset — the + // same resolution the meeting join path uses, so the picker shows what + // the mascot will actually speak with. + const overrideVoiceId = useAppSelector(selectMascotVoiceFor(mascotId)); + const effectiveVoiceId = useAppSelector(selectEffectiveMascotVoiceId); + const currentVoiceId = overrideVoiceId ?? effectiveVoiceId; + // Reuse the global gender bucket to filter the preset dropdown — the + // per-mascot control only overrides the voice id, not the gender filter. + const voiceGender = useAppSelector(selectMascotVoiceGender); + + // Paste-mode is sticky for the same reason as the primary control: a + // curated preset id and a mid-paste custom id both leave the stored + // value looking like a known id, so we can't derive the mode from it. + const [voiceDraft, setVoiceDraft] = useState(overrideVoiceId ?? ''); + const [voicePasteMode, setVoicePasteMode] = useState(false); + const [isPreviewingVoice, setIsPreviewingVoice] = useState(false); + const [voicePreviewError, setVoicePreviewError] = useState(null); + const previewAudioRef = useRef(null); + // Monotonically-bumped preview-request id, mirroring the primary + // control: unmount + each new preview both increment it so an in-flight + // `synthesizeSpeech(...)` whose resolve loses the race bails before it + // touches refs / state. + const previewRequestIdRef = useRef(0); + + // Stop any in-flight preview audio on unmount and invalidate a pending + // `synthesizeSpeech(...)` so a late resolve can't start audio for a row + // the user has already navigated away from. + useEffect(() => { + return () => { + previewRequestIdRef.current += 1; + if (previewAudioRef.current) { + previewAudioRef.current.pause(); + previewAudioRef.current.src = ''; + previewAudioRef.current = null; + } + }; + }, []); + + // Presets the dropdown should expose: always include the current voice + // (so the controlled select never points at an absent option) plus the + // active gender bucket and any '*' fallback voices. + const visiblePresets = ELEVENLABS_VOICE_PRESETS.filter( + p => p.id === currentVoiceId || p.gender === voiceGender || p.locales.includes('*') + ); + + // A custom (non-curated) override keeps the paste editor open so the + // stored id stays visible; the effective-voice fallback is never treated + // as "custom" because the mascot has no explicit override yet. + const isCustomVoice = + voicePasteMode || (overrideVoiceId != null && !isCuratedVoicePreset(overrideVoiceId)); + + const onPresetChange = (next: string) => { + if (next === '__custom__') { + setVoicePasteMode(true); + setVoiceDraft(overrideVoiceId ?? ''); + return; + } + setVoicePasteMode(false); + setVoicePreviewError(null); + setVoiceDraft(next); + dispatch(setMascotVoice({ mascotId, voiceId: next })); + }; + + const onSavePaste = () => { + setVoicePreviewError(null); + const trimmed = voiceDraft.trim(); + setVoiceDraft(trimmed); + dispatch(setMascotVoice({ mascotId, voiceId: trimmed.length > 0 ? trimmed : null })); + }; + + const onVoiceReset = () => { + setVoicePreviewError(null); + setVoicePasteMode(false); + setVoiceDraft(''); + dispatch(setMascotVoice({ mascotId, voiceId: null })); + }; + + const onVoicePreview = async () => { + // Same abort guard as the primary control: reserve a fresh id, and let + // a stale resolve detect that a newer preview (or unmount) superseded + // it before it mutates state or plays audio. + const requestId = ++previewRequestIdRef.current; + setIsPreviewingVoice(true); + setVoicePreviewError(null); + if (previewAudioRef.current) { + previewAudioRef.current.pause(); + previewAudioRef.current.src = ''; + previewAudioRef.current = null; + } + try { + const tts = await synthesizeSpeech(t('settings.mascot.voice.previewText'), { + voiceId: currentVoiceId, + }); + if (previewRequestIdRef.current !== requestId) return; + const src = `data:${tts.audio_mime || 'audio/mpeg'};base64,${tts.audio_base64}`; + const audio = new window.Audio(src); + previewAudioRef.current = audio; + await audio.play(); + } catch (err) { + if (previewRequestIdRef.current !== requestId) return; + const message = err instanceof Error ? err.message : t('settings.mascot.voice.previewError'); + setVoicePreviewError(message); + } finally { + if (previewRequestIdRef.current === requestId) setIsPreviewingVoice(false); + } + }; + + return ( +
+ + {label} + + + {/* Preset dropdown — mirrors the primary control's label + select combo */} + + + {isCustomVoice && ( + + )} + +
+ + + + {t('settings.mascot.voice.current')}: {currentVoiceId} + +
+ + {voicePreviewError && ( +
+ {t('settings.mascot.voice.previewError')}: {voicePreviewError} +
+ )} +
+ ); +}; + +export default PerMascotVoiceRow; diff --git a/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx b/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx index c73738659..30a8b3358 100644 --- a/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx +++ b/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx @@ -10,6 +10,7 @@ import mascotReducer, { setCustomMascotGifUrl, setMascotColor, setMascotVoiceId, + setSecondaryMascotId, setSelectedMascotId, } from '../../../../store/mascotSlice'; import MascotPanel from '../MascotPanel'; @@ -362,3 +363,90 @@ describe('MascotPanel — voice picker custom voice input (line 525)', () => { expect(store.getState().mascot.voiceId).toBe('new-voice-id'); }); }); + +// ── Dual mascots + per-mascot voice (issue #4277) ──────────────────────────── +describe('MascotPanel — meeting duo (second mascot)', () => { + const yellow = manifestEntry('yellow', 'Yellow'); + const toshi = manifestEntry('toshi', 'Toshi'); + + beforeEach(() => { + vi.clearAllMocks(); + useMascotManifestMock.mockReturnValue(manifestResult([yellow, toshi])); + mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0)); + }); + + it('dispatches setSecondaryMascotId when a second mascot is picked', () => { + const { store } = renderPanel(); + fireEvent.change(screen.getByTestId('mascot-secondary-select'), { target: { value: 'toshi' } }); + expect(store.getState().mascot.secondaryMascotId).toBe('toshi'); + }); + + it('clears the second mascot when None is selected', () => { + const store = buildStore(); + store.dispatch(setSecondaryMascotId('toshi')); + renderPanel(store); + fireEvent.change(screen.getByTestId('mascot-secondary-select'), { + target: { value: '__none__' }, + }); + expect(store.getState().mascot.secondaryMascotId).toBeNull(); + }); + + it('disables the primary mascot as a second-mascot option', () => { + const store = buildStore(); + store.dispatch(setSelectedMascotId('yellow')); + renderPanel(store); + const primaryOption = screen + .getByTestId('mascot-secondary-select') + .querySelector('option[value="yellow"]') as HTMLOptionElement | null; + expect(primaryOption).not.toBeNull(); + expect(primaryOption).toBeDisabled(); + }); + + it('hides the duo picker when a custom GIF avatar is set', () => { + const store = buildStore(); + store.dispatch(setCustomMascotGifUrl('https://example.com/avatar.gif')); + renderPanel(store); + expect(screen.queryByTestId('mascot-secondary-select')).not.toBeInTheDocument(); + }); + + it('dispatches setMascotVoice for the second mascot when its voice changes', () => { + const store = buildStore(); + store.dispatch(setSelectedMascotId('yellow')); + store.dispatch(setSecondaryMascotId('toshi')); + renderPanel(store); + + // The secondary voice row renders once a distinct duo mascot is set. + const select = screen.getByTestId('mascot-voice-secondary-select'); + fireEvent.change(select, { target: { value: 'pNInz6obpgDQGcFmaJgB' } }); + + expect(store.getState().mascot.mascotVoices.toshi).toBe('pNInz6obpgDQGcFmaJgB'); + }); + + it('dispatches setMascotVoice for the primary mascot when its voice changes', () => { + const store = buildStore(); + store.dispatch(setSelectedMascotId('yellow')); + store.dispatch(setSecondaryMascotId('toshi')); + renderPanel(store); + + const select = screen.getByTestId('mascot-voice-primary-select'); + fireEvent.change(select, { target: { value: 'pNInz6obpgDQGcFmaJgB' } }); + + expect(store.getState().mascot.mascotVoices.yellow).toBe('pNInz6obpgDQGcFmaJgB'); + }); + + it('clears a per-mascot voice via the row reset button', () => { + const store = buildStore(); + store.dispatch(setSelectedMascotId('yellow')); + store.dispatch(setSecondaryMascotId('toshi')); + renderPanel(store); + + // Seed an override, then reset it. + fireEvent.change(screen.getByTestId('mascot-voice-secondary-select'), { + target: { value: 'pNInz6obpgDQGcFmaJgB' }, + }); + expect(store.getState().mascot.mascotVoices.toshi).toBe('pNInz6obpgDQGcFmaJgB'); + + fireEvent.click(screen.getByTestId('mascot-voice-secondary-reset')); + expect(store.getState().mascot.mascotVoices.toshi).toBeUndefined(); + }); +}); diff --git a/app/src/components/settings/panels/__tests__/PerMascotVoiceRow.test.tsx b/app/src/components/settings/panels/__tests__/PerMascotVoiceRow.test.tsx new file mode 100644 index 000000000..ba28c3559 --- /dev/null +++ b/app/src/components/settings/panels/__tests__/PerMascotVoiceRow.test.tsx @@ -0,0 +1,203 @@ +import { configureStore } from '@reduxjs/toolkit'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { Provider } from 'react-redux'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import mascotReducer, { setMascotVoice } from '../../../../store/mascotSlice'; +import PerMascotVoiceRow from '../PerMascotVoiceRow'; + +const { mockSynthesizeSpeech } = vi.hoisted(() => ({ mockSynthesizeSpeech: vi.fn() })); + +vi.mock('../../../../features/human/voice/ttsClient', () => ({ + synthesizeSpeech: (...args: unknown[]) => mockSynthesizeSpeech(...args), +})); + +const TEST_ID = 'mascot-voice-primary'; +const MASCOT_ID = 'yellow'; + +function buildStore() { + return configureStore({ reducer: { mascot: mascotReducer } }); +} + +function renderRow(store = buildStore()) { + return { + store, + ...render( + + + + ), + }; +} + +/** Resolve helper to control the timing of the mocked synthesizeSpeech. */ +function deferredTts() { + let resolve!: (v: { audio_mime: string; audio_base64: string }) => void; + let reject!: (err: unknown) => void; + const promise = new Promise<{ audio_mime: string; audio_base64: string }>((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +describe('PerMascotVoiceRow', () => { + let playSpy: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + playSpy = vi.fn().mockResolvedValue(undefined); + // Stub window.Audio so `new window.Audio(src)` records the src and exposes + // a spy-able play()/pause() without touching the real audio pipeline. + vi.stubGlobal( + 'Audio', + class { + src: string; + constructor(src?: string) { + this.src = src ?? ''; + } + play = playSpy; + pause = vi.fn(); + } + ); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('reveals the paste input + Save when the custom option is selected and saves a new id', () => { + const { store } = renderRow(); + + // No custom input until the __custom__ option is chosen. + expect(screen.queryByTestId(`${TEST_ID}-input`)).not.toBeInTheDocument(); + + fireEvent.change(screen.getByTestId(`${TEST_ID}-select`), { target: { value: '__custom__' } }); + + const input = screen.getByTestId(`${TEST_ID}-input`); + expect(input).toBeInTheDocument(); + + const saveBtn = screen.getByTestId(`${TEST_ID}-save-paste`); + // Empty draft still equals the (empty) stored override → disabled. + expect(saveBtn).toBeDisabled(); + + // A whitespace-padded id enables Save and is trimmed on dispatch. + fireEvent.change(input, { target: { value: ' my-custom-voice ' } }); + expect(saveBtn).not.toBeDisabled(); + fireEvent.click(saveBtn); + + expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBe('my-custom-voice'); + }); + + it('plays a data:audio preview and toggles the previewing label when synthesizeSpeech resolves', async () => { + const d = deferredTts(); + mockSynthesizeSpeech.mockReturnValue(d.promise); + renderRow(); + + const previewBtn = screen.getByTestId(`${TEST_ID}-preview`); + fireEvent.click(previewBtn); + + // Previewing state toggles on immediately (button disabled + label swap). + await waitFor(() => expect(previewBtn).toBeDisabled()); + expect(previewBtn).toHaveTextContent('Previewing…'); + + d.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' }); + + await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(1)); + // A data:audio URI was constructed from the mime + base64 payload. + expect(playSpy.mock.instances[0].src).toBe('data:audio/mpeg;base64,QUJD'); + + // Previewing resets once the preview finishes. + await waitFor(() => expect(previewBtn).not.toBeDisabled()); + expect(previewBtn).toHaveTextContent('Preview voice'); + }); + + it('renders the preview-error text and resets previewing when synthesizeSpeech rejects', async () => { + const d = deferredTts(); + mockSynthesizeSpeech.mockReturnValue(d.promise); + renderRow(); + + const previewBtn = screen.getByTestId(`${TEST_ID}-preview`); + fireEvent.click(previewBtn); + await waitFor(() => expect(previewBtn).toBeDisabled()); + + d.reject(new Error('tts exploded')); + + const errorBox = await screen.findByTestId(`${TEST_ID}-preview-error`); + expect(errorBox).toHaveTextContent('tts exploded'); + expect(playSpy).not.toHaveBeenCalled(); + + // finally branch clears the previewing state. + await waitFor(() => expect(previewBtn).not.toBeDisabled()); + expect(previewBtn).toHaveTextContent('Preview voice'); + }); + + it('is a no-op when a preview resolves after the row has unmounted (previewRequestIdRef guard)', async () => { + const d = deferredTts(); + mockSynthesizeSpeech.mockReturnValue(d.promise); + const { unmount } = renderRow(); + + fireEvent.click(screen.getByTestId(`${TEST_ID}-preview`)); + await waitFor(() => expect(mockSynthesizeSpeech).toHaveBeenCalledTimes(1)); + + // Unmount bumps previewRequestIdRef, so the late resolve loses the race. + unmount(); + d.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' }); + + // Give the resolved promise a microtask/macrotask to flush. + await new Promise(r => setTimeout(r, 0)); + expect(playSpy).not.toHaveBeenCalled(); + }); + + it('exposes the current voice id set via setMascotVoice', () => { + const store = buildStore(); + store.dispatch(setMascotVoice({ mascotId: MASCOT_ID, voiceId: 'pNInz6obpgDQGcFmaJgB' })); + renderRow(store); + expect(screen.getByTestId(`${TEST_ID}-current`)).toHaveTextContent('pNInz6obpgDQGcFmaJgB'); + }); + + it('dispatches setMascotVoice when a curated preset is picked from the dropdown', () => { + const { store } = renderRow(); + fireEvent.change(screen.getByTestId(`${TEST_ID}-select`), { + target: { value: 'pNInz6obpgDQGcFmaJgB' }, + }); + expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBe('pNInz6obpgDQGcFmaJgB'); + }); + + it('clears the per-mascot override via the reset button', () => { + const store = buildStore(); + store.dispatch(setMascotVoice({ mascotId: MASCOT_ID, voiceId: 'pNInz6obpgDQGcFmaJgB' })); + renderRow(store); + + const resetBtn = screen.getByTestId(`${TEST_ID}-reset`); + expect(resetBtn).not.toBeDisabled(); + fireEvent.click(resetBtn); + + expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBeUndefined(); + }); + + it('stops the prior preview audio when a second preview starts', async () => { + const first = deferredTts(); + const second = deferredTts(); + mockSynthesizeSpeech.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise); + renderRow(); + + const previewBtn = screen.getByTestId(`${TEST_ID}-preview`); + + // First preview resolves and plays, seeding previewAudioRef. + fireEvent.click(previewBtn); + first.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' }); + await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(1)); + const firstAudio = playSpy.mock.instances[0]; + await waitFor(() => expect(previewBtn).not.toBeDisabled()); + + // Second preview should pause + clear the prior audio before synthesizing. + fireEvent.click(previewBtn); + await waitFor(() => expect(mockSynthesizeSpeech).toHaveBeenCalledTimes(2)); + expect(firstAudio.pause).toHaveBeenCalled(); + expect(firstAudio.src).toBe(''); + + second.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'WFla' }); + await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(2)); + }); +}); diff --git a/app/src/features/meet/MascotFrameProducer.tsx b/app/src/features/meet/MascotFrameProducer.tsx index 6acbdc9bb..332f4bc96 100644 --- a/app/src/features/meet/MascotFrameProducer.tsx +++ b/app/src/features/meet/MascotFrameProducer.tsx @@ -1,62 +1,69 @@ import { listen, type UnlistenFn } from '@tauri-apps/api/event'; -import { type FC, useCallback, useEffect, useRef, useState } from 'react'; +import { type FC, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { useSelector } from 'react-redux'; -import { RiveMascot } from '../human/Mascot'; +import { + selectCustomPrimaryColor, + selectCustomSecondaryColor, + selectMascotColor, + selectSecondaryMascotId, +} from '../../store/mascotSlice'; +import { + getMascotPalette, + hexToArgbInt, + ManifestRiveMascot, + type MascotFace, + type MascotManifestEntry, + RiveMascot, +} from '../human/Mascot'; +import { findMascot } from '../human/Mascot/manifest/manifestService'; +import { useMascotManifest } from '../human/Mascot/manifest/useMascotManifest'; +import { + drawMascotInCell, + FRAME_H, + FRAME_H_DUAL, + FRAME_W, + FRAME_W_DUAL, + MASCOT_INSET, + sampleCanvasPixels, +} from './mascotFrameCompositor'; +import { type ActiveMascotSlot, type MeetingPhase, useMeetingMascots } from './useMeetingMascots'; const PRODUCER_FPS = 24; -const FRAME_W = 320; -const FRAME_H = 240; const JPEG_QUALITY = 0.7; +/** + * How long the mascot(s) wave hello on join before settling into the live + * `active` face state (ms). Matches the greeting the participants see so the + * first frames read as "the bot is saying hi" rather than a cold stare. + */ +const GREETING_MS = 2500; +/** + * Teardown grace after `bus-stopped` (ms): keep the session mounted — WS + + * worker alive — long enough for the goodbye wave to stream before the frame + * pipeline is torn down. Without this the last thing the call sees is a hard + * cut mid-pose instead of a wave. + */ +const SIGNOFF_MS = 1500; + interface BusSession { requestId: string; port: number; } -export function sampleCanvasPixels( - ctx: OffscreenCanvasRenderingContext2D, - width: number, - height: number -) { - const cols = 7; - const rows = 5; - let sum = 0; - let min = 255; - let max = 0; - let count = 0; - let dark = 0; - let bright = 0; - - try { - for (let y = 0; y < rows; y++) { - for (let x = 0; x < cols; x++) { - const px = Math.max(0, Math.min(width - 1, Math.floor(((x + 0.5) * width) / cols))); - const py = Math.max(0, Math.min(height - 1, Math.floor(((y + 0.5) * height) / rows))); - const [r, g, b] = ctx.getImageData(px, py, 1, 1).data; - const luma = Math.round(r * 0.299 + g * 0.587 + b * 0.114); - sum += luma; - min = Math.min(min, luma); - max = Math.max(max, luma); - if (luma < 8) dark++; - if (luma > 32) bright++; - count++; - } - } - return { - avgLuma: Math.round(sum / Math.max(1, count)), - minLuma: min, - maxLuma: max, - darkSamples: dark, - brightSamples: bright, - sampleCount: count, - }; - } catch (err) { - return { error: String(err instanceof Error ? err.message : err) }; - } -} +// Re-export from the compositor for back-compat with existing importers +// (the producer test imports `sampleCanvasPixels` from here). The +// implementation moved to mascotFrameCompositor.ts (issue #4277). +export { sampleCanvasPixels }; export const MascotFrameProducer: FC = () => { const [session, setSession] = useState(null); + // Meeting lifecycle phase, owned here so it survives across the brief + // sign-off grace where `session` is still set but the bus has stopped. + const [phase, setPhase] = useState('greeting'); + // Set when `bus-stopped` fires; the session is cleared SIGNOFF_MS later so + // the goodbye wave can stream. A ref so the timers don't re-arm on render. + const signoffTimerRef = useRef | null>(null); useEffect(() => { let unlistenStarted: UnlistenFn | undefined; @@ -67,6 +74,11 @@ export const MascotFrameProducer: FC = () => { const payload = event.payload; if (!payload || !payload.port) return; console.log('[meet-video-producer] bus-started', payload); + if (signoffTimerRef.current) { + clearTimeout(signoffTimerRef.current); + signoffTimerRef.current = null; + } + setPhase('greeting'); setSession(payload); }) .then(stop => { @@ -77,7 +89,15 @@ export const MascotFrameProducer: FC = () => { listen<{ requestId?: string; request_id?: string }>('meet-video:bus-stopped', event => { console.log('[meet-video-producer] bus-stopped', event.payload); - setSession(null); + // Enter the goodbye wave and keep the pipeline alive for SIGNOFF_MS so + // the wave frames actually reach the call before we clear the session. + setPhase('signoff'); + if (signoffTimerRef.current) clearTimeout(signoffTimerRef.current); + signoffTimerRef.current = setTimeout(() => { + console.log('[meet-video-producer] sign-off grace elapsed, clearing session'); + signoffTimerRef.current = null; + setSession(null); + }, SIGNOFF_MS); }) .then(stop => { if (cancelled) stop(); @@ -89,69 +109,178 @@ export const MascotFrameProducer: FC = () => { cancelled = true; if (unlistenStarted) unlistenStarted(); if (unlistenStopped) unlistenStopped(); + if (signoffTimerRef.current) { + clearTimeout(signoffTimerRef.current); + signoffTimerRef.current = null; + } }; }, []); + // Advance greeting → active after GREETING_MS, per active session. Bound to + // requestId so a fresh session restarts the greeting. + useEffect(() => { + if (!session || phase !== 'greeting') return; + const id = setTimeout(() => { + console.log('[meet-video-producer] greeting elapsed → active', session.requestId); + setPhase('active'); + }, GREETING_MS); + return () => clearTimeout(id); + }, [session, phase]); + if (!session) return null; - return ; + return ; }; -const ProducerSession: FC<{ session: BusSession }> = ({ session }) => { +const ProducerSession: FC<{ session: BusSession; phase: MeetingPhase }> = ({ session, phase }) => { const hostRef = useRef(null); const wsRef = useRef(null); const wsReadyRef = useRef(false); const stoppedRef = useRef(false); const inflightRef = useRef(false); const lastDiagAtRef = useRef(0); - const isSpeakingRef = useRef(false); // True while the bot is actively producing PCM into the Meet call. // Drives the mascot face so the mouth animates in time with the audio // participants hear. Source of truth is the Rust speak_pump (edge-detected // from the RPC poll loop). Same requestId guards against stale events from // a previous session bleeding into this session's mascot state. const [isSpeaking, setIsSpeaking] = useState(false); + // Which mascot slot is speaking the current audio (issue #4277). Single- + // mascot calls always report slot 0. Read off the speaking-state event. + const [activeMascotSlot, setActiveMascotSlot] = useState(0); + // Per-slot render state (which mascot + which face) for this tick. + const render = useMeetingMascots({ speaking: isSpeaking, activeMascotSlot, phase }); + const { dualEnabled } = render; + + // Resolve the manifest entries for both slots. The primary follows the + // user's selection (same resolution the Human page uses); the secondary is + // looked up by its explicit id so the frame honors both selections and no + // longer always shows the bundled default (fixes the pre-existing single- + // mascot bug where the meeting camera ignored `selectedMascotId`). + const { manifest, entry: primaryEntry } = useMascotManifest(); + const secondaryMascotId = useSelector(selectSecondaryMascotId); + const secondaryEntry: MascotManifestEntry | null = + dualEnabled && manifest ? (findMascot(manifest, secondaryMascotId) ?? null) : null; + + // Mascot body colors, mirroring HumanPage so the meeting mascot matches the + // one on the Human stage. Per-mascot colors are out of scope (#4277) — both + // slots share the single selected color. + const mascotColor = useSelector(selectMascotColor); + const customPrimary = useSelector(selectCustomPrimaryColor); + const customSecondary = useSelector(selectCustomSecondaryColor); + const palette = getMascotPalette(mascotColor); + const primaryColor = useMemo( + () => hexToArgbInt(mascotColor === 'custom' ? customPrimary : palette.bodyFill), + [mascotColor, customPrimary, palette] + ); + const secondaryColor = useMemo( + () => hexToArgbInt(mascotColor === 'custom' ? customSecondary : palette.neckShadowColor), + [mascotColor, customSecondary, palette] + ); + + const isSpeakingRef = useRef(isSpeaking); useEffect(() => { isSpeakingRef.current = isSpeaking; }, [isSpeaking]); + // `dualEnabled` and `activeMascotSlot` are read inside captureFrame via refs + // so a speaker switch (activeMascotSlot 0↔1) or a dual-mode toggle does NOT + // change captureFrame's identity — otherwise the WS/worker effect below + // (which depends on captureFrame) would tear down and rebuild the socket + + // frame worker on every alternation. captureFrame stays keyed to the session + // only, matching the original single-mascot behavior. + const dualEnabledRef = useRef(dualEnabled); + useEffect(() => { + dualEnabledRef.current = dualEnabled; + }, [dualEnabled]); + const activeMascotSlotRef = useRef(activeMascotSlot); + useEffect(() => { + activeMascotSlotRef.current = activeMascotSlot; + }, [activeMascotSlot]); + const captureFrame = useCallback(async () => { if (stoppedRef.current || !wsReadyRef.current || inflightRef.current) return; const host = hostRef.current; if (!host) return; - const canvas = host.querySelector('canvas'); - if (!canvas) return; + const dualEnabledNow = dualEnabledRef.current; + // Look the source canvases up by slot so a face change (which re-renders + // the mascot) never changes which canvas we sample. + const primaryCanvas = host.querySelector( + '[data-mascot-slot="primary"] canvas' + ); + if (!primaryCanvas) return; + const secondaryCanvas = dualEnabledNow + ? host.querySelector('[data-mascot-slot="secondary"] canvas') + : null; + + const frameW = dualEnabledNow ? FRAME_W_DUAL : FRAME_W; + // The dual frame is taller (16:9) than the single frame so the fake-camera + // bridge's cover-scale fills the 1280×720 canvas without cropping — see + // FRAME_H_DUAL in mascotFrameCompositor.ts. + const frameH = dualEnabledNow ? FRAME_H_DUAL : FRAME_H; inflightRef.current = true; try { - const offscreen = new OffscreenCanvas(FRAME_W, FRAME_H); + const offscreen = new OffscreenCanvas(frameW, frameH); const ctx = offscreen.getContext('2d'); if (!ctx) return; const grad = ctx.createRadialGradient( - FRAME_W / 2, - FRAME_H / 2, + frameW / 2, + frameH / 2, 0, - FRAME_W / 2, - FRAME_H / 2, - Math.max(FRAME_W, FRAME_H) * 0.7 + frameW / 2, + frameH / 2, + Math.max(frameW, frameH) * 0.7 ); grad.addColorStop(0, '#FBF3D9'); grad.addColorStop(1, '#EFE3B8'); ctx.fillStyle = grad; - ctx.fillRect(0, 0, FRAME_W, FRAME_H); + ctx.fillRect(0, 0, frameW, frameH); - const inset = 0.06; - const fitW = FRAME_W * (1 - 2 * inset); - const fitH = FRAME_H * (1 - 2 * inset); - const scale = Math.min(fitW / canvas.width, fitH / canvas.height); - const dw = canvas.width * scale; - const dh = canvas.height * scale; - const dx = (FRAME_W - dw) / 2; - const dy = (FRAME_H - dh) / 2; - ctx.drawImage(canvas, dx, dy, dw, dh); + if (dualEnabledNow && secondaryCanvas) { + // Two half-cells: [0..half] primary, [half..frameW] secondary. + const half = frameW / 2; + drawMascotInCell( + ctx, + primaryCanvas, + 0, + 0, + half, + frameH, + MASCOT_INSET, + primaryCanvas.width, + primaryCanvas.height + ); + drawMascotInCell( + ctx, + secondaryCanvas, + half, + 0, + half, + frameH, + MASCOT_INSET, + secondaryCanvas.width, + secondaryCanvas.height + ); + } else { + // Single-cell draw. This also covers the dual-but-secondary-not-yet- + // mounted tick (a 2.2MB mascot still decoding): rather than emit a + // black half we draw the primary across the whole frame for this tick. + drawMascotInCell( + ctx, + primaryCanvas, + 0, + 0, + frameW, + frameH, + MASCOT_INSET, + primaryCanvas.width, + primaryCanvas.height + ); + } - const probe = sampleCanvasPixels(ctx, FRAME_W, FRAME_H); + const probe = sampleCanvasPixels(ctx, frameW, frameH); const blob = await offscreen.convertToBlob({ type: 'image/jpeg', quality: JPEG_QUALITY }); const buffer = await blob.arrayBuffer(); const ws = wsRef.current; @@ -163,12 +292,17 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => { JSON.stringify({ kind: 'producer-pixel-probe', requestId: session.requestId, - canvasWidth: canvas.width, - canvasHeight: canvas.height, - frameWidth: FRAME_W, - frameHeight: FRAME_H, + canvasWidth: primaryCanvas.width, + canvasHeight: primaryCanvas.height, + frameWidth: frameW, + frameHeight: frameH, jpegBytes: blob.size, isSpeaking: isSpeakingRef.current, + // Dual-mascot diagnostics (issue #4277): whether we drew two + // cells this tick, and which slot the audio is on. + dualEnabled: dualEnabledNow, + secondaryMounted: dualEnabledNow ? Boolean(secondaryCanvas) : undefined, + activeMascotSlot: activeMascotSlotRef.current, probe, }) ); @@ -241,13 +375,19 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => { // session — a remount tears it down with the rest of the pipeline. let unlistenSpeaking: UnlistenFn | undefined; let speakingListenerCancelled = false; - listen<{ requestId?: string; speaking?: boolean }>('meet-video:speaking-state', event => { - const payload = event.payload; - if (!payload) return; - // Ignore events from a different session during teardown / restart. - if (payload.requestId && payload.requestId !== session.requestId) return; - setIsSpeaking(!!payload.speaking); - }) + listen<{ requestId?: string; speaking?: boolean; activeMascotSlot?: number }>( + 'meet-video:speaking-state', + event => { + const payload = event.payload; + if (!payload) return; + // Ignore events from a different session during teardown / restart. + if (payload.requestId && payload.requestId !== session.requestId) return; + setIsSpeaking(!!payload.speaking); + // `activeMascotSlot` names which mascot is speaking this audio (0|1); + // default to slot 0 for single-mascot / older core builds that omit it. + setActiveMascotSlot(payload.activeMascotSlot === 1 ? 1 : 0); + } + ) .then(stop => { if (speakingListenerCancelled) stop(); else unlistenSpeaking = stop; @@ -289,14 +429,70 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => { position: 'fixed', left: '-99999px', top: 0, - width: FRAME_H, + width: dualEnabled ? FRAME_H * 2 : FRAME_H, height: FRAME_H, pointerEvents: 'none', opacity: 0, }}> - + {/* Slot 0 (primary). Stable key per mascot id so a face change updates in + place instead of remounting the Rive/WebGL context. */} +
+ +
+ {/* Slot 1 (secondary) — only mounted when a distinct second mascot is + enabled. Its 2.2MB asset may still be decoding for the first frames; + captureFrame falls back to a single-cell draw until its canvas exists. */} + {dualEnabled && render.secondary && ( +
+ +
+ )} ); }; +/** + * Render one mascot slot. Prefers the manifest mascot (honoring the user's + * selection) and falls back to the bundled `RiveMascot` while the manifest is + * still resolving so the frame never blanks. Keyed by mascot id upstream so a + * *selection* change remounts, while a *face* change updates in place. + */ +const MascotStage: FC<{ + entry: MascotManifestEntry | null; + face: MascotFace; + primaryColor: number; + secondaryColor: number; +}> = ({ entry, face, primaryColor, secondaryColor }) => { + if (entry) { + return ( + + ); + } + return ( + + ); +}; + export default MascotFrameProducer; diff --git a/app/src/features/meet/__tests__/MascotFrameProducer.test.tsx b/app/src/features/meet/__tests__/MascotFrameProducer.test.tsx index 05127bf1c..1cc4fd6f5 100644 --- a/app/src/features/meet/__tests__/MascotFrameProducer.test.tsx +++ b/app/src/features/meet/__tests__/MascotFrameProducer.test.tsx @@ -1,17 +1,204 @@ -import { cleanup } from '@testing-library/react'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { listen } from '@tauri-apps/api/event'; +import { act, cleanup, waitFor } from '@testing-library/react'; +import { createElement } from 'react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { renderWithProviders } from '../../../test/test-utils'; import { MascotFrameProducer, sampleCanvasPixels } from '../MascotFrameProducer'; -// @tauri-apps/api/event is already mocked in setup.ts (listen → vi.fn()) +// @tauri-apps/api/event is mocked in setup.ts (listen → vi.fn()). Here we +// override it per-test to capture the event handlers so we can drive a fake +// `meet-video:bus-started` and assert how many mascot hosts mount. +type Listener = (event: { payload: unknown }) => void; + +// The producer renders ManifestRiveMascot / RiveMascot (WebGL). Mock both leaf +// renderers to a plain host so the frame's slot structure is +// assertable without the Rive runtime. Each records the face it was asked to +// render via a data attribute. +vi.mock('../../human/Mascot', async () => { + const actual = await vi.importActual('../../human/Mascot'); + const stub = (props: { face?: string }) => + createElement('canvas', { 'data-face': props.face, width: 200, height: 200 }); + return { ...actual, ManifestRiveMascot: stub, RiveMascot: stub }; +}); + +// Keep the manifest resolution deterministic + synchronous — the host tree +// mounts regardless (MascotStage falls back to RiveMascot when entry is null), +// so we just avoid a real network fetch. +vi.mock('../../human/Mascot/manifest/useMascotManifest', () => ({ + useMascotManifest: () => ({ manifest: null, entry: null, loading: false, error: null }), +})); + +vi.mock('../../human/Mascot/manifest/manifestService', () => ({ findMascot: () => null })); + +// Timings the component uses (kept in sync with MascotFrameProducer.tsx). +const GREETING_MS = 2500; +const SIGNOFF_MS = 1500; + +/** + * A fake `Worker` whose `postMessage({cmd:'start'})` records itself so the test + * can fire a single tick on demand (invoking the producer's `onmessage`, which + * calls `captureFrame`). Real timers/intervals are avoided — the test drives + * ticks explicitly so the capture path runs deterministically under fake + * timers. + */ +const workers: FakeWorker[] = []; +class FakeWorker { + onmessage: ((e: { data: unknown }) => void) | null = null; + started = false; + terminated = false; + constructor() { + workers.push(this); + } + postMessage(msg: { cmd?: string }) { + if (msg?.cmd === 'start') this.started = true; + else if (msg?.cmd === 'stop') this.started = false; + } + terminate() { + this.terminated = true; + } + /** Deliver one 'tick' to the producer, as the interval would. */ + tick() { + this.onmessage?.({ data: 'tick' }); + } +} + +/** + * A fake `WebSocket` that opens synchronously (so `wsReadyRef` flips true) and + * records every `send()` so the capture path's binary frame + JSON probe are + * assertable. + */ +const sockets: FakeWebSocket[] = []; +class FakeWebSocket { + static OPEN = 1; + readyState = FakeWebSocket.OPEN; + binaryType = 'arraybuffer'; + onopen: (() => void) | null = null; + onclose: (() => void) | null = null; + onerror: ((e: unknown) => void) | null = null; + sent: unknown[] = []; + closed = false; + constructor() { + sockets.push(this); + // Fire onopen on the next microtask so the effect that assigns + // `ws.onopen = ...` has run before we invoke it. + queueMicrotask(() => this.onopen?.()); + } + send(data: unknown) { + this.sent.push(data); + } + close() { + this.closed = true; + this.onclose?.(); + } +} + +/** + * A fake 2D context + OffscreenCanvas that satisfies the capture path: + * gradient fill, per-cell drawImage, a pixel probe read, and a JPEG blob whose + * `arrayBuffer()` resolves so the buffer is `send()`-able. + */ +class FakeOffscreenCanvas { + drawImageCalls = 0; + constructor( + public width: number, + public height: number + ) { + offscreens.push(this); + } + getContext() { + const self = this; + return { + createRadialGradient: () => ({ addColorStop() {} }), + fillStyle: '' as unknown, + fillRect() {}, + drawImage() { + self.drawImageCalls++; + }, + getImageData: () => ({ data: [128, 128, 128, 255] }), + }; + } + convertToBlob() { + return Promise.resolve({ size: 1234, arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)) }); + } +} +const offscreens: FakeOffscreenCanvas[] = []; + +/** Install the browser globals the ProducerSession effect touches. */ +function installBrowserStubs() { + workers.length = 0; + sockets.length = 0; + offscreens.length = 0; + vi.stubGlobal('Worker', FakeWorker); + vi.stubGlobal('WebSocket', FakeWebSocket); + vi.stubGlobal('OffscreenCanvas', FakeOffscreenCanvas); + if (!('createObjectURL' in URL)) { + (URL as unknown as { createObjectURL: () => string }).createObjectURL = () => 'blob:x'; + } + if (!('revokeObjectURL' in URL)) { + (URL as unknown as { revokeObjectURL: () => void }).revokeObjectURL = () => {}; + } + // jsdom does not implement HTMLMediaElement.play(); the silent keep-alive + // audio the producer creates calls `.play().catch(...)`, so give it a + // resolving stub. + vi.spyOn(HTMLMediaElement.prototype, 'play').mockResolvedValue(undefined); +} + +/** + * Wire the mocked `listen` so each event name captures its handler. Returns a + * fn to fire a fake payload for a given event. + */ +function captureListeners() { + const handlers = new Map(); + vi.mocked(listen).mockImplementation((event: string, handler: unknown) => { + handlers.set(event, handler as Listener); + return Promise.resolve(vi.fn()); + }); + return { + fire(event: string, payload: unknown) { + const h = handlers.get(event); + if (!h) throw new Error(`no listener registered for ${event}`); + h({ payload }); + }, + has(event: string) { + return handlers.has(event); + }, + }; +} + +const SINGLE_MASCOT_STATE = { + mascot: { + color: 'yellow', + voiceId: null, + voiceGender: 'male', + voiceUseLocaleDefault: false, + selectedMascotId: 'tiny-mascot', + secondaryMascotId: null, + mascotVoices: {}, + customMascotGifUrl: null, + customPrimaryColor: '#F7D145', + customSecondaryColor: '#B23C05', + }, +}; + +const DUAL_MASCOT_STATE = { mascot: { ...SINGLE_MASCOT_STATE.mascot, secondaryMascotId: 'toshi' } }; describe('MascotFrameProducer', () => { - afterEach(() => cleanup()); + beforeEach(() => { + installBrowserStubs(); + }); + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + // Restore the setup.ts default so other files that rely on the shared + // `listen` mock (resolving to an unlisten fn) are unaffected. + vi.mocked(listen).mockReset(); + vi.mocked(listen).mockResolvedValue(vi.fn()); + }); it('renders nothing when no bus session is active', () => { const { container } = renderWithProviders(); - // Component returns null until a meet-video:bus-started Tauri event fires expect(container.firstChild).toBeNull(); }); @@ -21,44 +208,332 @@ describe('MascotFrameProducer', () => { unmount(); }).not.toThrow(); }); + + it('renders ONE mascot host for a single mascot', async () => { + const bus = captureListeners(); + const { container } = renderWithProviders(, { + preloadedState: SINGLE_MASCOT_STATE, + }); + + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'r1', port: 55555 }); + }); + + await waitFor(() => { + expect(container.querySelectorAll('[data-mascot-slot]').length).toBe(1); + }); + expect(container.querySelector('[data-mascot-slot="primary"]')).not.toBeNull(); + expect(container.querySelector('[data-mascot-slot="secondary"]')).toBeNull(); + }); + + it('renders TWO mascot hosts for two distinct mascots', async () => { + const bus = captureListeners(); + const { container } = renderWithProviders(, { + preloadedState: DUAL_MASCOT_STATE, + }); + + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'r2', port: 55556 }); + }); + + await waitFor(() => { + expect(container.querySelectorAll('[data-mascot-slot]').length).toBe(2); + }); + expect(container.querySelector('[data-mascot-slot="primary"]')).not.toBeNull(); + expect(container.querySelector('[data-mascot-slot="secondary"]')).not.toBeNull(); + }); + + it('ignores a bus-started payload with no port', async () => { + const bus = captureListeners(); + const { container } = renderWithProviders(, { + preloadedState: SINGLE_MASCOT_STATE, + }); + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'r0' }); + }); + // No port → guard returns early, session never set, nothing renders. + expect(container.querySelector('[data-mascot-slot]')).toBeNull(); + }); + + it('waves during greeting then transitions to active after GREETING_MS', async () => { + vi.useFakeTimers(); + try { + const bus = captureListeners(); + const { container } = renderWithProviders(, { + preloadedState: DUAL_MASCOT_STATE, + }); + + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'g1', port: 4100 }); + await vi.advanceTimersByTimeAsync(0); + }); + + // Greeting phase: both mascots wave. + const facesDuringGreeting = Array.from( + container.querySelectorAll('[data-mascot-slot] canvas') + ).map(c => c.getAttribute('data-face')); + expect(facesDuringGreeting).toEqual(['waving', 'waving']); + + // Advance past the greeting window → active phase. With no speaking + // event yet, the active-slot mascot rests (listening) and the other + // shows thinking — no longer both waving. + await act(async () => { + await vi.advanceTimersByTimeAsync(GREETING_MS + 10); + }); + + const facesAfter = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map( + c => c.getAttribute('data-face') + ); + expect(facesAfter).not.toEqual(['waving', 'waving']); + expect(facesAfter).toContain('listening'); + } finally { + vi.useRealTimers(); + } + }); + + it('holds the sign-off wave for SIGNOFF_MS then clears the session', async () => { + vi.useFakeTimers(); + try { + const bus = captureListeners(); + const { container } = renderWithProviders(, { + preloadedState: DUAL_MASCOT_STATE, + }); + + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 's1', port: 4200 }); + await vi.advanceTimersByTimeAsync(GREETING_MS + 10); + }); + // Session mounted (active phase). + expect(container.querySelector('[data-mascot-slot]')).not.toBeNull(); + + // bus-stopped → signoff phase; both mascots wave goodbye and the + // session stays mounted through the grace window. + await act(async () => { + bus.fire('meet-video:bus-stopped', { requestId: 's1' }); + await vi.advanceTimersByTimeAsync(0); + }); + const signoffFaces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map( + c => c.getAttribute('data-face') + ); + expect(signoffFaces).toEqual(['waving', 'waving']); + // Still mounted just before the grace elapses. + await act(async () => { + await vi.advanceTimersByTimeAsync(SIGNOFF_MS - 50); + }); + expect(container.querySelector('[data-mascot-slot]')).not.toBeNull(); + + // Grace elapsed → session cleared → producer renders nothing. + await act(async () => { + await vi.advanceTimersByTimeAsync(100); + }); + expect(container.querySelector('[data-mascot-slot]')).toBeNull(); + } finally { + vi.useRealTimers(); + } + }); + + it('re-arms a fresh greeting when bus-started fires during the sign-off grace', async () => { + vi.useFakeTimers(); + try { + const bus = captureListeners(); + const { container } = renderWithProviders(, { + preloadedState: DUAL_MASCOT_STATE, + }); + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'a1', port: 4300 }); + await vi.advanceTimersByTimeAsync(GREETING_MS + 10); + // Stop → enters signoff + arms the clear timer. + bus.fire('meet-video:bus-stopped', { requestId: 'a1' }); + await vi.advanceTimersByTimeAsync(SIGNOFF_MS - 200); + // A new session starts before the clear fires → clears the signoff + // timer and restarts greeting. + bus.fire('meet-video:bus-started', { requestId: 'a2', port: 4301 }); + await vi.advanceTimersByTimeAsync(0); + }); + const faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c => + c.getAttribute('data-face') + ); + expect(faces).toEqual(['waving', 'waving']); + + // The old signoff clear must NOT fire now; session stays mounted. + await act(async () => { + await vi.advanceTimersByTimeAsync(SIGNOFF_MS); + }); + expect(container.querySelector('[data-mascot-slot]')).not.toBeNull(); + } finally { + vi.useRealTimers(); + } + }); + + it('updates speaking state only for a matching requestId (gate)', async () => { + vi.useFakeTimers(); + try { + const bus = captureListeners(); + const { container } = renderWithProviders(, { + preloadedState: DUAL_MASCOT_STATE, + }); + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'sp1', port: 4400 }); + await vi.advanceTimersByTimeAsync(0); + }); + // Reach active phase so the face reflects speaking/slot rather than + // the greeting wave. + await act(async () => { + await vi.advanceTimersByTimeAsync(GREETING_MS + 10); + }); + expect( + Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c => + c.getAttribute('data-face') + ) + ).not.toEqual(['waving', 'waving']); + + // Non-matching requestId is ignored: slot 1 speaking with the wrong id + // must NOT flip any mascot to the speaking face. + await act(async () => { + bus.fire('meet-video:speaking-state', { + requestId: 'STALE', + speaking: true, + activeMascotSlot: 1, + }); + await vi.advanceTimersByTimeAsync(0); + }); + let faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c => + c.getAttribute('data-face') + ); + expect(faces).not.toContain('speaking'); + + // Matching requestId with slot 1 speaking → the secondary slot animates + // (speaking), primary shows thinking. + await act(async () => { + bus.fire('meet-video:speaking-state', { + requestId: 'sp1', + speaking: true, + activeMascotSlot: 1, + }); + await vi.advanceTimersByTimeAsync(0); + }); + faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c => + c.getAttribute('data-face') + ); + expect(faces).toEqual(['thinking', 'speaking']); + + // Slot 0 speaking → primary animates, secondary shows thinking. Also + // exercises the `activeMascotSlot === 1 ? 1 : 0` default-to-0 branch. + await act(async () => { + bus.fire('meet-video:speaking-state', { + requestId: 'sp1', + speaking: true, + activeMascotSlot: 0, + }); + await vi.advanceTimersByTimeAsync(0); + }); + faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c => + c.getAttribute('data-face') + ); + expect(faces).toEqual(['speaking', 'thinking']); + } finally { + vi.useRealTimers(); + } + }); + + it('ignores a speaking-state event with no payload', async () => { + const bus = captureListeners(); + renderWithProviders(, { preloadedState: SINGLE_MASCOT_STATE }); + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'np1', port: 4500 }); + }); + await waitFor(() => expect(bus.has('meet-video:speaking-state')).toBe(true)); + expect(() => + act(() => { + bus.fire('meet-video:speaking-state', null); + }) + ).not.toThrow(); + }); + + it('captures and sends a single-mascot frame over the websocket on a worker tick', async () => { + const bus = captureListeners(); + renderWithProviders(, { preloadedState: SINGLE_MASCOT_STATE }); + + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'cap1', port: 4600 }); + }); + // Wait for the WS/worker effect to wire up (worker registered, socket + // open). + await waitFor(() => expect(workers.length).toBeGreaterThan(0)); + await act(async () => { + // let the queued microtask fire ws.onopen + await Promise.resolve(); + }); + + await act(async () => { + workers[workers.length - 1].tick(); + // captureFrame is async (blob → arrayBuffer); flush its microtasks. + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + const ws = sockets[sockets.length - 1]; + // A binary ArrayBuffer frame was sent. + const binary = ws.sent.filter(m => m instanceof ArrayBuffer); + expect(binary.length).toBeGreaterThan(0); + // The diagnostic JSON probe was also sent, and reports single-mascot. + const jsonMsgs = ws.sent + .filter((m): m is string => typeof m === 'string') + .map(m => JSON.parse(m)); + expect(jsonMsgs.some(p => p.kind === 'producer-pixel-probe' && p.dualEnabled === false)).toBe( + true + ); + // Single-cell draw → drawMascotInCell called once. + expect(offscreens[offscreens.length - 1].drawImageCalls).toBe(1); + }); + + it('captures and sends a dual-mascot frame (two-cell composite) on a worker tick', async () => { + const bus = captureListeners(); + renderWithProviders(, { preloadedState: DUAL_MASCOT_STATE }); + + await act(async () => { + bus.fire('meet-video:bus-started', { requestId: 'cap2', port: 4700 }); + }); + await waitFor(() => expect(workers.length).toBeGreaterThan(0)); + // Ensure both mascot slots are mounted so the dual (two-cell) branch runs. + await waitFor(() => + expect(document.querySelectorAll('[data-mascot-slot="secondary"] canvas').length).toBe(1) + ); + await act(async () => { + await Promise.resolve(); + }); + + await act(async () => { + workers[workers.length - 1].tick(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + const ws = sockets[sockets.length - 1]; + const jsonMsgs = ws.sent + .filter((m): m is string => typeof m === 'string') + .map(m => JSON.parse(m)); + expect( + jsonMsgs.some( + p => p.kind === 'producer-pixel-probe' && p.dualEnabled === true && p.secondaryMounted + ) + ).toBe(true); + // Two-cell draw → drawMascotInCell called twice (primary + secondary). + expect(offscreens[offscreens.length - 1].drawImageCalls).toBe(2); + expect(ws.sent.filter(m => m instanceof ArrayBuffer).length).toBeGreaterThan(0); + }); }); -describe('sampleCanvasPixels', () => { - it('returns pixel stats for a canvas with mid-range luma', () => { - // luma = 0.299*128 + 0.587*128 + 0.114*128 ≈ 128 +// sampleCanvasPixels is still exported from the producer (re-exported from the +// compositor for back-compat); a light smoke check keeps that surface covered +// here. Full assertions live in mascotFrameCompositor.test.ts. +describe('sampleCanvasPixels (re-export)', () => { + it('is re-exported and returns pixel stats', () => { const mockCtx = { getImageData: vi.fn().mockReturnValue({ data: [128, 128, 128, 255] }), } as unknown as OffscreenCanvasRenderingContext2D; - - const result = sampleCanvasPixels(mockCtx, 320, 240); - expect(result).toMatchObject({ - avgLuma: 128, - minLuma: 128, - maxLuma: 128, - darkSamples: 0, - brightSamples: 35, // all 35 samples have luma > 32 - sampleCount: 35, // 7 cols × 5 rows - }); - }); - - it('counts dark samples correctly for near-black pixels', () => { - // luma ≈ 0.299*4 + 0.587*4 + 0.114*4 ≈ 4 → dark (< 8), not bright (> 32) - const mockCtx = { - getImageData: vi.fn().mockReturnValue({ data: [4, 4, 4, 255] }), - } as unknown as OffscreenCanvasRenderingContext2D; - - const result = sampleCanvasPixels(mockCtx, 320, 240); - expect(result).toMatchObject({ darkSamples: 35, brightSamples: 0 }); - }); - - it('returns an error object when getImageData throws', () => { - const mockCtx = { - getImageData: vi.fn().mockImplementation(() => { - throw new Error('canvas tainted'); - }), - } as unknown as OffscreenCanvasRenderingContext2D; - - const result = sampleCanvasPixels(mockCtx, 320, 240); - expect(result).toMatchObject({ error: 'canvas tainted' }); + expect(sampleCanvasPixels(mockCtx, 320, 240)).toMatchObject({ avgLuma: 128, sampleCount: 35 }); }); }); diff --git a/app/src/features/meet/__tests__/mascotFrameCompositor.test.ts b/app/src/features/meet/__tests__/mascotFrameCompositor.test.ts new file mode 100644 index 000000000..e1fa5072a --- /dev/null +++ b/app/src/features/meet/__tests__/mascotFrameCompositor.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + drawMascotInCell, + FRAME_H, + FRAME_H_DUAL, + FRAME_W, + FRAME_W_DUAL, + MASCOT_INSET, + sampleCanvasPixels, +} from '../mascotFrameCompositor'; + +/** A drawImage-only ctx stub that records the last destination rect. */ +function makeCtx() { + const drawImage = + vi.fn<(image: CanvasImageSource, dx: number, dy: number, dw: number, dh: number) => void>(); + return { drawImage }; +} + +describe('mascotFrameCompositor geometry constants', () => { + it('exposes the locked single/dual frame geometry', () => { + expect(FRAME_W).toBe(320); + expect(FRAME_W_DUAL).toBe(480); + expect(FRAME_H).toBe(240); + expect(FRAME_H_DUAL).toBe(270); + expect(MASCOT_INSET).toBeCloseTo(0.06); + }); + + it('keeps the dual frame at 16:9 so the camera bridge cover-scale never crops', () => { + // The Tauri camera bridge cover-scales the frame onto a 1280×720 (16:9) + // canvas; any non-16:9 frame loses its overflowing axis. Locking the dual + // frame to 16:9 keeps both mascots' outer edges intact. + expect(FRAME_W_DUAL / FRAME_H_DUAL).toBeCloseTo(16 / 9); + }); +}); + +describe('drawMascotInCell — containment (AC#6)', () => { + it('contain-scales a TALL source and centers it (fits height, letterboxed x)', () => { + const ctx = makeCtx(); + // Cell = full single frame; source is taller than wide. + const srcW = 100; + const srcH = 400; + const rect = drawMascotInCell( + ctx, + {} as CanvasImageSource, + 0, + 0, + FRAME_W, + FRAME_H, + MASCOT_INSET, + srcW, + srcH + ); + + const fitW = FRAME_W * (1 - 2 * MASCOT_INSET); // 281.6 + const fitH = FRAME_H * (1 - 2 * MASCOT_INSET); // 211.2 + const scale = Math.min(fitW / srcW, fitH / srcH); // height-bound → fitH/400 + const dw = srcW * scale; + const dh = srcH * scale; + const dx = (FRAME_W - dw) / 2; + const dy = (FRAME_H - dh) / 2; + + // Height-bound: dh must equal the padded fit height, and never exceed the cell. + expect(dh).toBeCloseTo(fitH); + expect(rect.dw).toBeCloseTo(dw); + expect(rect.dh).toBeCloseTo(dh); + expect(rect.dx).toBeCloseTo(dx); + expect(rect.dy).toBeCloseTo(dy); + // No clipping: the drawn rect stays fully inside the cell bounds. + expect(rect.dx).toBeGreaterThanOrEqual(0); + expect(rect.dy).toBeGreaterThanOrEqual(0); + expect(rect.dx + rect.dw).toBeLessThanOrEqual(FRAME_W + 1e-6); + expect(rect.dy + rect.dh).toBeLessThanOrEqual(FRAME_H + 1e-6); + expect(ctx.drawImage).toHaveBeenCalledWith(expect.anything(), dx, dy, dw, dh); + }); + + it('contain-scales a WIDE source and centers it (fits width, letterboxed y)', () => { + const ctx = makeCtx(); + const srcW = 400; + const srcH = 100; + const rect = drawMascotInCell( + ctx, + {} as CanvasImageSource, + 0, + 0, + FRAME_W, + FRAME_H, + MASCOT_INSET, + srcW, + srcH + ); + + const fitW = FRAME_W * (1 - 2 * MASCOT_INSET); + const fitH = FRAME_H * (1 - 2 * MASCOT_INSET); + const scale = Math.min(fitW / srcW, fitH / srcH); // width-bound → fitW/400 + const dw = srcW * scale; + const dh = srcH * scale; + + // Width-bound: dw must equal the padded fit width. + expect(dw).toBeCloseTo(fitW); + expect(rect.dw).toBeCloseTo(dw); + expect(rect.dh).toBeCloseTo(dh); + // Still fully contained. + expect(rect.dx).toBeGreaterThanOrEqual(0); + expect(rect.dx + rect.dw).toBeLessThanOrEqual(FRAME_W + 1e-6); + expect(rect.dy + rect.dh).toBeLessThanOrEqual(FRAME_H + 1e-6); + }); + + it('draws into each half-cell in dual mode without crossing the divider', () => { + const half = FRAME_W_DUAL / 2; // 240 + // Square source so scale is symmetric and easy to reason about. + const srcW = 200; + const srcH = 200; + + const leftCtx = makeCtx(); + const left = drawMascotInCell( + leftCtx, + {} as CanvasImageSource, + 0, + 0, + half, + FRAME_H, + MASCOT_INSET, + srcW, + srcH + ); + const rightCtx = makeCtx(); + const right = drawMascotInCell( + rightCtx, + {} as CanvasImageSource, + half, + 0, + half, + FRAME_H, + MASCOT_INSET, + srcW, + srcH + ); + + // Left cell stays entirely left of the divider. + expect(left.dx).toBeGreaterThanOrEqual(0); + expect(left.dx + left.dw).toBeLessThanOrEqual(half + 1e-6); + // Right cell stays entirely right of the divider. + expect(right.dx).toBeGreaterThanOrEqual(half - 1e-6); + expect(right.dx + right.dw).toBeLessThanOrEqual(FRAME_W_DUAL + 1e-6); + // Both cells share the same size (identical source + cell dims). + expect(right.dw).toBeCloseTo(left.dw); + expect(right.dh).toBeCloseTo(left.dh); + // The right cell is offset by exactly `half` from the left one. + expect(right.dx - left.dx).toBeCloseTo(half); + }); + + it('never divides by zero for a not-yet-laid-out (0×0) source', () => { + const ctx = makeCtx(); + const rect = drawMascotInCell( + ctx, + {} as CanvasImageSource, + 0, + 0, + FRAME_W, + FRAME_H, + MASCOT_INSET, + 0, + 0 + ); + // With the 1px guard the rect is finite and contained, not NaN. + expect(Number.isFinite(rect.dw)).toBe(true); + expect(Number.isFinite(rect.dh)).toBe(true); + expect(rect.dx).toBeGreaterThanOrEqual(0); + expect(rect.dy).toBeGreaterThanOrEqual(0); + }); +}); + +// Moved from MascotFrameProducer.test.tsx (issue #4277) — sampleCanvasPixels +// now lives in the compositor module. +describe('sampleCanvasPixels', () => { + it('returns pixel stats for a canvas with mid-range luma', () => { + // luma = 0.299*128 + 0.587*128 + 0.114*128 ≈ 128 + const mockCtx = { + getImageData: vi.fn().mockReturnValue({ data: [128, 128, 128, 255] }), + } as unknown as OffscreenCanvasRenderingContext2D; + + const result = sampleCanvasPixels(mockCtx, 320, 240); + expect(result).toMatchObject({ + avgLuma: 128, + minLuma: 128, + maxLuma: 128, + darkSamples: 0, + brightSamples: 35, // all 35 samples have luma > 32 + sampleCount: 35, // 7 cols × 5 rows + }); + }); + + it('counts dark samples correctly for near-black pixels', () => { + // luma ≈ 0.299*4 + 0.587*4 + 0.114*4 ≈ 4 → dark (< 8), not bright (> 32) + const mockCtx = { + getImageData: vi.fn().mockReturnValue({ data: [4, 4, 4, 255] }), + } as unknown as OffscreenCanvasRenderingContext2D; + + const result = sampleCanvasPixels(mockCtx, 320, 240); + expect(result).toMatchObject({ darkSamples: 35, brightSamples: 0 }); + }); + + it('returns an error object when getImageData throws', () => { + const mockCtx = { + getImageData: vi.fn().mockImplementation(() => { + throw new Error('canvas tainted'); + }), + } as unknown as OffscreenCanvasRenderingContext2D; + + const result = sampleCanvasPixels(mockCtx, 320, 240); + expect(result).toMatchObject({ error: 'canvas tainted' }); + }); +}); diff --git a/app/src/features/meet/__tests__/useMeetingMascots.test.ts b/app/src/features/meet/__tests__/useMeetingMascots.test.ts new file mode 100644 index 000000000..9d4f6e2f1 --- /dev/null +++ b/app/src/features/meet/__tests__/useMeetingMascots.test.ts @@ -0,0 +1,136 @@ +import { renderHook } from '@testing-library/react'; +import { createElement, type PropsWithChildren } from 'react'; +import { Provider } from 'react-redux'; +import { describe, expect, it } from 'vitest'; + +import { createTestStore } from '../../../test/test-utils'; +import { + type MeetingPhase, + useMeetingMascots, + type UseMeetingMascotsInput, +} from '../useMeetingMascots'; + +/** + * Seed the mascot slice via preloadedState. `secondaryMascotId` distinct from + * `selectedMascotId` is what flips `selectDualMascotEnabled` on. + */ +function makeWrapper(mascot: Record) { + const store = createTestStore({ + mascot: { + color: 'yellow', + voiceId: null, + voiceGender: 'male', + voiceUseLocaleDefault: false, + selectedMascotId: null, + secondaryMascotId: null, + mascotVoices: {}, + customMascotGifUrl: null, + customPrimaryColor: '#F7D145', + customSecondaryColor: '#B23C05', + ...mascot, + }, + }); + return function Wrapper({ children }: PropsWithChildren) { + return createElement(Provider, { store, children }); + }; +} + +function run(mascot: Record, input: UseMeetingMascotsInput) { + const { result } = renderHook(() => useMeetingMascots(input), { wrapper: makeWrapper(mascot) }); + return result.current; +} + +const SINGLE = { selectedMascotId: 'tiny-mascot', secondaryMascotId: null }; +const DUAL = { selectedMascotId: 'tiny-mascot', secondaryMascotId: 'toshi' }; + +describe('useMeetingMascots — dualEnabled gating', () => { + it('is single when no secondary mascot is set', () => { + const state = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase: 'active' }); + expect(state.dualEnabled).toBe(false); + expect(state.secondary).toBeNull(); + expect(state.primary.mascotId).toBe('tiny-mascot'); + }); + + it('is single when the secondary equals the primary (same mascot picked twice)', () => { + const state = run( + { selectedMascotId: 'tiny-mascot', secondaryMascotId: 'tiny-mascot' }, + { speaking: false, activeMascotSlot: 0, phase: 'active' } + ); + expect(state.dualEnabled).toBe(false); + expect(state.secondary).toBeNull(); + }); + + it('is dual when a distinct secondary mascot is set', () => { + const state = run(DUAL, { speaking: false, activeMascotSlot: 0, phase: 'active' }); + expect(state.dualEnabled).toBe(true); + expect(state.primary.mascotId).toBe('tiny-mascot'); + expect(state.secondary?.mascotId).toBe('toshi'); + }); +}); + +describe('useMeetingMascots — single-mascot face (preserves original behavior)', () => { + it('primary follows speaking → speaking, else idle; secondary null', () => { + const speaking = run(SINGLE, { speaking: true, activeMascotSlot: 0, phase: 'active' }); + expect(speaking.primary.face).toBe('speaking'); + expect(speaking.secondary).toBeNull(); + + const silent = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase: 'active' }); + expect(silent.primary.face).toBe('idle'); + }); + + it('single-mascot ignores phase for the face (no greeting/signoff wave)', () => { + // The single path deliberately keeps the legacy speaking/idle mapping. + for (const phase of ['greeting', 'active', 'signoff'] as MeetingPhase[]) { + const state = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase }); + expect(state.primary.face).toBe('idle'); + } + }); +}); + +describe('useMeetingMascots — dual face table', () => { + it('greeting → both slots wave, regardless of speaking/activeSlot', () => { + for (const activeMascotSlot of [0, 1] as const) { + for (const speaking of [false, true]) { + const state = run(DUAL, { speaking, activeMascotSlot, phase: 'greeting' }); + expect(state.primary.face).toBe('waving'); + expect(state.secondary?.face).toBe('waving'); + } + } + }); + + it('signoff → both slots wave, regardless of speaking/activeSlot', () => { + for (const activeMascotSlot of [0, 1] as const) { + for (const speaking of [false, true]) { + const state = run(DUAL, { speaking, activeMascotSlot, phase: 'signoff' }); + expect(state.primary.face).toBe('waving'); + expect(state.secondary?.face).toBe('waving'); + } + } + }); + + describe('active phase — activeSlot × speaking', () => { + it('slot 0 active + speaking → primary speaking, secondary thinking', () => { + const s = run(DUAL, { speaking: true, activeMascotSlot: 0, phase: 'active' }); + expect(s.primary.face).toBe('speaking'); + expect(s.secondary?.face).toBe('thinking'); + }); + + it('slot 0 active + not speaking → primary listening, secondary thinking', () => { + const s = run(DUAL, { speaking: false, activeMascotSlot: 0, phase: 'active' }); + expect(s.primary.face).toBe('listening'); + expect(s.secondary?.face).toBe('thinking'); + }); + + it('slot 1 active + speaking → secondary speaking, primary thinking', () => { + const s = run(DUAL, { speaking: true, activeMascotSlot: 1, phase: 'active' }); + expect(s.secondary?.face).toBe('speaking'); + expect(s.primary.face).toBe('thinking'); + }); + + it('slot 1 active + not speaking → secondary listening, primary thinking', () => { + const s = run(DUAL, { speaking: false, activeMascotSlot: 1, phase: 'active' }); + expect(s.secondary?.face).toBe('listening'); + expect(s.primary.face).toBe('thinking'); + }); + }); +}); diff --git a/app/src/features/meet/mascotFrameCompositor.ts b/app/src/features/meet/mascotFrameCompositor.ts new file mode 100644 index 000000000..7dca882c5 --- /dev/null +++ b/app/src/features/meet/mascotFrameCompositor.ts @@ -0,0 +1,129 @@ +/** + * Pure canvas-geometry helpers for the Meet video producer (issue #4277). + * + * Kept free of any WebGL / DOM-mount dependency so the containment math is + * cheap to unit-test in isolation. The producer (`MascotFrameProducer.tsx`) + * owns the OffscreenCanvas + WebSocket pipeline; this module owns only the + * "where does each mascot get drawn" arithmetic, so the no-clip guarantee + * (AC#6) is proven by the compositor tests rather than a WebGL render. + */ + +/** Frame width for a single mascot (unchanged from the original producer). */ +export const FRAME_W = 320; +/** Frame height for a single mascot (unchanged from the original producer). */ +export const FRAME_H = 240; +/** + * Frame dimensions when two mascots share the frame side-by-side (issue #4277). + * Wider than the single frame so each mascot keeps roughly the single-frame + * cell width instead of being squeezed to half. + * + * 480×270 is deliberately 16:9 — the same aspect as the fake-camera capture + * canvas (1280×720). The Tauri camera bridge cover-scales the received frame + * onto that canvas (`scale = Math.max(W/bw, H/bh)` in `camera_bridge.js`), so a + * frame whose aspect differs from 16:9 gets its overflowing axis cropped. A + * 480×240 (2:1) dual frame would be scaled to 1440×720 and lose ~27 source px + * off each side — clipping the two mascots' outer edges. Matching 16:9 makes the + * cover-scale a pure fit with no crop. + */ +export const FRAME_W_DUAL = 480; +export const FRAME_H_DUAL = 270; +/** + * Fraction of each cell reserved as padding on every side before the mascot + * is scaled to fit. Matches the original single-mascot inset so the framing + * is visually identical in the single path. + */ +export const MASCOT_INSET = 0.06; + +/** + * Draw `sourceCanvas` scaled to *contain* (never crop) inside the cell at + * `(cellX, cellY)` of size `cellW × cellH`, centred, with `inset` padding on + * every side. + * + * Contain-scaling (`min` of the two axis ratios) guarantees the mascot always + * fits within the padded cell, so it can never be clipped by the cell edge + * (AC#6) regardless of the source canvas aspect ratio. The mascot is centred + * within the cell so any leftover space is split evenly. + * + * Returns the computed destination rect for the caller's diagnostics (and so + * the geometry is directly assertable in tests) — the draw itself is the side + * effect. + */ +export function drawMascotInCell( + ctx: { + drawImage: (image: CanvasImageSource, dx: number, dy: number, dw: number, dh: number) => void; + }, + sourceCanvas: CanvasImageSource, + cellX: number, + cellY: number, + cellW: number, + cellH: number, + inset: number, + srcW: number, + srcH: number +): { dx: number; dy: number; dw: number; dh: number } { + // Guard against a zero-sized source (a canvas that hasn't laid out yet): + // scaling by it would produce NaN and a silent no-op draw. + const safeSrcW = srcW > 0 ? srcW : 1; + const safeSrcH = srcH > 0 ? srcH : 1; + + const fitW = cellW * (1 - 2 * inset); + const fitH = cellH * (1 - 2 * inset); + const scale = Math.min(fitW / safeSrcW, fitH / safeSrcH); + const dw = safeSrcW * scale; + const dh = safeSrcH * scale; + const dx = cellX + (cellW - dw) / 2; + const dy = cellY + (cellH - dh) / 2; + ctx.drawImage(sourceCanvas, dx, dy, dw, dh); + return { dx, dy, dw, dh }; +} + +/** + * Sample a coarse 7×5 grid of luma values from a rendered frame. Used as a + * cheap "is the mascot actually on the frame or is it black?" diagnostic that + * the producer streams over the debug WebSocket every couple of seconds. + * + * Moved here from `MascotFrameProducer.tsx` (issue #4277) so it lives next to + * the rest of the frame geometry; the producer re-exports it for back-compat + * with existing importers. + */ +export function sampleCanvasPixels( + ctx: OffscreenCanvasRenderingContext2D, + width: number, + height: number +) { + const cols = 7; + const rows = 5; + let sum = 0; + let min = 255; + let max = 0; + let count = 0; + let dark = 0; + let bright = 0; + + try { + for (let y = 0; y < rows; y++) { + for (let x = 0; x < cols; x++) { + const px = Math.max(0, Math.min(width - 1, Math.floor(((x + 0.5) * width) / cols))); + const py = Math.max(0, Math.min(height - 1, Math.floor(((y + 0.5) * height) / rows))); + const [r, g, b] = ctx.getImageData(px, py, 1, 1).data; + const luma = Math.round(r * 0.299 + g * 0.587 + b * 0.114); + sum += luma; + min = Math.min(min, luma); + max = Math.max(max, luma); + if (luma < 8) dark++; + if (luma > 32) bright++; + count++; + } + } + return { + avgLuma: Math.round(sum / Math.max(1, count)), + minLuma: min, + maxLuma: max, + darkSamples: dark, + brightSamples: bright, + sampleCount: count, + }; + } catch (err) { + return { error: String(err instanceof Error ? err.message : err) }; + } +} diff --git a/app/src/features/meet/useMeetingMascots.ts b/app/src/features/meet/useMeetingMascots.ts new file mode 100644 index 000000000..787846f30 --- /dev/null +++ b/app/src/features/meet/useMeetingMascots.ts @@ -0,0 +1,150 @@ +/** + * Resolve per-slot mascot render state for the Meet video frame (issue #4277). + * + * The producer (`MascotFrameProducer.tsx`) composites up to two mascots into + * the outgoing camera frame. This hook is the single source of truth for + * *which* mascot each slot shows and *what face* it wears, given the live + * speaking-state event + the meeting phase. Kept as a thin, pure-ish selector + * hook (redux in → render state out) so the face table is unit-testable via + * `renderHook` + `preloadedState` without mounting any WebGL. + * + * Slot mapping (locked by the mascotSlice contract): + * - slot 0 = primary = `selectedMascotId` + * - slot 1 = secondary = `secondaryMascotId` + * and `activeMascotSlot` on the speaking-state event names which slot is + * currently speaking the audio participants hear. + */ +import debug from 'debug'; + +import { useAppSelector } from '../../store/hooks'; +import { + selectDualMascotEnabled, + selectMeetingMascotVoicePair, + selectSecondaryMascotId, + selectSelectedMascotId, +} from '../../store/mascotSlice'; +import type { MascotFace } from '../human/Mascot'; + +const log = debug('meet:mascots'); + +/** + * Coarse lifecycle phase of the on-camera mascot(s): + * - `greeting` — just joined; both mascots wave hello. + * - `active` — the call is live; faces track who is speaking. + * - `signoff` — the session is tearing down; both wave goodbye. + */ +export type MeetingPhase = 'greeting' | 'active' | 'signoff'; + +/** Which slot is producing the audio the call currently hears. */ +export type ActiveMascotSlot = 0 | 1; + +export interface MeetingMascotSlotRender { + /** Manifest mascot id, or `null` for the primary on the default mascot. */ + mascotId: string | null; + /** MascotFace to render this tick. */ + face: MascotFace; +} + +export interface MeetingMascotsRenderState { + /** True when a distinct second mascot is enabled (drives dual composite). */ + dualEnabled: boolean; + primary: MeetingMascotSlotRender; + /** Non-null only when `dualEnabled`. */ + secondary: MeetingMascotSlotRender | null; +} + +export interface UseMeetingMascotsInput { + /** Live from `meet-video:speaking-state` — is audio currently streaming. */ + speaking: boolean; + /** Live from `meet-video:speaking-state` — which slot owns that audio. */ + activeMascotSlot: ActiveMascotSlot; + /** Meeting lifecycle phase; drives the greeting / sign-off wave. */ + phase: MeetingPhase; +} + +/** + * Face the *speaking* slot wears while the call is active: mouth-animating + * `speaking` when audio is streaming, otherwise `listening` (attentive rest). + * Both map to the `idle` body pose in `FACE_TO_POSE`, but `speaking` is what + * drives the viseme mouth on the producer, so the distinction is load-bearing. + */ +function activeSpeakerFace(speaking: boolean): MascotFace { + return speaking ? 'speaking' : 'listening'; +} + +/** + * The face for a single slot in the two-mascot layout. + * - greeting / signoff → both wave (`waving` → `hand_wave` pose). + * - active: the slot that owns the current audio follows + * {@link activeSpeakerFace}; the other slot shows `thinking` — the only + * asset-distinct "reacting / listening" body pose (verified against + * `FACE_TO_POSE` in riveMaps: `listening` collapses to `idle`, so it would + * be visually indistinguishable from the speaker's rest state; `thinking` + * is the distinct pose that reads as "the other mascot is paying + * attention"). + */ +function dualSlotFace(slot: ActiveMascotSlot, input: UseMeetingMascotsInput): MascotFace { + if (input.phase === 'greeting' || input.phase === 'signoff') return 'waving'; + if (slot === input.activeMascotSlot) return activeSpeakerFace(input.speaking); + return 'thinking'; +} + +/** + * Compute the per-slot render state. Exposed as a standalone pure function so + * the face table can be exercised directly (the hook is a thin redux wrapper + * over it). + */ +export function computeMeetingMascotsRenderState( + dualEnabled: boolean, + primaryMascotId: string | null, + secondaryMascotId: string | null, + input: UseMeetingMascotsInput +): MeetingMascotsRenderState { + if (!dualEnabled) { + // Single-mascot path — preserves the original producer behavior exactly: + // primary follows speaking → speaking/idle, no secondary slot. + return { + dualEnabled: false, + primary: { mascotId: primaryMascotId, face: input.speaking ? 'speaking' : 'idle' }, + secondary: null, + }; + } + return { + dualEnabled: true, + primary: { mascotId: primaryMascotId, face: dualSlotFace(0, input) }, + secondary: { mascotId: secondaryMascotId, face: dualSlotFace(1, input) }, + }; +} + +/** + * Hook form: reads the mascot slice and folds in the live speaking-state + + * phase to produce the render state the producer composites from. + */ +export function useMeetingMascots(input: UseMeetingMascotsInput): MeetingMascotsRenderState { + const dualEnabled = useAppSelector(selectDualMascotEnabled); + const primaryMascotId = useAppSelector(selectSelectedMascotId); + const secondaryMascotId = useAppSelector(selectSecondaryMascotId); + // Resolve the voice pair too so the primary slot's mascotId matches exactly + // what the join payload sent — the pair is the single source of truth and + // reading it here keeps the on-camera mascot and the spoken voice aligned. + const pair = useAppSelector(selectMeetingMascotVoicePair); + + const state = computeMeetingMascotsRenderState( + dualEnabled, + pair.primary.mascotId ?? primaryMascotId, + pair.secondary?.mascotId ?? secondaryMascotId, + input + ); + + log( + 'render state dual=%s phase=%s speaking=%s activeSlot=%d primaryFace=%s secondaryFace=%s', + state.dualEnabled, + input.phase, + input.speaking, + input.activeMascotSlot, + state.primary.face, + state.secondary?.face ?? 'none' + ); + + return state; +} diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index fb3fe3616..8d1cdb59b 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -5267,6 +5267,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': 'اختر لون التميمة المستخدم في جميع أنحاء التطبيق', 'settings.mascot.noCharacters': 'لا توجد شخصيات OpenHuman متاحة بعد', 'settings.mascot.noColorVariants': 'لا توجد ألوان متاحة', + 'settings.mascot.secondaryHeading': 'ثنائي الاجتماع (تميمة ثانية)', + 'settings.mascot.secondaryDesc': + 'أضِف تميمة ثانية للاجتماعات. عند تعيين اثنتين تظهران معًا وتتناوبان في الحديث. اتركها بلا لتميمة واحدة.', + 'settings.mascot.secondaryNone': 'بلا (تميمة واحدة)', + 'settings.mascot.perMascotVoiceHeading': 'أصوات لكل تميمة', + 'settings.mascot.primaryVoiceLabel': 'صوت التميمة الأولى', + 'settings.mascot.secondaryVoiceLabel': 'صوت التميمة الثانية', 'settings.mascot.voice.current': 'الحالي', 'settings.mascot.voice.customDesc': 'ابحث عن معرّفات الصوت في api.elevenlabs.io/v1/voices أو لوحة تحكم ElevenLabs الخاصة بك. يُخزَّن المعرّف فقط — يبقى مفتاح API الخاص بك على الخادم.', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index ac6a63e28..a19456a98 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -5386,6 +5386,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': 'অ্যাপ জুড়ে ব্যবহৃত মাসকটের রঙ বেছে নিন', 'settings.mascot.noCharacters': 'কোনো OpenHuman ক্যারেক্টার এখনও উপলব্ধ নেই', 'settings.mascot.noColorVariants': 'কোনো রঙের ভেরিয়েন্ট নেই', + 'settings.mascot.secondaryHeading': 'মিটিং জুটি (দ্বিতীয় ম্যাসকট)', + 'settings.mascot.secondaryDesc': + 'মিটিংয়ের জন্য একটি দ্বিতীয় ম্যাসকট যোগ করুন। দুটি সেট করা থাকলে তারা একসাথে দেখা দেয় এবং পালা করে কথা বলে। একটি ম্যাসকটের জন্য কোনোটি নয় রাখুন।', + 'settings.mascot.secondaryNone': 'কোনোটি নয় (একটি ম্যাসকট)', + 'settings.mascot.perMascotVoiceHeading': 'প্রতি ম্যাসকটের কণ্ঠস্বর', + 'settings.mascot.primaryVoiceLabel': 'প্রথম ম্যাসকটের কণ্ঠস্বর', + 'settings.mascot.secondaryVoiceLabel': 'দ্বিতীয় ম্যাসকটের কণ্ঠস্বর', 'settings.mascot.voice.current': 'বর্তমান', 'settings.mascot.voice.customDesc': 'ভয়েস আইডি খুঁজুন api.elevenlabs.io/v1/voices বা আপনার ElevenLabs ড্যাশবোর্ডে। শুধু আইডি সংরক্ষিত থাকে — আপনার API কী ব্যাকএন্ডে থাকে।', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index 0210c6c34..8945148ca 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -5528,6 +5528,13 @@ const messages: TranslationMap = { 'Wähle die Maskottchenfarbe aus, die in der gesamten App verwendet wird', 'settings.mascot.noCharacters': 'Es sind noch keine OpenHuman Zeichen verfügbar', 'settings.mascot.noColorVariants': 'Keine Farbvarianten', + 'settings.mascot.secondaryHeading': 'Meeting-Duo (zweites Maskottchen)', + 'settings.mascot.secondaryDesc': + 'Füge ein zweites Maskottchen für Meetings hinzu. Sind zwei gesetzt, erscheinen sie gemeinsam und sprechen abwechselnd. Für ein einzelnes Maskottchen auf Keines lassen.', + 'settings.mascot.secondaryNone': 'Keines (einzelnes Maskottchen)', + 'settings.mascot.perMascotVoiceHeading': 'Stimmen pro Maskottchen', + 'settings.mascot.primaryVoiceLabel': 'Stimme des ersten Maskottchens', + 'settings.mascot.secondaryVoiceLabel': 'Stimme des zweiten Maskottchens', 'settings.mascot.voice.current': 'aktuell', 'settings.mascot.voice.customDesc': 'Sprach-IDs findest du unter api.elevenlabs.io/v1/voices oder in deinem ElevenLabs-Dashboard. Es wird nur die ID gespeichert – dein API-Schlüssel bleibt im Backend.', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 67f45bf0b..bdac4e103 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -6069,6 +6069,13 @@ const en: TranslationMap = { 'settings.mascot.menuDesc': 'Pick the mascot color used across the app', 'settings.mascot.noCharacters': 'No OpenHuman characters are available yet', 'settings.mascot.noColorVariants': 'No color variants', + 'settings.mascot.secondaryHeading': 'Meeting duo (second mascot)', + 'settings.mascot.secondaryDesc': + 'Add a second mascot for meetings. When two are set they appear together and take turns speaking. Leave as None for a single mascot.', + 'settings.mascot.secondaryNone': 'None (single mascot)', + 'settings.mascot.perMascotVoiceHeading': 'Per-mascot voices', + 'settings.mascot.primaryVoiceLabel': 'First mascot voice', + 'settings.mascot.secondaryVoiceLabel': 'Second mascot voice', 'settings.mascot.voice.current': 'current', 'settings.mascot.voice.customDesc': 'Find voice ids at api.elevenlabs.io/v1/voices or your ElevenLabs dashboard. Only the id is stored — your API key stays on the backend.', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 0211d3f87..7168645fe 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -5484,6 +5484,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': 'Elige el color de la mascota usado en toda la app', 'settings.mascot.noCharacters': 'Aún no hay personajes de OpenHuman disponibles', 'settings.mascot.noColorVariants': 'Sin variantes de color', + 'settings.mascot.secondaryHeading': 'Dúo de reunión (segunda mascota)', + 'settings.mascot.secondaryDesc': + 'Añade una segunda mascota para las reuniones. Al fijar dos, aparecen juntas y hablan por turnos. Déjalo en Ninguna para una sola mascota.', + 'settings.mascot.secondaryNone': 'Ninguna (una sola mascota)', + 'settings.mascot.perMascotVoiceHeading': 'Voces por mascota', + 'settings.mascot.primaryVoiceLabel': 'Voz de la primera mascota', + 'settings.mascot.secondaryVoiceLabel': 'Voz de la segunda mascota', 'settings.mascot.voice.current': 'actual', 'settings.mascot.voice.customDesc': 'Encuentra los ID de voz en api.elevenlabs.io/v1/voices o en tu panel de ElevenLabs. Solo se almacena el ID — tu clave de API permanece en el backend.', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index 0a68fa89d..a0a73379d 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -5504,6 +5504,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': "Choisis la couleur de la mascotte utilisée dans toute l'application", 'settings.mascot.noCharacters': "Aucun personnage OpenHuman n'est encore disponible", 'settings.mascot.noColorVariants': 'Aucune variante de couleur', + 'settings.mascot.secondaryHeading': 'Duo de réunion (seconde mascotte)', + 'settings.mascot.secondaryDesc': + 'Ajoute une seconde mascotte pour les réunions. Lorsque deux sont définies, elles apparaissent ensemble et parlent à tour de rôle. Laisse sur Aucune pour une seule mascotte.', + 'settings.mascot.secondaryNone': 'Aucune (une seule mascotte)', + 'settings.mascot.perMascotVoiceHeading': 'Voix par mascotte', + 'settings.mascot.primaryVoiceLabel': 'Voix de la première mascotte', + 'settings.mascot.secondaryVoiceLabel': 'Voix de la seconde mascotte', 'settings.mascot.voice.current': 'actuel', 'settings.mascot.voice.customDesc': "Trouvez les identifiants vocaux sur api.elevenlabs.io/v1/voices ou dans votre tableau de bord ElevenLabs. Seul l'identifiant est stocké — votre clé API reste sur le backend.", diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index 7f3a2f97d..50b427a0c 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -5386,6 +5386,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': 'ऐप में उपयोग होने वाला मास्कॉट रंग चुनें', 'settings.mascot.noCharacters': 'अभी तक कोई OpenHuman कैरेक्टर उपलब्ध नहीं है', 'settings.mascot.noColorVariants': 'कोई कलर वेरिएंट नहीं', + 'settings.mascot.secondaryHeading': 'मीटिंग जोड़ी (दूसरा मैस्कट)', + 'settings.mascot.secondaryDesc': + 'मीटिंग के लिए एक दूसरा मैस्कट जोड़ें। दो सेट होने पर वे साथ दिखते हैं और बारी-बारी से बोलते हैं। एक ही मैस्कट के लिए कोई नहीं पर छोड़ें।', + 'settings.mascot.secondaryNone': 'कोई नहीं (एकल मैस्कट)', + 'settings.mascot.perMascotVoiceHeading': 'प्रति-मैस्कट आवाज़ें', + 'settings.mascot.primaryVoiceLabel': 'पहले मैस्कट की आवाज़', + 'settings.mascot.secondaryVoiceLabel': 'दूसरे मैस्कट की आवाज़', 'settings.mascot.voice.current': 'वर्तमान', 'settings.mascot.voice.customDesc': 'वॉइस आईडी api.elevenlabs.io/v1/voices या अपने ElevenLabs डैशबोर्ड पर खोजें। केवल आईडी संग्रहीत होती है — आपकी API कुंजी बैकएंड पर रहती है।', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index 28e7e06d7..3d2a02548 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -5402,6 +5402,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': 'Pilih warna maskot yang digunakan di seluruh aplikasi', 'settings.mascot.noCharacters': 'Belum ada karakter OpenHuman yang tersedia', 'settings.mascot.noColorVariants': 'Tidak ada varian warna', + 'settings.mascot.secondaryHeading': 'Duo rapat (maskot kedua)', + 'settings.mascot.secondaryDesc': + 'Tambahkan maskot kedua untuk rapat. Bila dua diatur, keduanya muncul bersama dan berbicara bergantian. Biarkan Tidak ada untuk satu maskot saja.', + 'settings.mascot.secondaryNone': 'Tidak ada (maskot tunggal)', + 'settings.mascot.perMascotVoiceHeading': 'Suara per maskot', + 'settings.mascot.primaryVoiceLabel': 'Suara maskot pertama', + 'settings.mascot.secondaryVoiceLabel': 'Suara maskot kedua', 'settings.mascot.voice.current': 'saat ini', 'settings.mascot.voice.customDesc': 'Temukan ID suara di api.elevenlabs.io/v1/voices atau dasbor ElevenLabs Anda. Hanya ID yang disimpan — kunci API Anda tetap di backend.', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index e48e92b6b..5ec29a5f5 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -5473,6 +5473,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': "Scegli il colore della mascotte usato in tutta l'app", 'settings.mascot.noCharacters': 'Nessun personaggio OpenHuman disponibile', 'settings.mascot.noColorVariants': 'Nessuna variante di colore', + 'settings.mascot.secondaryHeading': 'Duo riunione (seconda mascotte)', + 'settings.mascot.secondaryDesc': + 'Aggiungi una seconda mascotte per le riunioni. Quando ne imposti due, compaiono insieme e parlano a turno. Lascia su Nessuna per una sola mascotte.', + 'settings.mascot.secondaryNone': 'Nessuna (una sola mascotte)', + 'settings.mascot.perMascotVoiceHeading': 'Voci per mascotte', + 'settings.mascot.primaryVoiceLabel': 'Voce della prima mascotte', + 'settings.mascot.secondaryVoiceLabel': 'Voce della seconda mascotte', 'settings.mascot.voice.current': 'attuale', 'settings.mascot.voice.customDesc': "Trova gli ID vocali su api.elevenlabs.io/v1/voices o nel tuo dashboard ElevenLabs. Viene salvato solo l'ID — la tua chiave API rimane sul backend.", diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 72aa35c74..e6e420416 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -5324,6 +5324,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': '앱 전체에서 사용되는 마스코트 색상 선택', 'settings.mascot.noCharacters': '아직 사용할 수 있는 OpenHuman 캐릭터가 없습니다', 'settings.mascot.noColorVariants': '색상 변형 없음', + 'settings.mascot.secondaryHeading': '회의 듀오 (두 번째 마스코트)', + 'settings.mascot.secondaryDesc': + '회의에 두 번째 마스코트를 추가하세요. 둘을 설정하면 함께 나타나 번갈아 말합니다. 마스코트를 하나만 쓰려면 없음으로 두세요.', + 'settings.mascot.secondaryNone': '없음 (단일 마스코트)', + 'settings.mascot.perMascotVoiceHeading': '마스코트별 음성', + 'settings.mascot.primaryVoiceLabel': '첫 번째 마스코트 음성', + 'settings.mascot.secondaryVoiceLabel': '두 번째 마스코트 음성', 'settings.mascot.voice.current': '현재', 'settings.mascot.voice.customDesc': 'api.elevenlabs.io/v1/voices 또는 ElevenLabs 대시보드에서 음성 ID를 찾으세요. ID만 저장되며 API 키는 백엔드에 유지됩니다.', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index f2a7b8ded..81e5ec2be 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -5464,6 +5464,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': 'Wybierz kolor maskotki używany w aplikacji', 'settings.mascot.noCharacters': 'Nie ma jeszcze dostępnych postaci OpenHuman', 'settings.mascot.noColorVariants': 'Brak wariantów kolorystycznych', + 'settings.mascot.secondaryHeading': 'Duet spotkania (druga maskotka)', + 'settings.mascot.secondaryDesc': + 'Dodaj drugą maskotkę do spotkań. Gdy ustawisz dwie, pojawiają się razem i mówią na zmianę. Pozostaw Brak, aby korzystać z jednej maskotki.', + 'settings.mascot.secondaryNone': 'Brak (pojedyncza maskotka)', + 'settings.mascot.perMascotVoiceHeading': 'Głosy poszczególnych maskotek', + 'settings.mascot.primaryVoiceLabel': 'Głos pierwszej maskotki', + 'settings.mascot.secondaryVoiceLabel': 'Głos drugiej maskotki', 'settings.mascot.voice.current': 'bieżący', 'settings.mascot.voice.customDesc': 'Znajdź ID głosów w api.elevenlabs.io/v1/voices lub w panelu ElevenLabs. Przechowywane jest tylko ID — Twój klucz API pozostaje na backendzie.', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index dab7f0cb5..0583b7195 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -5473,6 +5473,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': 'Escolha a cor do mascote usada em todo o app', 'settings.mascot.noCharacters': 'Nenhum personagem do OpenHuman disponível ainda', 'settings.mascot.noColorVariants': 'Sem variantes de cor', + 'settings.mascot.secondaryHeading': 'Dupla de reunião (segundo mascote)', + 'settings.mascot.secondaryDesc': + 'Adicione um segundo mascote para as reuniões. Com dois definidos, eles aparecem juntos e falam em turnos. Deixe como Nenhum para um único mascote.', + 'settings.mascot.secondaryNone': 'Nenhum (mascote único)', + 'settings.mascot.perMascotVoiceHeading': 'Vozes por mascote', + 'settings.mascot.primaryVoiceLabel': 'Voz do primeiro mascote', + 'settings.mascot.secondaryVoiceLabel': 'Voz do segundo mascote', 'settings.mascot.voice.current': 'atual', 'settings.mascot.voice.customDesc': 'Encontre IDs de voz em api.elevenlabs.io/v1/voices ou no seu painel da ElevenLabs. Apenas o ID é armazenado — sua chave de API permanece no backend.', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index f39d6d69d..d52f00a3d 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -5439,6 +5439,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': 'Выберите цвет маскота, используемый во всем приложении', 'settings.mascot.noCharacters': 'Персонажи OpenHuman пока недоступны', 'settings.mascot.noColorVariants': 'Нет цветовых вариантов', + 'settings.mascot.secondaryHeading': 'Дуэт для встреч (второй маскот)', + 'settings.mascot.secondaryDesc': + 'Добавьте второго маскота для встреч. Когда заданы двое, они появляются вместе и говорят по очереди. Оставьте Нет для одного маскота.', + 'settings.mascot.secondaryNone': 'Нет (один маскот)', + 'settings.mascot.perMascotVoiceHeading': 'Голоса для каждого маскота', + 'settings.mascot.primaryVoiceLabel': 'Голос первого маскота', + 'settings.mascot.secondaryVoiceLabel': 'Голос второго маскота', 'settings.mascot.voice.current': 'текущий', 'settings.mascot.voice.customDesc': 'Идентификаторы голосов можно найти на api.elevenlabs.io/v1/voices или в вашей панели ElevenLabs. Сохраняется только идентификатор — ваш API-ключ остаётся на бэкенде.', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 2c7943948..f115fe884 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -5103,6 +5103,13 @@ const messages: TranslationMap = { 'settings.mascot.menuDesc': '选择应用内使用的吉祥物颜色', 'settings.mascot.noCharacters': '暂无可用的 OpenHuman 角色', 'settings.mascot.noColorVariants': '无颜色变体', + 'settings.mascot.secondaryHeading': '会议搭档(第二个吉祥物)', + 'settings.mascot.secondaryDesc': + '为会议添加第二个吉祥物。设置两个后,它们会一起出现并轮流说话。若只用一个吉祥物,请保持为无。', + 'settings.mascot.secondaryNone': '无(单个吉祥物)', + 'settings.mascot.perMascotVoiceHeading': '每个吉祥物的语音', + 'settings.mascot.primaryVoiceLabel': '第一个吉祥物的语音', + 'settings.mascot.secondaryVoiceLabel': '第二个吉祥物的语音', 'settings.mascot.voice.current': '当前', 'settings.mascot.voice.customDesc': '在 api.elevenlabs.io/v1/voices 或您的 ElevenLabs 仪表板中查找语音 ID。仅存储 ID — 您的 API 密钥保留在后端。', diff --git a/app/src/services/__tests__/meetCallService.test.ts b/app/src/services/__tests__/meetCallService.test.ts index e9f05422b..10efd84bd 100644 --- a/app/src/services/__tests__/meetCallService.test.ts +++ b/app/src/services/__tests__/meetCallService.test.ts @@ -77,6 +77,35 @@ describe('joinMeetCall', () => { }); }); + it('forwards per-mascot voices to the shell when provided (issue #4277)', async () => { + vi.mocked(callCoreRpc).mockResolvedValueOnce({ + ok: true, + request_id: 'req-2', + meet_url: 'https://meet.google.com/abc-defg-hij', + display_name: 'Agent Alice', + } as never); + vi.mocked(invoke).mockResolvedValueOnce('meet-call-req-2'); + + await joinMeetCall({ + meetUrl: 'https://meet.google.com/abc-defg-hij', + displayName: 'Agent Alice', + ownerDisplayName: 'Owner Bob', + primaryVoiceId: ' voice-a ', + secondaryVoiceId: 'voice-b', + }); + + expect(invoke).toHaveBeenCalledWith('meet_call_open_window', { + args: { + request_id: 'req-2', + meet_url: 'https://meet.google.com/abc-defg-hij', + display_name: 'Agent Alice', + owner_display_name: 'Owner Bob', + primary_voice_id: 'voice-a', + secondary_voice_id: 'voice-b', + }, + }); + }); + it('throws if core rejects the request', async () => { vi.mocked(callCoreRpc).mockResolvedValueOnce({ ok: false } as never); await expect( @@ -233,6 +262,60 @@ describe('joinMeetViaBackendBot', () => { expect(result).toEqual({ meetUrl: 'https://meet.google.com/abc-defg-hij', platform: 'gmeet' }); }); + it('omits mascots[] for a single-mascot join (unchanged wire shape)', async () => { + vi.mocked(callCoreRpc).mockResolvedValueOnce({ + ok: true, + meet_url: 'https://meet.google.com/abc-defg-hij', + platform: 'gmeet', + } as never); + + await joinMeetViaBackendBot({ + meetUrl: 'https://meet.google.com/abc-defg-hij', + mascotId: 'yellow', + }); + + // No `mascots` key on the params → the single `mascot_id` path is + // byte-identical to before (undefined is dropped by the RPC boundary). + const params = vi.mocked(callCoreRpc).mock.calls[0][0].params as Record; + expect(params.mascots).toBeUndefined(); + expect(params.mascot_id).toBe('yellow'); + }); + + it('maps dual mascots[] to snake_case slots and drops blank ids (issue #4277)', async () => { + vi.mocked(callCoreRpc).mockResolvedValueOnce({ + ok: true, + meet_url: 'https://meet.google.com/abc-defg-hij', + platform: 'gmeet', + } as never); + + await joinMeetViaBackendBot({ + meetUrl: 'https://meet.google.com/abc-defg-hij', + mascots: [ + { + mascotId: ' tiny-mascot ', + name: ' Tiny ', + voiceId: ' voice-a ', + riveColors: { primaryColor: '#111', secondaryColor: '#222' }, + }, + { mascotId: 'toshi', voiceId: 'voice-b' }, + { mascotId: ' ', voiceId: 'ignored' }, + ], + }); + + const params = vi.mocked(callCoreRpc).mock.calls[0][0].params as Record; + expect(params.mascots).toEqual([ + { + mascot_id: 'tiny-mascot', + // Name-addressed routing (#4277 follow-up): trimmed + forwarded. + name: 'Tiny', + voice_id: 'voice-a', + rive_colors: { primary_color: '#111', secondary_color: '#222' }, + }, + // Slot 1 supplies no name → `name` omitted (undefined). + { mascot_id: 'toshi', name: undefined, voice_id: 'voice-b', rive_colors: undefined }, + ]); + }); + it('rejects an empty meeting link before contacting core', async () => { await expect(joinMeetViaBackendBot({ meetUrl: ' ' })).rejects.toThrow(/meeting link/i); expect(callCoreRpc).not.toHaveBeenCalled(); diff --git a/app/src/services/meetCallService.ts b/app/src/services/meetCallService.ts index 11b3463e7..953c925d0 100644 --- a/app/src/services/meetCallService.ts +++ b/app/src/services/meetCallService.ts @@ -10,11 +10,32 @@ // Splitting it this way keeps platform-specific window code in the shell // while the validation rules live (and are tested) in the core. import { invoke } from '@tauri-apps/api/core'; +import debug from 'debug'; import { isTauri } from '../utils/tauriCommands/common'; import { apiClient } from './apiClient'; import { callCoreRpc } from './coreRpcClient'; +// Shares the sibling hook's namespace (`useMeetingMascots.ts`) so the whole +// dual-mascot flow can be traced under one prefix. +const log = debug('meet:mascots'); + +/** + * Map optional Rive colors to the backend's snake_case wire shape, trimming + * blanks and collapsing an all-empty pair to `undefined`. Shared by the + * top-level and per-slot color payloads so the two can't drift. + */ +function mapRiveColors(colors?: { + primaryColor?: string; + secondaryColor?: string; +}): { primary_color?: string; secondary_color?: string } | undefined { + if (!colors) return undefined; + const primary = colors.primaryColor?.trim() || undefined; + const secondary = colors.secondaryColor?.trim() || undefined; + if (!primary && !secondary) return undefined; + return { primary_color: primary, secondary_color: secondary }; +} + export type MeetJoinCallInput = { meetUrl: string; /** Bot's display name in Meet's "Your name" prompt. */ @@ -27,6 +48,14 @@ export type MeetJoinCallInput = { * (no wakes fire) which is the safe default during the rollout. */ ownerDisplayName?: string; + /** + * ElevenLabs voice id for the primary mascot (issue #4277). When two + * mascots are enabled the core alternates the speaking voice per reply. + * Omit for single-mascot calls (core keeps its default voice). + */ + primaryVoiceId?: string; + /** Voice id for the secondary mascot; present only in two-mascot calls. */ + secondaryVoiceId?: string; }; export type MeetJoinCallResult = { @@ -86,6 +115,10 @@ export async function joinMeetCall(input: MeetJoinCallInput): Promise; /** Only respond to messages from this participant name (empty = respond to all). */ respondToParticipant?: string; /** Wake phrase the participant must say before the bot responds (empty = no wake phrase). */ @@ -291,6 +343,29 @@ export async function joinMeetViaBackendBot( const meetUrl = input.meetUrl.trim(); if (!meetUrl) throw new Error('Please paste a meeting link.'); + // Dual-mascot slots (issue #4277), mapped to the backend's snake_case wire + // shape. Absent → backend falls back to `mascot_id`. + const slots = input.mascots?.filter(m => m.mascotId?.trim()); + const mascots = + slots && slots.length > 0 + ? slots.map(m => ({ + mascot_id: m.mascotId.trim(), + name: m.name?.trim() || undefined, + voice_id: m.voiceId?.trim() || undefined, + rive_colors: mapRiveColors(m.riveColors), + })) + : undefined; + + // Flow/state metadata only — no participant names, voices, or the meet URL. + log( + 'backend bot join corr=%s dual=%s slots=%d singleMascot=%s riveColors=%s', + input.correlationId?.trim() || '-', + Boolean(input.mascots?.length), + mascots?.length ?? 0, + Boolean(input.mascotId?.trim()), + Boolean(mapRiveColors(input.riveColors)) + ); + const result = await callCoreRpc({ method: 'openhuman.agent_meetings_join', params: { @@ -304,13 +379,8 @@ export async function joinMeetViaBackendBot( wake_phrase: input.wakePhrase?.trim() || undefined, correlation_id: input.correlationId?.trim() || undefined, listen_only: input.listenOnly ?? undefined, - rive_colors: (() => { - if (!input.riveColors) return undefined; - const primary = input.riveColors.primaryColor?.trim() || undefined; - const secondary = input.riveColors.secondaryColor?.trim() || undefined; - if (!primary && !secondary) return undefined; - return { primary_color: primary, secondary_color: secondary }; - })(), + rive_colors: mapRiveColors(input.riveColors), + mascots, }, }); diff --git a/app/src/store/__tests__/mascotSlice.test.ts b/app/src/store/__tests__/mascotSlice.test.ts index 6cf87cc55..bc671bf4e 100644 --- a/app/src/store/__tests__/mascotSlice.test.ts +++ b/app/src/store/__tests__/mascotSlice.test.ts @@ -6,13 +6,21 @@ import reducer, { isCustomMascotGifUrl, MAX_CUSTOM_MASCOT_GIF_URL_LEN, MAX_MASCOT_VOICE_ID_LEN, + MAX_MASCOT_VOICES, selectCustomMascotGifUrl, + selectDualMascotEnabled, selectMascotColor, + selectMascotVoiceFor, selectMascotVoiceId, + selectMascotVoices, + selectMeetingMascotVoicePair, + selectSecondaryMascotId, selectSelectedMascotId, setCustomMascotGifUrl, setMascotColor, + setMascotVoice, setMascotVoiceId, + setSecondaryMascotId, setSelectedMascotId, SUPPORTED_MASCOT_COLORS, } from '../mascotSlice'; @@ -277,4 +285,168 @@ describe('mascotSlice', () => { expect(state.customMascotGifUrl).toBeNull(); }); }); + + // Issue #4277 — second meeting mascot + per-mascot voices. + describe('secondary mascot id', () => { + it('defaults to null', () => { + const state = reducer(undefined, { type: '@@INIT' }); + expect(state.secondaryMascotId).toBeNull(); + expect(selectSecondaryMascotId({ mascot: state })).toBeNull(); + }); + + it('setSecondaryMascotId trims and stores a valid id', () => { + const state = reducer(undefined, setSecondaryMascotId(' toshi ')); + expect(state.secondaryMascotId).toBe('toshi'); + }); + + it('null / whitespace / oversize input clears it', () => { + const set = reducer(undefined, setSecondaryMascotId('toshi')); + expect(reducer(set, setSecondaryMascotId(null)).secondaryMascotId).toBeNull(); + expect(reducer(set, setSecondaryMascotId(' ')).secondaryMascotId).toBeNull(); + const tooLong = 'x'.repeat(MAX_MASCOT_VOICE_ID_LEN + 1); + expect(reducer(set, setSecondaryMascotId(tooLong)).secondaryMascotId).toBeNull(); + }); + + it('is cleared when a custom GIF avatar is set (mutually exclusive)', () => { + let state = reducer(undefined, setSecondaryMascotId('toshi')); + state = reducer(state, setCustomMascotGifUrl('https://example.com/avatar.gif')); + expect(state.secondaryMascotId).toBeNull(); + }); + + it('resetUserScopedState clears it', () => { + let state = reducer(undefined, setSecondaryMascotId('toshi')); + state = reducer(state, resetUserScopedState()); + expect(state.secondaryMascotId).toBeNull(); + }); + + const rehydrate = (key: string, payload?: unknown) => ({ type: REHYDRATE, key, payload }); + + it('restores a valid persisted id; scrubs invalid; tolerates missing (older blobs)', () => { + expect( + reducer(undefined, rehydrate('mascot', { secondaryMascotId: 'toshi' })).secondaryMascotId + ).toBe('toshi'); + expect( + reducer(undefined, rehydrate('mascot', { secondaryMascotId: ' ' })).secondaryMascotId + ).toBeNull(); + expect( + reducer(undefined, rehydrate('mascot', { color: 'navy' })).secondaryMascotId + ).toBeNull(); + }); + }); + + describe('per-mascot voices', () => { + it('defaults to an empty map', () => { + const state = reducer(undefined, { type: '@@INIT' }); + expect(state.mascotVoices).toEqual({}); + expect(selectMascotVoices({ mascot: state })).toEqual({}); + expect(selectMascotVoiceFor('toshi')({ mascot: state })).toBeNull(); + }); + + it('setMascotVoice records a trimmed mascotId → voiceId entry', () => { + const state = reducer(undefined, setMascotVoice({ mascotId: ' toshi ', voiceId: ' v-1 ' })); + expect(state.mascotVoices).toEqual({ toshi: 'v-1' }); + expect(selectMascotVoiceFor('toshi')({ mascot: state })).toBe('v-1'); + }); + + it('setMascotVoice with null / invalid voiceId removes the entry', () => { + const set = reducer(undefined, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-1' })); + expect( + reducer(set, setMascotVoice({ mascotId: 'toshi', voiceId: null })).mascotVoices + ).toEqual({}); + expect( + reducer(set, setMascotVoice({ mascotId: 'toshi', voiceId: ' ' })).mascotVoices + ).toEqual({}); + }); + + it('ignores an invalid mascotId key', () => { + const state = reducer(undefined, setMascotVoice({ mascotId: ' ', voiceId: 'v-1' })); + expect(state.mascotVoices).toEqual({}); + }); + + it('caps new keys at MAX_MASCOT_VOICES but still updates existing ones', () => { + let state = reducer(undefined, { type: '@@INIT' }); + for (let i = 0; i < MAX_MASCOT_VOICES; i += 1) { + state = reducer(state, setMascotVoice({ mascotId: `m-${i}`, voiceId: `v-${i}` })); + } + expect(Object.keys(state.mascotVoices)).toHaveLength(MAX_MASCOT_VOICES); + // A brand-new key over the cap is refused… + const overflow = reducer(state, setMascotVoice({ mascotId: 'm-extra', voiceId: 'v-x' })); + expect(overflow.mascotVoices['m-extra']).toBeUndefined(); + // …but re-voicing an existing mascot is always allowed. + const updated = reducer(state, setMascotVoice({ mascotId: 'm-0', voiceId: 'v-new' })); + expect(updated.mascotVoices['m-0']).toBe('v-new'); + }); + + it('resetUserScopedState clears the map', () => { + const dirty = reducer(undefined, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-1' })); + expect(reducer(dirty, resetUserScopedState()).mascotVoices).toEqual({}); + }); + + const rehydrate = (key: string, payload?: unknown) => ({ type: REHYDRATE, key, payload }); + + it('REHYDRATE keeps only valid entries and tolerates a missing/garbage map', () => { + const restored = reducer( + undefined, + rehydrate('mascot', { mascotVoices: { toshi: 'v-1', bad: ' ', ' ': 'v-2', ok: 'v-3' } }) + ); + expect(restored.mascotVoices).toEqual({ toshi: 'v-1', ok: 'v-3' }); + expect(reducer(undefined, rehydrate('mascot', { color: 'navy' })).mascotVoices).toEqual({}); + expect( + reducer(undefined, rehydrate('mascot', { mascotVoices: 'not-an-object' })).mascotVoices + ).toEqual({}); + }); + }); + + describe('dual-mascot resolution', () => { + it('selectDualMascotEnabled is false with one or duplicate mascots, true with two distinct', () => { + const one = reducer(undefined, setSelectedMascotId('tiny-mascot')); + expect(selectDualMascotEnabled({ mascot: one })).toBe(false); + const dup = reducer(one, setSecondaryMascotId('tiny-mascot')); + expect(selectDualMascotEnabled({ mascot: dup })).toBe(false); + const two = reducer(one, setSecondaryMascotId('toshi')); + expect(selectDualMascotEnabled({ mascot: two })).toBe(true); + }); + + it('selectMeetingMascotVoicePair returns null secondary when single', () => { + let state = reducer(undefined, setSelectedMascotId('tiny-mascot')); + state = reducer(state, setMascotVoiceId('v-primary')); + const pair = selectMeetingMascotVoicePair({ mascot: state }); + expect(pair.secondary).toBeNull(); + expect(pair.primary.mascotId).toBe('tiny-mascot'); + expect(pair.primary.voiceId).toBe('v-primary'); + }); + + it('selectMeetingMascotVoicePair resolves each slot to its per-mascot voice', () => { + let state = reducer(undefined, setSelectedMascotId('tiny-mascot')); + state = reducer(state, setSecondaryMascotId('toshi')); + state = reducer(state, setMascotVoice({ mascotId: 'tiny-mascot', voiceId: 'v-tiny' })); + state = reducer(state, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-toshi' })); + const pair = selectMeetingMascotVoicePair({ mascot: state }); + expect(pair.primary).toEqual({ mascotId: 'tiny-mascot', voiceId: 'v-tiny' }); + expect(pair.secondary).toEqual({ mascotId: 'toshi', voiceId: 'v-toshi' }); + }); + + it('per-mascot voice falls back to the effective single voice when unset', () => { + let state = reducer(undefined, setSelectedMascotId('tiny-mascot')); + state = reducer(state, setSecondaryMascotId('toshi')); + state = reducer(state, setMascotVoiceId('v-effective')); + const pair = selectMeetingMascotVoicePair({ mascot: state }); + // Neither mascot has an explicit override → both use the effective voice. + expect(pair.primary.voiceId).toBe('v-effective'); + expect(pair.secondary?.voiceId).toBe('v-effective'); + }); + + it('tolerates a legacy mascot state missing mascotVoices without throwing', () => { + // A pre-migration persisted blob or a partial preloadedState can omit + // `mascotVoices`; the meeting selectors must default it, not crash on + // `mascotVoices[selectedMascotId]`. + const legacy = { + mascot: { selectedMascotId: 'yellow', secondaryMascotId: null }, + } as unknown as Parameters[0]; + expect(() => selectMeetingMascotVoicePair(legacy)).not.toThrow(); + expect(selectMeetingMascotVoicePair(legacy).primary.mascotId).toBe('yellow'); + expect(selectMascotVoiceFor('yellow')(legacy)).toBeNull(); + expect(selectMascotVoices(legacy)).toEqual({}); + }); + }); }); diff --git a/app/src/store/mascotSlice.ts b/app/src/store/mascotSlice.ts index 4aec33c6e..7a001cfb0 100644 --- a/app/src/store/mascotSlice.ts +++ b/app/src/store/mascotSlice.ts @@ -40,6 +40,17 @@ export const DEFAULT_MASCOT_VOICE_GENDER: MascotVoiceGender = 'male'; export const MAX_MASCOT_VOICE_ID_LEN = 128; export const MAX_CUSTOM_MASCOT_GIF_URL_LEN = 2048; +/** + * Upper bound on how many per-mascot voice overrides we persist (issue + * #4277). A user only ever drives two mascots in a meeting, but they may + * try several before settling; the cap keeps the persisted map bounded + * against a runaway writer while comfortably covering real use. Once the + * cap is reached the reducer refuses NEW keys (an existing mascot can + * still be re-voiced); on rehydrate the first `MAX_MASCOT_VOICES` valid + * entries are kept and the rest dropped. + */ +export const MAX_MASCOT_VOICES = 16; + /** * Loose shape check for a stored mascot voice id. Issue #1762 lets users * paste a custom ElevenLabs voice id, so we cannot enumerate the valid @@ -118,6 +129,21 @@ export interface MascotState { * the persisted blob bounded. */ selectedMascotId: string | null; + /** + * Second mascot enabled for meetings (issue #4277). When set (and + * distinct from `selectedMascotId`) the meeting bot shows both mascots + * together and alternates who speaks each reply. `null` = single-mascot + * behavior, unchanged. Same validation/length cap as `selectedMascotId`. + */ + secondaryMascotId: string | null; + /** + * Per-mascot reply-voice overrides (issue #4277), keyed by manifest + * mascot id → ElevenLabs voice id. Lets each mascot in a two-mascot + * meeting speak in its own voice. Empty map = no per-mascot override, + * so every mascot falls back to `selectEffectiveMascotVoiceId` (the + * single-voice behavior). Bounded by `MAX_MASCOT_VOICES`. + */ + mascotVoices: Record; /** * User-supplied animated avatar source. Kept as a plain validated * string so the renderer can fall back to YellowMascot whenever the @@ -134,11 +160,31 @@ const initialState: MascotState = { voiceGender: DEFAULT_MASCOT_VOICE_GENDER, voiceUseLocaleDefault: false, selectedMascotId: null, + secondaryMascotId: null, + mascotVoices: {}, customMascotGifUrl: null, customPrimaryColor: '#F7D145', customSecondaryColor: '#B23C05', }; +/** + * Scrub a persisted / raw `mascotVoices` blob down to valid + * `mascotId → voiceId` entries under the size cap. Non-object inputs and + * any entry whose key or value fails `isMascotVoiceId` are dropped, so a + * corrupted localStorage blob can never poison the meeting TTS payload. + */ +function sanitizeMascotVoices(value: unknown): Record { + if (value == null || typeof value !== 'object' || Array.isArray(value)) return {}; + const out: Record = {}; + for (const [key, val] of Object.entries(value as Record)) { + if (Object.keys(out).length >= MAX_MASCOT_VOICES) break; + if (isMascotVoiceId(key) && isMascotVoiceId(val)) { + out[key.trim()] = (val as string).trim(); + } + } + return out; +} + function isMascotColor(value: unknown): value is MascotColor { return ( typeof value === 'string' && (SUPPORTED_MASCOT_COLORS as readonly string[]).includes(value) @@ -170,6 +216,49 @@ const mascotSlice = createSlice({ state.selectedMascotId = null; } }, + /** + * Enable / clear the second meeting mascot (issue #4277). Trimmed; + * empty / oversize / null clears it (back to single-mascot). A custom + * GIF avatar and a second Rive mascot are mutually exclusive, so + * setting one clears the GIF override — mirroring `setSelectedMascotId`. + */ + setSecondaryMascotId(state, action: PayloadAction) { + if (action.payload == null) { + state.secondaryMascotId = null; + return; + } + if (isMascotVoiceId(action.payload)) { + state.secondaryMascotId = action.payload.trim(); + state.customMascotGifUrl = null; + } else { + state.secondaryMascotId = null; + } + }, + /** + * Set or clear a per-mascot reply voice (issue #4277). A non-empty + * valid `voiceId` records `mascotId → voiceId`; a `null`/invalid + * `voiceId` removes the entry (that mascot falls back to the effective + * single voice). Both key and value are validated + trimmed so junk + * can't grow the persisted map. Over-cap writes are ignored. + */ + setMascotVoice(state, action: PayloadAction<{ mascotId: string; voiceId: string | null }>) { + const { mascotId, voiceId } = action.payload; + if (!isMascotVoiceId(mascotId)) return; + const key = mascotId.trim(); + if (voiceId == null || !isMascotVoiceId(voiceId)) { + delete state.mascotVoices[key]; + return; + } + // Only enforce the cap when introducing a NEW key — updating an + // existing mascot's voice must always be allowed. + if ( + !(key in state.mascotVoices) && + Object.keys(state.mascotVoices).length >= MAX_MASCOT_VOICES + ) { + return; + } + state.mascotVoices[key] = voiceId.trim(); + }, setCustomMascotGifUrl(state, action: PayloadAction) { if (action.payload == null) { state.customMascotGifUrl = null; @@ -178,6 +267,7 @@ const mascotSlice = createSlice({ if (isCustomMascotGifUrl(action.payload)) { state.customMascotGifUrl = action.payload.trim(); state.selectedMascotId = null; + state.secondaryMascotId = null; } else { state.customMascotGifUrl = null; } @@ -231,6 +321,8 @@ const mascotSlice = createSlice({ voiceGender?: unknown; voiceUseLocaleDefault?: unknown; selectedMascotId?: unknown; + secondaryMascotId?: unknown; + mascotVoices?: unknown; customMascotGifUrl?: unknown; customPrimaryColor?: unknown; customSecondaryColor?: unknown; @@ -246,6 +338,17 @@ const mascotSlice = createSlice({ : isMascotVoiceId(restoredSelectedMascotId) ? (restoredSelectedMascotId as string).trim() : null; + // Second mascot + per-mascot voices are absent in pre-#4277 blobs; + // the `null` / `{}` fallbacks match a fresh install and keep + // single-mascot users unchanged. Invalid values are scrubbed. + const restoredSecondaryMascotId = rehydrateAction.payload?.secondaryMascotId; + state.secondaryMascotId = + restoredSecondaryMascotId == null + ? null + : isMascotVoiceId(restoredSecondaryMascotId) + ? (restoredSecondaryMascotId as string).trim() + : null; + state.mascotVoices = sanitizeMascotVoices(rehydrateAction.payload?.mascotVoices); const restoredCustomMascotGifUrl = rehydrateAction.payload?.customMascotGifUrl; state.customMascotGifUrl = restoredCustomMascotGifUrl == null @@ -253,7 +356,12 @@ const mascotSlice = createSlice({ : isCustomMascotGifUrl(restoredCustomMascotGifUrl) ? (restoredCustomMascotGifUrl as string).trim() : null; - if (state.customMascotGifUrl) state.selectedMascotId = null; + // A custom GIF avatar is mutually exclusive with Rive mascots — + // drop both mascot selections if a GIF override survived. + if (state.customMascotGifUrl) { + state.selectedMascotId = null; + state.secondaryMascotId = null; + } // `voiceId` is optional in older persisted blobs (pre-#1762) — the // `null` fallback is the intended default and matches a fresh // install. Invalid values are scrubbed so a corrupted localStorage @@ -289,6 +397,8 @@ export const { setMascotVoiceGender, setMascotVoiceUseLocaleDefault, setSelectedMascotId, + setSecondaryMascotId, + setMascotVoice, setCustomMascotGifUrl, setCustomPrimaryColor, setCustomSecondaryColor, @@ -309,6 +419,32 @@ export const selectMascotVoiceUseLocaleDefault = (state: { mascot: MascotState } export const selectSelectedMascotId = (state: { mascot: MascotState }): string | null => state.mascot.selectedMascotId; +export const selectSecondaryMascotId = (state: { mascot: MascotState }): string | null => + state.mascot.secondaryMascotId; + +export const selectMascotVoices = (state: { mascot: MascotState }): Record => + state.mascot.mascotVoices ?? {}; + +/** + * Explicit per-mascot voice override for `mascotId`, or `null` when none + * is set (caller falls back to the effective single voice). Curried so it + * reads like the other parameterised selectors at call sites. + */ +export const selectMascotVoiceFor = + (mascotId: string | null) => + (state: { mascot: MascotState }): string | null => + mascotId ? (state.mascot.mascotVoices?.[mascotId] ?? null) : null; + +/** + * True when a distinct second mascot is enabled — the single gate the + * meeting render + join paths use to decide dual vs single behavior. + * Guards against the same mascot being picked twice. + */ +export const selectDualMascotEnabled = (state: { mascot: MascotState }): boolean => { + const { selectedMascotId, secondaryMascotId } = state.mascot; + return secondaryMascotId != null && secondaryMascotId !== selectedMascotId; +}; + export const selectCustomMascotGifUrl = (state: { mascot: MascotState }): string | null => state.mascot.customMascotGifUrl; @@ -354,5 +490,51 @@ export const selectEffectiveMascotVoiceId = (state: { return MASCOT_VOICE_ID || ELEVENLABS_VOICE_PRESETS[0].id; }; +export interface MeetingMascotSlot { + /** Manifest mascot id, or `null` for the primary when the user is on + * the default (first-`ready`) mascot. */ + mascotId: string | null; + /** Resolved voice id: the per-mascot override, else the effective + * single voice — never empty, so the join payload always carries one. */ + voiceId: string; +} + +export interface MeetingMascotVoicePair { + primary: MeetingMascotSlot; + secondary: MeetingMascotSlot | null; +} + +/** + * Resolve the (up to two) mascots + voices a meeting join should use + * (issue #4277). Single source of truth for both join paths — the CEF + * `meet_call_open_window` sender and the backend `agent_meetings_join` + * sender — and for tests, so they can't drift. + * + * `secondary` is non-null only when a distinct second mascot is enabled + * (`selectDualMascotEnabled`). Each slot's voice is its per-mascot + * override, falling back to `selectEffectiveMascotVoiceId`; when the user + * hasn't set distinct voices both slots resolve to that same voice + * (harmless — alternation still works, it just sounds the same). + */ +export const selectMeetingMascotVoicePair = (state: { + mascot: MascotState; + locale?: { current: Locale }; +}): MeetingMascotVoicePair => { + const effective = selectEffectiveMascotVoiceId(state); + const { selectedMascotId, secondaryMascotId } = state.mascot; + // Tolerate a partial / pre-migration mascot slice (e.g. a legacy persisted + // blob or a test's preloadedState) that predates `mascotVoices`. + const mascotVoices = state.mascot.mascotVoices ?? {}; + const primary: MeetingMascotSlot = { + mascotId: selectedMascotId, + voiceId: (selectedMascotId && mascotVoices[selectedMascotId]) || effective, + }; + const dualEnabled = secondaryMascotId != null && secondaryMascotId !== selectedMascotId; + const secondary: MeetingMascotSlot | null = dualEnabled + ? { mascotId: secondaryMascotId, voiceId: mascotVoices[secondaryMascotId] || effective } + : null; + return { primary, secondary }; +}; + export { mascotSlice }; export default mascotSlice.reducer; diff --git a/src/core/event_bus/events.rs b/src/core/event_bus/events.rs index dc3ace350..4644cfc18 100644 --- a/src/core/event_bus/events.rs +++ b/src/core/event_bus/events.rs @@ -1203,6 +1203,10 @@ pub enum DomainEvent { command_text: String, recent_transcript: Vec, timestamp_ms: u64, + /// Dual-mascot name addressing (#4277 follow-up): slot (0 = primary, + /// 1 = secondary) whose mascot name was addressed, or `None` when no + /// specific mascot was named. Forwarded to `bot:speak` as `mascotSlot`. + mascot_slot: Option, }, /// Core asked the backend bot to speak into the call (`bot:speak`). /// Published for observability after the Socket.IO emit succeeds. diff --git a/src/openhuman/agent_meetings/bus.rs b/src/openhuman/agent_meetings/bus.rs index ba3bf19f0..f40e0b605 100644 --- a/src/openhuman/agent_meetings/bus.rs +++ b/src/openhuman/agent_meetings/bus.rs @@ -257,11 +257,13 @@ impl EventHandler for MeetingEventSubscriber { command_text, recent_transcript, timestamp_ms, + mascot_slot, } => { tracing::info!( correlation_id = ?correlation_id, speaker = %speaker, cmd_len = command_text.len(), + mascot_slot = ?mascot_slot, "{LOG_PREFIX} in-call request received" ); // The orchestrator turn can run for tens of seconds (tools, @@ -271,6 +273,7 @@ impl EventHandler for MeetingEventSubscriber { let command_text = command_text.clone(); let recent_transcript = recent_transcript.clone(); let timestamp_ms = *timestamp_ms; + let mascot_slot = *mascot_slot; tokio::spawn(async move { super::in_call::handle_in_call_request( correlation_id, @@ -278,6 +281,7 @@ impl EventHandler for MeetingEventSubscriber { command_text, recent_transcript, timestamp_ms, + mascot_slot, ) .await; }); diff --git a/src/openhuman/agent_meetings/in_call.rs b/src/openhuman/agent_meetings/in_call.rs index f46ee2036..595a84aef 100644 --- a/src/openhuman/agent_meetings/in_call.rs +++ b/src/openhuman/agent_meetings/in_call.rs @@ -161,6 +161,7 @@ pub async fn handle_in_call_request( command_text: String, recent_transcript: Vec, timestamp_ms: u64, + mascot_slot: Option, ) { let command = command_text.trim(); if command.is_empty() { @@ -201,12 +202,21 @@ pub async fn handle_in_call_request( return; } + // Name-addressed routing (#4277 follow-up): `mascot_slot` is decided by the + // backend's wake matcher (it sees the un-stripped caption and knows which + // mascot name was said) and pins this whole turn — the speculative ack, the + // streamed chunks, and the final reply — to that slot so the named mascot's + // voice + face own the answer. `None` (no name / single mascot / follow-up) + // leaves the backend's mechanical alternation untouched, i.e. today's + // behavior. + tracing::info!( correlation_id = ?correlation_id, speaker = %speaker, cmd_len = command.len(), transcript_turns = recent_transcript.len(), timestamp_ms = timestamp_ms, + mascot_slot = ?mascot_slot, "{LOG_PREFIX} dispatching in-call command to orchestrator" ); @@ -220,7 +230,7 @@ pub async fn handle_in_call_request( let ack_task = tokio::spawn(async move { tokio::select! { _ = tokio::time::sleep(Duration::from_secs(ACK_AFTER_SECS)) => { - if let Err(e) = emit_bot_filler(ACK_PHRASE, ack_cid.as_deref()).await { + if let Err(e) = emit_bot_filler(ACK_PHRASE, ack_cid.as_deref(), mascot_slot).await { tracing::debug!("[agent_meetings::in_call] ack emit failed: {e}"); } } @@ -235,6 +245,7 @@ pub async fn handle_in_call_request( &recent_transcript, streaming, ack_cancel.clone(), + mascot_slot, ) .await; @@ -253,7 +264,15 @@ pub async fn handle_in_call_request( // In streaming mode the reply was already spoken sentence-by- // sentence during the turn; only the buffered path emits here. if !streaming { - if let Err(e) = emit_bot_speak(&text, correlation_id.as_deref()).await { + if let Err(e) = emit_bot_speak_inner( + &text, + correlation_id.as_deref(), + None, + "reply", + mascot_slot, + ) + .await + { tracing::warn!("{LOG_PREFIX} bot:speak emit failed: {e}"); } } @@ -264,7 +283,18 @@ pub async fn handle_in_call_request( } Err(e) => { tracing::warn!("{LOG_PREFIX} in-call turn failed: {e}"); - if let Err(e2) = emit_bot_speak(FAILURE_PHRASE, correlation_id.as_deref()).await { + // Speak the failure reply from the same mascot that was addressed + // (name-addressed routing, #4277 follow-up) — a terminal reply must + // carry `mascot_slot`, matching the success path above. + if let Err(e2) = emit_bot_speak_inner( + FAILURE_PHRASE, + correlation_id.as_deref(), + None, + "reply", + mascot_slot, + ) + .await + { tracing::debug!("{LOG_PREFIX} failure phrase emit failed: {e2}"); } // The failure phrase promises a note in the thread — keep it. @@ -363,8 +393,9 @@ pub(super) async fn speak_approval_prompt(action_summary: &str, correlation_id: or — Hey Tiny, deny — to cancel." ); // Filler, not a terminal reply: the bot is now waiting for a spoken - // decision, so the mascot should stay in its thinking cue. - if let Err(e) = emit_bot_filler(&prompt, correlation_id).await { + // decision, so the mascot should stay in its thinking cue. Slot-agnostic + // (an approval prompt isn't name-addressed) → default alternation. + if let Err(e) = emit_bot_filler(&prompt, correlation_id, None).await { tracing::warn!("{LOG_PREFIX} approval prompt emit failed: {e}"); } } @@ -378,6 +409,7 @@ async fn run_orchestrator_turn( recent_transcript: &[BackendMeetTurn], streaming: bool, ack_cancel: Arc, + mascot_slot: Option, ) -> Result { let agent_lock = get_or_build_agent(correlation_id).await?; let mut agent = agent_lock.lock().await; @@ -408,7 +440,12 @@ async fn run_orchestrator_turn( let (tx, rx) = mpsc::channel::(64); agent.set_on_progress(Some(tx)); let cid_owned = correlation_id.map(String::from); - Some(tokio::spawn(stream_sentences(rx, cid_owned, ack_cancel))) + Some(tokio::spawn(stream_sentences( + rx, + cid_owned, + ack_cancel, + mascot_slot, + ))) } else { None }; @@ -487,6 +524,7 @@ async fn stream_sentences( mut rx: mpsc::Receiver, correlation_id: Option, ack_cancel: Arc, + mascot_slot: Option, ) { let mut buf = String::new(); let mut seq: u32 = 0; @@ -501,6 +539,7 @@ async fn stream_sentences( &mut seq, &ack_cancel, &mut spoke, + mascot_slot, ) .await; } @@ -517,6 +556,7 @@ async fn stream_sentences( &mut seq, &ack_cancel, &mut spoke, + mascot_slot, ) .await; } @@ -528,6 +568,7 @@ async fn speak_stream_chunk( seq: &mut u32, ack_cancel: &Arc, spoke: &mut bool, + mascot_slot: Option, ) { if text.is_empty() { return; @@ -536,7 +577,9 @@ async fn speak_stream_chunk( *spoke = true; ack_cancel.notify_one(); // first real audio — no need for the filler ack } - if let Err(e) = emit_bot_speak_inner(text, correlation_id, Some(*seq), "reply").await { + if let Err(e) = + emit_bot_speak_inner(text, correlation_id, Some(*seq), "reply", mascot_slot).await + { tracing::debug!("{LOG_PREFIX} streamed chunk emit failed: {e}"); } *seq += 1; @@ -799,26 +842,34 @@ async fn get_or_create_meeting_thread( /// The backend's SpeakOrchestrator handles streaming TTS + the audio /// politeness gate from there. async fn emit_bot_speak(text: &str, correlation_id: Option<&str>) -> Result<(), String> { - emit_bot_speak_inner(text, correlation_id, None, "reply").await + emit_bot_speak_inner(text, correlation_id, None, "reply", None).await } /// Emit a non-terminal *filler* utterance (the speculative "On it" ack, an /// approval prompt): tagged `kind="ack"` so the backend mascot returns to its /// thinking cue afterwards instead of settling to idle — the real reply is -/// still on the way. -async fn emit_bot_filler(text: &str, correlation_id: Option<&str>) -> Result<(), String> { - emit_bot_speak_inner(text, correlation_id, None, "ack").await +/// still on the way. `mascot_slot` pins the filler to the name-addressed slot +/// so the ack comes from the same mascot that will answer. +async fn emit_bot_filler( + text: &str, + correlation_id: Option<&str>, + mascot_slot: Option, +) -> Result<(), String> { + emit_bot_speak_inner(text, correlation_id, None, "ack", mascot_slot).await } /// As [`emit_bot_speak`], plus an optional `seq` so the backend can order -/// the per-sentence chunks of a streamed reply, and a `kind` ("ack" filler vs -/// terminal "reply") so the backend can drive the mascot's post-speech pose -/// (both additive; older backends ignore them). +/// the per-sentence chunks of a streamed reply, a `kind` ("ack" filler vs +/// terminal "reply") so the backend can drive the mascot's post-speech pose, +/// and an optional `mascot_slot` (0|1) so a name-addressed turn is spoken by a +/// specific mascot instead of the backend's default alternation (all additive; +/// older backends ignore them). async fn emit_bot_speak_inner( text: &str, correlation_id: Option<&str>, seq: Option, kind: &str, + mascot_slot: Option, ) -> Result<(), String> { let mgr = global_socket_manager() .ok_or_else(|| format!("{LOG_PREFIX} socket manager not initialized"))?; @@ -835,11 +886,20 @@ async fn emit_bot_speak_inner( map.insert("seq".to_string(), json!(s)); } map.insert("kind".to_string(), json!(kind)); + // Name-addressed routing (#4277 follow-up): pin this utterance to a + // specific mascot slot. Maps to `MeetingBotSpeakPayload.mascotSlot`, + // which the backend prefers over its alternation cursor. + if let Some(slot) = mascot_slot { + map.insert("mascotSlot".to_string(), json!(slot)); + } } tracing::info!( text_len = text.len(), correlation_id = ?correlation_id, + kind = kind, + seq = ?seq, + mascot_slot = ?mascot_slot, "{LOG_PREFIX} emitting bot:speak" ); mgr.emit("bot:speak", payload) @@ -1019,6 +1079,7 @@ mod tests { " ".into(), vec![], 0, + None, ) .await; } @@ -1036,6 +1097,7 @@ mod tests { "what time is it".into(), vec![], 0, + None, ) .await; // No agent should have been built for the meeting. diff --git a/src/openhuman/agent_meetings/ops.rs b/src/openhuman/agent_meetings/ops.rs index 7ddf39cc5..6fb6d9727 100644 --- a/src/openhuman/agent_meetings/ops.rs +++ b/src/openhuman/agent_meetings/ops.rs @@ -761,6 +761,38 @@ fn build_join_payload( }), ); } + // Dual-mascot payload (issue #4277). Emitted only when the caller + // supplied a `mascots` array; the backend bot renders both slots, + // alternates the speaker per reply, and uses each slot's voice. + // Absent → the backend falls back to the single `mascotId` above, + // so one-mascot callers are unchanged. + if let Some(mascots) = &req.mascots { + let slots: Vec = mascots + .iter() + .map(|m| { + let mut slot = json!({ "mascotId": m.mascot_id }); + if let Some(obj) = slot.as_object_mut() { + if let Some(name) = &m.name { + obj.insert("name".to_string(), json!(name)); + } + if let Some(colors) = &m.rive_colors { + obj.insert( + "riveColors".to_string(), + json!({ + "primaryColor": colors.primary_color, + "secondaryColor": colors.secondary_color, + }), + ); + } + if let Some(vid) = &m.voice_id { + obj.insert("voiceId".to_string(), json!(vid)); + } + } + slot + }) + .collect(); + map.insert("mascots".to_string(), json!(slots)); + } if let Some(respond_to) = &req.respond_to_participant { map.insert("respondToParticipant".to_string(), json!(respond_to)); } @@ -872,10 +904,23 @@ pub async fn handle_join(params: Map) -> Result { return Err("[agent_meetings] socket not connected to backend".to_string()); } + // Name-addressing (#4277 follow-up) trace: the mascot ids + names forwarded + // to the backend. A `None` name on a slot means the backend can't route + // that mascot by name ("Hey Toshi"). + let mascot_trace: Vec<(String, Option)> = req + .mascots + .as_ref() + .map(|ms| { + ms.iter() + .map(|m| (m.mascot_id.clone(), m.name.clone())) + .collect() + }) + .unwrap_or_default(); tracing::info!( meet_url_host = %normalized_url.host_str().unwrap_or(""), platform = %platform, display_name_len = display_name.len(), + mascots = ?mascot_trace, "[agent_meetings] emitting bot:join" ); @@ -1783,6 +1828,8 @@ mod tests { assert!(payload.get("systemPrompt").is_none()); assert!(payload.get("mascotId").is_none()); assert!(payload.get("riveColors").is_none()); + // Single-mascot join carries no `mascots` array (issue #4277). + assert!(payload.get("mascots").is_none()); assert!(payload.get("respondToParticipant").is_none()); assert!(payload.get("wakePhrase").is_none()); } @@ -1839,6 +1886,50 @@ mod tests { assert_eq!(payload["riveColors"]["secondaryColor"], "#00ff00"); assert_eq!(payload["respondToParticipant"], "Bob"); assert_eq!(payload["wakePhrase"], "Hello bot"); + // A single `mascot_id` (no `mascots` array) must NOT emit `mascots` + // — the legacy single-mascot wire shape is unchanged (issue #4277). + assert!(payload.get("mascots").is_none()); + } + + #[test] + fn build_join_payload_with_dual_mascots() { + // Two-mascot join (issue #4277): the `mascots` array is emitted with + // camelCase slot fields the backend expects; slot 0 = primary. + let req: BackendMeetJoinRequest = serde_json::from_value(json!({ + "meet_url": "https://meet.google.com/abc-defg-hij", + "mascot_id": "tiny-mascot", + "mascots": [ + { + "mascot_id": "tiny-mascot", + "name": "Tiny", + "voice_id": "voice-a", + "rive_colors": { "primary_color": "#111", "secondary_color": "#222" } + }, + { "mascot_id": "toshi", "voice_id": "voice-b" } + ] + })) + .unwrap(); + let payload = build_join_payload( + "https://meet.google.com/abc-defg-hij", + "OpenHuman", + "gmeet", + &req, + ); + let mascots = payload["mascots"].as_array().expect("mascots array"); + assert_eq!(mascots.len(), 2); + assert_eq!(mascots[0]["mascotId"], "tiny-mascot"); + // Name-addressed routing (#4277 follow-up): the display name is + // forwarded so the bot can route "Hey Tiny …" to slot 0. + assert_eq!(mascots[0]["name"], "Tiny"); + assert_eq!(mascots[0]["voiceId"], "voice-a"); + assert_eq!(mascots[0]["riveColors"]["primaryColor"], "#111"); + assert_eq!(mascots[0]["riveColors"]["secondaryColor"], "#222"); + assert_eq!(mascots[1]["mascotId"], "toshi"); + assert_eq!(mascots[1]["voiceId"], "voice-b"); + // No name supplied for slot 1 → no `name` key. + assert!(mascots[1].get("name").is_none()); + // No colors supplied for slot 1 → no `riveColors` key. + assert!(mascots[1].get("riveColors").is_none()); } #[test] diff --git a/src/openhuman/agent_meetings/schemas.rs b/src/openhuman/agent_meetings/schemas.rs index 413134c47..722db18d3 100644 --- a/src/openhuman/agent_meetings/schemas.rs +++ b/src/openhuman/agent_meetings/schemas.rs @@ -126,6 +126,16 @@ fn schema_join() -> ControllerSchema { comment: "Optional Rive mascot color overrides forwarded to the backend bot.", required: false, }, + FieldSchema { + name: "mascots", + ty: TypeSchema::Json, + comment: "Optional dual-mascot config (issue #4277): array of up to 2 slots, each \ + { mascotId, name?, riveColors?, voiceId? }. When present the backend renders \ + both mascots and alternates the speaker per reply. `name` (from the manifest) \ + enables name-addressed routing (\"Hey Toshi …\" → that slot). Absent falls \ + back to mascot_id.", + required: false, + }, FieldSchema { name: "respond_to_participant", ty: TypeSchema::String, diff --git a/src/openhuman/agent_meetings/types.rs b/src/openhuman/agent_meetings/types.rs index c6ba09204..f2344e736 100644 --- a/src/openhuman/agent_meetings/types.rs +++ b/src/openhuman/agent_meetings/types.rs @@ -90,6 +90,33 @@ pub struct RiveColors { pub secondary_color: Option, } +/// One mascot slot for a multi-mascot backend meeting (issue #4277). +/// Slot 0 = primary speaker, slot 1 = secondary. The backend bot renders +/// both mascots side-by-side, alternates the speaking slot per reply, and +/// synthesizes each slot's replies with `voice_id` (falling back to its +/// configured default when absent). +/// +/// Deserialize-only: the RPC input arrives snake_case (`mascot_id`, +/// `rive_colors`, `voice_id`); the outbound `bot:join` payload is built by +/// hand in `ops.rs` (camelCase), so this struct is never serialized. +#[derive(Debug, Clone, Deserialize)] +pub struct BackendMascotSlot { + /// Rive mascot id (e.g. "yellow", "toshi"). + pub mascot_id: String, + /// Optional human-facing mascot name (e.g. "Toshi", "Tiny"), taken from the + /// manifest. Drives name-addressed routing (#4277 follow-up): a participant + /// who says "Hey Toshi …" is routed to this slot instead of the mechanical + /// alternation. Absent → that slot is not name-addressable. + #[serde(default)] + pub name: Option, + /// Optional per-mascot color palette overrides. + #[serde(default)] + pub rive_colors: Option, + /// Optional per-mascot ElevenLabs voice id. + #[serde(default)] + pub voice_id: Option, +} + /// Inputs to `openhuman.agent_meetings_join`. #[derive(Debug, Clone, Deserialize)] pub struct BackendMeetJoinRequest { @@ -105,11 +132,18 @@ pub struct BackendMeetJoinRequest { #[serde(default)] pub system_prompt: Option, /// Selects which Rive mascot appears in the meeting (e.g. "yellow", "blue"). + /// Legacy single-mascot field; still honored when `mascots` is absent. #[serde(default)] pub mascot_id: Option, - /// Optional Rive mascot color palette overrides. + /// Optional Rive mascot color palette overrides (single-mascot path). #[serde(default)] pub rive_colors: Option, + /// Dual-mascot config (issue #4277). When present (2 slots) the backend + /// renders both mascots and alternates the speaker per reply, using each + /// slot's `voice_id`. Absent / single-element falls back to the legacy + /// single-mascot `mascot_id` + `rive_colors` behavior. + #[serde(default)] + pub mascots: Option>, /// Only respond to this participant's messages (empty/absent = respond to everyone). #[serde(default)] pub respond_to_participant: Option, diff --git a/src/openhuman/meet_agent/brain/access.rs b/src/openhuman/meet_agent/brain/access.rs index c3d0a74c6..95d9bc5b1 100644 --- a/src/openhuman/meet_agent/brain/access.rs +++ b/src/openhuman/meet_agent/brain/access.rs @@ -221,7 +221,7 @@ pub async fn run_grant_turn(request_id: &str, grantee: &str) -> Result samples, Err(err) => { log::warn!("[meet-agent] grant TTS failed request_id={request_id} err={err}"); @@ -301,7 +301,7 @@ pub async fn run_soft_deny_turn( // Cancel any prior outbound so the refusal doesn't queue behind a // half-drained reply from a previous turn. let _ = registry().with_session(request_id, |s| s.cancel_outbound()); - let samples = match tts(&message).await { + let samples = match tts(&message, None).await { Ok(samples) => samples, Err(err) => { log::warn!("[meet-agent] soft-deny TTS failed request_id={request_id} err={err}"); diff --git a/src/openhuman/meet_agent/brain/speech.rs b/src/openhuman/meet_agent/brain/speech.rs index 67d003674..88eb70f53 100644 --- a/src/openhuman/meet_agent/brain/speech.rs +++ b/src/openhuman/meet_agent/brain/speech.rs @@ -26,7 +26,11 @@ pub(super) async fn stt(samples: &[i16]) -> Result { // ─── Real TTS adapter ─────────────────────────────────────────────── -pub(super) async fn tts(text: &str) -> Result, String> { +/// Synthesize `text` to PCM16LE @ 16 kHz. `voice_id` selects the +/// ElevenLabs voice for this utterance (the speaker-alternation slot's +/// voice); `None` lets the reply-speech backend pick its own default — +/// the exact prior behavior for single-mascot calls. +pub(super) async fn tts(text: &str, voice_id: Option<&str>) -> Result, String> { use crate::openhuman::voice::reply_speech::{synthesize_reply, ReplySpeechOptions}; let config = crate::openhuman::config::ops::load_config_with_timeout().await?; @@ -51,6 +55,9 @@ pub(super) async fn tts(text: &str) -> Result, String> { output_format: Some("pcm_16000".to_string()), model_id: Some(TTS_MODEL_ID.to_string()), voice_settings: Some(voice_settings), + // Per-mascot voice for speaker alternation. `None` preserves the + // backend's default-voice pick (single-mascot behavior). + voice_id: voice_id.map(str::to_owned), ..Default::default() }; let outcome = synthesize_reply(&config, text, &opts).await?; diff --git a/src/openhuman/meet_agent/brain/turns.rs b/src/openhuman/meet_agent/brain/turns.rs index f80f4618a..969bd6eb5 100644 --- a/src/openhuman/meet_agent/brain/turns.rs +++ b/src/openhuman/meet_agent/brain/turns.rs @@ -96,7 +96,13 @@ pub async fn run_turn(request_id: &str) -> Result { let synthesized = if reply_text.trim().is_empty() { Vec::new() } else { - match tts(&reply_text).await { + // Pick this reply's speaker voice. For two-mascot calls this + // alternates voice + active slot once per spoken reply; a + // single-mascot call gets `None` (backend default voice) and the + // slot stays 0. Advanced only when we actually speak so declined + // turns don't rotate the speaker out of sync. + let voice_id = registry().with_session(request_id, |s| s.advance_speaker())?; + match tts(&reply_text, voice_id.as_deref()).await { Ok(samples) => samples, Err(err) => { log::warn!("[meet-agent] TTS failed request_id={request_id} err={err}"); @@ -229,8 +235,20 @@ pub async fn run_caption_turn(request_id: &str) -> Result { // can hear you" sounds redundant. The 50-char threshold is a // rough proxy; real second-brain questions ("am I free Friday // afternoon for a 30 min slot") are almost always longer. - if !was_bare_wake && prompt.chars().count() > PREROLL_SKIP_PROMPT_CHARS { - if let Ok(ack_pcm) = tts(PREROLL_ACK_PHRASE).await { + // One speaker per caption turn: the pre-roll ack and the final reply + // come from the SAME mascot, and the active slot advances exactly once + // per turn — but ONLY when the turn actually speaks. A turn that fires + // no ack and then declines must not rotate the speaker (mirrors + // `run_turn`, which advances only on a non-empty reply). `turn_voice` + // stays `None` for single-mascot calls (backend default voice, slot 0). + let will_ack = !was_bare_wake && prompt.chars().count() > PREROLL_SKIP_PROMPT_CHARS; + let mut turn_voice: Option = None; + let mut speaker_advanced = false; + + if will_ack { + turn_voice = registry().with_session(request_id, |s| s.advance_speaker())?; + speaker_advanced = true; + if let Ok(ack_pcm) = tts(PREROLL_ACK_PHRASE, turn_voice.as_deref()).await { let _ = registry().with_session(request_id, |s| { s.enqueue_outbound_pcm(&ack_pcm, false); }); @@ -279,7 +297,13 @@ pub async fn run_caption_turn(request_id: &str) -> Result { let synthesized = if reply_text.trim().is_empty() { Vec::new() } else { - match tts(&reply_text).await { + // If no pre-roll ack fired, this reply is the turn's first (and + // only) speech — advance the speaker now so a declined turn never + // rotated the slot, and a spoken turn rotates exactly once. + if !speaker_advanced { + turn_voice = registry().with_session(request_id, |s| s.advance_speaker())?; + } + match tts(&reply_text, turn_voice.as_deref()).await { Ok(samples) => samples, Err(err) => { log::warn!( diff --git a/src/openhuman/meet_agent/rpc.rs b/src/openhuman/meet_agent/rpc.rs index 319f7e609..cdb03759d 100644 --- a/src/openhuman/meet_agent/rpc.rs +++ b/src/openhuman/meet_agent/rpc.rs @@ -49,14 +49,22 @@ pub async fn handle_start_session(params: Map) -> Result) -> Result) -> Result, + /// Per-mascot TTS voice ids in speaker-slot order (slot 0 = primary, + /// slot 1 = secondary). 0, 1, or 2 entries. Empty preserves the old + /// single-default-voice behavior (the reply-speech backend picks its + /// own default). Set once at `start_session` from the shell payload. + voices: Vec, + /// Index into `voices` of the mascot speaking the *current* turn. + /// Advanced once per assistant reply by [`Self::advance_speaker`] so + /// two-mascot calls alternate which voice speaks (and which mascot + /// the frontend lip-syncs). Meaningless when `voices.len() <= 1`. + active_voice_ix: usize, + /// False until the first [`Self::advance_speaker`] of a two-mascot call. + /// Keeps the first reply on the primary (slot 0) — so an idle session also + /// reports slot 0 — while every subsequent reply rotates the speaker. + speaker_primed: bool, } impl MeetAgentSession { @@ -228,6 +242,9 @@ impl MeetAgentSession { pending_unauthorized_speaker: None, pending_unauthorized_at_ms: 0, allowlist: HashSet::new(), + voices: Vec::new(), + active_voice_ix: 0, + speaker_primed: false, } } @@ -305,6 +322,64 @@ impl MeetAgentSession { self.bot_display_name = bot_display_name.trim().to_string(); } + /// Configure the per-mascot TTS voices for speaker alternation. + /// `primary` is slot 0, `secondary` is slot 1. Empty / whitespace + /// ids are dropped, so: + /// * both present → two-voice call, speaker alternates per reply; + /// * one present → that single voice speaks every turn; + /// * none present → `advance_speaker` yields `None` and the + /// reply-speech backend keeps picking its own default voice + /// (exact previous behavior). + /// The active slot starts at 0 so an idle session reports the primary + /// mascot; [`Self::advance_speaker`] keeps the first reply there and + /// rotates thereafter. + pub fn set_voices(&mut self, primary: Option, secondary: Option) { + let mut voices = Vec::new(); + for id in [primary, secondary].into_iter().flatten() { + let trimmed = id.trim(); + if !trimmed.is_empty() { + voices.push(trimmed.to_string()); + } + } + self.active_voice_ix = 0; + self.speaker_primed = false; + self.voices = voices; + } + + /// Pick the voice for the turn that is about to start and rotate the + /// active speaker for two-mascot calls. Returns `None` when no voices + /// were configured (single default-voice behavior). Call exactly once + /// per assistant reply, before synthesizing, so a multi-chunk reply + /// keeps one voice for its whole duration. + pub fn advance_speaker(&mut self) -> Option { + match self.voices.len() { + 0 => None, + 1 => Some(self.voices[0].clone()), + n => { + // First reply stays on the primary (slot 0); each later reply + // rotates. This keeps an idle session on slot 0 too. + if self.speaker_primed { + self.active_voice_ix = (self.active_voice_ix + 1) % n; + } else { + self.speaker_primed = true; + } + Some(self.voices[self.active_voice_ix].clone()) + } + } + } + + /// Slot (0 = primary, 1 = secondary) of the mascot speaking the + /// current turn. The shell forwards this on the `meet-video:speaking-state` + /// edge so the frontend lip-syncs the right mascot and reacts the + /// other. Defaults to 0 (single-mascot rendering ignores it). + pub fn active_slot(&self) -> u8 { + if self.voices.is_empty() { + 0 + } else { + (self.active_voice_ix % self.voices.len()) as u8 + } + } + /// Read accessor used by audit logging. Empty when set_identities /// has not been called for this session. pub fn owner_display_name(&self) -> &str { @@ -950,6 +1025,57 @@ mod tests { .unwrap(); } + // ─── Speaker alternation (issue #4277) ────────────────────────── + + #[test] + fn set_voices_empty_disables_alternation() { + let mut s = MeetAgentSession::new("p".into(), 16_000); + s.set_voices(None, None); + // No voices → brain keeps the backend default voice, slot pinned 0. + assert_eq!(s.advance_speaker(), None); + assert_eq!(s.active_slot(), 0); + } + + #[test] + fn single_voice_speaks_every_turn_on_slot_zero() { + let mut s = MeetAgentSession::new("p".into(), 16_000); + s.set_voices(Some("voice-a".into()), None); + assert_eq!(s.advance_speaker().as_deref(), Some("voice-a")); + assert_eq!(s.active_slot(), 0); + // A single voice never rotates. + assert_eq!(s.advance_speaker().as_deref(), Some("voice-a")); + assert_eq!(s.active_slot(), 0); + } + + #[test] + fn two_voices_alternate_speaker_and_slot_per_turn() { + let mut s = MeetAgentSession::new("p".into(), 16_000); + s.set_voices(Some("voice-a".into()), Some("voice-b".into())); + // First reply = primary (slot 0)… + assert_eq!(s.advance_speaker().as_deref(), Some("voice-a")); + assert_eq!(s.active_slot(), 0); + // …second = secondary (slot 1)… + assert_eq!(s.advance_speaker().as_deref(), Some("voice-b")); + assert_eq!(s.active_slot(), 1); + // …third wraps back to primary. + assert_eq!(s.advance_speaker().as_deref(), Some("voice-a")); + assert_eq!(s.active_slot(), 0); + } + + #[test] + fn set_voices_filters_blank_and_trims() { + let mut s = MeetAgentSession::new("p".into(), 16_000); + // Blank primary collapses to a single (secondary) voice on slot 0… + s.set_voices(Some(" ".into()), Some(" voice-b ".into())); + assert_eq!(s.advance_speaker().as_deref(), Some("voice-b")); + assert_eq!(s.advance_speaker().as_deref(), Some("voice-b")); + assert_eq!(s.active_slot(), 0); + // …and two real ids are trimmed before use. + s.set_voices(Some(" voice-a ".into()), Some(" voice-b ".into())); + assert_eq!(s.advance_speaker().as_deref(), Some("voice-a")); + assert_eq!(s.advance_speaker().as_deref(), Some("voice-b")); + } + /// Build a session pre-configured for the wake-word tests: Alice /// is the call owner, "OpenHuman" is the bot's Meet tile. Every /// wake-path test goes through this helper so the owner gate diff --git a/src/openhuman/meet_agent/types.rs b/src/openhuman/meet_agent/types.rs index ceadd85b3..c094ee3a6 100644 --- a/src/openhuman/meet_agent/types.rs +++ b/src/openhuman/meet_agent/types.rs @@ -45,6 +45,19 @@ pub struct StartSessionRequest { /// updated to forward the URL still parse the payload. #[serde(default)] pub meet_url: String, + /// ElevenLabs voice id for the PRIMARY mascot (speaker slot 0). When + /// two mascots are enabled the session alternates the speaking voice + /// once per assistant reply (see [`super::session::MeetAgentSession::advance_speaker`]). + /// `None`/empty preserves the previous single-default-voice behavior + /// (the reply-speech backend picks its own default voice). Defaulted + /// so older shells / smoke tests still parse the payload. + #[serde(default)] + pub primary_voice_id: Option, + /// ElevenLabs voice id for the SECONDARY mascot (speaker slot 1). + /// Absent when the user has only one mascot enabled — in that case no + /// alternation happens and slot 0 speaks every turn. + #[serde(default)] + pub secondary_voice_id: Option, } fn default_sample_rate() -> u32 { diff --git a/src/openhuman/socket/event_handlers.rs b/src/openhuman/socket/event_handlers.rs index 426aa2c5d..461efd82f 100644 --- a/src/openhuman/socket/event_handlers.rs +++ b/src/openhuman/socket/event_handlers.rs @@ -390,10 +390,18 @@ pub(super) fn handle_sio_event( .get("timestampMs") .and_then(|v| v.as_u64()) .unwrap_or(0); + // Dual-mascot name addressing (#4277 follow-up): which slot the + // backend's wake matcher decided was addressed (0|1), if any. + let mascot_slot = data + .get("mascotSlot") + .and_then(|v| v.as_u64()) + .filter(|s| *s <= 1) + .map(|s| s as u8); log::info!( - "[socket] bot:in_call_request speaker={} cmd_len={}", + "[socket] bot:in_call_request speaker={} cmd_len={} mascot_slot={:?}", speaker, - command_text.len() + command_text.len(), + mascot_slot ); publish_global(DomainEvent::BackendMeetInCallRequest { correlation_id, @@ -401,6 +409,7 @@ pub(super) fn handle_sio_event( command_text, recent_transcript, timestamp_ms, + mascot_slot, }); } "bot:error" => {