From 796468c0344b25ba6360fc8e4bff5403b3eef112 Mon Sep 17 00:00:00 2001
From: YellowSnnowmann <167776381+YellowSnnowmann@users.noreply.github.com>
Date: Wed, 8 Jul 2026 02:26:57 +0530
Subject: [PATCH] =?UTF-8?q?feat:=20dual=20mascots=20in=20meetings=20?=
=?UTF-8?q?=E2=80=94=20per-mascot=20voice=20+=20speaker=20alternation=20(#?=
=?UTF-8?q?4277)=20(#4585)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
app/src-tauri/src/lib.rs | 4 +
app/src-tauri/src/meet_audio/mod.rs | 15 +-
app/src-tauri/src/meet_audio/speak_pump.rs | 137 ++++-
app/src-tauri/src/meet_call/mod.rs | 42 ++
app/src/components/meetings/MeetComposer.tsx | 34 ++
app/src/components/meetings/UpcomingTable.tsx | 54 +-
.../meetings/__tests__/meetingUtils.test.ts | 132 ++++-
app/src/components/meetings/meetingUtils.ts | 75 ++-
.../settings/panels/MascotPanel.tsx | 95 +++
.../settings/panels/PerMascotVoiceRow.tsx | 245 ++++++++
.../panels/__tests__/MascotPanel.test.tsx | 88 +++
.../__tests__/PerMascotVoiceRow.test.tsx | 203 +++++++
app/src/features/meet/MascotFrameProducer.tsx | 358 +++++++++---
.../__tests__/MascotFrameProducer.test.tsx | 553 ++++++++++++++++--
.../__tests__/mascotFrameCompositor.test.ts | 214 +++++++
.../meet/__tests__/useMeetingMascots.test.ts | 136 +++++
.../features/meet/mascotFrameCompositor.ts | 129 ++++
app/src/features/meet/useMeetingMascots.ts | 150 +++++
app/src/lib/i18n/ar.ts | 7 +
app/src/lib/i18n/bn.ts | 7 +
app/src/lib/i18n/de.ts | 7 +
app/src/lib/i18n/en.ts | 7 +
app/src/lib/i18n/es.ts | 7 +
app/src/lib/i18n/fr.ts | 7 +
app/src/lib/i18n/hi.ts | 7 +
app/src/lib/i18n/id.ts | 7 +
app/src/lib/i18n/it.ts | 7 +
app/src/lib/i18n/ko.ts | 7 +
app/src/lib/i18n/pl.ts | 7 +
app/src/lib/i18n/pt.ts | 7 +
app/src/lib/i18n/ru.ts | 7 +
app/src/lib/i18n/zh-CN.ts | 7 +
.../__tests__/meetCallService.test.ts | 83 +++
app/src/services/meetCallService.ts | 84 ++-
app/src/store/__tests__/mascotSlice.test.ts | 172 ++++++
app/src/store/mascotSlice.ts | 184 +++++-
src/core/event_bus/events.rs | 4 +
src/openhuman/agent_meetings/bus.rs | 4 +
src/openhuman/agent_meetings/in_call.rs | 90 ++-
src/openhuman/agent_meetings/ops.rs | 91 +++
src/openhuman/agent_meetings/schemas.rs | 10 +
src/openhuman/agent_meetings/types.rs | 36 +-
src/openhuman/meet_agent/brain/access.rs | 4 +-
src/openhuman/meet_agent/brain/speech.rs | 9 +-
src/openhuman/meet_agent/brain/turns.rs | 32 +-
src/openhuman/meet_agent/rpc.rs | 50 +-
src/openhuman/meet_agent/session.rs | 126 ++++
src/openhuman/meet_agent/types.rs | 13 +
src/openhuman/socket/event_handlers.rs | 13 +-
49 files changed, 3581 insertions(+), 186 deletions(-)
create mode 100644 app/src/components/settings/panels/PerMascotVoiceRow.tsx
create mode 100644 app/src/components/settings/panels/__tests__/PerMascotVoiceRow.test.tsx
create mode 100644 app/src/features/meet/__tests__/mascotFrameCompositor.test.ts
create mode 100644 app/src/features/meet/__tests__/useMeetingMascots.test.ts
create mode 100644 app/src/features/meet/mascotFrameCompositor.ts
create mode 100644 app/src/features/meet/useMeetingMascots.ts
diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs
index 4aae5e711..1840d17fc 100644
--- a/app/src-tauri/src/lib.rs
+++ b/app/src-tauri/src/lib.rs
@@ -3678,6 +3678,10 @@ pub fn run() {
// wake gate will fail-closed (no wakes fire) which
// is the safe posture for an automated harness.
owner_display_name: String::new(),
+ // No mascot config on the dev-auto path → single
+ // default voice (issue #4277).
+ primary_voice_id: None,
+ secondary_voice_id: None,
};
match meet_call::meet_call_open_window(app_handle.clone(), state, args)
.await
diff --git a/app/src-tauri/src/meet_audio/mod.rs b/app/src-tauri/src/meet_audio/mod.rs
index fbd0769d0..a60f582b9 100644
--- a/app/src-tauri/src/meet_audio/mod.rs
+++ b/app/src-tauri/src/meet_audio/mod.rs
@@ -89,13 +89,19 @@ pub async fn start(
meet_url: String,
owner_display_name: String,
bot_display_name: String,
+ primary_voice_id: Option,
+ secondary_voice_id: Option,
) -> Result<(), String> {
log::info!(
"[meet-audio] start request_id={request_id} url_prefix={} \
- owner_chars={} bot_chars={}",
+ owner_chars={} bot_chars={} voices={}",
truncate_for_log(&meet_url, 64),
owner_display_name.chars().count(),
- bot_display_name.chars().count()
+ bot_display_name.chars().count(),
+ [&primary_voice_id, &secondary_voice_id]
+ .into_iter()
+ .filter(|v| v.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false))
+ .count()
);
if let Some(state) = app.try_state::() {
@@ -125,6 +131,11 @@ pub async fn start(
// " in the history list. The URL the shell built
// the CEF window with is the canonical value.
"meet_url": meet_url,
+ // Per-mascot TTS voices for speaker alternation (issue #4277).
+ // Null for single-mascot calls → core keeps the backend
+ // default voice (unchanged behavior).
+ "primary_voice_id": primary_voice_id,
+ "secondary_voice_id": secondary_voice_id,
}),
)
.await?;
diff --git a/app/src-tauri/src/meet_audio/speak_pump.rs b/app/src-tauri/src/meet_audio/speak_pump.rs
index 4b83a15ce..2cda72a79 100644
--- a/app/src-tauri/src/meet_audio/speak_pump.rs
+++ b/app/src-tauri/src/meet_audio/speak_pump.rs
@@ -98,10 +98,10 @@ pub fn start(
break;
}
_ = tick.tick() => {
- let had_pcm = match poll_and_feed(&request_id_for_task, &mut cdp, &session_id).await {
- Ok(had) => {
+ let (had_pcm, active_slot) = match poll_and_feed(&request_id_for_task, &mut cdp, &session_id).await {
+ Ok((had, slot)) => {
feed_errors = 0;
- had
+ (had, Some(slot))
}
Err(err) => {
feed_errors += 1;
@@ -118,11 +118,12 @@ pub fn start(
// A failed tick is *not* evidence the bot
// stopped speaking — leave the hangover to
// expire naturally so transient CDP errors
- // don't flicker the mascot's mouth shut.
- false
+ // don't flicker the mascot's mouth shut. No
+ // fresh slot data → keep the last-known slot.
+ (false, None)
}
};
- speaking_state.tick(had_pcm, &app, &request_id_for_task);
+ speaking_state.tick(had_pcm, active_slot, &app, &request_id_for_task);
}
}
}
@@ -148,6 +149,14 @@ struct SpeakingTracker {
/// every tick that carries PCM; the state flips back to `false`
/// only once `now > hangover_until` AND a tick with no PCM lands.
hangover_until: Option,
+ /// Which mascot (0 = primary, 1 = secondary) is speaking the current
+ /// outbound audio, as reported by `meet_agent_poll_speech`. For
+ /// two-mascot calls the brain alternates this per reply; the frontend
+ /// lip-syncs this slot and reacts the other. Emitted on the
+ /// speaking-state edge AND whenever the slot changes mid-speech (so a
+ /// back-to-back reply from the other mascot still switches lip-sync
+ /// even if the hangover bridged the gap). 0 for single-mascot calls.
+ active_slot: u8,
}
impl SpeakingTracker {
@@ -155,6 +164,7 @@ impl SpeakingTracker {
Self {
reported: false,
hangover_until: None,
+ active_slot: 0,
}
}
@@ -162,12 +172,21 @@ impl SpeakingTracker {
/// is whether `poll_and_feed` saw a non-empty `pcm_base64` for
/// this tick. Emits the Tauri event only when the reported
/// state actually flips.
- fn tick(&mut self, had_pcm: bool, app: &AppHandle, request_id: &str) {
+ /// `active_slot` is the speaking mascot slot for this tick, or `None`
+ /// when the tick had no fresh poll data (e.g. a CDP error) — in that
+ /// case the last-known slot is retained.
+ fn tick(
+ &mut self,
+ had_pcm: bool,
+ active_slot: Option,
+ app: &AppHandle,
+ request_id: &str,
+ ) {
if had_pcm {
// Extend the hangover. If we were idle, flip up to
// speaking — the user hears audio starting now.
self.hangover_until = Some(Instant::now() + SPEAKING_HANGOVER);
- self.set_reported(true, app, request_id);
+ self.update(true, active_slot, app, request_id);
return;
}
// No PCM this tick. If the hangover hasn't expired, stay in
@@ -182,7 +201,7 @@ impl SpeakingTracker {
self.hangover_until = None;
}
// Hangover expired or never armed → bot is genuinely idle.
- self.set_reported(false, app, request_id);
+ self.update(false, active_slot, app, request_id);
}
/// Force the reported state to `false` and emit an event if that's
@@ -190,30 +209,71 @@ impl SpeakingTracker {
/// can't get stuck mid-talk.
fn force_off(&mut self, app: &AppHandle, request_id: &str) {
self.hangover_until = None;
- self.set_reported(false, app, request_id);
+ self.update(false, None, app, request_id);
}
- fn set_reported(&mut self, next: bool, app: &AppHandle, request_id: &str) {
- if self.reported == next {
+ /// Update reported speaking state + active slot, emitting the Tauri
+ /// event when either the speaking flag flips OR (while speaking) the
+ /// active mascot slot changes. `slot = None` keeps the last-known
+ /// slot (used when a tick carried no fresh poll data).
+ fn update(
+ &mut self,
+ next: bool,
+ slot: Option,
+ app: &AppHandle,
+ request_id: &str,
+ ) {
+ let (should_emit, resolved_slot) =
+ next_speaking_state(self.reported, self.active_slot, next, slot);
+ self.reported = next;
+ self.active_slot = resolved_slot;
+ if !should_emit {
return;
}
- self.reported = next;
let payload = serde_json::json!({
"requestId": request_id,
"speaking": next,
+ // Which mascot is speaking this audio (0 = primary, 1 =
+ // secondary). Frontend lip-syncs this slot, reacts the other.
+ "activeMascotSlot": self.active_slot,
});
if let Err(err) = app.emit(SPEAKING_STATE_EVENT, payload) {
// Best-effort: a missing renderer (closed window mid-tick)
// is the common case and not worth raising the log level.
log::debug!(
- "[meet-audio] speaking-state emit failed request_id={request_id} speaking={next} err={err}"
+ "[meet-audio] speaking-state emit failed request_id={request_id} speaking={next} slot={} err={err}",
+ self.active_slot
);
} else {
- log::debug!("[meet-audio] speaking-state -> {next} request_id={request_id}");
+ log::debug!(
+ "[meet-audio] speaking-state -> {next} slot={} request_id={request_id}",
+ self.active_slot
+ );
}
}
}
+/// Pure decision for [`SpeakingTracker::update`]: given the previously
+/// reported speaking flag + slot and the incoming `next`/`slot`, compute
+/// whether the Tauri event should be emitted and which slot to resolve to.
+///
+/// Extracted so the emit logic can be unit-tested without a live
+/// `AppHandle`. `slot = None` retains the previous slot (a tick with no
+/// fresh poll data). We emit when the speaking flag flips OR (while
+/// speaking) the active slot changes — the latter switches lip-sync on a
+/// back-to-back reply from the other mascot even if the hangover bridged
+/// the gap.
+fn next_speaking_state(
+ prev_reported: bool,
+ prev_slot: u8,
+ next: bool,
+ slot: Option,
+) -> (bool, u8) {
+ let resolved = slot.unwrap_or(prev_slot);
+ let should_emit = prev_reported != next || (next && prev_slot != resolved);
+ (should_emit, resolved)
+}
+
/// No-op pump used when bridge install failed at session start. Keeps
/// the rest of the session lifecycle uniform — `MeetAudioSession` can
/// still hold a `SpeakPump` regardless of speak-path readiness.
@@ -231,7 +291,7 @@ async fn poll_and_feed(
request_id: &str,
cdp: &mut CdpConn,
session_id: &str,
-) -> Result {
+) -> Result<(bool, u8), String> {
let v = super::rpc_call(
"openhuman.meet_agent_poll_speech",
serde_json::json!({ "request_id": request_id }),
@@ -249,6 +309,12 @@ async fn poll_and_feed(
.get("flush_pending")
.and_then(|x| x.as_bool())
.unwrap_or(false);
+ // Which mascot slot is speaking this audio (0 = primary, 1 =
+ // secondary). Absent on older cores / single-mascot → 0.
+ let active_slot = v
+ .get("active_mascot_slot")
+ .and_then(|x| x.as_u64())
+ .unwrap_or(0) as u8;
// Barge-in: brain set flush_pending when it cancelled the previous
// outbound. Stop in-flight playback inside the JS bridge BEFORE we
@@ -278,10 +344,45 @@ async fn poll_and_feed(
bytes.len()
);
inject::feed_pcm_chunk(cdp, session_id, pcm_b64).await?;
- return Ok(true);
+ return Ok((true, active_slot));
}
if utterance_done {
log::info!("[meet-audio] speak pump utterance complete request_id={request_id}");
}
- Ok(false)
+ Ok((false, active_slot))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn speaking_edge_and_slot_change_emit_logic() {
+ // idle -> speaking: a real edge, emits.
+ let (emit, slot) = next_speaking_state(false, 0, true, Some(0));
+ assert!(emit, "idle -> speaking must emit");
+ assert_eq!(slot, 0);
+
+ // speaking -> speaking, same slot: no edge, no slot change, no emit.
+ let (emit, slot) = next_speaking_state(true, 0, true, Some(0));
+ assert!(!emit, "speaking -> speaking same slot must not emit");
+ assert_eq!(slot, 0);
+
+ // speaking(slot 0) -> speaking(slot 1): mid-speech mascot switch emits
+ // and resolves to the new slot so lip-sync moves to the other mascot.
+ let (emit, slot) = next_speaking_state(true, 0, true, Some(1));
+ assert!(emit, "speaking slot change must emit");
+ assert_eq!(slot, 1, "resolved slot must be the new slot 1");
+
+ // slot = None retains the previous slot; with the same reported flag
+ // that is neither an edge nor a slot change, so no emit.
+ let (emit, slot) = next_speaking_state(true, 1, true, None);
+ assert!(!emit, "no fresh slot + same reported must not emit");
+ assert_eq!(slot, 1, "None retains the previous slot");
+
+ // speaking -> idle: a real edge, emits.
+ let (emit, slot) = next_speaking_state(true, 1, false, None);
+ assert!(emit, "speaking -> idle must emit");
+ assert_eq!(slot, 1, "None retains the previous slot on the way down");
+ }
}
diff --git a/app/src-tauri/src/meet_call/mod.rs b/app/src-tauri/src/meet_call/mod.rs
index eb9e24aa4..592aa2789 100644
--- a/app/src-tauri/src/meet_call/mod.rs
+++ b/app/src-tauri/src/meet_call/mod.rs
@@ -93,6 +93,15 @@ pub struct OpenWindowArgs {
/// fails closed in core (no wakes fire).
#[serde(default)]
pub owner_display_name: String,
+ /// ElevenLabs voice for the primary mascot (issue #4277). When the
+ /// user has two mascots enabled the core alternates the speaking
+ /// voice per reply. Absent/empty → core keeps its default voice.
+ #[serde(default)]
+ pub primary_voice_id: Option,
+ /// ElevenLabs voice for the secondary mascot. Absent when only one
+ /// mascot is enabled.
+ #[serde(default)]
+ pub secondary_voice_id: Option,
}
/// Open a dedicated top-level CEF webview window pointed at the Meet URL.
@@ -262,6 +271,8 @@ pub async fn meet_call_open_window(
let url_for_audio = parsed.to_string();
let bot_for_audio = args.display_name.clone();
let owner_for_audio = args.owner_display_name.clone();
+ let primary_voice_for_audio = args.primary_voice_id.clone();
+ let secondary_voice_for_audio = args.secondary_voice_id.clone();
tauri::async_runtime::spawn(async move {
if let Err(err) = crate::meet_audio::start(
app_for_audio,
@@ -269,6 +280,8 @@ pub async fn meet_call_open_window(
url_for_audio,
owner_for_audio,
bot_for_audio,
+ primary_voice_for_audio,
+ secondary_voice_for_audio,
)
.await
{
@@ -551,6 +564,35 @@ mod tests {
);
}
+ #[test]
+ fn open_window_args_deserializes_voice_ids() {
+ use serde_json::json;
+
+ // Both voice ids present → parse into Some(...). Proves the
+ // #4277 per-mascot voice fields ride through the command payload.
+ let args: OpenWindowArgs = serde_json::from_value(json!({
+ "request_id": "r",
+ "meet_url": "u",
+ "display_name": "d",
+ "primary_voice_id": "va",
+ "secondary_voice_id": "vb",
+ }))
+ .unwrap();
+ assert_eq!(args.primary_voice_id.as_deref(), Some("va"));
+ assert_eq!(args.secondary_voice_id.as_deref(), Some("vb"));
+
+ // Both omitted → #[serde(default)] yields None (single-mascot
+ // callers and older shells that don't forward voices still parse).
+ let args: OpenWindowArgs = serde_json::from_value(json!({
+ "request_id": "r",
+ "meet_url": "u",
+ "display_name": "d",
+ }))
+ .unwrap();
+ assert_eq!(args.primary_voice_id, None);
+ assert_eq!(args.secondary_voice_id, None);
+ }
+
#[test]
fn meet_call_state_default_is_empty() {
let state = MeetCallState::default();
diff --git a/app/src/components/meetings/MeetComposer.tsx b/app/src/components/meetings/MeetComposer.tsx
index 5b69b6361..a64af50b4 100644
--- a/app/src/components/meetings/MeetComposer.tsx
+++ b/app/src/components/meetings/MeetComposer.tsx
@@ -9,6 +9,7 @@
import debug from 'debug';
import { type RefObject, useEffect, useRef, useState } from 'react';
+import { useMascotManifest } from '../../features/human/Mascot/manifest/useMascotManifest';
import { useComposioIntegrations } from '../../lib/composio/hooks';
import { useT } from '../../lib/i18n/I18nContext';
import {
@@ -25,12 +26,15 @@ import { useAppDispatch, useAppSelector } from '../../store/hooks';
import {
selectCustomPrimaryColor,
selectCustomSecondaryColor,
+ selectDualMascotEnabled,
selectMascotColor,
+ selectMeetingMascotVoicePair,
selectSelectedMascotId,
} from '../../store/mascotSlice';
import { selectPersonaDescription, selectPersonaDisplayName } from '../../store/personaSlice';
import Button from '../ui/Button';
import {
+ buildMeetingMascots,
platformLabel,
platformUrlPlaceholder,
resolveMeetingBotMascotId,
@@ -76,6 +80,15 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
const mascotColor = useAppSelector(selectMascotColor);
const customPrimaryColor = useAppSelector(selectCustomPrimaryColor);
const customSecondaryColor = useAppSelector(selectCustomSecondaryColor);
+ // Dual-mascot config (issue #4277): when a distinct second mascot is enabled
+ // we send both slots (each with its own voice) so the backend bot renders
+ // two mascots and alternates who speaks. Single-mascot keeps the legacy
+ // `mascotId` path below untouched.
+ const dualMascotEnabled = useAppSelector(selectDualMascotEnabled);
+ const mascotVoicePair = useAppSelector(selectMeetingMascotVoicePair);
+ // Manifest drives name-addressed routing (#4277 follow-up): each dual slot is
+ // tagged with its display name so "Hey Toshi …" routes to that mascot.
+ const { manifest } = useMascotManifest();
// ── Meet slice ───────────────────────────────────────────────────────────
const meetStatus = useAppSelector(selectBackendMeetStatus);
@@ -129,6 +142,17 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
mascotColor === 'custom'
? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor }
: undefined;
+ // Two-mascot slots for the backend bot (issue #4277) — built via the shared
+ // helper so this live-join path and the UpcomingTable scheduled-join path stay
+ // behaviorally identical.
+ const mascots = buildMeetingMascots({
+ dualMascotEnabled,
+ mascotVoicePair,
+ manifest,
+ mascotId,
+ riveColors,
+ agentName,
+ });
const wakePhrase = listenOnly ? undefined : `Hey ${agentName}`;
const handleSubmit = async (event: React.FormEvent) => {
@@ -143,6 +167,14 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
!listenOnly,
meetingId
);
+ // Name-addressing (#4277 follow-up) trace: the exact mascot ids + names sent
+ // to the backend. If `mascots` is undefined or a slot's `name` is empty,
+ // name addressing ("Hey Toshi") can't work — the backend needs both names.
+ log(
+ '[composer] join mascots=%o wakePhrase=%s',
+ mascots?.map(m => ({ mascotId: m.mascotId, name: m.name })),
+ wakePhrase
+ );
try {
// Await the RPC BEFORE dispatching setBackendMeetJoining so that a
// synchronous rejection (bad URL, auth failure) can be shown inline
@@ -159,6 +191,8 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
systemPrompt,
mascotId,
riveColors,
+ // Dual-mascot slots (issue #4277); undefined for single-mascot calls.
+ mascots,
correlationId: meetingId,
respondToParticipant: displayedRespondTo.trim() || undefined,
wakePhrase,
diff --git a/app/src/components/meetings/UpcomingTable.tsx b/app/src/components/meetings/UpcomingTable.tsx
index 971f70400..7ff6bcc6d 100644
--- a/app/src/components/meetings/UpcomingTable.tsx
+++ b/app/src/components/meetings/UpcomingTable.tsx
@@ -14,6 +14,7 @@
import debug from 'debug';
import { useState } from 'react';
+import { useMascotManifest } from '../../features/human/Mascot/manifest/useMascotManifest';
import { useT } from '../../lib/i18n/I18nContext';
import {
joinMeetViaBackendBot,
@@ -26,13 +27,21 @@ import { useAppSelector } from '../../store/hooks';
import {
selectCustomPrimaryColor,
selectCustomSecondaryColor,
+ selectDualMascotEnabled,
selectMascotColor,
+ selectMeetingMascotVoicePair,
selectSelectedMascotId,
} from '../../store/mascotSlice';
import { selectPersonaDescription, selectPersonaDisplayName } from '../../store/personaSlice';
import Button from '../ui/Button';
import { type JoinPolicy, JoinPolicyToggle } from './JoinPolicyToggle';
-import { inferPlatformFromUrl, platformLabel, platformLogoUrl } from './meetingUtils';
+import {
+ buildMeetingMascots,
+ inferPlatformFromUrl,
+ platformLabel,
+ platformLogoUrl,
+ resolveMeetingBotMascotId,
+} from './meetingUtils';
import { useUpcomingMeetings } from './useUpcomingMeetings';
const log = debug('meetings:upcoming-table');
@@ -359,6 +368,14 @@ export function UpcomingTable({
const mascotColor = useAppSelector(selectMascotColor);
const customPrimaryColor = useAppSelector(selectCustomPrimaryColor);
const customSecondaryColor = useAppSelector(selectCustomSecondaryColor);
+ // Dual-mascot config (issue #4277) — see MeetComposer for the rationale;
+ // both join sites resolve the two-mascot slots the same way so a "Join now"
+ // and a manual join behave identically.
+ const dualMascotEnabled = useAppSelector(selectDualMascotEnabled);
+ const mascotVoicePair = useAppSelector(selectMeetingMascotVoicePair);
+ // Manifest drives name-addressed routing (#4277 follow-up): tag each dual slot
+ // with its display name so "Hey Toshi …" routes to that mascot.
+ const { manifest } = useMascotManifest();
// Live in-call state — lets a row detect that its meeting is already joined
// and suppress the "Join now" button. correlationId is a fresh per-join UUID
@@ -371,12 +388,29 @@ export function UpcomingTable({
return Boolean(backendMeetUrl && m.meet_url && backendMeetUrl === m.meet_url);
};
- // Resolve bot join params the same way MeetComposer does.
- const mascotId = selectedMascotId ?? (mascotColor === 'custom' ? undefined : mascotColor);
+ // Resolve bot join params the same way MeetComposer does — via
+ // resolveMeetingBotMascotId, so a manifest-only id the backend bot doesn't
+ // recognize is dropped here too (a raw `selectedMascotId` fallback would let
+ // it through and diverge from the MeetComposer join).
+ const mascotId = resolveMeetingBotMascotId(selectedMascotId, mascotColor);
const riveColors =
mascotColor === 'custom'
? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor }
: undefined;
+ // Bot join name — hoisted to component-body scope because both the `mascots`
+ // array here and the wake phrase inside handleJoin need it.
+ const agentName = personaDisplayName.trim() || 'Tiny';
+ // Two-mascot slots (issue #4277) — built via the shared helper so this
+ // scheduled-join path and the MeetComposer live-join path stay behaviorally
+ // identical.
+ const mascots = buildMeetingMascots({
+ dualMascotEnabled,
+ mascotVoicePair,
+ manifest,
+ mascotId,
+ riveColors,
+ agentName,
+ });
const handleJoin = async (meeting: UpcomingMeeting) => {
if (!meeting.meet_url) return;
@@ -388,9 +422,8 @@ export function UpcomingTable({
const anchor = replyDisplayName.trim();
// Reply mode gates the bot behind a wake phrase so it only reacts when
// addressed ("Hey Alex, …"), never to every caption from the anchor —
- // mirroring MeetComposer. The bot joins as `agentName`, so the phrase must
- // match it. Listen-only joins (no anchor) send no wake phrase.
- const agentName = personaDisplayName.trim() || 'Tiny';
+ // mirroring MeetComposer. The bot joins as `agentName` (hoisted above), so
+ // the phrase must match it. Listen-only joins (no anchor) send no wake phrase.
const wakePhrase = anchor ? `Hey ${agentName}` : undefined;
// Mint a fresh correlation id per join. It becomes the call record's
// `request_id` (recent-calls list key + per-call detail filename), so it
@@ -408,6 +441,13 @@ export function UpcomingTable({
correlationId
);
setJoiningId(meeting.calendar_event_id);
+ // Name-addressing (#4277 follow-up) trace: mascot ids + names sent to the
+ // backend. Empty `name` on a slot ⇒ name addressing can't route to it.
+ log(
+ '[upcoming] join mascots=%o wakePhrase=%s',
+ mascots?.map(m => ({ mascotId: m.mascotId, name: m.name })),
+ wakePhrase
+ );
try {
await joinMeetViaBackendBot({
meetUrl: meeting.meet_url,
@@ -415,6 +455,8 @@ export function UpcomingTable({
agentName,
systemPrompt: personaDescription || undefined,
mascotId: mascotId || undefined,
+ // Dual-mascot slots (issue #4277); undefined for single-mascot calls.
+ mascots,
respondToParticipant: anchor || undefined,
wakePhrase,
listenOnly: !anchor,
diff --git a/app/src/components/meetings/__tests__/meetingUtils.test.ts b/app/src/components/meetings/__tests__/meetingUtils.test.ts
index 4bb4d224a..ec3c2094a 100644
--- a/app/src/components/meetings/__tests__/meetingUtils.test.ts
+++ b/app/src/components/meetings/__tests__/meetingUtils.test.ts
@@ -1,7 +1,9 @@
import { describe, expect, it } from 'vitest';
+import type { MascotManifest } from '../../../features/human/Mascot/manifest/types';
import type { ComposioConnection } from '../../../lib/composio/types';
import {
+ buildMeetingMascots,
deriveDisplayNameFromEmail,
inferPlatformFromUrl,
MEETING_PLATFORMS,
@@ -257,8 +259,16 @@ describe('resolveMeetingBotMascotId', () => {
expect(resolveMeetingBotMascotId('navy', 'yellow')).toBe('navy');
});
- it('falls back to the legacy mascot color for a manifest-only mascot id', () => {
- expect(resolveMeetingBotMascotId('river-guide', 'yellow')).toBe('yellow');
+ it('keeps the "toshi" manifest mascot id the backend now ships as an asset', () => {
+ expect(resolveMeetingBotMascotId('toshi', 'yellow')).toBe('toshi');
+ });
+
+ it('keeps the "tiny-mascot" manifest mascot id the backend now ships as an asset', () => {
+ expect(resolveMeetingBotMascotId('tiny-mascot', 'navy')).toBe('tiny-mascot');
+ });
+
+ it('falls back to the legacy mascot color for a manifest-only mascot id the backend still lacks', () => {
+ expect(resolveMeetingBotMascotId('jarvis', 'yellow')).toBe('yellow');
});
it('uses the mascot color when no mascot id is selected', () => {
@@ -269,3 +279,121 @@ describe('resolveMeetingBotMascotId', () => {
expect(resolveMeetingBotMascotId('river-guide', 'custom')).toBeUndefined();
});
});
+
+// ---------------------------------------------------------------------------
+// buildMeetingMascots
+// ---------------------------------------------------------------------------
+
+describe('buildMeetingMascots', () => {
+ // findMascot only reads `.id`/`.name`, so a partial manifest is enough.
+ const manifest = {
+ mascots: [
+ { id: 'toshi', name: 'Toshi' },
+ { id: 'tiny-mascot', name: 'Tiny' },
+ ],
+ } as unknown as MascotManifest;
+
+ const pair = {
+ primary: { mascotId: 'toshi', voiceId: 'voice-a' },
+ secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
+ };
+
+ it('returns undefined when dual mode is off', () => {
+ expect(
+ buildMeetingMascots({
+ dualMascotEnabled: false,
+ mascotVoicePair: pair,
+ manifest,
+ mascotId: 'toshi',
+ agentName: 'Tiny',
+ })
+ ).toBeUndefined();
+ });
+
+ it('returns undefined when no secondary is configured', () => {
+ expect(
+ buildMeetingMascots({
+ dualMascotEnabled: true,
+ mascotVoicePair: { primary: { mascotId: 'toshi', voiceId: 'voice-a' } },
+ manifest,
+ mascotId: 'toshi',
+ agentName: 'Tiny',
+ })
+ ).toBeUndefined();
+ });
+
+ it('returns undefined when the primary slot id cannot be resolved', () => {
+ // primary has no explicit mascot AND the resolved single-path id is undefined.
+ expect(
+ buildMeetingMascots({
+ dualMascotEnabled: true,
+ mascotVoicePair: {
+ primary: { voiceId: 'voice-a' },
+ secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
+ },
+ manifest,
+ mascotId: undefined,
+ agentName: 'Tiny',
+ })
+ ).toBeUndefined();
+ });
+
+ it('falls back to the resolved single-path mascotId for the primary slot', () => {
+ const mascots = buildMeetingMascots({
+ dualMascotEnabled: true,
+ mascotVoicePair: {
+ primary: { voiceId: 'voice-a' },
+ secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
+ },
+ manifest,
+ mascotId: 'toshi',
+ agentName: 'Tiny',
+ });
+ expect(mascots?.[0].mascotId).toBe('toshi');
+ });
+
+ it('builds two slots with manifest names and per-slot voices', () => {
+ const riveColors = { primaryColor: '#111', secondaryColor: '#222' };
+ const mascots = buildMeetingMascots({
+ dualMascotEnabled: true,
+ mascotVoicePair: pair,
+ manifest,
+ mascotId: 'toshi',
+ riveColors,
+ agentName: 'Tiny',
+ });
+ expect(mascots).toEqual([
+ { mascotId: 'toshi', name: 'Toshi', voiceId: 'voice-a', riveColors },
+ { mascotId: 'tiny-mascot', name: 'Tiny', voiceId: 'voice-b', riveColors },
+ ]);
+ });
+
+ it('resolves names by slot so either mascot order works', () => {
+ const mascots = buildMeetingMascots({
+ dualMascotEnabled: true,
+ mascotVoicePair: {
+ primary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
+ secondary: { mascotId: 'toshi', voiceId: 'voice-a' },
+ },
+ manifest,
+ mascotId: 'tiny-mascot',
+ agentName: 'Tiny',
+ });
+ expect(mascots?.map(m => m.name)).toEqual(['Tiny', 'Toshi']);
+ });
+
+ it('falls back to agentName for the primary when its manifest entry is missing', () => {
+ const mascots = buildMeetingMascots({
+ dualMascotEnabled: true,
+ mascotVoicePair: {
+ primary: { mascotId: 'river-guide', voiceId: 'voice-a' },
+ secondary: { mascotId: 'toshi', voiceId: 'voice-a' },
+ },
+ manifest,
+ mascotId: 'river-guide',
+ agentName: 'Persona',
+ });
+ expect(mascots?.[0].name).toBe('Persona');
+ expect(mascots?.[1].name).toBe('Toshi');
+ });
+});
diff --git a/app/src/components/meetings/meetingUtils.ts b/app/src/components/meetings/meetingUtils.ts
index 441d274d6..86f158d00 100644
--- a/app/src/components/meetings/meetingUtils.ts
+++ b/app/src/components/meetings/meetingUtils.ts
@@ -5,17 +5,31 @@
* previously embedded inside MeetingBotsCard so they can be unit-tested in
* isolation and shared across the split composer components.
*/
+import { findMascot } from '../../features/human/Mascot/manifest/manifestService';
+import type { MascotManifest } from '../../features/human/Mascot/manifest/types';
import type { MascotColor } from '../../features/human/Mascot/mascotPalette';
import type { ComposioConnection } from '../../lib/composio/types';
-import type { MeetingPlatform } from '../../services/meetCallService';
+import type { BackendMeetJoinInput, MeetingPlatform } from '../../services/meetCallService';
import { composioLogoUrl } from '../composio/toolkitMeta';
/**
* Mascot ids the meeting-bot backend recognizes. Newer manifest-only mascot
* ids (e.g. "river-guide") aren't supported there, so the bot falls back to the
* legacy mascot color for them.
+ *
+ * "toshi" and "tiny-mascot" are the tinyhumansai/mascots manifest ids that the
+ * Recall meeting-bot backend now ships as real mascot assets, so they pass
+ * through unchanged instead of being collapsed to a color.
*/
-const MEETING_BOT_MASCOT_IDS = new Set(['yellow', 'blue', 'burgundy', 'black', 'navy']);
+const MEETING_BOT_MASCOT_IDS = new Set([
+ 'yellow',
+ 'blue',
+ 'burgundy',
+ 'black',
+ 'navy',
+ 'toshi',
+ 'tiny-mascot',
+]);
/**
* Resolve the mascot id to send to the meeting bot: the selected mascot id when
@@ -31,6 +45,63 @@ export function resolveMeetingBotMascotId(
return undefined;
}
+/** Voice/mascot pair the composer selects for the two speaking slots. */
+type MeetingMascotVoicePair = {
+ primary: { mascotId?: string | null; voiceId?: string };
+ secondary?: { mascotId?: string | null; voiceId?: string } | null;
+};
+
+/**
+ * Build the backend bot's dual-mascot `mascots` array (issue #4277), shared by
+ * both join sites — MeetComposer (live join) and UpcomingTable (scheduled join)
+ * — so the two paths stay behaviorally identical. They had already begun to
+ * drift, which is exactly the class of bug centralising this prevents.
+ *
+ * Slot 0 (primary) reuses the resolved single-path `mascotId` when the voice
+ * pair's primary carries no explicit mascot; slot 1 is the secondary. Slot names
+ * come from the manifest so name-addressed routing works in either order ("Hey
+ * Toshi" → whichever slot Toshi occupies); the primary falls back to `agentName`
+ * only when its manifest entry is unavailable. Per-mascot colors are out of
+ * scope — both slots reuse the single `riveColors`.
+ *
+ * Returns `undefined` (→ the single-mascot `mascotId` join) unless dual mode is
+ * on, a secondary is configured, AND both slot ids resolve to a concrete value:
+ * a blank slot-0 id would make the backend drop it and render the secondary
+ * alone, mismatching the on-camera primary.
+ */
+export function buildMeetingMascots(input: {
+ dualMascotEnabled: boolean;
+ mascotVoicePair: MeetingMascotVoicePair;
+ manifest: MascotManifest | null;
+ mascotId: string | undefined;
+ riveColors?: { primaryColor?: string; secondaryColor?: string };
+ agentName: string;
+}): BackendMeetJoinInput['mascots'] {
+ const { dualMascotEnabled, mascotVoicePair, manifest, mascotId, riveColors, agentName } = input;
+ const primarySlotId = mascotVoicePair.primary.mascotId ?? mascotId;
+ const secondarySlotId = mascotVoicePair.secondary?.mascotId ?? undefined;
+ if (!dualMascotEnabled || !mascotVoicePair.secondary || !primarySlotId || !secondarySlotId) {
+ return undefined;
+ }
+ const primaryName =
+ (manifest ? findMascot(manifest, primarySlotId)?.name : undefined) ?? agentName;
+ const secondaryName = manifest ? findMascot(manifest, secondarySlotId)?.name : undefined;
+ return [
+ {
+ mascotId: primarySlotId,
+ name: primaryName,
+ voiceId: mascotVoicePair.primary.voiceId,
+ riveColors,
+ },
+ {
+ mascotId: secondarySlotId,
+ name: secondaryName,
+ voiceId: mascotVoicePair.secondary.voiceId,
+ riveColors,
+ },
+ ];
+}
+
// ---------------------------------------------------------------------------
// Platform registry
// ---------------------------------------------------------------------------
diff --git a/app/src/components/settings/panels/MascotPanel.tsx b/app/src/components/settings/panels/MascotPanel.tsx
index a97be584c..cc2b42381 100644
--- a/app/src/components/settings/panels/MascotPanel.tsx
+++ b/app/src/components/settings/panels/MascotPanel.tsx
@@ -22,6 +22,7 @@ import {
selectMascotVoiceGender,
selectMascotVoiceId,
selectMascotVoiceUseLocaleDefault,
+ selectSecondaryMascotId,
selectSelectedMascotId,
setCustomMascotGifUrl,
setCustomPrimaryColor,
@@ -30,6 +31,7 @@ import {
setMascotVoiceGender,
setMascotVoiceId,
setMascotVoiceUseLocaleDefault,
+ setSecondaryMascotId,
setSelectedMascotId,
SUPPORTED_MASCOT_COLORS,
} from '../../../store/mascotSlice';
@@ -41,6 +43,7 @@ import {
ELEVENLABS_VOICE_PRESETS,
isCuratedVoicePreset,
} from './elevenlabsVoicePresets';
+import PerMascotVoiceRow from './PerMascotVoiceRow';
interface ColorOption {
id: MascotColor;
@@ -70,6 +73,7 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
const customPrimary = useAppSelector(selectCustomPrimaryColor);
const customSecondary = useAppSelector(selectCustomSecondaryColor);
const selectedMascotId = useAppSelector(selectSelectedMascotId);
+ const secondaryMascotId = useAppSelector(selectSecondaryMascotId);
const customMascotGifUrl = useAppSelector(selectCustomMascotGifUrl);
const storedVoiceId = useAppSelector(selectMascotVoiceId);
const voiceGender = useAppSelector(selectMascotVoiceGender);
@@ -126,6 +130,21 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
// null ("default") case has to clear it here so the stage falls back to
// the default manifest mascot rather than the GIF.
if (id == null) dispatch(setCustomMascotGifUrl(null));
+ // A newly-picked primary that collides with the current secondary would
+ // leave both slots pointing at the same mascot; clear the secondary so
+ // the duo never duplicates. The reducer's `selectDualMascotEnabled`
+ // guard already treats a collision as single-mascot, but clearing here
+ // keeps the picker's rendered state honest.
+ if (id != null && id === secondaryMascotId) dispatch(setSecondaryMascotId(null));
+ };
+
+ // ── Second-mascot picker (issue #4277) ───────────────────────────
+ // Enable / clear the meeting duo's second mascot. `null` (the "None"
+ // option) drops back to single-mascot. Picking the primary's id is
+ // disabled in the dropdown, so this only ever dispatches a distinct id
+ // or null.
+ const handleSelectSecondaryMascot = (id: string | null) => {
+ dispatch(setSecondaryMascotId(id));
};
const onSaveCustomGif = () => {
@@ -683,6 +702,82 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
{t('settings.mascot.characterDesc')}
+
+ {/* ── Meeting duo: second mascot + per-mascot voices (issue #4277) ─
+ Only meaningful for manifest mascots — a custom GIF avatar is a
+ single-figure path, so the whole block hides while one is set. */}
+ {manifest && manifest.mascots.length > 0 && !customMascotGifUrl && (
+
+
+ {t('settings.mascot.secondaryHeading')}
+
+
+ {/* Second-mascot picker — bespoke label + select combo mirroring
+ the voice preset dropdown. The primary's id is disabled so the
+ duo can never duplicate a single mascot. */}
+
+
+
+
+ {t('settings.mascot.secondaryDesc')}
+
+
+ {/* Per-mascot voices — a row per mascot whose voice is actually
+ addressable in `mascotVoices` (keyed by a concrete manifest
+ id). The join path (`selectMeetingMascotVoicePair`) resolves
+ the primary slot's voice from `mascotVoices[selectedMascotId]`,
+ so the primary row only appears once a specific primary mascot
+ is pinned; on the default mascot the effective single voice
+ (governed by the Voice section above) is what plays. Each row
+ writes its own `mascotVoices` entry and owns a guarded preview. */}
+ {secondaryMascotId != null && secondaryMascotId !== selectedMascotId && (
+
+
+ {t('settings.mascot.perMascotVoiceHeading')}
+
+ {selectedMascotId != null && (
+
+ )}
+
+
+ )}
+
+ )}
>
);
diff --git a/app/src/components/settings/panels/PerMascotVoiceRow.tsx b/app/src/components/settings/panels/PerMascotVoiceRow.tsx
new file mode 100644
index 000000000..c5ddca6c2
--- /dev/null
+++ b/app/src/components/settings/panels/PerMascotVoiceRow.tsx
@@ -0,0 +1,245 @@
+import { useEffect, useRef, useState } from 'react';
+
+import { synthesizeSpeech } from '../../../features/human/voice/ttsClient';
+import { useT } from '../../../lib/i18n/I18nContext';
+import { useAppDispatch, useAppSelector } from '../../../store/hooks';
+import {
+ selectEffectiveMascotVoiceId,
+ selectMascotVoiceFor,
+ selectMascotVoiceGender,
+ setMascotVoice,
+} from '../../../store/mascotSlice';
+import Button from '../../ui/Button';
+import { SettingsSelect, SettingsTextField } from '../controls';
+import { ELEVENLABS_VOICE_PRESETS, isCuratedVoicePreset } from './elevenlabsVoicePresets';
+
+interface PerMascotVoiceRowProps {
+ /** Manifest mascot id this row controls the voice for. Writes land in
+ * `mascotVoices[mascotId]` via `setMascotVoice`. */
+ mascotId: string;
+ /** Human-readable heading for the row (e.g. the mascot's name), already
+ * localized by the caller. */
+ label: string;
+ /** Stable test hook so both the primary and secondary rows expose
+ * distinct `data-testid`s (`mascot-voice-{primary,secondary}-*`). */
+ testIdPrefix: string;
+}
+
+/**
+ * Per-mascot reply-voice control (issue #4277). A trimmed sibling of the
+ * primary voice section in `MascotPanel`: it reuses the same preset
+ * dropdown + custom-paste + guarded `synthesizeSpeech` preview, but writes
+ * to `mascotVoices[mascotId]` via `setMascotVoice` instead of the single
+ * `voiceId`. The current value is the per-mascot override
+ * (`selectMascotVoiceFor`) falling back to the effective single voice
+ * (`selectEffectiveMascotVoiceId`), so a mascot with no override sounds
+ * exactly like the single-voice behaviour until the user picks one.
+ *
+ * Extracted from `MascotPanel` to keep that file within the ~500-line
+ * budget while the per-mascot map (primary + secondary) doubles the voice
+ * UI. Each instance owns its own preview-abort guard (`previewRequestIdRef`)
+ * so the two rows never share an in-flight preview.
+ */
+const PerMascotVoiceRow = ({ mascotId, label, testIdPrefix }: PerMascotVoiceRowProps) => {
+ const { t } = useT();
+ const dispatch = useAppDispatch();
+ // Per-mascot override, or the effective single voice when unset — the
+ // same resolution the meeting join path uses, so the picker shows what
+ // the mascot will actually speak with.
+ const overrideVoiceId = useAppSelector(selectMascotVoiceFor(mascotId));
+ const effectiveVoiceId = useAppSelector(selectEffectiveMascotVoiceId);
+ const currentVoiceId = overrideVoiceId ?? effectiveVoiceId;
+ // Reuse the global gender bucket to filter the preset dropdown — the
+ // per-mascot control only overrides the voice id, not the gender filter.
+ const voiceGender = useAppSelector(selectMascotVoiceGender);
+
+ // Paste-mode is sticky for the same reason as the primary control: a
+ // curated preset id and a mid-paste custom id both leave the stored
+ // value looking like a known id, so we can't derive the mode from it.
+ const [voiceDraft, setVoiceDraft] = useState(overrideVoiceId ?? '');
+ const [voicePasteMode, setVoicePasteMode] = useState(false);
+ const [isPreviewingVoice, setIsPreviewingVoice] = useState(false);
+ const [voicePreviewError, setVoicePreviewError] = useState(null);
+ const previewAudioRef = useRef(null);
+ // Monotonically-bumped preview-request id, mirroring the primary
+ // control: unmount + each new preview both increment it so an in-flight
+ // `synthesizeSpeech(...)` whose resolve loses the race bails before it
+ // touches refs / state.
+ const previewRequestIdRef = useRef(0);
+
+ // Stop any in-flight preview audio on unmount and invalidate a pending
+ // `synthesizeSpeech(...)` so a late resolve can't start audio for a row
+ // the user has already navigated away from.
+ useEffect(() => {
+ return () => {
+ previewRequestIdRef.current += 1;
+ if (previewAudioRef.current) {
+ previewAudioRef.current.pause();
+ previewAudioRef.current.src = '';
+ previewAudioRef.current = null;
+ }
+ };
+ }, []);
+
+ // Presets the dropdown should expose: always include the current voice
+ // (so the controlled select never points at an absent option) plus the
+ // active gender bucket and any '*' fallback voices.
+ const visiblePresets = ELEVENLABS_VOICE_PRESETS.filter(
+ p => p.id === currentVoiceId || p.gender === voiceGender || p.locales.includes('*')
+ );
+
+ // A custom (non-curated) override keeps the paste editor open so the
+ // stored id stays visible; the effective-voice fallback is never treated
+ // as "custom" because the mascot has no explicit override yet.
+ const isCustomVoice =
+ voicePasteMode || (overrideVoiceId != null && !isCuratedVoicePreset(overrideVoiceId));
+
+ const onPresetChange = (next: string) => {
+ if (next === '__custom__') {
+ setVoicePasteMode(true);
+ setVoiceDraft(overrideVoiceId ?? '');
+ return;
+ }
+ setVoicePasteMode(false);
+ setVoicePreviewError(null);
+ setVoiceDraft(next);
+ dispatch(setMascotVoice({ mascotId, voiceId: next }));
+ };
+
+ const onSavePaste = () => {
+ setVoicePreviewError(null);
+ const trimmed = voiceDraft.trim();
+ setVoiceDraft(trimmed);
+ dispatch(setMascotVoice({ mascotId, voiceId: trimmed.length > 0 ? trimmed : null }));
+ };
+
+ const onVoiceReset = () => {
+ setVoicePreviewError(null);
+ setVoicePasteMode(false);
+ setVoiceDraft('');
+ dispatch(setMascotVoice({ mascotId, voiceId: null }));
+ };
+
+ const onVoicePreview = async () => {
+ // Same abort guard as the primary control: reserve a fresh id, and let
+ // a stale resolve detect that a newer preview (or unmount) superseded
+ // it before it mutates state or plays audio.
+ const requestId = ++previewRequestIdRef.current;
+ setIsPreviewingVoice(true);
+ setVoicePreviewError(null);
+ if (previewAudioRef.current) {
+ previewAudioRef.current.pause();
+ previewAudioRef.current.src = '';
+ previewAudioRef.current = null;
+ }
+ try {
+ const tts = await synthesizeSpeech(t('settings.mascot.voice.previewText'), {
+ voiceId: currentVoiceId,
+ });
+ if (previewRequestIdRef.current !== requestId) return;
+ const src = `data:${tts.audio_mime || 'audio/mpeg'};base64,${tts.audio_base64}`;
+ const audio = new window.Audio(src);
+ previewAudioRef.current = audio;
+ await audio.play();
+ } catch (err) {
+ if (previewRequestIdRef.current !== requestId) return;
+ const message = err instanceof Error ? err.message : t('settings.mascot.voice.previewError');
+ setVoicePreviewError(message);
+ } finally {
+ if (previewRequestIdRef.current === requestId) setIsPreviewingVoice(false);
+ }
+ };
+
+ return (
+
+ );
+};
+
+export default PerMascotVoiceRow;
diff --git a/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx b/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx
index c73738659..30a8b3358 100644
--- a/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx
+++ b/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx
@@ -10,6 +10,7 @@ import mascotReducer, {
setCustomMascotGifUrl,
setMascotColor,
setMascotVoiceId,
+ setSecondaryMascotId,
setSelectedMascotId,
} from '../../../../store/mascotSlice';
import MascotPanel from '../MascotPanel';
@@ -362,3 +363,90 @@ describe('MascotPanel — voice picker custom voice input (line 525)', () => {
expect(store.getState().mascot.voiceId).toBe('new-voice-id');
});
});
+
+// ── Dual mascots + per-mascot voice (issue #4277) ────────────────────────────
+describe('MascotPanel — meeting duo (second mascot)', () => {
+ const yellow = manifestEntry('yellow', 'Yellow');
+ const toshi = manifestEntry('toshi', 'Toshi');
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ useMascotManifestMock.mockReturnValue(manifestResult([yellow, toshi]));
+ mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0));
+ });
+
+ it('dispatches setSecondaryMascotId when a second mascot is picked', () => {
+ const { store } = renderPanel();
+ fireEvent.change(screen.getByTestId('mascot-secondary-select'), { target: { value: 'toshi' } });
+ expect(store.getState().mascot.secondaryMascotId).toBe('toshi');
+ });
+
+ it('clears the second mascot when None is selected', () => {
+ const store = buildStore();
+ store.dispatch(setSecondaryMascotId('toshi'));
+ renderPanel(store);
+ fireEvent.change(screen.getByTestId('mascot-secondary-select'), {
+ target: { value: '__none__' },
+ });
+ expect(store.getState().mascot.secondaryMascotId).toBeNull();
+ });
+
+ it('disables the primary mascot as a second-mascot option', () => {
+ const store = buildStore();
+ store.dispatch(setSelectedMascotId('yellow'));
+ renderPanel(store);
+ const primaryOption = screen
+ .getByTestId('mascot-secondary-select')
+ .querySelector('option[value="yellow"]') as HTMLOptionElement | null;
+ expect(primaryOption).not.toBeNull();
+ expect(primaryOption).toBeDisabled();
+ });
+
+ it('hides the duo picker when a custom GIF avatar is set', () => {
+ const store = buildStore();
+ store.dispatch(setCustomMascotGifUrl('https://example.com/avatar.gif'));
+ renderPanel(store);
+ expect(screen.queryByTestId('mascot-secondary-select')).not.toBeInTheDocument();
+ });
+
+ it('dispatches setMascotVoice for the second mascot when its voice changes', () => {
+ const store = buildStore();
+ store.dispatch(setSelectedMascotId('yellow'));
+ store.dispatch(setSecondaryMascotId('toshi'));
+ renderPanel(store);
+
+ // The secondary voice row renders once a distinct duo mascot is set.
+ const select = screen.getByTestId('mascot-voice-secondary-select');
+ fireEvent.change(select, { target: { value: 'pNInz6obpgDQGcFmaJgB' } });
+
+ expect(store.getState().mascot.mascotVoices.toshi).toBe('pNInz6obpgDQGcFmaJgB');
+ });
+
+ it('dispatches setMascotVoice for the primary mascot when its voice changes', () => {
+ const store = buildStore();
+ store.dispatch(setSelectedMascotId('yellow'));
+ store.dispatch(setSecondaryMascotId('toshi'));
+ renderPanel(store);
+
+ const select = screen.getByTestId('mascot-voice-primary-select');
+ fireEvent.change(select, { target: { value: 'pNInz6obpgDQGcFmaJgB' } });
+
+ expect(store.getState().mascot.mascotVoices.yellow).toBe('pNInz6obpgDQGcFmaJgB');
+ });
+
+ it('clears a per-mascot voice via the row reset button', () => {
+ const store = buildStore();
+ store.dispatch(setSelectedMascotId('yellow'));
+ store.dispatch(setSecondaryMascotId('toshi'));
+ renderPanel(store);
+
+ // Seed an override, then reset it.
+ fireEvent.change(screen.getByTestId('mascot-voice-secondary-select'), {
+ target: { value: 'pNInz6obpgDQGcFmaJgB' },
+ });
+ expect(store.getState().mascot.mascotVoices.toshi).toBe('pNInz6obpgDQGcFmaJgB');
+
+ fireEvent.click(screen.getByTestId('mascot-voice-secondary-reset'));
+ expect(store.getState().mascot.mascotVoices.toshi).toBeUndefined();
+ });
+});
diff --git a/app/src/components/settings/panels/__tests__/PerMascotVoiceRow.test.tsx b/app/src/components/settings/panels/__tests__/PerMascotVoiceRow.test.tsx
new file mode 100644
index 000000000..ba28c3559
--- /dev/null
+++ b/app/src/components/settings/panels/__tests__/PerMascotVoiceRow.test.tsx
@@ -0,0 +1,203 @@
+import { configureStore } from '@reduxjs/toolkit';
+import { fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { Provider } from 'react-redux';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import mascotReducer, { setMascotVoice } from '../../../../store/mascotSlice';
+import PerMascotVoiceRow from '../PerMascotVoiceRow';
+
+const { mockSynthesizeSpeech } = vi.hoisted(() => ({ mockSynthesizeSpeech: vi.fn() }));
+
+vi.mock('../../../../features/human/voice/ttsClient', () => ({
+ synthesizeSpeech: (...args: unknown[]) => mockSynthesizeSpeech(...args),
+}));
+
+const TEST_ID = 'mascot-voice-primary';
+const MASCOT_ID = 'yellow';
+
+function buildStore() {
+ return configureStore({ reducer: { mascot: mascotReducer } });
+}
+
+function renderRow(store = buildStore()) {
+ return {
+ store,
+ ...render(
+
+
+
+ ),
+ };
+}
+
+/** Resolve helper to control the timing of the mocked synthesizeSpeech. */
+function deferredTts() {
+ let resolve!: (v: { audio_mime: string; audio_base64: string }) => void;
+ let reject!: (err: unknown) => void;
+ const promise = new Promise<{ audio_mime: string; audio_base64: string }>((res, rej) => {
+ resolve = res;
+ reject = rej;
+ });
+ return { promise, resolve, reject };
+}
+
+describe('PerMascotVoiceRow', () => {
+ let playSpy: ReturnType;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ playSpy = vi.fn().mockResolvedValue(undefined);
+ // Stub window.Audio so `new window.Audio(src)` records the src and exposes
+ // a spy-able play()/pause() without touching the real audio pipeline.
+ vi.stubGlobal(
+ 'Audio',
+ class {
+ src: string;
+ constructor(src?: string) {
+ this.src = src ?? '';
+ }
+ play = playSpy;
+ pause = vi.fn();
+ }
+ );
+ });
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it('reveals the paste input + Save when the custom option is selected and saves a new id', () => {
+ const { store } = renderRow();
+
+ // No custom input until the __custom__ option is chosen.
+ expect(screen.queryByTestId(`${TEST_ID}-input`)).not.toBeInTheDocument();
+
+ fireEvent.change(screen.getByTestId(`${TEST_ID}-select`), { target: { value: '__custom__' } });
+
+ const input = screen.getByTestId(`${TEST_ID}-input`);
+ expect(input).toBeInTheDocument();
+
+ const saveBtn = screen.getByTestId(`${TEST_ID}-save-paste`);
+ // Empty draft still equals the (empty) stored override → disabled.
+ expect(saveBtn).toBeDisabled();
+
+ // A whitespace-padded id enables Save and is trimmed on dispatch.
+ fireEvent.change(input, { target: { value: ' my-custom-voice ' } });
+ expect(saveBtn).not.toBeDisabled();
+ fireEvent.click(saveBtn);
+
+ expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBe('my-custom-voice');
+ });
+
+ it('plays a data:audio preview and toggles the previewing label when synthesizeSpeech resolves', async () => {
+ const d = deferredTts();
+ mockSynthesizeSpeech.mockReturnValue(d.promise);
+ renderRow();
+
+ const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
+ fireEvent.click(previewBtn);
+
+ // Previewing state toggles on immediately (button disabled + label swap).
+ await waitFor(() => expect(previewBtn).toBeDisabled());
+ expect(previewBtn).toHaveTextContent('Previewing…');
+
+ d.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
+
+ await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(1));
+ // A data:audio URI was constructed from the mime + base64 payload.
+ expect(playSpy.mock.instances[0].src).toBe('data:audio/mpeg;base64,QUJD');
+
+ // Previewing resets once the preview finishes.
+ await waitFor(() => expect(previewBtn).not.toBeDisabled());
+ expect(previewBtn).toHaveTextContent('Preview voice');
+ });
+
+ it('renders the preview-error text and resets previewing when synthesizeSpeech rejects', async () => {
+ const d = deferredTts();
+ mockSynthesizeSpeech.mockReturnValue(d.promise);
+ renderRow();
+
+ const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
+ fireEvent.click(previewBtn);
+ await waitFor(() => expect(previewBtn).toBeDisabled());
+
+ d.reject(new Error('tts exploded'));
+
+ const errorBox = await screen.findByTestId(`${TEST_ID}-preview-error`);
+ expect(errorBox).toHaveTextContent('tts exploded');
+ expect(playSpy).not.toHaveBeenCalled();
+
+ // finally branch clears the previewing state.
+ await waitFor(() => expect(previewBtn).not.toBeDisabled());
+ expect(previewBtn).toHaveTextContent('Preview voice');
+ });
+
+ it('is a no-op when a preview resolves after the row has unmounted (previewRequestIdRef guard)', async () => {
+ const d = deferredTts();
+ mockSynthesizeSpeech.mockReturnValue(d.promise);
+ const { unmount } = renderRow();
+
+ fireEvent.click(screen.getByTestId(`${TEST_ID}-preview`));
+ await waitFor(() => expect(mockSynthesizeSpeech).toHaveBeenCalledTimes(1));
+
+ // Unmount bumps previewRequestIdRef, so the late resolve loses the race.
+ unmount();
+ d.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
+
+ // Give the resolved promise a microtask/macrotask to flush.
+ await new Promise(r => setTimeout(r, 0));
+ expect(playSpy).not.toHaveBeenCalled();
+ });
+
+ it('exposes the current voice id set via setMascotVoice', () => {
+ const store = buildStore();
+ store.dispatch(setMascotVoice({ mascotId: MASCOT_ID, voiceId: 'pNInz6obpgDQGcFmaJgB' }));
+ renderRow(store);
+ expect(screen.getByTestId(`${TEST_ID}-current`)).toHaveTextContent('pNInz6obpgDQGcFmaJgB');
+ });
+
+ it('dispatches setMascotVoice when a curated preset is picked from the dropdown', () => {
+ const { store } = renderRow();
+ fireEvent.change(screen.getByTestId(`${TEST_ID}-select`), {
+ target: { value: 'pNInz6obpgDQGcFmaJgB' },
+ });
+ expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBe('pNInz6obpgDQGcFmaJgB');
+ });
+
+ it('clears the per-mascot override via the reset button', () => {
+ const store = buildStore();
+ store.dispatch(setMascotVoice({ mascotId: MASCOT_ID, voiceId: 'pNInz6obpgDQGcFmaJgB' }));
+ renderRow(store);
+
+ const resetBtn = screen.getByTestId(`${TEST_ID}-reset`);
+ expect(resetBtn).not.toBeDisabled();
+ fireEvent.click(resetBtn);
+
+ expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBeUndefined();
+ });
+
+ it('stops the prior preview audio when a second preview starts', async () => {
+ const first = deferredTts();
+ const second = deferredTts();
+ mockSynthesizeSpeech.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
+ renderRow();
+
+ const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
+
+ // First preview resolves and plays, seeding previewAudioRef.
+ fireEvent.click(previewBtn);
+ first.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
+ await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(1));
+ const firstAudio = playSpy.mock.instances[0];
+ await waitFor(() => expect(previewBtn).not.toBeDisabled());
+
+ // Second preview should pause + clear the prior audio before synthesizing.
+ fireEvent.click(previewBtn);
+ await waitFor(() => expect(mockSynthesizeSpeech).toHaveBeenCalledTimes(2));
+ expect(firstAudio.pause).toHaveBeenCalled();
+ expect(firstAudio.src).toBe('');
+
+ second.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'WFla' });
+ await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(2));
+ });
+});
diff --git a/app/src/features/meet/MascotFrameProducer.tsx b/app/src/features/meet/MascotFrameProducer.tsx
index 6acbdc9bb..332f4bc96 100644
--- a/app/src/features/meet/MascotFrameProducer.tsx
+++ b/app/src/features/meet/MascotFrameProducer.tsx
@@ -1,62 +1,69 @@
import { listen, type UnlistenFn } from '@tauri-apps/api/event';
-import { type FC, useCallback, useEffect, useRef, useState } from 'react';
+import { type FC, useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { useSelector } from 'react-redux';
-import { RiveMascot } from '../human/Mascot';
+import {
+ selectCustomPrimaryColor,
+ selectCustomSecondaryColor,
+ selectMascotColor,
+ selectSecondaryMascotId,
+} from '../../store/mascotSlice';
+import {
+ getMascotPalette,
+ hexToArgbInt,
+ ManifestRiveMascot,
+ type MascotFace,
+ type MascotManifestEntry,
+ RiveMascot,
+} from '../human/Mascot';
+import { findMascot } from '../human/Mascot/manifest/manifestService';
+import { useMascotManifest } from '../human/Mascot/manifest/useMascotManifest';
+import {
+ drawMascotInCell,
+ FRAME_H,
+ FRAME_H_DUAL,
+ FRAME_W,
+ FRAME_W_DUAL,
+ MASCOT_INSET,
+ sampleCanvasPixels,
+} from './mascotFrameCompositor';
+import { type ActiveMascotSlot, type MeetingPhase, useMeetingMascots } from './useMeetingMascots';
const PRODUCER_FPS = 24;
-const FRAME_W = 320;
-const FRAME_H = 240;
const JPEG_QUALITY = 0.7;
+/**
+ * How long the mascot(s) wave hello on join before settling into the live
+ * `active` face state (ms). Matches the greeting the participants see so the
+ * first frames read as "the bot is saying hi" rather than a cold stare.
+ */
+const GREETING_MS = 2500;
+/**
+ * Teardown grace after `bus-stopped` (ms): keep the session mounted — WS +
+ * worker alive — long enough for the goodbye wave to stream before the frame
+ * pipeline is torn down. Without this the last thing the call sees is a hard
+ * cut mid-pose instead of a wave.
+ */
+const SIGNOFF_MS = 1500;
+
interface BusSession {
requestId: string;
port: number;
}
-export function sampleCanvasPixels(
- ctx: OffscreenCanvasRenderingContext2D,
- width: number,
- height: number
-) {
- const cols = 7;
- const rows = 5;
- let sum = 0;
- let min = 255;
- let max = 0;
- let count = 0;
- let dark = 0;
- let bright = 0;
-
- try {
- for (let y = 0; y < rows; y++) {
- for (let x = 0; x < cols; x++) {
- const px = Math.max(0, Math.min(width - 1, Math.floor(((x + 0.5) * width) / cols)));
- const py = Math.max(0, Math.min(height - 1, Math.floor(((y + 0.5) * height) / rows)));
- const [r, g, b] = ctx.getImageData(px, py, 1, 1).data;
- const luma = Math.round(r * 0.299 + g * 0.587 + b * 0.114);
- sum += luma;
- min = Math.min(min, luma);
- max = Math.max(max, luma);
- if (luma < 8) dark++;
- if (luma > 32) bright++;
- count++;
- }
- }
- return {
- avgLuma: Math.round(sum / Math.max(1, count)),
- minLuma: min,
- maxLuma: max,
- darkSamples: dark,
- brightSamples: bright,
- sampleCount: count,
- };
- } catch (err) {
- return { error: String(err instanceof Error ? err.message : err) };
- }
-}
+// Re-export from the compositor for back-compat with existing importers
+// (the producer test imports `sampleCanvasPixels` from here). The
+// implementation moved to mascotFrameCompositor.ts (issue #4277).
+export { sampleCanvasPixels };
export const MascotFrameProducer: FC = () => {
const [session, setSession] = useState(null);
+ // Meeting lifecycle phase, owned here so it survives across the brief
+ // sign-off grace where `session` is still set but the bus has stopped.
+ const [phase, setPhase] = useState('greeting');
+ // Set when `bus-stopped` fires; the session is cleared SIGNOFF_MS later so
+ // the goodbye wave can stream. A ref so the timers don't re-arm on render.
+ const signoffTimerRef = useRef | null>(null);
useEffect(() => {
let unlistenStarted: UnlistenFn | undefined;
@@ -67,6 +74,11 @@ export const MascotFrameProducer: FC = () => {
const payload = event.payload;
if (!payload || !payload.port) return;
console.log('[meet-video-producer] bus-started', payload);
+ if (signoffTimerRef.current) {
+ clearTimeout(signoffTimerRef.current);
+ signoffTimerRef.current = null;
+ }
+ setPhase('greeting');
setSession(payload);
})
.then(stop => {
@@ -77,7 +89,15 @@ export const MascotFrameProducer: FC = () => {
listen<{ requestId?: string; request_id?: string }>('meet-video:bus-stopped', event => {
console.log('[meet-video-producer] bus-stopped', event.payload);
- setSession(null);
+ // Enter the goodbye wave and keep the pipeline alive for SIGNOFF_MS so
+ // the wave frames actually reach the call before we clear the session.
+ setPhase('signoff');
+ if (signoffTimerRef.current) clearTimeout(signoffTimerRef.current);
+ signoffTimerRef.current = setTimeout(() => {
+ console.log('[meet-video-producer] sign-off grace elapsed, clearing session');
+ signoffTimerRef.current = null;
+ setSession(null);
+ }, SIGNOFF_MS);
})
.then(stop => {
if (cancelled) stop();
@@ -89,69 +109,178 @@ export const MascotFrameProducer: FC = () => {
cancelled = true;
if (unlistenStarted) unlistenStarted();
if (unlistenStopped) unlistenStopped();
+ if (signoffTimerRef.current) {
+ clearTimeout(signoffTimerRef.current);
+ signoffTimerRef.current = null;
+ }
};
}, []);
+ // Advance greeting → active after GREETING_MS, per active session. Bound to
+ // requestId so a fresh session restarts the greeting.
+ useEffect(() => {
+ if (!session || phase !== 'greeting') return;
+ const id = setTimeout(() => {
+ console.log('[meet-video-producer] greeting elapsed → active', session.requestId);
+ setPhase('active');
+ }, GREETING_MS);
+ return () => clearTimeout(id);
+ }, [session, phase]);
+
if (!session) return null;
- return ;
+ return ;
};
-const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
+const ProducerSession: FC<{ session: BusSession; phase: MeetingPhase }> = ({ session, phase }) => {
const hostRef = useRef(null);
const wsRef = useRef(null);
const wsReadyRef = useRef(false);
const stoppedRef = useRef(false);
const inflightRef = useRef(false);
const lastDiagAtRef = useRef(0);
- const isSpeakingRef = useRef(false);
// True while the bot is actively producing PCM into the Meet call.
// Drives the mascot face so the mouth animates in time with the audio
// participants hear. Source of truth is the Rust speak_pump (edge-detected
// from the RPC poll loop). Same requestId guards against stale events from
// a previous session bleeding into this session's mascot state.
const [isSpeaking, setIsSpeaking] = useState(false);
+ // Which mascot slot is speaking the current audio (issue #4277). Single-
+ // mascot calls always report slot 0. Read off the speaking-state event.
+ const [activeMascotSlot, setActiveMascotSlot] = useState(0);
+ // Per-slot render state (which mascot + which face) for this tick.
+ const render = useMeetingMascots({ speaking: isSpeaking, activeMascotSlot, phase });
+ const { dualEnabled } = render;
+
+ // Resolve the manifest entries for both slots. The primary follows the
+ // user's selection (same resolution the Human page uses); the secondary is
+ // looked up by its explicit id so the frame honors both selections and no
+ // longer always shows the bundled default (fixes the pre-existing single-
+ // mascot bug where the meeting camera ignored `selectedMascotId`).
+ const { manifest, entry: primaryEntry } = useMascotManifest();
+ const secondaryMascotId = useSelector(selectSecondaryMascotId);
+ const secondaryEntry: MascotManifestEntry | null =
+ dualEnabled && manifest ? (findMascot(manifest, secondaryMascotId) ?? null) : null;
+
+ // Mascot body colors, mirroring HumanPage so the meeting mascot matches the
+ // one on the Human stage. Per-mascot colors are out of scope (#4277) — both
+ // slots share the single selected color.
+ const mascotColor = useSelector(selectMascotColor);
+ const customPrimary = useSelector(selectCustomPrimaryColor);
+ const customSecondary = useSelector(selectCustomSecondaryColor);
+ const palette = getMascotPalette(mascotColor);
+ const primaryColor = useMemo(
+ () => hexToArgbInt(mascotColor === 'custom' ? customPrimary : palette.bodyFill),
+ [mascotColor, customPrimary, palette]
+ );
+ const secondaryColor = useMemo(
+ () => hexToArgbInt(mascotColor === 'custom' ? customSecondary : palette.neckShadowColor),
+ [mascotColor, customSecondary, palette]
+ );
+
+ const isSpeakingRef = useRef(isSpeaking);
useEffect(() => {
isSpeakingRef.current = isSpeaking;
}, [isSpeaking]);
+ // `dualEnabled` and `activeMascotSlot` are read inside captureFrame via refs
+ // so a speaker switch (activeMascotSlot 0↔1) or a dual-mode toggle does NOT
+ // change captureFrame's identity — otherwise the WS/worker effect below
+ // (which depends on captureFrame) would tear down and rebuild the socket +
+ // frame worker on every alternation. captureFrame stays keyed to the session
+ // only, matching the original single-mascot behavior.
+ const dualEnabledRef = useRef(dualEnabled);
+ useEffect(() => {
+ dualEnabledRef.current = dualEnabled;
+ }, [dualEnabled]);
+ const activeMascotSlotRef = useRef(activeMascotSlot);
+ useEffect(() => {
+ activeMascotSlotRef.current = activeMascotSlot;
+ }, [activeMascotSlot]);
+
const captureFrame = useCallback(async () => {
if (stoppedRef.current || !wsReadyRef.current || inflightRef.current) return;
const host = hostRef.current;
if (!host) return;
- const canvas = host.querySelector('canvas');
- if (!canvas) return;
+ const dualEnabledNow = dualEnabledRef.current;
+ // Look the source canvases up by slot so a face change (which re-renders
+ // the mascot) never changes which canvas we sample.
+ const primaryCanvas = host.querySelector(
+ '[data-mascot-slot="primary"] canvas'
+ );
+ if (!primaryCanvas) return;
+ const secondaryCanvas = dualEnabledNow
+ ? host.querySelector('[data-mascot-slot="secondary"] canvas')
+ : null;
+
+ const frameW = dualEnabledNow ? FRAME_W_DUAL : FRAME_W;
+ // The dual frame is taller (16:9) than the single frame so the fake-camera
+ // bridge's cover-scale fills the 1280×720 canvas without cropping — see
+ // FRAME_H_DUAL in mascotFrameCompositor.ts.
+ const frameH = dualEnabledNow ? FRAME_H_DUAL : FRAME_H;
inflightRef.current = true;
try {
- const offscreen = new OffscreenCanvas(FRAME_W, FRAME_H);
+ const offscreen = new OffscreenCanvas(frameW, frameH);
const ctx = offscreen.getContext('2d');
if (!ctx) return;
const grad = ctx.createRadialGradient(
- FRAME_W / 2,
- FRAME_H / 2,
+ frameW / 2,
+ frameH / 2,
0,
- FRAME_W / 2,
- FRAME_H / 2,
- Math.max(FRAME_W, FRAME_H) * 0.7
+ frameW / 2,
+ frameH / 2,
+ Math.max(frameW, frameH) * 0.7
);
grad.addColorStop(0, '#FBF3D9');
grad.addColorStop(1, '#EFE3B8');
ctx.fillStyle = grad;
- ctx.fillRect(0, 0, FRAME_W, FRAME_H);
+ ctx.fillRect(0, 0, frameW, frameH);
- const inset = 0.06;
- const fitW = FRAME_W * (1 - 2 * inset);
- const fitH = FRAME_H * (1 - 2 * inset);
- const scale = Math.min(fitW / canvas.width, fitH / canvas.height);
- const dw = canvas.width * scale;
- const dh = canvas.height * scale;
- const dx = (FRAME_W - dw) / 2;
- const dy = (FRAME_H - dh) / 2;
- ctx.drawImage(canvas, dx, dy, dw, dh);
+ if (dualEnabledNow && secondaryCanvas) {
+ // Two half-cells: [0..half] primary, [half..frameW] secondary.
+ const half = frameW / 2;
+ drawMascotInCell(
+ ctx,
+ primaryCanvas,
+ 0,
+ 0,
+ half,
+ frameH,
+ MASCOT_INSET,
+ primaryCanvas.width,
+ primaryCanvas.height
+ );
+ drawMascotInCell(
+ ctx,
+ secondaryCanvas,
+ half,
+ 0,
+ half,
+ frameH,
+ MASCOT_INSET,
+ secondaryCanvas.width,
+ secondaryCanvas.height
+ );
+ } else {
+ // Single-cell draw. This also covers the dual-but-secondary-not-yet-
+ // mounted tick (a 2.2MB mascot still decoding): rather than emit a
+ // black half we draw the primary across the whole frame for this tick.
+ drawMascotInCell(
+ ctx,
+ primaryCanvas,
+ 0,
+ 0,
+ frameW,
+ frameH,
+ MASCOT_INSET,
+ primaryCanvas.width,
+ primaryCanvas.height
+ );
+ }
- const probe = sampleCanvasPixels(ctx, FRAME_W, FRAME_H);
+ const probe = sampleCanvasPixels(ctx, frameW, frameH);
const blob = await offscreen.convertToBlob({ type: 'image/jpeg', quality: JPEG_QUALITY });
const buffer = await blob.arrayBuffer();
const ws = wsRef.current;
@@ -163,12 +292,17 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
JSON.stringify({
kind: 'producer-pixel-probe',
requestId: session.requestId,
- canvasWidth: canvas.width,
- canvasHeight: canvas.height,
- frameWidth: FRAME_W,
- frameHeight: FRAME_H,
+ canvasWidth: primaryCanvas.width,
+ canvasHeight: primaryCanvas.height,
+ frameWidth: frameW,
+ frameHeight: frameH,
jpegBytes: blob.size,
isSpeaking: isSpeakingRef.current,
+ // Dual-mascot diagnostics (issue #4277): whether we drew two
+ // cells this tick, and which slot the audio is on.
+ dualEnabled: dualEnabledNow,
+ secondaryMounted: dualEnabledNow ? Boolean(secondaryCanvas) : undefined,
+ activeMascotSlot: activeMascotSlotRef.current,
probe,
})
);
@@ -241,13 +375,19 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
// session — a remount tears it down with the rest of the pipeline.
let unlistenSpeaking: UnlistenFn | undefined;
let speakingListenerCancelled = false;
- listen<{ requestId?: string; speaking?: boolean }>('meet-video:speaking-state', event => {
- const payload = event.payload;
- if (!payload) return;
- // Ignore events from a different session during teardown / restart.
- if (payload.requestId && payload.requestId !== session.requestId) return;
- setIsSpeaking(!!payload.speaking);
- })
+ listen<{ requestId?: string; speaking?: boolean; activeMascotSlot?: number }>(
+ 'meet-video:speaking-state',
+ event => {
+ const payload = event.payload;
+ if (!payload) return;
+ // Ignore events from a different session during teardown / restart.
+ if (payload.requestId && payload.requestId !== session.requestId) return;
+ setIsSpeaking(!!payload.speaking);
+ // `activeMascotSlot` names which mascot is speaking this audio (0|1);
+ // default to slot 0 for single-mascot / older core builds that omit it.
+ setActiveMascotSlot(payload.activeMascotSlot === 1 ? 1 : 0);
+ }
+ )
.then(stop => {
if (speakingListenerCancelled) stop();
else unlistenSpeaking = stop;
@@ -289,14 +429,70 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
position: 'fixed',
left: '-99999px',
top: 0,
- width: FRAME_H,
+ width: dualEnabled ? FRAME_H * 2 : FRAME_H,
height: FRAME_H,
pointerEvents: 'none',
opacity: 0,
}}>
-
+ {/* Slot 0 (primary). Stable key per mascot id so a face change updates in
+ place instead of remounting the Rive/WebGL context. */}
+
+
+
+ {/* Slot 1 (secondary) — only mounted when a distinct second mascot is
+ enabled. Its 2.2MB asset may still be decoding for the first frames;
+ captureFrame falls back to a single-cell draw until its canvas exists. */}
+ {dualEnabled && render.secondary && (
+
+
+
+ )}
);
};
+/**
+ * Render one mascot slot. Prefers the manifest mascot (honoring the user's
+ * selection) and falls back to the bundled `RiveMascot` while the manifest is
+ * still resolving so the frame never blanks. Keyed by mascot id upstream so a
+ * *selection* change remounts, while a *face* change updates in place.
+ */
+const MascotStage: FC<{
+ entry: MascotManifestEntry | null;
+ face: MascotFace;
+ primaryColor: number;
+ secondaryColor: number;
+}> = ({ entry, face, primaryColor, secondaryColor }) => {
+ if (entry) {
+ return (
+
+ );
+ }
+ return (
+
+ );
+};
+
export default MascotFrameProducer;
diff --git a/app/src/features/meet/__tests__/MascotFrameProducer.test.tsx b/app/src/features/meet/__tests__/MascotFrameProducer.test.tsx
index 05127bf1c..1cc4fd6f5 100644
--- a/app/src/features/meet/__tests__/MascotFrameProducer.test.tsx
+++ b/app/src/features/meet/__tests__/MascotFrameProducer.test.tsx
@@ -1,17 +1,204 @@
-import { cleanup } from '@testing-library/react';
-import { afterEach, describe, expect, it, vi } from 'vitest';
+import { listen } from '@tauri-apps/api/event';
+import { act, cleanup, waitFor } from '@testing-library/react';
+import { createElement } from 'react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { renderWithProviders } from '../../../test/test-utils';
import { MascotFrameProducer, sampleCanvasPixels } from '../MascotFrameProducer';
-// @tauri-apps/api/event is already mocked in setup.ts (listen → vi.fn())
+// @tauri-apps/api/event is mocked in setup.ts (listen → vi.fn()). Here we
+// override it per-test to capture the event handlers so we can drive a fake
+// `meet-video:bus-started` and assert how many mascot hosts mount.
+type Listener = (event: { payload: unknown }) => void;
+
+// The producer renders ManifestRiveMascot / RiveMascot (WebGL). Mock both leaf
+// renderers to a plain