feat: dual mascots in meetings — per-mascot voice + speaker alternation (#4277) (#4585)

This commit is contained in:
YellowSnnowmann
2026-07-07 13:56:57 -07:00
committed by GitHub
parent c0e7b5158a
commit 796468c034
49 changed files with 3581 additions and 186 deletions
+4
View File
@@ -3678,6 +3678,10 @@ pub fn run() {
// wake gate will fail-closed (no wakes fire) which
// is the safe posture for an automated harness.
owner_display_name: String::new(),
// No mascot config on the dev-auto path → single
// default voice (issue #4277).
primary_voice_id: None,
secondary_voice_id: None,
};
match meet_call::meet_call_open_window(app_handle.clone(), state, args)
.await
+13 -2
View File
@@ -89,13 +89,19 @@ pub async fn start<R: Runtime>(
meet_url: String,
owner_display_name: String,
bot_display_name: String,
primary_voice_id: Option<String>,
secondary_voice_id: Option<String>,
) -> Result<(), String> {
log::info!(
"[meet-audio] start request_id={request_id} url_prefix={} \
owner_chars={} bot_chars={}",
owner_chars={} bot_chars={} voices={}",
truncate_for_log(&meet_url, 64),
owner_display_name.chars().count(),
bot_display_name.chars().count()
bot_display_name.chars().count(),
[&primary_voice_id, &secondary_voice_id]
.into_iter()
.filter(|v| v.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false))
.count()
);
if let Some(state) = app.try_state::<MeetAudioState>() {
@@ -125,6 +131,11 @@ pub async fn start<R: Runtime>(
// <code>" in the history list. The URL the shell built
// the CEF window with is the canonical value.
"meet_url": meet_url,
// Per-mascot TTS voices for speaker alternation (issue #4277).
// Null for single-mascot calls → core keeps the backend
// default voice (unchanged behavior).
"primary_voice_id": primary_voice_id,
"secondary_voice_id": secondary_voice_id,
}),
)
.await?;
+119 -18
View File
@@ -98,10 +98,10 @@ pub fn start<R: Runtime>(
break;
}
_ = tick.tick() => {
let had_pcm = match poll_and_feed(&request_id_for_task, &mut cdp, &session_id).await {
Ok(had) => {
let (had_pcm, active_slot) = match poll_and_feed(&request_id_for_task, &mut cdp, &session_id).await {
Ok((had, slot)) => {
feed_errors = 0;
had
(had, Some(slot))
}
Err(err) => {
feed_errors += 1;
@@ -118,11 +118,12 @@ pub fn start<R: Runtime>(
// A failed tick is *not* evidence the bot
// stopped speaking — leave the hangover to
// expire naturally so transient CDP errors
// don't flicker the mascot's mouth shut.
false
// don't flicker the mascot's mouth shut. No
// fresh slot data → keep the last-known slot.
(false, None)
}
};
speaking_state.tick(had_pcm, &app, &request_id_for_task);
speaking_state.tick(had_pcm, active_slot, &app, &request_id_for_task);
}
}
}
@@ -148,6 +149,14 @@ struct SpeakingTracker {
/// every tick that carries PCM; the state flips back to `false`
/// only once `now > hangover_until` AND a tick with no PCM lands.
hangover_until: Option<Instant>,
/// Which mascot (0 = primary, 1 = secondary) is speaking the current
/// outbound audio, as reported by `meet_agent_poll_speech`. For
/// two-mascot calls the brain alternates this per reply; the frontend
/// lip-syncs this slot and reacts the other. Emitted on the
/// speaking-state edge AND whenever the slot changes mid-speech (so a
/// back-to-back reply from the other mascot still switches lip-sync
/// even if the hangover bridged the gap). 0 for single-mascot calls.
active_slot: u8,
}
impl SpeakingTracker {
@@ -155,6 +164,7 @@ impl SpeakingTracker {
Self {
reported: false,
hangover_until: None,
active_slot: 0,
}
}
@@ -162,12 +172,21 @@ impl SpeakingTracker {
/// is whether `poll_and_feed` saw a non-empty `pcm_base64` for
/// this tick. Emits the Tauri event only when the reported
/// state actually flips.
fn tick<R: Runtime>(&mut self, had_pcm: bool, app: &AppHandle<R>, request_id: &str) {
/// `active_slot` is the speaking mascot slot for this tick, or `None`
/// when the tick had no fresh poll data (e.g. a CDP error) — in that
/// case the last-known slot is retained.
fn tick<R: Runtime>(
&mut self,
had_pcm: bool,
active_slot: Option<u8>,
app: &AppHandle<R>,
request_id: &str,
) {
if had_pcm {
// Extend the hangover. If we were idle, flip up to
// speaking — the user hears audio starting now.
self.hangover_until = Some(Instant::now() + SPEAKING_HANGOVER);
self.set_reported(true, app, request_id);
self.update(true, active_slot, app, request_id);
return;
}
// No PCM this tick. If the hangover hasn't expired, stay in
@@ -182,7 +201,7 @@ impl SpeakingTracker {
self.hangover_until = None;
}
// Hangover expired or never armed → bot is genuinely idle.
self.set_reported(false, app, request_id);
self.update(false, active_slot, app, request_id);
}
/// Force the reported state to `false` and emit an event if that's
@@ -190,30 +209,71 @@ impl SpeakingTracker {
/// can't get stuck mid-talk.
fn force_off<R: Runtime>(&mut self, app: &AppHandle<R>, request_id: &str) {
self.hangover_until = None;
self.set_reported(false, app, request_id);
self.update(false, None, app, request_id);
}
fn set_reported<R: Runtime>(&mut self, next: bool, app: &AppHandle<R>, request_id: &str) {
if self.reported == next {
/// Update reported speaking state + active slot, emitting the Tauri
/// event when either the speaking flag flips OR (while speaking) the
/// active mascot slot changes. `slot = None` keeps the last-known
/// slot (used when a tick carried no fresh poll data).
fn update<R: Runtime>(
&mut self,
next: bool,
slot: Option<u8>,
app: &AppHandle<R>,
request_id: &str,
) {
let (should_emit, resolved_slot) =
next_speaking_state(self.reported, self.active_slot, next, slot);
self.reported = next;
self.active_slot = resolved_slot;
if !should_emit {
return;
}
self.reported = next;
let payload = serde_json::json!({
"requestId": request_id,
"speaking": next,
// Which mascot is speaking this audio (0 = primary, 1 =
// secondary). Frontend lip-syncs this slot, reacts the other.
"activeMascotSlot": self.active_slot,
});
if let Err(err) = app.emit(SPEAKING_STATE_EVENT, payload) {
// Best-effort: a missing renderer (closed window mid-tick)
// is the common case and not worth raising the log level.
log::debug!(
"[meet-audio] speaking-state emit failed request_id={request_id} speaking={next} err={err}"
"[meet-audio] speaking-state emit failed request_id={request_id} speaking={next} slot={} err={err}",
self.active_slot
);
} else {
log::debug!("[meet-audio] speaking-state -> {next} request_id={request_id}");
log::debug!(
"[meet-audio] speaking-state -> {next} slot={} request_id={request_id}",
self.active_slot
);
}
}
}
/// Pure decision for [`SpeakingTracker::update`]: given the previously
/// reported speaking flag + slot and the incoming `next`/`slot`, compute
/// whether the Tauri event should be emitted and which slot to resolve to.
///
/// Extracted so the emit logic can be unit-tested without a live
/// `AppHandle`. `slot = None` retains the previous slot (a tick with no
/// fresh poll data). We emit when the speaking flag flips OR (while
/// speaking) the active slot changes — the latter switches lip-sync on a
/// back-to-back reply from the other mascot even if the hangover bridged
/// the gap.
fn next_speaking_state(
prev_reported: bool,
prev_slot: u8,
next: bool,
slot: Option<u8>,
) -> (bool, u8) {
let resolved = slot.unwrap_or(prev_slot);
let should_emit = prev_reported != next || (next && prev_slot != resolved);
(should_emit, resolved)
}
/// No-op pump used when bridge install failed at session start. Keeps
/// the rest of the session lifecycle uniform — `MeetAudioSession` can
/// still hold a `SpeakPump` regardless of speak-path readiness.
@@ -231,7 +291,7 @@ async fn poll_and_feed(
request_id: &str,
cdp: &mut CdpConn,
session_id: &str,
) -> Result<bool, String> {
) -> Result<(bool, u8), String> {
let v = super::rpc_call(
"openhuman.meet_agent_poll_speech",
serde_json::json!({ "request_id": request_id }),
@@ -249,6 +309,12 @@ async fn poll_and_feed(
.get("flush_pending")
.and_then(|x| x.as_bool())
.unwrap_or(false);
// Which mascot slot is speaking this audio (0 = primary, 1 =
// secondary). Absent on older cores / single-mascot → 0.
let active_slot = v
.get("active_mascot_slot")
.and_then(|x| x.as_u64())
.unwrap_or(0) as u8;
// Barge-in: brain set flush_pending when it cancelled the previous
// outbound. Stop in-flight playback inside the JS bridge BEFORE we
@@ -278,10 +344,45 @@ async fn poll_and_feed(
bytes.len()
);
inject::feed_pcm_chunk(cdp, session_id, pcm_b64).await?;
return Ok(true);
return Ok((true, active_slot));
}
if utterance_done {
log::info!("[meet-audio] speak pump utterance complete request_id={request_id}");
}
Ok(false)
Ok((false, active_slot))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn speaking_edge_and_slot_change_emit_logic() {
// idle -> speaking: a real edge, emits.
let (emit, slot) = next_speaking_state(false, 0, true, Some(0));
assert!(emit, "idle -> speaking must emit");
assert_eq!(slot, 0);
// speaking -> speaking, same slot: no edge, no slot change, no emit.
let (emit, slot) = next_speaking_state(true, 0, true, Some(0));
assert!(!emit, "speaking -> speaking same slot must not emit");
assert_eq!(slot, 0);
// speaking(slot 0) -> speaking(slot 1): mid-speech mascot switch emits
// and resolves to the new slot so lip-sync moves to the other mascot.
let (emit, slot) = next_speaking_state(true, 0, true, Some(1));
assert!(emit, "speaking slot change must emit");
assert_eq!(slot, 1, "resolved slot must be the new slot 1");
// slot = None retains the previous slot; with the same reported flag
// that is neither an edge nor a slot change, so no emit.
let (emit, slot) = next_speaking_state(true, 1, true, None);
assert!(!emit, "no fresh slot + same reported must not emit");
assert_eq!(slot, 1, "None retains the previous slot");
// speaking -> idle: a real edge, emits.
let (emit, slot) = next_speaking_state(true, 1, false, None);
assert!(emit, "speaking -> idle must emit");
assert_eq!(slot, 1, "None retains the previous slot on the way down");
}
}
+42
View File
@@ -93,6 +93,15 @@ pub struct OpenWindowArgs {
/// fails closed in core (no wakes fire).
#[serde(default)]
pub owner_display_name: String,
/// ElevenLabs voice for the primary mascot (issue #4277). When the
/// user has two mascots enabled the core alternates the speaking
/// voice per reply. Absent/empty → core keeps its default voice.
#[serde(default)]
pub primary_voice_id: Option<String>,
/// ElevenLabs voice for the secondary mascot. Absent when only one
/// mascot is enabled.
#[serde(default)]
pub secondary_voice_id: Option<String>,
}
/// Open a dedicated top-level CEF webview window pointed at the Meet URL.
@@ -262,6 +271,8 @@ pub async fn meet_call_open_window<R: Runtime>(
let url_for_audio = parsed.to_string();
let bot_for_audio = args.display_name.clone();
let owner_for_audio = args.owner_display_name.clone();
let primary_voice_for_audio = args.primary_voice_id.clone();
let secondary_voice_for_audio = args.secondary_voice_id.clone();
tauri::async_runtime::spawn(async move {
if let Err(err) = crate::meet_audio::start(
app_for_audio,
@@ -269,6 +280,8 @@ pub async fn meet_call_open_window<R: Runtime>(
url_for_audio,
owner_for_audio,
bot_for_audio,
primary_voice_for_audio,
secondary_voice_for_audio,
)
.await
{
@@ -551,6 +564,35 @@ mod tests {
);
}
#[test]
fn open_window_args_deserializes_voice_ids() {
use serde_json::json;
// Both voice ids present → parse into Some(...). Proves the
// #4277 per-mascot voice fields ride through the command payload.
let args: OpenWindowArgs = serde_json::from_value(json!({
"request_id": "r",
"meet_url": "u",
"display_name": "d",
"primary_voice_id": "va",
"secondary_voice_id": "vb",
}))
.unwrap();
assert_eq!(args.primary_voice_id.as_deref(), Some("va"));
assert_eq!(args.secondary_voice_id.as_deref(), Some("vb"));
// Both omitted → #[serde(default)] yields None (single-mascot
// callers and older shells that don't forward voices still parse).
let args: OpenWindowArgs = serde_json::from_value(json!({
"request_id": "r",
"meet_url": "u",
"display_name": "d",
}))
.unwrap();
assert_eq!(args.primary_voice_id, None);
assert_eq!(args.secondary_voice_id, None);
}
#[test]
fn meet_call_state_default_is_empty() {
let state = MeetCallState::default();
@@ -9,6 +9,7 @@
import debug from 'debug';
import { type RefObject, useEffect, useRef, useState } from 'react';
import { useMascotManifest } from '../../features/human/Mascot/manifest/useMascotManifest';
import { useComposioIntegrations } from '../../lib/composio/hooks';
import { useT } from '../../lib/i18n/I18nContext';
import {
@@ -25,12 +26,15 @@ import { useAppDispatch, useAppSelector } from '../../store/hooks';
import {
selectCustomPrimaryColor,
selectCustomSecondaryColor,
selectDualMascotEnabled,
selectMascotColor,
selectMeetingMascotVoicePair,
selectSelectedMascotId,
} from '../../store/mascotSlice';
import { selectPersonaDescription, selectPersonaDisplayName } from '../../store/personaSlice';
import Button from '../ui/Button';
import {
buildMeetingMascots,
platformLabel,
platformUrlPlaceholder,
resolveMeetingBotMascotId,
@@ -76,6 +80,15 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
const mascotColor = useAppSelector(selectMascotColor);
const customPrimaryColor = useAppSelector(selectCustomPrimaryColor);
const customSecondaryColor = useAppSelector(selectCustomSecondaryColor);
// Dual-mascot config (issue #4277): when a distinct second mascot is enabled
// we send both slots (each with its own voice) so the backend bot renders
// two mascots and alternates who speaks. Single-mascot keeps the legacy
// `mascotId` path below untouched.
const dualMascotEnabled = useAppSelector(selectDualMascotEnabled);
const mascotVoicePair = useAppSelector(selectMeetingMascotVoicePair);
// Manifest drives name-addressed routing (#4277 follow-up): each dual slot is
// tagged with its display name so "Hey Toshi …" routes to that mascot.
const { manifest } = useMascotManifest();
// ── Meet slice ───────────────────────────────────────────────────────────
const meetStatus = useAppSelector(selectBackendMeetStatus);
@@ -129,6 +142,17 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
mascotColor === 'custom'
? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor }
: undefined;
// Two-mascot slots for the backend bot (issue #4277) — built via the shared
// helper so this live-join path and the UpcomingTable scheduled-join path stay
// behaviorally identical.
const mascots = buildMeetingMascots({
dualMascotEnabled,
mascotVoicePair,
manifest,
mascotId,
riveColors,
agentName,
});
const wakePhrase = listenOnly ? undefined : `Hey ${agentName}`;
const handleSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
@@ -143,6 +167,14 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
!listenOnly,
meetingId
);
// Name-addressing (#4277 follow-up) trace: the exact mascot ids + names sent
// to the backend. If `mascots` is undefined or a slot's `name` is empty,
// name addressing ("Hey Toshi") can't work — the backend needs both names.
log(
'[composer] join mascots=%o wakePhrase=%s',
mascots?.map(m => ({ mascotId: m.mascotId, name: m.name })),
wakePhrase
);
try {
// Await the RPC BEFORE dispatching setBackendMeetJoining so that a
// synchronous rejection (bad URL, auth failure) can be shown inline
@@ -159,6 +191,8 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
systemPrompt,
mascotId,
riveColors,
// Dual-mascot slots (issue #4277); undefined for single-mascot calls.
mascots,
correlationId: meetingId,
respondToParticipant: displayedRespondTo.trim() || undefined,
wakePhrase,
+48 -6
View File
@@ -14,6 +14,7 @@
import debug from 'debug';
import { useState } from 'react';
import { useMascotManifest } from '../../features/human/Mascot/manifest/useMascotManifest';
import { useT } from '../../lib/i18n/I18nContext';
import {
joinMeetViaBackendBot,
@@ -26,13 +27,21 @@ import { useAppSelector } from '../../store/hooks';
import {
selectCustomPrimaryColor,
selectCustomSecondaryColor,
selectDualMascotEnabled,
selectMascotColor,
selectMeetingMascotVoicePair,
selectSelectedMascotId,
} from '../../store/mascotSlice';
import { selectPersonaDescription, selectPersonaDisplayName } from '../../store/personaSlice';
import Button from '../ui/Button';
import { type JoinPolicy, JoinPolicyToggle } from './JoinPolicyToggle';
import { inferPlatformFromUrl, platformLabel, platformLogoUrl } from './meetingUtils';
import {
buildMeetingMascots,
inferPlatformFromUrl,
platformLabel,
platformLogoUrl,
resolveMeetingBotMascotId,
} from './meetingUtils';
import { useUpcomingMeetings } from './useUpcomingMeetings';
const log = debug('meetings:upcoming-table');
@@ -359,6 +368,14 @@ export function UpcomingTable({
const mascotColor = useAppSelector(selectMascotColor);
const customPrimaryColor = useAppSelector(selectCustomPrimaryColor);
const customSecondaryColor = useAppSelector(selectCustomSecondaryColor);
// Dual-mascot config (issue #4277) — see MeetComposer for the rationale;
// both join sites resolve the two-mascot slots the same way so a "Join now"
// and a manual join behave identically.
const dualMascotEnabled = useAppSelector(selectDualMascotEnabled);
const mascotVoicePair = useAppSelector(selectMeetingMascotVoicePair);
// Manifest drives name-addressed routing (#4277 follow-up): tag each dual slot
// with its display name so "Hey Toshi …" routes to that mascot.
const { manifest } = useMascotManifest();
// Live in-call state — lets a row detect that its meeting is already joined
// and suppress the "Join now" button. correlationId is a fresh per-join UUID
@@ -371,12 +388,29 @@ export function UpcomingTable({
return Boolean(backendMeetUrl && m.meet_url && backendMeetUrl === m.meet_url);
};
// Resolve bot join params the same way MeetComposer does.
const mascotId = selectedMascotId ?? (mascotColor === 'custom' ? undefined : mascotColor);
// Resolve bot join params the same way MeetComposer does — via
// resolveMeetingBotMascotId, so a manifest-only id the backend bot doesn't
// recognize is dropped here too (a raw `selectedMascotId` fallback would let
// it through and diverge from the MeetComposer join).
const mascotId = resolveMeetingBotMascotId(selectedMascotId, mascotColor);
const riveColors =
mascotColor === 'custom'
? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor }
: undefined;
// Bot join name — hoisted to component-body scope because both the `mascots`
// array here and the wake phrase inside handleJoin need it.
const agentName = personaDisplayName.trim() || 'Tiny';
// Two-mascot slots (issue #4277) — built via the shared helper so this
// scheduled-join path and the MeetComposer live-join path stay behaviorally
// identical.
const mascots = buildMeetingMascots({
dualMascotEnabled,
mascotVoicePair,
manifest,
mascotId,
riveColors,
agentName,
});
const handleJoin = async (meeting: UpcomingMeeting) => {
if (!meeting.meet_url) return;
@@ -388,9 +422,8 @@ export function UpcomingTable({
const anchor = replyDisplayName.trim();
// Reply mode gates the bot behind a wake phrase so it only reacts when
// addressed ("Hey Alex, …"), never to every caption from the anchor —
// mirroring MeetComposer. The bot joins as `agentName`, so the phrase must
// match it. Listen-only joins (no anchor) send no wake phrase.
const agentName = personaDisplayName.trim() || 'Tiny';
// mirroring MeetComposer. The bot joins as `agentName` (hoisted above), so
// the phrase must match it. Listen-only joins (no anchor) send no wake phrase.
const wakePhrase = anchor ? `Hey ${agentName}` : undefined;
// Mint a fresh correlation id per join. It becomes the call record's
// `request_id` (recent-calls list key + per-call detail filename), so it
@@ -408,6 +441,13 @@ export function UpcomingTable({
correlationId
);
setJoiningId(meeting.calendar_event_id);
// Name-addressing (#4277 follow-up) trace: mascot ids + names sent to the
// backend. Empty `name` on a slot ⇒ name addressing can't route to it.
log(
'[upcoming] join mascots=%o wakePhrase=%s',
mascots?.map(m => ({ mascotId: m.mascotId, name: m.name })),
wakePhrase
);
try {
await joinMeetViaBackendBot({
meetUrl: meeting.meet_url,
@@ -415,6 +455,8 @@ export function UpcomingTable({
agentName,
systemPrompt: personaDescription || undefined,
mascotId: mascotId || undefined,
// Dual-mascot slots (issue #4277); undefined for single-mascot calls.
mascots,
respondToParticipant: anchor || undefined,
wakePhrase,
listenOnly: !anchor,
@@ -1,7 +1,9 @@
import { describe, expect, it } from 'vitest';
import type { MascotManifest } from '../../../features/human/Mascot/manifest/types';
import type { ComposioConnection } from '../../../lib/composio/types';
import {
buildMeetingMascots,
deriveDisplayNameFromEmail,
inferPlatformFromUrl,
MEETING_PLATFORMS,
@@ -257,8 +259,16 @@ describe('resolveMeetingBotMascotId', () => {
expect(resolveMeetingBotMascotId('navy', 'yellow')).toBe('navy');
});
it('falls back to the legacy mascot color for a manifest-only mascot id', () => {
expect(resolveMeetingBotMascotId('river-guide', 'yellow')).toBe('yellow');
it('keeps the "toshi" manifest mascot id the backend now ships as an asset', () => {
expect(resolveMeetingBotMascotId('toshi', 'yellow')).toBe('toshi');
});
it('keeps the "tiny-mascot" manifest mascot id the backend now ships as an asset', () => {
expect(resolveMeetingBotMascotId('tiny-mascot', 'navy')).toBe('tiny-mascot');
});
it('falls back to the legacy mascot color for a manifest-only mascot id the backend still lacks', () => {
expect(resolveMeetingBotMascotId('jarvis', 'yellow')).toBe('yellow');
});
it('uses the mascot color when no mascot id is selected', () => {
@@ -269,3 +279,121 @@ describe('resolveMeetingBotMascotId', () => {
expect(resolveMeetingBotMascotId('river-guide', 'custom')).toBeUndefined();
});
});
// ---------------------------------------------------------------------------
// buildMeetingMascots
// ---------------------------------------------------------------------------
describe('buildMeetingMascots', () => {
// findMascot only reads `.id`/`.name`, so a partial manifest is enough.
const manifest = {
mascots: [
{ id: 'toshi', name: 'Toshi' },
{ id: 'tiny-mascot', name: 'Tiny' },
],
} as unknown as MascotManifest;
const pair = {
primary: { mascotId: 'toshi', voiceId: 'voice-a' },
secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
};
it('returns undefined when dual mode is off', () => {
expect(
buildMeetingMascots({
dualMascotEnabled: false,
mascotVoicePair: pair,
manifest,
mascotId: 'toshi',
agentName: 'Tiny',
})
).toBeUndefined();
});
it('returns undefined when no secondary is configured', () => {
expect(
buildMeetingMascots({
dualMascotEnabled: true,
mascotVoicePair: { primary: { mascotId: 'toshi', voiceId: 'voice-a' } },
manifest,
mascotId: 'toshi',
agentName: 'Tiny',
})
).toBeUndefined();
});
it('returns undefined when the primary slot id cannot be resolved', () => {
// primary has no explicit mascot AND the resolved single-path id is undefined.
expect(
buildMeetingMascots({
dualMascotEnabled: true,
mascotVoicePair: {
primary: { voiceId: 'voice-a' },
secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
},
manifest,
mascotId: undefined,
agentName: 'Tiny',
})
).toBeUndefined();
});
it('falls back to the resolved single-path mascotId for the primary slot', () => {
const mascots = buildMeetingMascots({
dualMascotEnabled: true,
mascotVoicePair: {
primary: { voiceId: 'voice-a' },
secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
},
manifest,
mascotId: 'toshi',
agentName: 'Tiny',
});
expect(mascots?.[0].mascotId).toBe('toshi');
});
it('builds two slots with manifest names and per-slot voices', () => {
const riveColors = { primaryColor: '#111', secondaryColor: '#222' };
const mascots = buildMeetingMascots({
dualMascotEnabled: true,
mascotVoicePair: pair,
manifest,
mascotId: 'toshi',
riveColors,
agentName: 'Tiny',
});
expect(mascots).toEqual([
{ mascotId: 'toshi', name: 'Toshi', voiceId: 'voice-a', riveColors },
{ mascotId: 'tiny-mascot', name: 'Tiny', voiceId: 'voice-b', riveColors },
]);
});
it('resolves names by slot so either mascot order works', () => {
const mascots = buildMeetingMascots({
dualMascotEnabled: true,
mascotVoicePair: {
primary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
secondary: { mascotId: 'toshi', voiceId: 'voice-a' },
},
manifest,
mascotId: 'tiny-mascot',
agentName: 'Tiny',
});
expect(mascots?.map(m => m.name)).toEqual(['Tiny', 'Toshi']);
});
it('falls back to agentName for the primary when its manifest entry is missing', () => {
const mascots = buildMeetingMascots({
dualMascotEnabled: true,
mascotVoicePair: {
primary: { mascotId: 'river-guide', voiceId: 'voice-a' },
secondary: { mascotId: 'toshi', voiceId: 'voice-a' },
},
manifest,
mascotId: 'river-guide',
agentName: 'Persona',
});
expect(mascots?.[0].name).toBe('Persona');
expect(mascots?.[1].name).toBe('Toshi');
});
});
+73 -2
View File
@@ -5,17 +5,31 @@
* previously embedded inside MeetingBotsCard so they can be unit-tested in
* isolation and shared across the split composer components.
*/
import { findMascot } from '../../features/human/Mascot/manifest/manifestService';
import type { MascotManifest } from '../../features/human/Mascot/manifest/types';
import type { MascotColor } from '../../features/human/Mascot/mascotPalette';
import type { ComposioConnection } from '../../lib/composio/types';
import type { MeetingPlatform } from '../../services/meetCallService';
import type { BackendMeetJoinInput, MeetingPlatform } from '../../services/meetCallService';
import { composioLogoUrl } from '../composio/toolkitMeta';
/**
* Mascot ids the meeting-bot backend recognizes. Newer manifest-only mascot
* ids (e.g. "river-guide") aren't supported there, so the bot falls back to the
* legacy mascot color for them.
*
* "toshi" and "tiny-mascot" are the tinyhumansai/mascots manifest ids that the
* Recall meeting-bot backend now ships as real mascot assets, so they pass
* through unchanged instead of being collapsed to a color.
*/
const MEETING_BOT_MASCOT_IDS = new Set(['yellow', 'blue', 'burgundy', 'black', 'navy']);
const MEETING_BOT_MASCOT_IDS = new Set([
'yellow',
'blue',
'burgundy',
'black',
'navy',
'toshi',
'tiny-mascot',
]);
/**
* Resolve the mascot id to send to the meeting bot: the selected mascot id when
@@ -31,6 +45,63 @@ export function resolveMeetingBotMascotId(
return undefined;
}
/** Voice/mascot pair the composer selects for the two speaking slots. */
type MeetingMascotVoicePair = {
primary: { mascotId?: string | null; voiceId?: string };
secondary?: { mascotId?: string | null; voiceId?: string } | null;
};
/**
* Build the backend bot's dual-mascot `mascots` array (issue #4277), shared by
* both join sites — MeetComposer (live join) and UpcomingTable (scheduled join)
* — so the two paths stay behaviorally identical. They had already begun to
* drift, which is exactly the class of bug centralising this prevents.
*
* Slot 0 (primary) reuses the resolved single-path `mascotId` when the voice
* pair's primary carries no explicit mascot; slot 1 is the secondary. Slot names
* come from the manifest so name-addressed routing works in either order ("Hey
* Toshi" → whichever slot Toshi occupies); the primary falls back to `agentName`
* only when its manifest entry is unavailable. Per-mascot colors are out of
* scope — both slots reuse the single `riveColors`.
*
* Returns `undefined` (→ the single-mascot `mascotId` join) unless dual mode is
* on, a secondary is configured, AND both slot ids resolve to a concrete value:
* a blank slot-0 id would make the backend drop it and render the secondary
* alone, mismatching the on-camera primary.
*/
export function buildMeetingMascots(input: {
dualMascotEnabled: boolean;
mascotVoicePair: MeetingMascotVoicePair;
manifest: MascotManifest | null;
mascotId: string | undefined;
riveColors?: { primaryColor?: string; secondaryColor?: string };
agentName: string;
}): BackendMeetJoinInput['mascots'] {
const { dualMascotEnabled, mascotVoicePair, manifest, mascotId, riveColors, agentName } = input;
const primarySlotId = mascotVoicePair.primary.mascotId ?? mascotId;
const secondarySlotId = mascotVoicePair.secondary?.mascotId ?? undefined;
if (!dualMascotEnabled || !mascotVoicePair.secondary || !primarySlotId || !secondarySlotId) {
return undefined;
}
const primaryName =
(manifest ? findMascot(manifest, primarySlotId)?.name : undefined) ?? agentName;
const secondaryName = manifest ? findMascot(manifest, secondarySlotId)?.name : undefined;
return [
{
mascotId: primarySlotId,
name: primaryName,
voiceId: mascotVoicePair.primary.voiceId,
riveColors,
},
{
mascotId: secondarySlotId,
name: secondaryName,
voiceId: mascotVoicePair.secondary.voiceId,
riveColors,
},
];
}
// ---------------------------------------------------------------------------
// Platform registry
// ---------------------------------------------------------------------------
@@ -22,6 +22,7 @@ import {
selectMascotVoiceGender,
selectMascotVoiceId,
selectMascotVoiceUseLocaleDefault,
selectSecondaryMascotId,
selectSelectedMascotId,
setCustomMascotGifUrl,
setCustomPrimaryColor,
@@ -30,6 +31,7 @@ import {
setMascotVoiceGender,
setMascotVoiceId,
setMascotVoiceUseLocaleDefault,
setSecondaryMascotId,
setSelectedMascotId,
SUPPORTED_MASCOT_COLORS,
} from '../../../store/mascotSlice';
@@ -41,6 +43,7 @@ import {
ELEVENLABS_VOICE_PRESETS,
isCuratedVoicePreset,
} from './elevenlabsVoicePresets';
import PerMascotVoiceRow from './PerMascotVoiceRow';
interface ColorOption {
id: MascotColor;
@@ -70,6 +73,7 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
const customPrimary = useAppSelector(selectCustomPrimaryColor);
const customSecondary = useAppSelector(selectCustomSecondaryColor);
const selectedMascotId = useAppSelector(selectSelectedMascotId);
const secondaryMascotId = useAppSelector(selectSecondaryMascotId);
const customMascotGifUrl = useAppSelector(selectCustomMascotGifUrl);
const storedVoiceId = useAppSelector(selectMascotVoiceId);
const voiceGender = useAppSelector(selectMascotVoiceGender);
@@ -126,6 +130,21 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
// null ("default") case has to clear it here so the stage falls back to
// the default manifest mascot rather than the GIF.
if (id == null) dispatch(setCustomMascotGifUrl(null));
// A newly-picked primary that collides with the current secondary would
// leave both slots pointing at the same mascot; clear the secondary so
// the duo never duplicates. The reducer's `selectDualMascotEnabled`
// guard already treats a collision as single-mascot, but clearing here
// keeps the picker's rendered state honest.
if (id != null && id === secondaryMascotId) dispatch(setSecondaryMascotId(null));
};
// ── Second-mascot picker (issue #4277) ───────────────────────────
// Enable / clear the meeting duo's second mascot. `null` (the "None"
// option) drops back to single-mascot. Picking the primary's id is
// disabled in the dropdown, so this only ever dispatches a distinct id
// or null.
const handleSelectSecondaryMascot = (id: string | null) => {
dispatch(setSecondaryMascotId(id));
};
const onSaveCustomGif = () => {
@@ -683,6 +702,82 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
{t('settings.mascot.characterDesc')}
</p>
</div>
{/* ── Meeting duo: second mascot + per-mascot voices (issue #4277) ─
Only meaningful for manifest mascots — a custom GIF avatar is a
single-figure path, so the whole block hides while one is set. */}
{manifest && manifest.mascots.length > 0 && !customMascotGifUrl && (
<div>
<h3 className="text-xs font-semibold uppercase tracking-wider text-content-faint mb-2 px-1">
{t('settings.mascot.secondaryHeading')}
</h3>
{/* Second-mascot picker — bespoke label + select combo mirroring
the voice preset dropdown. The primary's id is disabled so the
duo can never duplicate a single mascot. */}
<div className="bg-surface rounded-xl border border-line p-4 space-y-1">
<label className="block space-y-1">
<span className="sr-only">{t('settings.mascot.secondaryHeading')}</span>
<SettingsSelect
aria-label={t('settings.mascot.secondaryHeading')}
data-testid="mascot-secondary-select"
value={secondaryMascotId ?? '__none__'}
onChange={e =>
handleSelectSecondaryMascot(e.target.value === '__none__' ? null : e.target.value)
}
className="w-full">
<option value="__none__">{t('settings.mascot.secondaryNone')}</option>
{manifest.mascots.map(mascot => (
<option
key={mascot.id}
value={mascot.id}
// Skip the primary's id — it already speaks as the first
// mascot, so offering it as the second is a no-op the
// reducer would reject anyway. When the primary is still the
// default (selectedMascotId is null), the effective primary
// is the resolved default entry (activeEntry), so disable
// that too — otherwise the same mascot could be picked for
// both slots and the meeting would render two identical ones.
disabled={mascot.id === (selectedMascotId ?? activeEntry?.id)}>
{mascot.name}
</option>
))}
</SettingsSelect>
</label>
</div>
<p className="text-xs text-content-muted leading-relaxed px-1 mt-2">
{t('settings.mascot.secondaryDesc')}
</p>
{/* Per-mascot voices — a row per mascot whose voice is actually
addressable in `mascotVoices` (keyed by a concrete manifest
id). The join path (`selectMeetingMascotVoicePair`) resolves
the primary slot's voice from `mascotVoices[selectedMascotId]`,
so the primary row only appears once a specific primary mascot
is pinned; on the default mascot the effective single voice
(governed by the Voice section above) is what plays. Each row
writes its own `mascotVoices` entry and owns a guarded preview. */}
{secondaryMascotId != null && secondaryMascotId !== selectedMascotId && (
<div className="mt-3 space-y-3">
<h4 className="text-[11px] font-medium uppercase tracking-wide text-content-muted px-1">
{t('settings.mascot.perMascotVoiceHeading')}
</h4>
{selectedMascotId != null && (
<PerMascotVoiceRow
mascotId={selectedMascotId}
label={t('settings.mascot.primaryVoiceLabel')}
testIdPrefix="mascot-voice-primary"
/>
)}
<PerMascotVoiceRow
mascotId={secondaryMascotId}
label={t('settings.mascot.secondaryVoiceLabel')}
testIdPrefix="mascot-voice-secondary"
/>
</div>
)}
</div>
)}
</>
);
@@ -0,0 +1,245 @@
import { useEffect, useRef, useState } from 'react';
import { synthesizeSpeech } from '../../../features/human/voice/ttsClient';
import { useT } from '../../../lib/i18n/I18nContext';
import { useAppDispatch, useAppSelector } from '../../../store/hooks';
import {
selectEffectiveMascotVoiceId,
selectMascotVoiceFor,
selectMascotVoiceGender,
setMascotVoice,
} from '../../../store/mascotSlice';
import Button from '../../ui/Button';
import { SettingsSelect, SettingsTextField } from '../controls';
import { ELEVENLABS_VOICE_PRESETS, isCuratedVoicePreset } from './elevenlabsVoicePresets';
interface PerMascotVoiceRowProps {
/** Manifest mascot id this row controls the voice for. Writes land in
* `mascotVoices[mascotId]` via `setMascotVoice`. */
mascotId: string;
/** Human-readable heading for the row (e.g. the mascot's name), already
* localized by the caller. */
label: string;
/** Stable test hook so both the primary and secondary rows expose
* distinct `data-testid`s (`mascot-voice-{primary,secondary}-*`). */
testIdPrefix: string;
}
/**
* Per-mascot reply-voice control (issue #4277). A trimmed sibling of the
* primary voice section in `MascotPanel`: it reuses the same preset
* dropdown + custom-paste + guarded `synthesizeSpeech` preview, but writes
* to `mascotVoices[mascotId]` via `setMascotVoice` instead of the single
* `voiceId`. The current value is the per-mascot override
* (`selectMascotVoiceFor`) falling back to the effective single voice
* (`selectEffectiveMascotVoiceId`), so a mascot with no override sounds
* exactly like the single-voice behaviour until the user picks one.
*
* Extracted from `MascotPanel` to keep that file within the ~500-line
* budget while the per-mascot map (primary + secondary) doubles the voice
* UI. Each instance owns its own preview-abort guard (`previewRequestIdRef`)
* so the two rows never share an in-flight preview.
*/
const PerMascotVoiceRow = ({ mascotId, label, testIdPrefix }: PerMascotVoiceRowProps) => {
const { t } = useT();
const dispatch = useAppDispatch();
// Per-mascot override, or the effective single voice when unset — the
// same resolution the meeting join path uses, so the picker shows what
// the mascot will actually speak with.
const overrideVoiceId = useAppSelector(selectMascotVoiceFor(mascotId));
const effectiveVoiceId = useAppSelector(selectEffectiveMascotVoiceId);
const currentVoiceId = overrideVoiceId ?? effectiveVoiceId;
// Reuse the global gender bucket to filter the preset dropdown — the
// per-mascot control only overrides the voice id, not the gender filter.
const voiceGender = useAppSelector(selectMascotVoiceGender);
// Paste-mode is sticky for the same reason as the primary control: a
// curated preset id and a mid-paste custom id both leave the stored
// value looking like a known id, so we can't derive the mode from it.
const [voiceDraft, setVoiceDraft] = useState<string>(overrideVoiceId ?? '');
const [voicePasteMode, setVoicePasteMode] = useState<boolean>(false);
const [isPreviewingVoice, setIsPreviewingVoice] = useState(false);
const [voicePreviewError, setVoicePreviewError] = useState<string | null>(null);
const previewAudioRef = useRef<HTMLAudioElement | null>(null);
// Monotonically-bumped preview-request id, mirroring the primary
// control: unmount + each new preview both increment it so an in-flight
// `synthesizeSpeech(...)` whose resolve loses the race bails before it
// touches refs / state.
const previewRequestIdRef = useRef(0);
// Stop any in-flight preview audio on unmount and invalidate a pending
// `synthesizeSpeech(...)` so a late resolve can't start audio for a row
// the user has already navigated away from.
useEffect(() => {
return () => {
previewRequestIdRef.current += 1;
if (previewAudioRef.current) {
previewAudioRef.current.pause();
previewAudioRef.current.src = '';
previewAudioRef.current = null;
}
};
}, []);
// Presets the dropdown should expose: always include the current voice
// (so the controlled select never points at an absent option) plus the
// active gender bucket and any '*' fallback voices.
const visiblePresets = ELEVENLABS_VOICE_PRESETS.filter(
p => p.id === currentVoiceId || p.gender === voiceGender || p.locales.includes('*')
);
// A custom (non-curated) override keeps the paste editor open so the
// stored id stays visible; the effective-voice fallback is never treated
// as "custom" because the mascot has no explicit override yet.
const isCustomVoice =
voicePasteMode || (overrideVoiceId != null && !isCuratedVoicePreset(overrideVoiceId));
const onPresetChange = (next: string) => {
if (next === '__custom__') {
setVoicePasteMode(true);
setVoiceDraft(overrideVoiceId ?? '');
return;
}
setVoicePasteMode(false);
setVoicePreviewError(null);
setVoiceDraft(next);
dispatch(setMascotVoice({ mascotId, voiceId: next }));
};
const onSavePaste = () => {
setVoicePreviewError(null);
const trimmed = voiceDraft.trim();
setVoiceDraft(trimmed);
dispatch(setMascotVoice({ mascotId, voiceId: trimmed.length > 0 ? trimmed : null }));
};
const onVoiceReset = () => {
setVoicePreviewError(null);
setVoicePasteMode(false);
setVoiceDraft('');
dispatch(setMascotVoice({ mascotId, voiceId: null }));
};
const onVoicePreview = async () => {
// Same abort guard as the primary control: reserve a fresh id, and let
// a stale resolve detect that a newer preview (or unmount) superseded
// it before it mutates state or plays audio.
const requestId = ++previewRequestIdRef.current;
setIsPreviewingVoice(true);
setVoicePreviewError(null);
if (previewAudioRef.current) {
previewAudioRef.current.pause();
previewAudioRef.current.src = '';
previewAudioRef.current = null;
}
try {
const tts = await synthesizeSpeech(t('settings.mascot.voice.previewText'), {
voiceId: currentVoiceId,
});
if (previewRequestIdRef.current !== requestId) return;
const src = `data:${tts.audio_mime || 'audio/mpeg'};base64,${tts.audio_base64}`;
const audio = new window.Audio(src);
previewAudioRef.current = audio;
await audio.play();
} catch (err) {
if (previewRequestIdRef.current !== requestId) return;
const message = err instanceof Error ? err.message : t('settings.mascot.voice.previewError');
setVoicePreviewError(message);
} finally {
if (previewRequestIdRef.current === requestId) setIsPreviewingVoice(false);
}
};
return (
<div className="bg-surface rounded-xl border border-line p-4 space-y-3">
<span className="text-xs font-medium text-content-muted dark:text-content-secondary">
{label}
</span>
{/* Preset dropdown — mirrors the primary control's label + select combo */}
<label className="block space-y-1">
<span className="sr-only">{t('settings.mascot.voice.presetHeading')}</span>
<SettingsSelect
aria-label={`${label}${t('settings.mascot.voice.presetHeading')}`}
data-testid={`${testIdPrefix}-select`}
value={isCustomVoice ? '__custom__' : currentVoiceId}
onChange={e => onPresetChange(e.target.value)}
className="w-full">
{visiblePresets.map(v => (
<option key={v.id} value={v.id}>
{v.label}
</option>
))}
<option value="__custom__">{t('settings.mascot.voice.customOption')}</option>
</SettingsSelect>
</label>
{isCustomVoice && (
<label className="block space-y-1">
<span className="text-xs font-medium text-content-muted dark:text-content-secondary">
{t('settings.mascot.voice.customHeading')}
</span>
<div className="flex gap-2">
<SettingsTextField
aria-label={`${label}${t('settings.mascot.voice.customHeading')}`}
data-testid={`${testIdPrefix}-input`}
value={voiceDraft}
placeholder={t('settings.mascot.voice.customPlaceholder')}
onChange={e => setVoiceDraft(e.target.value)}
className="flex-1"
/>
<Button
type="button"
variant="primary"
size="xs"
data-testid={`${testIdPrefix}-save-paste`}
onClick={onSavePaste}
disabled={voiceDraft.trim() === (overrideVoiceId ?? '').trim()}>
{t('common.save')}
</Button>
</div>
</label>
)}
<div className="flex flex-wrap items-center gap-2">
<Button
type="button"
variant="primary"
size="xs"
data-testid={`${testIdPrefix}-preview`}
onClick={() => void onVoicePreview()}
disabled={isPreviewingVoice}
className="bg-emerald-600 hover:bg-emerald-700 dark:bg-emerald-600 dark:hover:bg-emerald-500">
{isPreviewingVoice
? t('settings.mascot.voice.previewing')
: t('settings.mascot.voice.preview')}
</Button>
<Button
type="button"
variant="secondary"
size="xs"
data-testid={`${testIdPrefix}-reset`}
onClick={onVoiceReset}
disabled={overrideVoiceId == null}>
{t('settings.mascot.voice.reset')}
</Button>
<span
data-testid={`${testIdPrefix}-current`}
className="ml-1 text-[11px] text-content-muted truncate max-w-[18rem]"
title={currentVoiceId}>
{t('settings.mascot.voice.current')}: <code className="font-mono">{currentVoiceId}</code>
</span>
</div>
{voicePreviewError && (
<div
data-testid={`${testIdPrefix}-preview-error`}
className="rounded-md border border-amber-200 dark:border-amber-500/30 bg-amber-50 dark:bg-amber-500/10 p-3 text-xs text-amber-800 dark:text-amber-200">
{t('settings.mascot.voice.previewError')}: {voicePreviewError}
</div>
)}
</div>
);
};
export default PerMascotVoiceRow;
@@ -10,6 +10,7 @@ import mascotReducer, {
setCustomMascotGifUrl,
setMascotColor,
setMascotVoiceId,
setSecondaryMascotId,
setSelectedMascotId,
} from '../../../../store/mascotSlice';
import MascotPanel from '../MascotPanel';
@@ -362,3 +363,90 @@ describe('MascotPanel — voice picker custom voice input (line 525)', () => {
expect(store.getState().mascot.voiceId).toBe('new-voice-id');
});
});
// ── Dual mascots + per-mascot voice (issue #4277) ────────────────────────────
describe('MascotPanel — meeting duo (second mascot)', () => {
const yellow = manifestEntry('yellow', 'Yellow');
const toshi = manifestEntry('toshi', 'Toshi');
beforeEach(() => {
vi.clearAllMocks();
useMascotManifestMock.mockReturnValue(manifestResult([yellow, toshi]));
mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0));
});
it('dispatches setSecondaryMascotId when a second mascot is picked', () => {
const { store } = renderPanel();
fireEvent.change(screen.getByTestId('mascot-secondary-select'), { target: { value: 'toshi' } });
expect(store.getState().mascot.secondaryMascotId).toBe('toshi');
});
it('clears the second mascot when None is selected', () => {
const store = buildStore();
store.dispatch(setSecondaryMascotId('toshi'));
renderPanel(store);
fireEvent.change(screen.getByTestId('mascot-secondary-select'), {
target: { value: '__none__' },
});
expect(store.getState().mascot.secondaryMascotId).toBeNull();
});
it('disables the primary mascot as a second-mascot option', () => {
const store = buildStore();
store.dispatch(setSelectedMascotId('yellow'));
renderPanel(store);
const primaryOption = screen
.getByTestId('mascot-secondary-select')
.querySelector('option[value="yellow"]') as HTMLOptionElement | null;
expect(primaryOption).not.toBeNull();
expect(primaryOption).toBeDisabled();
});
it('hides the duo picker when a custom GIF avatar is set', () => {
const store = buildStore();
store.dispatch(setCustomMascotGifUrl('https://example.com/avatar.gif'));
renderPanel(store);
expect(screen.queryByTestId('mascot-secondary-select')).not.toBeInTheDocument();
});
it('dispatches setMascotVoice for the second mascot when its voice changes', () => {
const store = buildStore();
store.dispatch(setSelectedMascotId('yellow'));
store.dispatch(setSecondaryMascotId('toshi'));
renderPanel(store);
// The secondary voice row renders once a distinct duo mascot is set.
const select = screen.getByTestId('mascot-voice-secondary-select');
fireEvent.change(select, { target: { value: 'pNInz6obpgDQGcFmaJgB' } });
expect(store.getState().mascot.mascotVoices.toshi).toBe('pNInz6obpgDQGcFmaJgB');
});
it('dispatches setMascotVoice for the primary mascot when its voice changes', () => {
const store = buildStore();
store.dispatch(setSelectedMascotId('yellow'));
store.dispatch(setSecondaryMascotId('toshi'));
renderPanel(store);
const select = screen.getByTestId('mascot-voice-primary-select');
fireEvent.change(select, { target: { value: 'pNInz6obpgDQGcFmaJgB' } });
expect(store.getState().mascot.mascotVoices.yellow).toBe('pNInz6obpgDQGcFmaJgB');
});
it('clears a per-mascot voice via the row reset button', () => {
const store = buildStore();
store.dispatch(setSelectedMascotId('yellow'));
store.dispatch(setSecondaryMascotId('toshi'));
renderPanel(store);
// Seed an override, then reset it.
fireEvent.change(screen.getByTestId('mascot-voice-secondary-select'), {
target: { value: 'pNInz6obpgDQGcFmaJgB' },
});
expect(store.getState().mascot.mascotVoices.toshi).toBe('pNInz6obpgDQGcFmaJgB');
fireEvent.click(screen.getByTestId('mascot-voice-secondary-reset'));
expect(store.getState().mascot.mascotVoices.toshi).toBeUndefined();
});
});
@@ -0,0 +1,203 @@
import { configureStore } from '@reduxjs/toolkit';
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import { Provider } from 'react-redux';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import mascotReducer, { setMascotVoice } from '../../../../store/mascotSlice';
import PerMascotVoiceRow from '../PerMascotVoiceRow';
const { mockSynthesizeSpeech } = vi.hoisted(() => ({ mockSynthesizeSpeech: vi.fn() }));
vi.mock('../../../../features/human/voice/ttsClient', () => ({
synthesizeSpeech: (...args: unknown[]) => mockSynthesizeSpeech(...args),
}));
const TEST_ID = 'mascot-voice-primary';
const MASCOT_ID = 'yellow';
function buildStore() {
return configureStore({ reducer: { mascot: mascotReducer } });
}
function renderRow(store = buildStore()) {
return {
store,
...render(
<Provider store={store}>
<PerMascotVoiceRow mascotId={MASCOT_ID} label="Yellow" testIdPrefix={TEST_ID} />
</Provider>
),
};
}
/** Resolve helper to control the timing of the mocked synthesizeSpeech. */
function deferredTts() {
let resolve!: (v: { audio_mime: string; audio_base64: string }) => void;
let reject!: (err: unknown) => void;
const promise = new Promise<{ audio_mime: string; audio_base64: string }>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
describe('PerMascotVoiceRow', () => {
let playSpy: ReturnType<typeof vi.fn>;
beforeEach(() => {
vi.clearAllMocks();
playSpy = vi.fn().mockResolvedValue(undefined);
// Stub window.Audio so `new window.Audio(src)` records the src and exposes
// a spy-able play()/pause() without touching the real audio pipeline.
vi.stubGlobal(
'Audio',
class {
src: string;
constructor(src?: string) {
this.src = src ?? '';
}
play = playSpy;
pause = vi.fn();
}
);
});
afterEach(() => {
vi.unstubAllGlobals();
});
it('reveals the paste input + Save when the custom option is selected and saves a new id', () => {
const { store } = renderRow();
// No custom input until the __custom__ option is chosen.
expect(screen.queryByTestId(`${TEST_ID}-input`)).not.toBeInTheDocument();
fireEvent.change(screen.getByTestId(`${TEST_ID}-select`), { target: { value: '__custom__' } });
const input = screen.getByTestId(`${TEST_ID}-input`);
expect(input).toBeInTheDocument();
const saveBtn = screen.getByTestId(`${TEST_ID}-save-paste`);
// Empty draft still equals the (empty) stored override → disabled.
expect(saveBtn).toBeDisabled();
// A whitespace-padded id enables Save and is trimmed on dispatch.
fireEvent.change(input, { target: { value: ' my-custom-voice ' } });
expect(saveBtn).not.toBeDisabled();
fireEvent.click(saveBtn);
expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBe('my-custom-voice');
});
it('plays a data:audio preview and toggles the previewing label when synthesizeSpeech resolves', async () => {
const d = deferredTts();
mockSynthesizeSpeech.mockReturnValue(d.promise);
renderRow();
const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
fireEvent.click(previewBtn);
// Previewing state toggles on immediately (button disabled + label swap).
await waitFor(() => expect(previewBtn).toBeDisabled());
expect(previewBtn).toHaveTextContent('Previewing…');
d.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(1));
// A data:audio URI was constructed from the mime + base64 payload.
expect(playSpy.mock.instances[0].src).toBe('data:audio/mpeg;base64,QUJD');
// Previewing resets once the preview finishes.
await waitFor(() => expect(previewBtn).not.toBeDisabled());
expect(previewBtn).toHaveTextContent('Preview voice');
});
it('renders the preview-error text and resets previewing when synthesizeSpeech rejects', async () => {
const d = deferredTts();
mockSynthesizeSpeech.mockReturnValue(d.promise);
renderRow();
const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
fireEvent.click(previewBtn);
await waitFor(() => expect(previewBtn).toBeDisabled());
d.reject(new Error('tts exploded'));
const errorBox = await screen.findByTestId(`${TEST_ID}-preview-error`);
expect(errorBox).toHaveTextContent('tts exploded');
expect(playSpy).not.toHaveBeenCalled();
// finally branch clears the previewing state.
await waitFor(() => expect(previewBtn).not.toBeDisabled());
expect(previewBtn).toHaveTextContent('Preview voice');
});
it('is a no-op when a preview resolves after the row has unmounted (previewRequestIdRef guard)', async () => {
const d = deferredTts();
mockSynthesizeSpeech.mockReturnValue(d.promise);
const { unmount } = renderRow();
fireEvent.click(screen.getByTestId(`${TEST_ID}-preview`));
await waitFor(() => expect(mockSynthesizeSpeech).toHaveBeenCalledTimes(1));
// Unmount bumps previewRequestIdRef, so the late resolve loses the race.
unmount();
d.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
// Give the resolved promise a microtask/macrotask to flush.
await new Promise(r => setTimeout(r, 0));
expect(playSpy).not.toHaveBeenCalled();
});
it('exposes the current voice id set via setMascotVoice', () => {
const store = buildStore();
store.dispatch(setMascotVoice({ mascotId: MASCOT_ID, voiceId: 'pNInz6obpgDQGcFmaJgB' }));
renderRow(store);
expect(screen.getByTestId(`${TEST_ID}-current`)).toHaveTextContent('pNInz6obpgDQGcFmaJgB');
});
it('dispatches setMascotVoice when a curated preset is picked from the dropdown', () => {
const { store } = renderRow();
fireEvent.change(screen.getByTestId(`${TEST_ID}-select`), {
target: { value: 'pNInz6obpgDQGcFmaJgB' },
});
expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBe('pNInz6obpgDQGcFmaJgB');
});
it('clears the per-mascot override via the reset button', () => {
const store = buildStore();
store.dispatch(setMascotVoice({ mascotId: MASCOT_ID, voiceId: 'pNInz6obpgDQGcFmaJgB' }));
renderRow(store);
const resetBtn = screen.getByTestId(`${TEST_ID}-reset`);
expect(resetBtn).not.toBeDisabled();
fireEvent.click(resetBtn);
expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBeUndefined();
});
it('stops the prior preview audio when a second preview starts', async () => {
const first = deferredTts();
const second = deferredTts();
mockSynthesizeSpeech.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
renderRow();
const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
// First preview resolves and plays, seeding previewAudioRef.
fireEvent.click(previewBtn);
first.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(1));
const firstAudio = playSpy.mock.instances[0];
await waitFor(() => expect(previewBtn).not.toBeDisabled());
// Second preview should pause + clear the prior audio before synthesizing.
fireEvent.click(previewBtn);
await waitFor(() => expect(mockSynthesizeSpeech).toHaveBeenCalledTimes(2));
expect(firstAudio.pause).toHaveBeenCalled();
expect(firstAudio.src).toBe('');
second.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'WFla' });
await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(2));
});
});
+277 -81
View File
@@ -1,62 +1,69 @@
import { listen, type UnlistenFn } from '@tauri-apps/api/event';
import { type FC, useCallback, useEffect, useRef, useState } from 'react';
import { type FC, useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useSelector } from 'react-redux';
import { RiveMascot } from '../human/Mascot';
import {
selectCustomPrimaryColor,
selectCustomSecondaryColor,
selectMascotColor,
selectSecondaryMascotId,
} from '../../store/mascotSlice';
import {
getMascotPalette,
hexToArgbInt,
ManifestRiveMascot,
type MascotFace,
type MascotManifestEntry,
RiveMascot,
} from '../human/Mascot';
import { findMascot } from '../human/Mascot/manifest/manifestService';
import { useMascotManifest } from '../human/Mascot/manifest/useMascotManifest';
import {
drawMascotInCell,
FRAME_H,
FRAME_H_DUAL,
FRAME_W,
FRAME_W_DUAL,
MASCOT_INSET,
sampleCanvasPixels,
} from './mascotFrameCompositor';
import { type ActiveMascotSlot, type MeetingPhase, useMeetingMascots } from './useMeetingMascots';
const PRODUCER_FPS = 24;
const FRAME_W = 320;
const FRAME_H = 240;
const JPEG_QUALITY = 0.7;
/**
* How long the mascot(s) wave hello on join before settling into the live
* `active` face state (ms). Matches the greeting the participants see so the
* first frames read as "the bot is saying hi" rather than a cold stare.
*/
const GREETING_MS = 2500;
/**
* Teardown grace after `bus-stopped` (ms): keep the session mounted — WS +
* worker alive — long enough for the goodbye wave to stream before the frame
* pipeline is torn down. Without this the last thing the call sees is a hard
* cut mid-pose instead of a wave.
*/
const SIGNOFF_MS = 1500;
interface BusSession {
requestId: string;
port: number;
}
export function sampleCanvasPixels(
ctx: OffscreenCanvasRenderingContext2D,
width: number,
height: number
) {
const cols = 7;
const rows = 5;
let sum = 0;
let min = 255;
let max = 0;
let count = 0;
let dark = 0;
let bright = 0;
try {
for (let y = 0; y < rows; y++) {
for (let x = 0; x < cols; x++) {
const px = Math.max(0, Math.min(width - 1, Math.floor(((x + 0.5) * width) / cols)));
const py = Math.max(0, Math.min(height - 1, Math.floor(((y + 0.5) * height) / rows)));
const [r, g, b] = ctx.getImageData(px, py, 1, 1).data;
const luma = Math.round(r * 0.299 + g * 0.587 + b * 0.114);
sum += luma;
min = Math.min(min, luma);
max = Math.max(max, luma);
if (luma < 8) dark++;
if (luma > 32) bright++;
count++;
}
}
return {
avgLuma: Math.round(sum / Math.max(1, count)),
minLuma: min,
maxLuma: max,
darkSamples: dark,
brightSamples: bright,
sampleCount: count,
};
} catch (err) {
return { error: String(err instanceof Error ? err.message : err) };
}
}
// Re-export from the compositor for back-compat with existing importers
// (the producer test imports `sampleCanvasPixels` from here). The
// implementation moved to mascotFrameCompositor.ts (issue #4277).
export { sampleCanvasPixels };
export const MascotFrameProducer: FC = () => {
const [session, setSession] = useState<BusSession | null>(null);
// Meeting lifecycle phase, owned here so it survives across the brief
// sign-off grace where `session` is still set but the bus has stopped.
const [phase, setPhase] = useState<MeetingPhase>('greeting');
// Set when `bus-stopped` fires; the session is cleared SIGNOFF_MS later so
// the goodbye wave can stream. A ref so the timers don't re-arm on render.
const signoffTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
useEffect(() => {
let unlistenStarted: UnlistenFn | undefined;
@@ -67,6 +74,11 @@ export const MascotFrameProducer: FC = () => {
const payload = event.payload;
if (!payload || !payload.port) return;
console.log('[meet-video-producer] bus-started', payload);
if (signoffTimerRef.current) {
clearTimeout(signoffTimerRef.current);
signoffTimerRef.current = null;
}
setPhase('greeting');
setSession(payload);
})
.then(stop => {
@@ -77,7 +89,15 @@ export const MascotFrameProducer: FC = () => {
listen<{ requestId?: string; request_id?: string }>('meet-video:bus-stopped', event => {
console.log('[meet-video-producer] bus-stopped', event.payload);
setSession(null);
// Enter the goodbye wave and keep the pipeline alive for SIGNOFF_MS so
// the wave frames actually reach the call before we clear the session.
setPhase('signoff');
if (signoffTimerRef.current) clearTimeout(signoffTimerRef.current);
signoffTimerRef.current = setTimeout(() => {
console.log('[meet-video-producer] sign-off grace elapsed, clearing session');
signoffTimerRef.current = null;
setSession(null);
}, SIGNOFF_MS);
})
.then(stop => {
if (cancelled) stop();
@@ -89,69 +109,178 @@ export const MascotFrameProducer: FC = () => {
cancelled = true;
if (unlistenStarted) unlistenStarted();
if (unlistenStopped) unlistenStopped();
if (signoffTimerRef.current) {
clearTimeout(signoffTimerRef.current);
signoffTimerRef.current = null;
}
};
}, []);
// Advance greeting → active after GREETING_MS, per active session. Bound to
// requestId so a fresh session restarts the greeting.
useEffect(() => {
if (!session || phase !== 'greeting') return;
const id = setTimeout(() => {
console.log('[meet-video-producer] greeting elapsed → active', session.requestId);
setPhase('active');
}, GREETING_MS);
return () => clearTimeout(id);
}, [session, phase]);
if (!session) return null;
return <ProducerSession key={session.requestId} session={session} />;
return <ProducerSession key={session.requestId} session={session} phase={phase} />;
};
const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
const ProducerSession: FC<{ session: BusSession; phase: MeetingPhase }> = ({ session, phase }) => {
const hostRef = useRef<HTMLDivElement>(null);
const wsRef = useRef<WebSocket | null>(null);
const wsReadyRef = useRef(false);
const stoppedRef = useRef(false);
const inflightRef = useRef(false);
const lastDiagAtRef = useRef(0);
const isSpeakingRef = useRef(false);
// True while the bot is actively producing PCM into the Meet call.
// Drives the mascot face so the mouth animates in time with the audio
// participants hear. Source of truth is the Rust speak_pump (edge-detected
// from the RPC poll loop). Same requestId guards against stale events from
// a previous session bleeding into this session's mascot state.
const [isSpeaking, setIsSpeaking] = useState(false);
// Which mascot slot is speaking the current audio (issue #4277). Single-
// mascot calls always report slot 0. Read off the speaking-state event.
const [activeMascotSlot, setActiveMascotSlot] = useState<ActiveMascotSlot>(0);
// Per-slot render state (which mascot + which face) for this tick.
const render = useMeetingMascots({ speaking: isSpeaking, activeMascotSlot, phase });
const { dualEnabled } = render;
// Resolve the manifest entries for both slots. The primary follows the
// user's selection (same resolution the Human page uses); the secondary is
// looked up by its explicit id so the frame honors both selections and no
// longer always shows the bundled default (fixes the pre-existing single-
// mascot bug where the meeting camera ignored `selectedMascotId`).
const { manifest, entry: primaryEntry } = useMascotManifest();
const secondaryMascotId = useSelector(selectSecondaryMascotId);
const secondaryEntry: MascotManifestEntry | null =
dualEnabled && manifest ? (findMascot(manifest, secondaryMascotId) ?? null) : null;
// Mascot body colors, mirroring HumanPage so the meeting mascot matches the
// one on the Human stage. Per-mascot colors are out of scope (#4277) — both
// slots share the single selected color.
const mascotColor = useSelector(selectMascotColor);
const customPrimary = useSelector(selectCustomPrimaryColor);
const customSecondary = useSelector(selectCustomSecondaryColor);
const palette = getMascotPalette(mascotColor);
const primaryColor = useMemo(
() => hexToArgbInt(mascotColor === 'custom' ? customPrimary : palette.bodyFill),
[mascotColor, customPrimary, palette]
);
const secondaryColor = useMemo(
() => hexToArgbInt(mascotColor === 'custom' ? customSecondary : palette.neckShadowColor),
[mascotColor, customSecondary, palette]
);
const isSpeakingRef = useRef(isSpeaking);
useEffect(() => {
isSpeakingRef.current = isSpeaking;
}, [isSpeaking]);
// `dualEnabled` and `activeMascotSlot` are read inside captureFrame via refs
// so a speaker switch (activeMascotSlot 0↔1) or a dual-mode toggle does NOT
// change captureFrame's identity — otherwise the WS/worker effect below
// (which depends on captureFrame) would tear down and rebuild the socket +
// frame worker on every alternation. captureFrame stays keyed to the session
// only, matching the original single-mascot behavior.
const dualEnabledRef = useRef(dualEnabled);
useEffect(() => {
dualEnabledRef.current = dualEnabled;
}, [dualEnabled]);
const activeMascotSlotRef = useRef(activeMascotSlot);
useEffect(() => {
activeMascotSlotRef.current = activeMascotSlot;
}, [activeMascotSlot]);
const captureFrame = useCallback(async () => {
if (stoppedRef.current || !wsReadyRef.current || inflightRef.current) return;
const host = hostRef.current;
if (!host) return;
const canvas = host.querySelector('canvas');
if (!canvas) return;
const dualEnabledNow = dualEnabledRef.current;
// Look the source canvases up by slot so a face change (which re-renders
// the mascot) never changes which canvas we sample.
const primaryCanvas = host.querySelector<HTMLCanvasElement>(
'[data-mascot-slot="primary"] canvas'
);
if (!primaryCanvas) return;
const secondaryCanvas = dualEnabledNow
? host.querySelector<HTMLCanvasElement>('[data-mascot-slot="secondary"] canvas')
: null;
const frameW = dualEnabledNow ? FRAME_W_DUAL : FRAME_W;
// The dual frame is taller (16:9) than the single frame so the fake-camera
// bridge's cover-scale fills the 1280×720 canvas without cropping — see
// FRAME_H_DUAL in mascotFrameCompositor.ts.
const frameH = dualEnabledNow ? FRAME_H_DUAL : FRAME_H;
inflightRef.current = true;
try {
const offscreen = new OffscreenCanvas(FRAME_W, FRAME_H);
const offscreen = new OffscreenCanvas(frameW, frameH);
const ctx = offscreen.getContext('2d');
if (!ctx) return;
const grad = ctx.createRadialGradient(
FRAME_W / 2,
FRAME_H / 2,
frameW / 2,
frameH / 2,
0,
FRAME_W / 2,
FRAME_H / 2,
Math.max(FRAME_W, FRAME_H) * 0.7
frameW / 2,
frameH / 2,
Math.max(frameW, frameH) * 0.7
);
grad.addColorStop(0, '#FBF3D9');
grad.addColorStop(1, '#EFE3B8');
ctx.fillStyle = grad;
ctx.fillRect(0, 0, FRAME_W, FRAME_H);
ctx.fillRect(0, 0, frameW, frameH);
const inset = 0.06;
const fitW = FRAME_W * (1 - 2 * inset);
const fitH = FRAME_H * (1 - 2 * inset);
const scale = Math.min(fitW / canvas.width, fitH / canvas.height);
const dw = canvas.width * scale;
const dh = canvas.height * scale;
const dx = (FRAME_W - dw) / 2;
const dy = (FRAME_H - dh) / 2;
ctx.drawImage(canvas, dx, dy, dw, dh);
if (dualEnabledNow && secondaryCanvas) {
// Two half-cells: [0..half] primary, [half..frameW] secondary.
const half = frameW / 2;
drawMascotInCell(
ctx,
primaryCanvas,
0,
0,
half,
frameH,
MASCOT_INSET,
primaryCanvas.width,
primaryCanvas.height
);
drawMascotInCell(
ctx,
secondaryCanvas,
half,
0,
half,
frameH,
MASCOT_INSET,
secondaryCanvas.width,
secondaryCanvas.height
);
} else {
// Single-cell draw. This also covers the dual-but-secondary-not-yet-
// mounted tick (a 2.2MB mascot still decoding): rather than emit a
// black half we draw the primary across the whole frame for this tick.
drawMascotInCell(
ctx,
primaryCanvas,
0,
0,
frameW,
frameH,
MASCOT_INSET,
primaryCanvas.width,
primaryCanvas.height
);
}
const probe = sampleCanvasPixels(ctx, FRAME_W, FRAME_H);
const probe = sampleCanvasPixels(ctx, frameW, frameH);
const blob = await offscreen.convertToBlob({ type: 'image/jpeg', quality: JPEG_QUALITY });
const buffer = await blob.arrayBuffer();
const ws = wsRef.current;
@@ -163,12 +292,17 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
JSON.stringify({
kind: 'producer-pixel-probe',
requestId: session.requestId,
canvasWidth: canvas.width,
canvasHeight: canvas.height,
frameWidth: FRAME_W,
frameHeight: FRAME_H,
canvasWidth: primaryCanvas.width,
canvasHeight: primaryCanvas.height,
frameWidth: frameW,
frameHeight: frameH,
jpegBytes: blob.size,
isSpeaking: isSpeakingRef.current,
// Dual-mascot diagnostics (issue #4277): whether we drew two
// cells this tick, and which slot the audio is on.
dualEnabled: dualEnabledNow,
secondaryMounted: dualEnabledNow ? Boolean(secondaryCanvas) : undefined,
activeMascotSlot: activeMascotSlotRef.current,
probe,
})
);
@@ -241,13 +375,19 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
// session — a remount tears it down with the rest of the pipeline.
let unlistenSpeaking: UnlistenFn | undefined;
let speakingListenerCancelled = false;
listen<{ requestId?: string; speaking?: boolean }>('meet-video:speaking-state', event => {
const payload = event.payload;
if (!payload) return;
// Ignore events from a different session during teardown / restart.
if (payload.requestId && payload.requestId !== session.requestId) return;
setIsSpeaking(!!payload.speaking);
})
listen<{ requestId?: string; speaking?: boolean; activeMascotSlot?: number }>(
'meet-video:speaking-state',
event => {
const payload = event.payload;
if (!payload) return;
// Ignore events from a different session during teardown / restart.
if (payload.requestId && payload.requestId !== session.requestId) return;
setIsSpeaking(!!payload.speaking);
// `activeMascotSlot` names which mascot is speaking this audio (0|1);
// default to slot 0 for single-mascot / older core builds that omit it.
setActiveMascotSlot(payload.activeMascotSlot === 1 ? 1 : 0);
}
)
.then(stop => {
if (speakingListenerCancelled) stop();
else unlistenSpeaking = stop;
@@ -289,14 +429,70 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
position: 'fixed',
left: '-99999px',
top: 0,
width: FRAME_H,
width: dualEnabled ? FRAME_H * 2 : FRAME_H,
height: FRAME_H,
pointerEvents: 'none',
opacity: 0,
}}>
<RiveMascot face={isSpeaking ? 'speaking' : 'idle'} size={FRAME_H} />
{/* Slot 0 (primary). Stable key per mascot id so a face change updates in
place instead of remounting the Rive/WebGL context. */}
<div data-mascot-slot="primary" style={{ width: FRAME_H, height: FRAME_H }}>
<MascotStage
entry={primaryEntry}
face={render.primary.face}
primaryColor={primaryColor}
secondaryColor={secondaryColor}
/>
</div>
{/* Slot 1 (secondary) — only mounted when a distinct second mascot is
enabled. Its 2.2MB asset may still be decoding for the first frames;
captureFrame falls back to a single-cell draw until its canvas exists. */}
{dualEnabled && render.secondary && (
<div data-mascot-slot="secondary" style={{ width: FRAME_H, height: FRAME_H }}>
<MascotStage
entry={secondaryEntry}
face={render.secondary.face}
primaryColor={primaryColor}
secondaryColor={secondaryColor}
/>
</div>
)}
</div>
);
};
/**
* Render one mascot slot. Prefers the manifest mascot (honoring the user's
* selection) and falls back to the bundled `RiveMascot` while the manifest is
* still resolving so the frame never blanks. Keyed by mascot id upstream so a
* *selection* change remounts, while a *face* change updates in place.
*/
const MascotStage: FC<{
entry: MascotManifestEntry | null;
face: MascotFace;
primaryColor: number;
secondaryColor: number;
}> = ({ entry, face, primaryColor, secondaryColor }) => {
if (entry) {
return (
<ManifestRiveMascot
key={entry.id}
entry={entry}
face={face}
size={FRAME_H}
primaryColor={primaryColor}
secondaryColor={secondaryColor}
/>
);
}
return (
<RiveMascot
face={face}
size={FRAME_H}
primaryColor={primaryColor}
secondaryColor={secondaryColor}
/>
);
};
export default MascotFrameProducer;
@@ -1,17 +1,204 @@
import { cleanup } from '@testing-library/react';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { listen } from '@tauri-apps/api/event';
import { act, cleanup, waitFor } from '@testing-library/react';
import { createElement } from 'react';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { renderWithProviders } from '../../../test/test-utils';
import { MascotFrameProducer, sampleCanvasPixels } from '../MascotFrameProducer';
// @tauri-apps/api/event is already mocked in setup.ts (listen → vi.fn())
// @tauri-apps/api/event is mocked in setup.ts (listen → vi.fn()). Here we
// override it per-test to capture the event handlers so we can drive a fake
// `meet-video:bus-started` and assert how many mascot hosts mount.
type Listener = (event: { payload: unknown }) => void;
// The producer renders ManifestRiveMascot / RiveMascot (WebGL). Mock both leaf
// renderers to a plain <canvas> host so the frame's slot structure is
// assertable without the Rive runtime. Each records the face it was asked to
// render via a data attribute.
vi.mock('../../human/Mascot', async () => {
const actual = await vi.importActual<typeof import('../../human/Mascot')>('../../human/Mascot');
const stub = (props: { face?: string }) =>
createElement('canvas', { 'data-face': props.face, width: 200, height: 200 });
return { ...actual, ManifestRiveMascot: stub, RiveMascot: stub };
});
// Keep the manifest resolution deterministic + synchronous — the host tree
// mounts regardless (MascotStage falls back to RiveMascot when entry is null),
// so we just avoid a real network fetch.
vi.mock('../../human/Mascot/manifest/useMascotManifest', () => ({
useMascotManifest: () => ({ manifest: null, entry: null, loading: false, error: null }),
}));
vi.mock('../../human/Mascot/manifest/manifestService', () => ({ findMascot: () => null }));
// Timings the component uses (kept in sync with MascotFrameProducer.tsx).
const GREETING_MS = 2500;
const SIGNOFF_MS = 1500;
/**
* A fake `Worker` whose `postMessage({cmd:'start'})` records itself so the test
* can fire a single tick on demand (invoking the producer's `onmessage`, which
* calls `captureFrame`). Real timers/intervals are avoided — the test drives
* ticks explicitly so the capture path runs deterministically under fake
* timers.
*/
const workers: FakeWorker[] = [];
class FakeWorker {
onmessage: ((e: { data: unknown }) => void) | null = null;
started = false;
terminated = false;
constructor() {
workers.push(this);
}
postMessage(msg: { cmd?: string }) {
if (msg?.cmd === 'start') this.started = true;
else if (msg?.cmd === 'stop') this.started = false;
}
terminate() {
this.terminated = true;
}
/** Deliver one 'tick' to the producer, as the interval would. */
tick() {
this.onmessage?.({ data: 'tick' });
}
}
/**
* A fake `WebSocket` that opens synchronously (so `wsReadyRef` flips true) and
* records every `send()` so the capture path's binary frame + JSON probe are
* assertable.
*/
const sockets: FakeWebSocket[] = [];
class FakeWebSocket {
static OPEN = 1;
readyState = FakeWebSocket.OPEN;
binaryType = 'arraybuffer';
onopen: (() => void) | null = null;
onclose: (() => void) | null = null;
onerror: ((e: unknown) => void) | null = null;
sent: unknown[] = [];
closed = false;
constructor() {
sockets.push(this);
// Fire onopen on the next microtask so the effect that assigns
// `ws.onopen = ...` has run before we invoke it.
queueMicrotask(() => this.onopen?.());
}
send(data: unknown) {
this.sent.push(data);
}
close() {
this.closed = true;
this.onclose?.();
}
}
/**
* A fake 2D context + OffscreenCanvas that satisfies the capture path:
* gradient fill, per-cell drawImage, a pixel probe read, and a JPEG blob whose
* `arrayBuffer()` resolves so the buffer is `send()`-able.
*/
class FakeOffscreenCanvas {
drawImageCalls = 0;
constructor(
public width: number,
public height: number
) {
offscreens.push(this);
}
getContext() {
const self = this;
return {
createRadialGradient: () => ({ addColorStop() {} }),
fillStyle: '' as unknown,
fillRect() {},
drawImage() {
self.drawImageCalls++;
},
getImageData: () => ({ data: [128, 128, 128, 255] }),
};
}
convertToBlob() {
return Promise.resolve({ size: 1234, arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)) });
}
}
const offscreens: FakeOffscreenCanvas[] = [];
/** Install the browser globals the ProducerSession effect touches. */
function installBrowserStubs() {
workers.length = 0;
sockets.length = 0;
offscreens.length = 0;
vi.stubGlobal('Worker', FakeWorker);
vi.stubGlobal('WebSocket', FakeWebSocket);
vi.stubGlobal('OffscreenCanvas', FakeOffscreenCanvas);
if (!('createObjectURL' in URL)) {
(URL as unknown as { createObjectURL: () => string }).createObjectURL = () => 'blob:x';
}
if (!('revokeObjectURL' in URL)) {
(URL as unknown as { revokeObjectURL: () => void }).revokeObjectURL = () => {};
}
// jsdom does not implement HTMLMediaElement.play(); the silent keep-alive
// audio the producer creates calls `.play().catch(...)`, so give it a
// resolving stub.
vi.spyOn(HTMLMediaElement.prototype, 'play').mockResolvedValue(undefined);
}
/**
* Wire the mocked `listen` so each event name captures its handler. Returns a
* fn to fire a fake payload for a given event.
*/
function captureListeners() {
const handlers = new Map<string, Listener>();
vi.mocked(listen).mockImplementation((event: string, handler: unknown) => {
handlers.set(event, handler as Listener);
return Promise.resolve(vi.fn());
});
return {
fire(event: string, payload: unknown) {
const h = handlers.get(event);
if (!h) throw new Error(`no listener registered for ${event}`);
h({ payload });
},
has(event: string) {
return handlers.has(event);
},
};
}
const SINGLE_MASCOT_STATE = {
mascot: {
color: 'yellow',
voiceId: null,
voiceGender: 'male',
voiceUseLocaleDefault: false,
selectedMascotId: 'tiny-mascot',
secondaryMascotId: null,
mascotVoices: {},
customMascotGifUrl: null,
customPrimaryColor: '#F7D145',
customSecondaryColor: '#B23C05',
},
};
const DUAL_MASCOT_STATE = { mascot: { ...SINGLE_MASCOT_STATE.mascot, secondaryMascotId: 'toshi' } };
describe('MascotFrameProducer', () => {
afterEach(() => cleanup());
beforeEach(() => {
installBrowserStubs();
});
afterEach(() => {
cleanup();
vi.unstubAllGlobals();
vi.restoreAllMocks();
// Restore the setup.ts default so other files that rely on the shared
// `listen` mock (resolving to an unlisten fn) are unaffected.
vi.mocked(listen).mockReset();
vi.mocked(listen).mockResolvedValue(vi.fn());
});
it('renders nothing when no bus session is active', () => {
const { container } = renderWithProviders(<MascotFrameProducer />);
// Component returns null until a meet-video:bus-started Tauri event fires
expect(container.firstChild).toBeNull();
});
@@ -21,44 +208,332 @@ describe('MascotFrameProducer', () => {
unmount();
}).not.toThrow();
});
it('renders ONE mascot host for a single mascot', async () => {
const bus = captureListeners();
const { container } = renderWithProviders(<MascotFrameProducer />, {
preloadedState: SINGLE_MASCOT_STATE,
});
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'r1', port: 55555 });
});
await waitFor(() => {
expect(container.querySelectorAll('[data-mascot-slot]').length).toBe(1);
});
expect(container.querySelector('[data-mascot-slot="primary"]')).not.toBeNull();
expect(container.querySelector('[data-mascot-slot="secondary"]')).toBeNull();
});
it('renders TWO mascot hosts for two distinct mascots', async () => {
const bus = captureListeners();
const { container } = renderWithProviders(<MascotFrameProducer />, {
preloadedState: DUAL_MASCOT_STATE,
});
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'r2', port: 55556 });
});
await waitFor(() => {
expect(container.querySelectorAll('[data-mascot-slot]').length).toBe(2);
});
expect(container.querySelector('[data-mascot-slot="primary"]')).not.toBeNull();
expect(container.querySelector('[data-mascot-slot="secondary"]')).not.toBeNull();
});
it('ignores a bus-started payload with no port', async () => {
const bus = captureListeners();
const { container } = renderWithProviders(<MascotFrameProducer />, {
preloadedState: SINGLE_MASCOT_STATE,
});
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'r0' });
});
// No port → guard returns early, session never set, nothing renders.
expect(container.querySelector('[data-mascot-slot]')).toBeNull();
});
it('waves during greeting then transitions to active after GREETING_MS', async () => {
vi.useFakeTimers();
try {
const bus = captureListeners();
const { container } = renderWithProviders(<MascotFrameProducer />, {
preloadedState: DUAL_MASCOT_STATE,
});
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'g1', port: 4100 });
await vi.advanceTimersByTimeAsync(0);
});
// Greeting phase: both mascots wave.
const facesDuringGreeting = Array.from(
container.querySelectorAll('[data-mascot-slot] canvas')
).map(c => c.getAttribute('data-face'));
expect(facesDuringGreeting).toEqual(['waving', 'waving']);
// Advance past the greeting window → active phase. With no speaking
// event yet, the active-slot mascot rests (listening) and the other
// shows thinking — no longer both waving.
await act(async () => {
await vi.advanceTimersByTimeAsync(GREETING_MS + 10);
});
const facesAfter = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(
c => c.getAttribute('data-face')
);
expect(facesAfter).not.toEqual(['waving', 'waving']);
expect(facesAfter).toContain('listening');
} finally {
vi.useRealTimers();
}
});
it('holds the sign-off wave for SIGNOFF_MS then clears the session', async () => {
vi.useFakeTimers();
try {
const bus = captureListeners();
const { container } = renderWithProviders(<MascotFrameProducer />, {
preloadedState: DUAL_MASCOT_STATE,
});
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 's1', port: 4200 });
await vi.advanceTimersByTimeAsync(GREETING_MS + 10);
});
// Session mounted (active phase).
expect(container.querySelector('[data-mascot-slot]')).not.toBeNull();
// bus-stopped → signoff phase; both mascots wave goodbye and the
// session stays mounted through the grace window.
await act(async () => {
bus.fire('meet-video:bus-stopped', { requestId: 's1' });
await vi.advanceTimersByTimeAsync(0);
});
const signoffFaces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(
c => c.getAttribute('data-face')
);
expect(signoffFaces).toEqual(['waving', 'waving']);
// Still mounted just before the grace elapses.
await act(async () => {
await vi.advanceTimersByTimeAsync(SIGNOFF_MS - 50);
});
expect(container.querySelector('[data-mascot-slot]')).not.toBeNull();
// Grace elapsed → session cleared → producer renders nothing.
await act(async () => {
await vi.advanceTimersByTimeAsync(100);
});
expect(container.querySelector('[data-mascot-slot]')).toBeNull();
} finally {
vi.useRealTimers();
}
});
it('re-arms a fresh greeting when bus-started fires during the sign-off grace', async () => {
vi.useFakeTimers();
try {
const bus = captureListeners();
const { container } = renderWithProviders(<MascotFrameProducer />, {
preloadedState: DUAL_MASCOT_STATE,
});
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'a1', port: 4300 });
await vi.advanceTimersByTimeAsync(GREETING_MS + 10);
// Stop → enters signoff + arms the clear timer.
bus.fire('meet-video:bus-stopped', { requestId: 'a1' });
await vi.advanceTimersByTimeAsync(SIGNOFF_MS - 200);
// A new session starts before the clear fires → clears the signoff
// timer and restarts greeting.
bus.fire('meet-video:bus-started', { requestId: 'a2', port: 4301 });
await vi.advanceTimersByTimeAsync(0);
});
const faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
c.getAttribute('data-face')
);
expect(faces).toEqual(['waving', 'waving']);
// The old signoff clear must NOT fire now; session stays mounted.
await act(async () => {
await vi.advanceTimersByTimeAsync(SIGNOFF_MS);
});
expect(container.querySelector('[data-mascot-slot]')).not.toBeNull();
} finally {
vi.useRealTimers();
}
});
it('updates speaking state only for a matching requestId (gate)', async () => {
vi.useFakeTimers();
try {
const bus = captureListeners();
const { container } = renderWithProviders(<MascotFrameProducer />, {
preloadedState: DUAL_MASCOT_STATE,
});
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'sp1', port: 4400 });
await vi.advanceTimersByTimeAsync(0);
});
// Reach active phase so the face reflects speaking/slot rather than
// the greeting wave.
await act(async () => {
await vi.advanceTimersByTimeAsync(GREETING_MS + 10);
});
expect(
Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
c.getAttribute('data-face')
)
).not.toEqual(['waving', 'waving']);
// Non-matching requestId is ignored: slot 1 speaking with the wrong id
// must NOT flip any mascot to the speaking face.
await act(async () => {
bus.fire('meet-video:speaking-state', {
requestId: 'STALE',
speaking: true,
activeMascotSlot: 1,
});
await vi.advanceTimersByTimeAsync(0);
});
let faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
c.getAttribute('data-face')
);
expect(faces).not.toContain('speaking');
// Matching requestId with slot 1 speaking → the secondary slot animates
// (speaking), primary shows thinking.
await act(async () => {
bus.fire('meet-video:speaking-state', {
requestId: 'sp1',
speaking: true,
activeMascotSlot: 1,
});
await vi.advanceTimersByTimeAsync(0);
});
faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
c.getAttribute('data-face')
);
expect(faces).toEqual(['thinking', 'speaking']);
// Slot 0 speaking → primary animates, secondary shows thinking. Also
// exercises the `activeMascotSlot === 1 ? 1 : 0` default-to-0 branch.
await act(async () => {
bus.fire('meet-video:speaking-state', {
requestId: 'sp1',
speaking: true,
activeMascotSlot: 0,
});
await vi.advanceTimersByTimeAsync(0);
});
faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
c.getAttribute('data-face')
);
expect(faces).toEqual(['speaking', 'thinking']);
} finally {
vi.useRealTimers();
}
});
it('ignores a speaking-state event with no payload', async () => {
const bus = captureListeners();
renderWithProviders(<MascotFrameProducer />, { preloadedState: SINGLE_MASCOT_STATE });
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'np1', port: 4500 });
});
await waitFor(() => expect(bus.has('meet-video:speaking-state')).toBe(true));
expect(() =>
act(() => {
bus.fire('meet-video:speaking-state', null);
})
).not.toThrow();
});
it('captures and sends a single-mascot frame over the websocket on a worker tick', async () => {
const bus = captureListeners();
renderWithProviders(<MascotFrameProducer />, { preloadedState: SINGLE_MASCOT_STATE });
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'cap1', port: 4600 });
});
// Wait for the WS/worker effect to wire up (worker registered, socket
// open).
await waitFor(() => expect(workers.length).toBeGreaterThan(0));
await act(async () => {
// let the queued microtask fire ws.onopen
await Promise.resolve();
});
await act(async () => {
workers[workers.length - 1].tick();
// captureFrame is async (blob → arrayBuffer); flush its microtasks.
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
});
const ws = sockets[sockets.length - 1];
// A binary ArrayBuffer frame was sent.
const binary = ws.sent.filter(m => m instanceof ArrayBuffer);
expect(binary.length).toBeGreaterThan(0);
// The diagnostic JSON probe was also sent, and reports single-mascot.
const jsonMsgs = ws.sent
.filter((m): m is string => typeof m === 'string')
.map(m => JSON.parse(m));
expect(jsonMsgs.some(p => p.kind === 'producer-pixel-probe' && p.dualEnabled === false)).toBe(
true
);
// Single-cell draw → drawMascotInCell called once.
expect(offscreens[offscreens.length - 1].drawImageCalls).toBe(1);
});
it('captures and sends a dual-mascot frame (two-cell composite) on a worker tick', async () => {
const bus = captureListeners();
renderWithProviders(<MascotFrameProducer />, { preloadedState: DUAL_MASCOT_STATE });
await act(async () => {
bus.fire('meet-video:bus-started', { requestId: 'cap2', port: 4700 });
});
await waitFor(() => expect(workers.length).toBeGreaterThan(0));
// Ensure both mascot slots are mounted so the dual (two-cell) branch runs.
await waitFor(() =>
expect(document.querySelectorAll('[data-mascot-slot="secondary"] canvas').length).toBe(1)
);
await act(async () => {
await Promise.resolve();
});
await act(async () => {
workers[workers.length - 1].tick();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
});
const ws = sockets[sockets.length - 1];
const jsonMsgs = ws.sent
.filter((m): m is string => typeof m === 'string')
.map(m => JSON.parse(m));
expect(
jsonMsgs.some(
p => p.kind === 'producer-pixel-probe' && p.dualEnabled === true && p.secondaryMounted
)
).toBe(true);
// Two-cell draw → drawMascotInCell called twice (primary + secondary).
expect(offscreens[offscreens.length - 1].drawImageCalls).toBe(2);
expect(ws.sent.filter(m => m instanceof ArrayBuffer).length).toBeGreaterThan(0);
});
});
describe('sampleCanvasPixels', () => {
it('returns pixel stats for a canvas with mid-range luma', () => {
// luma = 0.299*128 + 0.587*128 + 0.114*128 ≈ 128
// sampleCanvasPixels is still exported from the producer (re-exported from the
// compositor for back-compat); a light smoke check keeps that surface covered
// here. Full assertions live in mascotFrameCompositor.test.ts.
describe('sampleCanvasPixels (re-export)', () => {
it('is re-exported and returns pixel stats', () => {
const mockCtx = {
getImageData: vi.fn().mockReturnValue({ data: [128, 128, 128, 255] }),
} as unknown as OffscreenCanvasRenderingContext2D;
const result = sampleCanvasPixels(mockCtx, 320, 240);
expect(result).toMatchObject({
avgLuma: 128,
minLuma: 128,
maxLuma: 128,
darkSamples: 0,
brightSamples: 35, // all 35 samples have luma > 32
sampleCount: 35, // 7 cols × 5 rows
});
});
it('counts dark samples correctly for near-black pixels', () => {
// luma ≈ 0.299*4 + 0.587*4 + 0.114*4 ≈ 4 → dark (< 8), not bright (> 32)
const mockCtx = {
getImageData: vi.fn().mockReturnValue({ data: [4, 4, 4, 255] }),
} as unknown as OffscreenCanvasRenderingContext2D;
const result = sampleCanvasPixels(mockCtx, 320, 240);
expect(result).toMatchObject({ darkSamples: 35, brightSamples: 0 });
});
it('returns an error object when getImageData throws', () => {
const mockCtx = {
getImageData: vi.fn().mockImplementation(() => {
throw new Error('canvas tainted');
}),
} as unknown as OffscreenCanvasRenderingContext2D;
const result = sampleCanvasPixels(mockCtx, 320, 240);
expect(result).toMatchObject({ error: 'canvas tainted' });
expect(sampleCanvasPixels(mockCtx, 320, 240)).toMatchObject({ avgLuma: 128, sampleCount: 35 });
});
});
@@ -0,0 +1,214 @@
import { describe, expect, it, vi } from 'vitest';
import {
drawMascotInCell,
FRAME_H,
FRAME_H_DUAL,
FRAME_W,
FRAME_W_DUAL,
MASCOT_INSET,
sampleCanvasPixels,
} from '../mascotFrameCompositor';
/** A drawImage-only ctx stub that records the last destination rect. */
function makeCtx() {
const drawImage =
vi.fn<(image: CanvasImageSource, dx: number, dy: number, dw: number, dh: number) => void>();
return { drawImage };
}
describe('mascotFrameCompositor geometry constants', () => {
it('exposes the locked single/dual frame geometry', () => {
expect(FRAME_W).toBe(320);
expect(FRAME_W_DUAL).toBe(480);
expect(FRAME_H).toBe(240);
expect(FRAME_H_DUAL).toBe(270);
expect(MASCOT_INSET).toBeCloseTo(0.06);
});
it('keeps the dual frame at 16:9 so the camera bridge cover-scale never crops', () => {
// The Tauri camera bridge cover-scales the frame onto a 1280×720 (16:9)
// canvas; any non-16:9 frame loses its overflowing axis. Locking the dual
// frame to 16:9 keeps both mascots' outer edges intact.
expect(FRAME_W_DUAL / FRAME_H_DUAL).toBeCloseTo(16 / 9);
});
});
describe('drawMascotInCell — containment (AC#6)', () => {
it('contain-scales a TALL source and centers it (fits height, letterboxed x)', () => {
const ctx = makeCtx();
// Cell = full single frame; source is taller than wide.
const srcW = 100;
const srcH = 400;
const rect = drawMascotInCell(
ctx,
{} as CanvasImageSource,
0,
0,
FRAME_W,
FRAME_H,
MASCOT_INSET,
srcW,
srcH
);
const fitW = FRAME_W * (1 - 2 * MASCOT_INSET); // 281.6
const fitH = FRAME_H * (1 - 2 * MASCOT_INSET); // 211.2
const scale = Math.min(fitW / srcW, fitH / srcH); // height-bound → fitH/400
const dw = srcW * scale;
const dh = srcH * scale;
const dx = (FRAME_W - dw) / 2;
const dy = (FRAME_H - dh) / 2;
// Height-bound: dh must equal the padded fit height, and never exceed the cell.
expect(dh).toBeCloseTo(fitH);
expect(rect.dw).toBeCloseTo(dw);
expect(rect.dh).toBeCloseTo(dh);
expect(rect.dx).toBeCloseTo(dx);
expect(rect.dy).toBeCloseTo(dy);
// No clipping: the drawn rect stays fully inside the cell bounds.
expect(rect.dx).toBeGreaterThanOrEqual(0);
expect(rect.dy).toBeGreaterThanOrEqual(0);
expect(rect.dx + rect.dw).toBeLessThanOrEqual(FRAME_W + 1e-6);
expect(rect.dy + rect.dh).toBeLessThanOrEqual(FRAME_H + 1e-6);
expect(ctx.drawImage).toHaveBeenCalledWith(expect.anything(), dx, dy, dw, dh);
});
it('contain-scales a WIDE source and centers it (fits width, letterboxed y)', () => {
const ctx = makeCtx();
const srcW = 400;
const srcH = 100;
const rect = drawMascotInCell(
ctx,
{} as CanvasImageSource,
0,
0,
FRAME_W,
FRAME_H,
MASCOT_INSET,
srcW,
srcH
);
const fitW = FRAME_W * (1 - 2 * MASCOT_INSET);
const fitH = FRAME_H * (1 - 2 * MASCOT_INSET);
const scale = Math.min(fitW / srcW, fitH / srcH); // width-bound → fitW/400
const dw = srcW * scale;
const dh = srcH * scale;
// Width-bound: dw must equal the padded fit width.
expect(dw).toBeCloseTo(fitW);
expect(rect.dw).toBeCloseTo(dw);
expect(rect.dh).toBeCloseTo(dh);
// Still fully contained.
expect(rect.dx).toBeGreaterThanOrEqual(0);
expect(rect.dx + rect.dw).toBeLessThanOrEqual(FRAME_W + 1e-6);
expect(rect.dy + rect.dh).toBeLessThanOrEqual(FRAME_H + 1e-6);
});
it('draws into each half-cell in dual mode without crossing the divider', () => {
const half = FRAME_W_DUAL / 2; // 240
// Square source so scale is symmetric and easy to reason about.
const srcW = 200;
const srcH = 200;
const leftCtx = makeCtx();
const left = drawMascotInCell(
leftCtx,
{} as CanvasImageSource,
0,
0,
half,
FRAME_H,
MASCOT_INSET,
srcW,
srcH
);
const rightCtx = makeCtx();
const right = drawMascotInCell(
rightCtx,
{} as CanvasImageSource,
half,
0,
half,
FRAME_H,
MASCOT_INSET,
srcW,
srcH
);
// Left cell stays entirely left of the divider.
expect(left.dx).toBeGreaterThanOrEqual(0);
expect(left.dx + left.dw).toBeLessThanOrEqual(half + 1e-6);
// Right cell stays entirely right of the divider.
expect(right.dx).toBeGreaterThanOrEqual(half - 1e-6);
expect(right.dx + right.dw).toBeLessThanOrEqual(FRAME_W_DUAL + 1e-6);
// Both cells share the same size (identical source + cell dims).
expect(right.dw).toBeCloseTo(left.dw);
expect(right.dh).toBeCloseTo(left.dh);
// The right cell is offset by exactly `half` from the left one.
expect(right.dx - left.dx).toBeCloseTo(half);
});
it('never divides by zero for a not-yet-laid-out (0×0) source', () => {
const ctx = makeCtx();
const rect = drawMascotInCell(
ctx,
{} as CanvasImageSource,
0,
0,
FRAME_W,
FRAME_H,
MASCOT_INSET,
0,
0
);
// With the 1px guard the rect is finite and contained, not NaN.
expect(Number.isFinite(rect.dw)).toBe(true);
expect(Number.isFinite(rect.dh)).toBe(true);
expect(rect.dx).toBeGreaterThanOrEqual(0);
expect(rect.dy).toBeGreaterThanOrEqual(0);
});
});
// Moved from MascotFrameProducer.test.tsx (issue #4277) — sampleCanvasPixels
// now lives in the compositor module.
describe('sampleCanvasPixels', () => {
it('returns pixel stats for a canvas with mid-range luma', () => {
// luma = 0.299*128 + 0.587*128 + 0.114*128 ≈ 128
const mockCtx = {
getImageData: vi.fn().mockReturnValue({ data: [128, 128, 128, 255] }),
} as unknown as OffscreenCanvasRenderingContext2D;
const result = sampleCanvasPixels(mockCtx, 320, 240);
expect(result).toMatchObject({
avgLuma: 128,
minLuma: 128,
maxLuma: 128,
darkSamples: 0,
brightSamples: 35, // all 35 samples have luma > 32
sampleCount: 35, // 7 cols × 5 rows
});
});
it('counts dark samples correctly for near-black pixels', () => {
// luma ≈ 0.299*4 + 0.587*4 + 0.114*4 ≈ 4 → dark (< 8), not bright (> 32)
const mockCtx = {
getImageData: vi.fn().mockReturnValue({ data: [4, 4, 4, 255] }),
} as unknown as OffscreenCanvasRenderingContext2D;
const result = sampleCanvasPixels(mockCtx, 320, 240);
expect(result).toMatchObject({ darkSamples: 35, brightSamples: 0 });
});
it('returns an error object when getImageData throws', () => {
const mockCtx = {
getImageData: vi.fn().mockImplementation(() => {
throw new Error('canvas tainted');
}),
} as unknown as OffscreenCanvasRenderingContext2D;
const result = sampleCanvasPixels(mockCtx, 320, 240);
expect(result).toMatchObject({ error: 'canvas tainted' });
});
});
@@ -0,0 +1,136 @@
import { renderHook } from '@testing-library/react';
import { createElement, type PropsWithChildren } from 'react';
import { Provider } from 'react-redux';
import { describe, expect, it } from 'vitest';
import { createTestStore } from '../../../test/test-utils';
import {
type MeetingPhase,
useMeetingMascots,
type UseMeetingMascotsInput,
} from '../useMeetingMascots';
/**
* Seed the mascot slice via preloadedState. `secondaryMascotId` distinct from
* `selectedMascotId` is what flips `selectDualMascotEnabled` on.
*/
function makeWrapper(mascot: Record<string, unknown>) {
const store = createTestStore({
mascot: {
color: 'yellow',
voiceId: null,
voiceGender: 'male',
voiceUseLocaleDefault: false,
selectedMascotId: null,
secondaryMascotId: null,
mascotVoices: {},
customMascotGifUrl: null,
customPrimaryColor: '#F7D145',
customSecondaryColor: '#B23C05',
...mascot,
},
});
return function Wrapper({ children }: PropsWithChildren) {
return createElement(Provider, { store, children });
};
}
function run(mascot: Record<string, unknown>, input: UseMeetingMascotsInput) {
const { result } = renderHook(() => useMeetingMascots(input), { wrapper: makeWrapper(mascot) });
return result.current;
}
const SINGLE = { selectedMascotId: 'tiny-mascot', secondaryMascotId: null };
const DUAL = { selectedMascotId: 'tiny-mascot', secondaryMascotId: 'toshi' };
describe('useMeetingMascots — dualEnabled gating', () => {
it('is single when no secondary mascot is set', () => {
const state = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase: 'active' });
expect(state.dualEnabled).toBe(false);
expect(state.secondary).toBeNull();
expect(state.primary.mascotId).toBe('tiny-mascot');
});
it('is single when the secondary equals the primary (same mascot picked twice)', () => {
const state = run(
{ selectedMascotId: 'tiny-mascot', secondaryMascotId: 'tiny-mascot' },
{ speaking: false, activeMascotSlot: 0, phase: 'active' }
);
expect(state.dualEnabled).toBe(false);
expect(state.secondary).toBeNull();
});
it('is dual when a distinct secondary mascot is set', () => {
const state = run(DUAL, { speaking: false, activeMascotSlot: 0, phase: 'active' });
expect(state.dualEnabled).toBe(true);
expect(state.primary.mascotId).toBe('tiny-mascot');
expect(state.secondary?.mascotId).toBe('toshi');
});
});
describe('useMeetingMascots — single-mascot face (preserves original behavior)', () => {
it('primary follows speaking → speaking, else idle; secondary null', () => {
const speaking = run(SINGLE, { speaking: true, activeMascotSlot: 0, phase: 'active' });
expect(speaking.primary.face).toBe('speaking');
expect(speaking.secondary).toBeNull();
const silent = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase: 'active' });
expect(silent.primary.face).toBe('idle');
});
it('single-mascot ignores phase for the face (no greeting/signoff wave)', () => {
// The single path deliberately keeps the legacy speaking/idle mapping.
for (const phase of ['greeting', 'active', 'signoff'] as MeetingPhase[]) {
const state = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase });
expect(state.primary.face).toBe('idle');
}
});
});
describe('useMeetingMascots — dual face table', () => {
it('greeting → both slots wave, regardless of speaking/activeSlot', () => {
for (const activeMascotSlot of [0, 1] as const) {
for (const speaking of [false, true]) {
const state = run(DUAL, { speaking, activeMascotSlot, phase: 'greeting' });
expect(state.primary.face).toBe('waving');
expect(state.secondary?.face).toBe('waving');
}
}
});
it('signoff → both slots wave, regardless of speaking/activeSlot', () => {
for (const activeMascotSlot of [0, 1] as const) {
for (const speaking of [false, true]) {
const state = run(DUAL, { speaking, activeMascotSlot, phase: 'signoff' });
expect(state.primary.face).toBe('waving');
expect(state.secondary?.face).toBe('waving');
}
}
});
describe('active phase — activeSlot × speaking', () => {
it('slot 0 active + speaking → primary speaking, secondary thinking', () => {
const s = run(DUAL, { speaking: true, activeMascotSlot: 0, phase: 'active' });
expect(s.primary.face).toBe('speaking');
expect(s.secondary?.face).toBe('thinking');
});
it('slot 0 active + not speaking → primary listening, secondary thinking', () => {
const s = run(DUAL, { speaking: false, activeMascotSlot: 0, phase: 'active' });
expect(s.primary.face).toBe('listening');
expect(s.secondary?.face).toBe('thinking');
});
it('slot 1 active + speaking → secondary speaking, primary thinking', () => {
const s = run(DUAL, { speaking: true, activeMascotSlot: 1, phase: 'active' });
expect(s.secondary?.face).toBe('speaking');
expect(s.primary.face).toBe('thinking');
});
it('slot 1 active + not speaking → secondary listening, primary thinking', () => {
const s = run(DUAL, { speaking: false, activeMascotSlot: 1, phase: 'active' });
expect(s.secondary?.face).toBe('listening');
expect(s.primary.face).toBe('thinking');
});
});
});
@@ -0,0 +1,129 @@
/**
* Pure canvas-geometry helpers for the Meet video producer (issue #4277).
*
* Kept free of any WebGL / DOM-mount dependency so the containment math is
* cheap to unit-test in isolation. The producer (`MascotFrameProducer.tsx`)
* owns the OffscreenCanvas + WebSocket pipeline; this module owns only the
* "where does each mascot get drawn" arithmetic, so the no-clip guarantee
* (AC#6) is proven by the compositor tests rather than a WebGL render.
*/
/** Frame width for a single mascot (unchanged from the original producer). */
export const FRAME_W = 320;
/** Frame height for a single mascot (unchanged from the original producer). */
export const FRAME_H = 240;
/**
* Frame dimensions when two mascots share the frame side-by-side (issue #4277).
* Wider than the single frame so each mascot keeps roughly the single-frame
* cell width instead of being squeezed to half.
*
* 480×270 is deliberately 16:9 — the same aspect as the fake-camera capture
* canvas (1280×720). The Tauri camera bridge cover-scales the received frame
* onto that canvas (`scale = Math.max(W/bw, H/bh)` in `camera_bridge.js`), so a
* frame whose aspect differs from 16:9 gets its overflowing axis cropped. A
* 480×240 (2:1) dual frame would be scaled to 1440×720 and lose ~27 source px
* off each side — clipping the two mascots' outer edges. Matching 16:9 makes the
* cover-scale a pure fit with no crop.
*/
export const FRAME_W_DUAL = 480;
export const FRAME_H_DUAL = 270;
/**
* Fraction of each cell reserved as padding on every side before the mascot
* is scaled to fit. Matches the original single-mascot inset so the framing
* is visually identical in the single path.
*/
export const MASCOT_INSET = 0.06;
/**
* Draw `sourceCanvas` scaled to *contain* (never crop) inside the cell at
* `(cellX, cellY)` of size `cellW × cellH`, centred, with `inset` padding on
* every side.
*
* Contain-scaling (`min` of the two axis ratios) guarantees the mascot always
* fits within the padded cell, so it can never be clipped by the cell edge
* (AC#6) regardless of the source canvas aspect ratio. The mascot is centred
* within the cell so any leftover space is split evenly.
*
* Returns the computed destination rect for the caller's diagnostics (and so
* the geometry is directly assertable in tests) — the draw itself is the side
* effect.
*/
export function drawMascotInCell(
ctx: {
drawImage: (image: CanvasImageSource, dx: number, dy: number, dw: number, dh: number) => void;
},
sourceCanvas: CanvasImageSource,
cellX: number,
cellY: number,
cellW: number,
cellH: number,
inset: number,
srcW: number,
srcH: number
): { dx: number; dy: number; dw: number; dh: number } {
// Guard against a zero-sized source (a canvas that hasn't laid out yet):
// scaling by it would produce NaN and a silent no-op draw.
const safeSrcW = srcW > 0 ? srcW : 1;
const safeSrcH = srcH > 0 ? srcH : 1;
const fitW = cellW * (1 - 2 * inset);
const fitH = cellH * (1 - 2 * inset);
const scale = Math.min(fitW / safeSrcW, fitH / safeSrcH);
const dw = safeSrcW * scale;
const dh = safeSrcH * scale;
const dx = cellX + (cellW - dw) / 2;
const dy = cellY + (cellH - dh) / 2;
ctx.drawImage(sourceCanvas, dx, dy, dw, dh);
return { dx, dy, dw, dh };
}
/**
* Sample a coarse 7×5 grid of luma values from a rendered frame. Used as a
* cheap "is the mascot actually on the frame or is it black?" diagnostic that
* the producer streams over the debug WebSocket every couple of seconds.
*
* Moved here from `MascotFrameProducer.tsx` (issue #4277) so it lives next to
* the rest of the frame geometry; the producer re-exports it for back-compat
* with existing importers.
*/
export function sampleCanvasPixels(
ctx: OffscreenCanvasRenderingContext2D,
width: number,
height: number
) {
const cols = 7;
const rows = 5;
let sum = 0;
let min = 255;
let max = 0;
let count = 0;
let dark = 0;
let bright = 0;
try {
for (let y = 0; y < rows; y++) {
for (let x = 0; x < cols; x++) {
const px = Math.max(0, Math.min(width - 1, Math.floor(((x + 0.5) * width) / cols)));
const py = Math.max(0, Math.min(height - 1, Math.floor(((y + 0.5) * height) / rows)));
const [r, g, b] = ctx.getImageData(px, py, 1, 1).data;
const luma = Math.round(r * 0.299 + g * 0.587 + b * 0.114);
sum += luma;
min = Math.min(min, luma);
max = Math.max(max, luma);
if (luma < 8) dark++;
if (luma > 32) bright++;
count++;
}
}
return {
avgLuma: Math.round(sum / Math.max(1, count)),
minLuma: min,
maxLuma: max,
darkSamples: dark,
brightSamples: bright,
sampleCount: count,
};
} catch (err) {
return { error: String(err instanceof Error ? err.message : err) };
}
}
+150
View File
@@ -0,0 +1,150 @@
/**
* Resolve per-slot mascot render state for the Meet video frame (issue #4277).
*
* The producer (`MascotFrameProducer.tsx`) composites up to two mascots into
* the outgoing camera frame. This hook is the single source of truth for
* *which* mascot each slot shows and *what face* it wears, given the live
* speaking-state event + the meeting phase. Kept as a thin, pure-ish selector
* hook (redux in → render state out) so the face table is unit-testable via
* `renderHook` + `preloadedState` without mounting any WebGL.
*
* Slot mapping (locked by the mascotSlice contract):
* - slot 0 = primary = `selectedMascotId`
* - slot 1 = secondary = `secondaryMascotId`
* and `activeMascotSlot` on the speaking-state event names which slot is
* currently speaking the audio participants hear.
*/
import debug from 'debug';
import { useAppSelector } from '../../store/hooks';
import {
selectDualMascotEnabled,
selectMeetingMascotVoicePair,
selectSecondaryMascotId,
selectSelectedMascotId,
} from '../../store/mascotSlice';
import type { MascotFace } from '../human/Mascot';
const log = debug('meet:mascots');
/**
* Coarse lifecycle phase of the on-camera mascot(s):
* - `greeting` — just joined; both mascots wave hello.
* - `active` — the call is live; faces track who is speaking.
* - `signoff` — the session is tearing down; both wave goodbye.
*/
export type MeetingPhase = 'greeting' | 'active' | 'signoff';
/** Which slot is producing the audio the call currently hears. */
export type ActiveMascotSlot = 0 | 1;
export interface MeetingMascotSlotRender {
/** Manifest mascot id, or `null` for the primary on the default mascot. */
mascotId: string | null;
/** MascotFace to render this tick. */
face: MascotFace;
}
export interface MeetingMascotsRenderState {
/** True when a distinct second mascot is enabled (drives dual composite). */
dualEnabled: boolean;
primary: MeetingMascotSlotRender;
/** Non-null only when `dualEnabled`. */
secondary: MeetingMascotSlotRender | null;
}
export interface UseMeetingMascotsInput {
/** Live from `meet-video:speaking-state` — is audio currently streaming. */
speaking: boolean;
/** Live from `meet-video:speaking-state` — which slot owns that audio. */
activeMascotSlot: ActiveMascotSlot;
/** Meeting lifecycle phase; drives the greeting / sign-off wave. */
phase: MeetingPhase;
}
/**
* Face the *speaking* slot wears while the call is active: mouth-animating
* `speaking` when audio is streaming, otherwise `listening` (attentive rest).
* Both map to the `idle` body pose in `FACE_TO_POSE`, but `speaking` is what
* drives the viseme mouth on the producer, so the distinction is load-bearing.
*/
function activeSpeakerFace(speaking: boolean): MascotFace {
return speaking ? 'speaking' : 'listening';
}
/**
* The face for a single slot in the two-mascot layout.
* - greeting / signoff → both wave (`waving` → `hand_wave` pose).
* - active: the slot that owns the current audio follows
* {@link activeSpeakerFace}; the other slot shows `thinking` — the only
* asset-distinct "reacting / listening" body pose (verified against
* `FACE_TO_POSE` in riveMaps: `listening` collapses to `idle`, so it would
* be visually indistinguishable from the speaker's rest state; `thinking`
* is the distinct pose that reads as "the other mascot is paying
* attention").
*/
function dualSlotFace(slot: ActiveMascotSlot, input: UseMeetingMascotsInput): MascotFace {
if (input.phase === 'greeting' || input.phase === 'signoff') return 'waving';
if (slot === input.activeMascotSlot) return activeSpeakerFace(input.speaking);
return 'thinking';
}
/**
* Compute the per-slot render state. Exposed as a standalone pure function so
* the face table can be exercised directly (the hook is a thin redux wrapper
* over it).
*/
export function computeMeetingMascotsRenderState(
dualEnabled: boolean,
primaryMascotId: string | null,
secondaryMascotId: string | null,
input: UseMeetingMascotsInput
): MeetingMascotsRenderState {
if (!dualEnabled) {
// Single-mascot path — preserves the original producer behavior exactly:
// primary follows speaking → speaking/idle, no secondary slot.
return {
dualEnabled: false,
primary: { mascotId: primaryMascotId, face: input.speaking ? 'speaking' : 'idle' },
secondary: null,
};
}
return {
dualEnabled: true,
primary: { mascotId: primaryMascotId, face: dualSlotFace(0, input) },
secondary: { mascotId: secondaryMascotId, face: dualSlotFace(1, input) },
};
}
/**
* Hook form: reads the mascot slice and folds in the live speaking-state +
* phase to produce the render state the producer composites from.
*/
export function useMeetingMascots(input: UseMeetingMascotsInput): MeetingMascotsRenderState {
const dualEnabled = useAppSelector(selectDualMascotEnabled);
const primaryMascotId = useAppSelector(selectSelectedMascotId);
const secondaryMascotId = useAppSelector(selectSecondaryMascotId);
// Resolve the voice pair too so the primary slot's mascotId matches exactly
// what the join payload sent — the pair is the single source of truth and
// reading it here keeps the on-camera mascot and the spoken voice aligned.
const pair = useAppSelector(selectMeetingMascotVoicePair);
const state = computeMeetingMascotsRenderState(
dualEnabled,
pair.primary.mascotId ?? primaryMascotId,
pair.secondary?.mascotId ?? secondaryMascotId,
input
);
log(
'render state dual=%s phase=%s speaking=%s activeSlot=%d primaryFace=%s secondaryFace=%s',
state.dualEnabled,
input.phase,
input.speaking,
input.activeMascotSlot,
state.primary.face,
state.secondary?.face ?? 'none'
);
return state;
}
+7
View File
@@ -5267,6 +5267,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': 'اختر لون التميمة المستخدم في جميع أنحاء التطبيق',
'settings.mascot.noCharacters': 'لا توجد شخصيات OpenHuman متاحة بعد',
'settings.mascot.noColorVariants': 'لا توجد ألوان متاحة',
'settings.mascot.secondaryHeading': 'ثنائي الاجتماع (تميمة ثانية)',
'settings.mascot.secondaryDesc':
'أضِف تميمة ثانية للاجتماعات. عند تعيين اثنتين تظهران معًا وتتناوبان في الحديث. اتركها بلا لتميمة واحدة.',
'settings.mascot.secondaryNone': 'بلا (تميمة واحدة)',
'settings.mascot.perMascotVoiceHeading': 'أصوات لكل تميمة',
'settings.mascot.primaryVoiceLabel': 'صوت التميمة الأولى',
'settings.mascot.secondaryVoiceLabel': 'صوت التميمة الثانية',
'settings.mascot.voice.current': 'الحالي',
'settings.mascot.voice.customDesc':
'ابحث عن معرّفات الصوت في api.elevenlabs.io/v1/voices أو لوحة تحكم ElevenLabs الخاصة بك. يُخزَّن المعرّف فقط — يبقى مفتاح API الخاص بك على الخادم.',
+7
View File
@@ -5386,6 +5386,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': 'অ্যাপ জুড়ে ব্যবহৃত মাসকটের রঙ বেছে নিন',
'settings.mascot.noCharacters': 'কোনো OpenHuman ক্যারেক্টার এখনও উপলব্ধ নেই',
'settings.mascot.noColorVariants': 'কোনো রঙের ভেরিয়েন্ট নেই',
'settings.mascot.secondaryHeading': 'মিটিং জুটি (দ্বিতীয় ম্যাসকট)',
'settings.mascot.secondaryDesc':
'মিটিংয়ের জন্য একটি দ্বিতীয় ম্যাসকট যোগ করুন। দুটি সেট করা থাকলে তারা একসাথে দেখা দেয় এবং পালা করে কথা বলে। একটি ম্যাসকটের জন্য কোনোটি নয় রাখুন।',
'settings.mascot.secondaryNone': 'কোনোটি নয় (একটি ম্যাসকট)',
'settings.mascot.perMascotVoiceHeading': 'প্রতি ম্যাসকটের কণ্ঠস্বর',
'settings.mascot.primaryVoiceLabel': 'প্রথম ম্যাসকটের কণ্ঠস্বর',
'settings.mascot.secondaryVoiceLabel': 'দ্বিতীয় ম্যাসকটের কণ্ঠস্বর',
'settings.mascot.voice.current': 'বর্তমান',
'settings.mascot.voice.customDesc':
'ভয়েস আইডি খুঁজুন api.elevenlabs.io/v1/voices বা আপনার ElevenLabs ড্যাশবোর্ডে। শুধু আইডি সংরক্ষিত থাকে — আপনার API কী ব্যাকএন্ডে থাকে।',
+7
View File
@@ -5528,6 +5528,13 @@ const messages: TranslationMap = {
'Wähle die Maskottchenfarbe aus, die in der gesamten App verwendet wird',
'settings.mascot.noCharacters': 'Es sind noch keine OpenHuman Zeichen verfügbar',
'settings.mascot.noColorVariants': 'Keine Farbvarianten',
'settings.mascot.secondaryHeading': 'Meeting-Duo (zweites Maskottchen)',
'settings.mascot.secondaryDesc':
'Füge ein zweites Maskottchen für Meetings hinzu. Sind zwei gesetzt, erscheinen sie gemeinsam und sprechen abwechselnd. Für ein einzelnes Maskottchen auf Keines lassen.',
'settings.mascot.secondaryNone': 'Keines (einzelnes Maskottchen)',
'settings.mascot.perMascotVoiceHeading': 'Stimmen pro Maskottchen',
'settings.mascot.primaryVoiceLabel': 'Stimme des ersten Maskottchens',
'settings.mascot.secondaryVoiceLabel': 'Stimme des zweiten Maskottchens',
'settings.mascot.voice.current': 'aktuell',
'settings.mascot.voice.customDesc':
'Sprach-IDs findest du unter api.elevenlabs.io/v1/voices oder in deinem ElevenLabs-Dashboard. Es wird nur die ID gespeichert dein API-Schlüssel bleibt im Backend.',
+7
View File
@@ -6069,6 +6069,13 @@ const en: TranslationMap = {
'settings.mascot.menuDesc': 'Pick the mascot color used across the app',
'settings.mascot.noCharacters': 'No OpenHuman characters are available yet',
'settings.mascot.noColorVariants': 'No color variants',
'settings.mascot.secondaryHeading': 'Meeting duo (second mascot)',
'settings.mascot.secondaryDesc':
'Add a second mascot for meetings. When two are set they appear together and take turns speaking. Leave as None for a single mascot.',
'settings.mascot.secondaryNone': 'None (single mascot)',
'settings.mascot.perMascotVoiceHeading': 'Per-mascot voices',
'settings.mascot.primaryVoiceLabel': 'First mascot voice',
'settings.mascot.secondaryVoiceLabel': 'Second mascot voice',
'settings.mascot.voice.current': 'current',
'settings.mascot.voice.customDesc':
'Find voice ids at api.elevenlabs.io/v1/voices or your ElevenLabs dashboard. Only the id is stored — your API key stays on the backend.',
+7
View File
@@ -5484,6 +5484,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': 'Elige el color de la mascota usado en toda la app',
'settings.mascot.noCharacters': 'Aún no hay personajes de OpenHuman disponibles',
'settings.mascot.noColorVariants': 'Sin variantes de color',
'settings.mascot.secondaryHeading': 'Dúo de reunión (segunda mascota)',
'settings.mascot.secondaryDesc':
'Añade una segunda mascota para las reuniones. Al fijar dos, aparecen juntas y hablan por turnos. Déjalo en Ninguna para una sola mascota.',
'settings.mascot.secondaryNone': 'Ninguna (una sola mascota)',
'settings.mascot.perMascotVoiceHeading': 'Voces por mascota',
'settings.mascot.primaryVoiceLabel': 'Voz de la primera mascota',
'settings.mascot.secondaryVoiceLabel': 'Voz de la segunda mascota',
'settings.mascot.voice.current': 'actual',
'settings.mascot.voice.customDesc':
'Encuentra los ID de voz en api.elevenlabs.io/v1/voices o en tu panel de ElevenLabs. Solo se almacena el ID — tu clave de API permanece en el backend.',
+7
View File
@@ -5504,6 +5504,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': "Choisis la couleur de la mascotte utilisée dans toute l'application",
'settings.mascot.noCharacters': "Aucun personnage OpenHuman n'est encore disponible",
'settings.mascot.noColorVariants': 'Aucune variante de couleur',
'settings.mascot.secondaryHeading': 'Duo de réunion (seconde mascotte)',
'settings.mascot.secondaryDesc':
'Ajoute une seconde mascotte pour les réunions. Lorsque deux sont définies, elles apparaissent ensemble et parlent à tour de rôle. Laisse sur Aucune pour une seule mascotte.',
'settings.mascot.secondaryNone': 'Aucune (une seule mascotte)',
'settings.mascot.perMascotVoiceHeading': 'Voix par mascotte',
'settings.mascot.primaryVoiceLabel': 'Voix de la première mascotte',
'settings.mascot.secondaryVoiceLabel': 'Voix de la seconde mascotte',
'settings.mascot.voice.current': 'actuel',
'settings.mascot.voice.customDesc':
"Trouvez les identifiants vocaux sur api.elevenlabs.io/v1/voices ou dans votre tableau de bord ElevenLabs. Seul l'identifiant est stocké — votre clé API reste sur le backend.",
+7
View File
@@ -5386,6 +5386,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': 'ऐप में उपयोग होने वाला मास्कॉट रंग चुनें',
'settings.mascot.noCharacters': 'अभी तक कोई OpenHuman कैरेक्टर उपलब्ध नहीं है',
'settings.mascot.noColorVariants': 'कोई कलर वेरिएंट नहीं',
'settings.mascot.secondaryHeading': 'मीटिंग जोड़ी (दूसरा मैस्कट)',
'settings.mascot.secondaryDesc':
'मीटिंग के लिए एक दूसरा मैस्कट जोड़ें। दो सेट होने पर वे साथ दिखते हैं और बारी-बारी से बोलते हैं। एक ही मैस्कट के लिए कोई नहीं पर छोड़ें।',
'settings.mascot.secondaryNone': 'कोई नहीं (एकल मैस्कट)',
'settings.mascot.perMascotVoiceHeading': 'प्रति-मैस्कट आवाज़ें',
'settings.mascot.primaryVoiceLabel': 'पहले मैस्कट की आवाज़',
'settings.mascot.secondaryVoiceLabel': 'दूसरे मैस्कट की आवाज़',
'settings.mascot.voice.current': 'वर्तमान',
'settings.mascot.voice.customDesc':
'वॉइस आईडी api.elevenlabs.io/v1/voices या अपने ElevenLabs डैशबोर्ड पर खोजें। केवल आईडी संग्रहीत होती है — आपकी API कुंजी बैकएंड पर रहती है।',
+7
View File
@@ -5402,6 +5402,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': 'Pilih warna maskot yang digunakan di seluruh aplikasi',
'settings.mascot.noCharacters': 'Belum ada karakter OpenHuman yang tersedia',
'settings.mascot.noColorVariants': 'Tidak ada varian warna',
'settings.mascot.secondaryHeading': 'Duo rapat (maskot kedua)',
'settings.mascot.secondaryDesc':
'Tambahkan maskot kedua untuk rapat. Bila dua diatur, keduanya muncul bersama dan berbicara bergantian. Biarkan Tidak ada untuk satu maskot saja.',
'settings.mascot.secondaryNone': 'Tidak ada (maskot tunggal)',
'settings.mascot.perMascotVoiceHeading': 'Suara per maskot',
'settings.mascot.primaryVoiceLabel': 'Suara maskot pertama',
'settings.mascot.secondaryVoiceLabel': 'Suara maskot kedua',
'settings.mascot.voice.current': 'saat ini',
'settings.mascot.voice.customDesc':
'Temukan ID suara di api.elevenlabs.io/v1/voices atau dasbor ElevenLabs Anda. Hanya ID yang disimpan — kunci API Anda tetap di backend.',
+7
View File
@@ -5473,6 +5473,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': "Scegli il colore della mascotte usato in tutta l'app",
'settings.mascot.noCharacters': 'Nessun personaggio OpenHuman disponibile',
'settings.mascot.noColorVariants': 'Nessuna variante di colore',
'settings.mascot.secondaryHeading': 'Duo riunione (seconda mascotte)',
'settings.mascot.secondaryDesc':
'Aggiungi una seconda mascotte per le riunioni. Quando ne imposti due, compaiono insieme e parlano a turno. Lascia su Nessuna per una sola mascotte.',
'settings.mascot.secondaryNone': 'Nessuna (una sola mascotte)',
'settings.mascot.perMascotVoiceHeading': 'Voci per mascotte',
'settings.mascot.primaryVoiceLabel': 'Voce della prima mascotte',
'settings.mascot.secondaryVoiceLabel': 'Voce della seconda mascotte',
'settings.mascot.voice.current': 'attuale',
'settings.mascot.voice.customDesc':
"Trova gli ID vocali su api.elevenlabs.io/v1/voices o nel tuo dashboard ElevenLabs. Viene salvato solo l'ID — la tua chiave API rimane sul backend.",
+7
View File
@@ -5324,6 +5324,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': '앱 전체에서 사용되는 마스코트 색상 선택',
'settings.mascot.noCharacters': '아직 사용할 수 있는 OpenHuman 캐릭터가 없습니다',
'settings.mascot.noColorVariants': '색상 변형 없음',
'settings.mascot.secondaryHeading': '회의 듀오 (두 번째 마스코트)',
'settings.mascot.secondaryDesc':
'회의에 두 번째 마스코트를 추가하세요. 둘을 설정하면 함께 나타나 번갈아 말합니다. 마스코트를 하나만 쓰려면 없음으로 두세요.',
'settings.mascot.secondaryNone': '없음 (단일 마스코트)',
'settings.mascot.perMascotVoiceHeading': '마스코트별 음성',
'settings.mascot.primaryVoiceLabel': '첫 번째 마스코트 음성',
'settings.mascot.secondaryVoiceLabel': '두 번째 마스코트 음성',
'settings.mascot.voice.current': '현재',
'settings.mascot.voice.customDesc':
'api.elevenlabs.io/v1/voices 또는 ElevenLabs 대시보드에서 음성 ID를 찾으세요. ID만 저장되며 API 키는 백엔드에 유지됩니다.',
+7
View File
@@ -5464,6 +5464,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': 'Wybierz kolor maskotki używany w aplikacji',
'settings.mascot.noCharacters': 'Nie ma jeszcze dostępnych postaci OpenHuman',
'settings.mascot.noColorVariants': 'Brak wariantów kolorystycznych',
'settings.mascot.secondaryHeading': 'Duet spotkania (druga maskotka)',
'settings.mascot.secondaryDesc':
'Dodaj drugą maskotkę do spotkań. Gdy ustawisz dwie, pojawiają się razem i mówią na zmianę. Pozostaw Brak, aby korzystać z jednej maskotki.',
'settings.mascot.secondaryNone': 'Brak (pojedyncza maskotka)',
'settings.mascot.perMascotVoiceHeading': 'Głosy poszczególnych maskotek',
'settings.mascot.primaryVoiceLabel': 'Głos pierwszej maskotki',
'settings.mascot.secondaryVoiceLabel': 'Głos drugiej maskotki',
'settings.mascot.voice.current': 'bieżący',
'settings.mascot.voice.customDesc':
'Znajdź ID głosów w api.elevenlabs.io/v1/voices lub w panelu ElevenLabs. Przechowywane jest tylko ID — Twój klucz API pozostaje na backendzie.',
+7
View File
@@ -5473,6 +5473,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': 'Escolha a cor do mascote usada em todo o app',
'settings.mascot.noCharacters': 'Nenhum personagem do OpenHuman disponível ainda',
'settings.mascot.noColorVariants': 'Sem variantes de cor',
'settings.mascot.secondaryHeading': 'Dupla de reunião (segundo mascote)',
'settings.mascot.secondaryDesc':
'Adicione um segundo mascote para as reuniões. Com dois definidos, eles aparecem juntos e falam em turnos. Deixe como Nenhum para um único mascote.',
'settings.mascot.secondaryNone': 'Nenhum (mascote único)',
'settings.mascot.perMascotVoiceHeading': 'Vozes por mascote',
'settings.mascot.primaryVoiceLabel': 'Voz do primeiro mascote',
'settings.mascot.secondaryVoiceLabel': 'Voz do segundo mascote',
'settings.mascot.voice.current': 'atual',
'settings.mascot.voice.customDesc':
'Encontre IDs de voz em api.elevenlabs.io/v1/voices ou no seu painel da ElevenLabs. Apenas o ID é armazenado — sua chave de API permanece no backend.',
+7
View File
@@ -5439,6 +5439,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': 'Выберите цвет маскота, используемый во всем приложении',
'settings.mascot.noCharacters': 'Персонажи OpenHuman пока недоступны',
'settings.mascot.noColorVariants': 'Нет цветовых вариантов',
'settings.mascot.secondaryHeading': 'Дуэт для встреч (второй маскот)',
'settings.mascot.secondaryDesc':
'Добавьте второго маскота для встреч. Когда заданы двое, они появляются вместе и говорят по очереди. Оставьте Нет для одного маскота.',
'settings.mascot.secondaryNone': 'Нет (один маскот)',
'settings.mascot.perMascotVoiceHeading': 'Голоса для каждого маскота',
'settings.mascot.primaryVoiceLabel': 'Голос первого маскота',
'settings.mascot.secondaryVoiceLabel': 'Голос второго маскота',
'settings.mascot.voice.current': 'текущий',
'settings.mascot.voice.customDesc':
'Идентификаторы голосов можно найти на api.elevenlabs.io/v1/voices или в вашей панели ElevenLabs. Сохраняется только идентификатор — ваш API-ключ остаётся на бэкенде.',
+7
View File
@@ -5103,6 +5103,13 @@ const messages: TranslationMap = {
'settings.mascot.menuDesc': '选择应用内使用的吉祥物颜色',
'settings.mascot.noCharacters': '暂无可用的 OpenHuman 角色',
'settings.mascot.noColorVariants': '无颜色变体',
'settings.mascot.secondaryHeading': '会议搭档(第二个吉祥物)',
'settings.mascot.secondaryDesc':
'为会议添加第二个吉祥物。设置两个后,它们会一起出现并轮流说话。若只用一个吉祥物,请保持为无。',
'settings.mascot.secondaryNone': '无(单个吉祥物)',
'settings.mascot.perMascotVoiceHeading': '每个吉祥物的语音',
'settings.mascot.primaryVoiceLabel': '第一个吉祥物的语音',
'settings.mascot.secondaryVoiceLabel': '第二个吉祥物的语音',
'settings.mascot.voice.current': '当前',
'settings.mascot.voice.customDesc':
'在 api.elevenlabs.io/v1/voices 或您的 ElevenLabs 仪表板中查找语音 ID。仅存储 ID — 您的 API 密钥保留在后端。',
@@ -77,6 +77,35 @@ describe('joinMeetCall', () => {
});
});
it('forwards per-mascot voices to the shell when provided (issue #4277)', async () => {
vi.mocked(callCoreRpc).mockResolvedValueOnce({
ok: true,
request_id: 'req-2',
meet_url: 'https://meet.google.com/abc-defg-hij',
display_name: 'Agent Alice',
} as never);
vi.mocked(invoke).mockResolvedValueOnce('meet-call-req-2');
await joinMeetCall({
meetUrl: 'https://meet.google.com/abc-defg-hij',
displayName: 'Agent Alice',
ownerDisplayName: 'Owner Bob',
primaryVoiceId: ' voice-a ',
secondaryVoiceId: 'voice-b',
});
expect(invoke).toHaveBeenCalledWith('meet_call_open_window', {
args: {
request_id: 'req-2',
meet_url: 'https://meet.google.com/abc-defg-hij',
display_name: 'Agent Alice',
owner_display_name: 'Owner Bob',
primary_voice_id: 'voice-a',
secondary_voice_id: 'voice-b',
},
});
});
it('throws if core rejects the request', async () => {
vi.mocked(callCoreRpc).mockResolvedValueOnce({ ok: false } as never);
await expect(
@@ -233,6 +262,60 @@ describe('joinMeetViaBackendBot', () => {
expect(result).toEqual({ meetUrl: 'https://meet.google.com/abc-defg-hij', platform: 'gmeet' });
});
it('omits mascots[] for a single-mascot join (unchanged wire shape)', async () => {
vi.mocked(callCoreRpc).mockResolvedValueOnce({
ok: true,
meet_url: 'https://meet.google.com/abc-defg-hij',
platform: 'gmeet',
} as never);
await joinMeetViaBackendBot({
meetUrl: 'https://meet.google.com/abc-defg-hij',
mascotId: 'yellow',
});
// No `mascots` key on the params → the single `mascot_id` path is
// byte-identical to before (undefined is dropped by the RPC boundary).
const params = vi.mocked(callCoreRpc).mock.calls[0][0].params as Record<string, unknown>;
expect(params.mascots).toBeUndefined();
expect(params.mascot_id).toBe('yellow');
});
it('maps dual mascots[] to snake_case slots and drops blank ids (issue #4277)', async () => {
vi.mocked(callCoreRpc).mockResolvedValueOnce({
ok: true,
meet_url: 'https://meet.google.com/abc-defg-hij',
platform: 'gmeet',
} as never);
await joinMeetViaBackendBot({
meetUrl: 'https://meet.google.com/abc-defg-hij',
mascots: [
{
mascotId: ' tiny-mascot ',
name: ' Tiny ',
voiceId: ' voice-a ',
riveColors: { primaryColor: '#111', secondaryColor: '#222' },
},
{ mascotId: 'toshi', voiceId: 'voice-b' },
{ mascotId: ' ', voiceId: 'ignored' },
],
});
const params = vi.mocked(callCoreRpc).mock.calls[0][0].params as Record<string, unknown>;
expect(params.mascots).toEqual([
{
mascot_id: 'tiny-mascot',
// Name-addressed routing (#4277 follow-up): trimmed + forwarded.
name: 'Tiny',
voice_id: 'voice-a',
rive_colors: { primary_color: '#111', secondary_color: '#222' },
},
// Slot 1 supplies no name → `name` omitted (undefined).
{ mascot_id: 'toshi', name: undefined, voice_id: 'voice-b', rive_colors: undefined },
]);
});
it('rejects an empty meeting link before contacting core', async () => {
await expect(joinMeetViaBackendBot({ meetUrl: ' ' })).rejects.toThrow(/meeting link/i);
expect(callCoreRpc).not.toHaveBeenCalled();
+77 -7
View File
@@ -10,11 +10,32 @@
// Splitting it this way keeps platform-specific window code in the shell
// while the validation rules live (and are tested) in the core.
import { invoke } from '@tauri-apps/api/core';
import debug from 'debug';
import { isTauri } from '../utils/tauriCommands/common';
import { apiClient } from './apiClient';
import { callCoreRpc } from './coreRpcClient';
// Shares the sibling hook's namespace (`useMeetingMascots.ts`) so the whole
// dual-mascot flow can be traced under one prefix.
const log = debug('meet:mascots');
/**
* Map optional Rive colors to the backend's snake_case wire shape, trimming
* blanks and collapsing an all-empty pair to `undefined`. Shared by the
* top-level and per-slot color payloads so the two can't drift.
*/
function mapRiveColors(colors?: {
primaryColor?: string;
secondaryColor?: string;
}): { primary_color?: string; secondary_color?: string } | undefined {
if (!colors) return undefined;
const primary = colors.primaryColor?.trim() || undefined;
const secondary = colors.secondaryColor?.trim() || undefined;
if (!primary && !secondary) return undefined;
return { primary_color: primary, secondary_color: secondary };
}
export type MeetJoinCallInput = {
meetUrl: string;
/** Bot's display name in Meet's "Your name" prompt. */
@@ -27,6 +48,14 @@ export type MeetJoinCallInput = {
* (no wakes fire) which is the safe default during the rollout.
*/
ownerDisplayName?: string;
/**
* ElevenLabs voice id for the primary mascot (issue #4277). When two
* mascots are enabled the core alternates the speaking voice per reply.
* Omit for single-mascot calls (core keeps its default voice).
*/
primaryVoiceId?: string;
/** Voice id for the secondary mascot; present only in two-mascot calls. */
secondaryVoiceId?: string;
};
export type MeetJoinCallResult = {
@@ -86,6 +115,10 @@ export async function joinMeetCall(input: MeetJoinCallInput): Promise<MeetJoinCa
// hand it to the wake-word gate. See feat/mascot-meet-flowA
// Plan C — owner-only privacy lock.
owner_display_name: ownerDisplayName,
// Per-mascot voices for speaker alternation (issue #4277). Absent
// → core keeps its single default voice (unchanged behavior).
primary_voice_id: input.primaryVoiceId?.trim() || undefined,
secondary_voice_id: input.secondaryVoiceId?.trim() || undefined,
},
});
} catch (err) {
@@ -264,6 +297,25 @@ export type BackendMeetJoinInput = {
systemPrompt?: string;
mascotId?: string;
riveColors?: { primaryColor?: string; secondaryColor?: string };
/**
* Dual-mascot config (issue #4277). Up to 2 slots; slot 0 = primary,
* slot 1 = secondary. When two are present the backend bot renders both
* mascots and alternates who speaks each reply (each with its own
* `voiceId`). Omit / single element = legacy single-mascot behavior via
* `mascotId`.
*/
mascots?: Array<{
mascotId: string;
/**
* Human-facing mascot name (from the manifest, e.g. "Toshi"). Enables
* name-addressed routing (#4277 follow-up): a participant who says
* "Hey Toshi …" is answered by this slot instead of the mechanical
* alternation. Omit → that slot is not name-addressable.
*/
name?: string;
riveColors?: { primaryColor?: string; secondaryColor?: string };
voiceId?: string;
}>;
/** Only respond to messages from this participant name (empty = respond to all). */
respondToParticipant?: string;
/** Wake phrase the participant must say before the bot responds (empty = no wake phrase). */
@@ -291,6 +343,29 @@ export async function joinMeetViaBackendBot(
const meetUrl = input.meetUrl.trim();
if (!meetUrl) throw new Error('Please paste a meeting link.');
// Dual-mascot slots (issue #4277), mapped to the backend's snake_case wire
// shape. Absent → backend falls back to `mascot_id`.
const slots = input.mascots?.filter(m => m.mascotId?.trim());
const mascots =
slots && slots.length > 0
? slots.map(m => ({
mascot_id: m.mascotId.trim(),
name: m.name?.trim() || undefined,
voice_id: m.voiceId?.trim() || undefined,
rive_colors: mapRiveColors(m.riveColors),
}))
: undefined;
// Flow/state metadata only — no participant names, voices, or the meet URL.
log(
'backend bot join corr=%s dual=%s slots=%d singleMascot=%s riveColors=%s',
input.correlationId?.trim() || '-',
Boolean(input.mascots?.length),
mascots?.length ?? 0,
Boolean(input.mascotId?.trim()),
Boolean(mapRiveColors(input.riveColors))
);
const result = await callCoreRpc<CoreBackendMeetJoinResponse>({
method: 'openhuman.agent_meetings_join',
params: {
@@ -304,13 +379,8 @@ export async function joinMeetViaBackendBot(
wake_phrase: input.wakePhrase?.trim() || undefined,
correlation_id: input.correlationId?.trim() || undefined,
listen_only: input.listenOnly ?? undefined,
rive_colors: (() => {
if (!input.riveColors) return undefined;
const primary = input.riveColors.primaryColor?.trim() || undefined;
const secondary = input.riveColors.secondaryColor?.trim() || undefined;
if (!primary && !secondary) return undefined;
return { primary_color: primary, secondary_color: secondary };
})(),
rive_colors: mapRiveColors(input.riveColors),
mascots,
},
});
+172
View File
@@ -6,13 +6,21 @@ import reducer, {
isCustomMascotGifUrl,
MAX_CUSTOM_MASCOT_GIF_URL_LEN,
MAX_MASCOT_VOICE_ID_LEN,
MAX_MASCOT_VOICES,
selectCustomMascotGifUrl,
selectDualMascotEnabled,
selectMascotColor,
selectMascotVoiceFor,
selectMascotVoiceId,
selectMascotVoices,
selectMeetingMascotVoicePair,
selectSecondaryMascotId,
selectSelectedMascotId,
setCustomMascotGifUrl,
setMascotColor,
setMascotVoice,
setMascotVoiceId,
setSecondaryMascotId,
setSelectedMascotId,
SUPPORTED_MASCOT_COLORS,
} from '../mascotSlice';
@@ -277,4 +285,168 @@ describe('mascotSlice', () => {
expect(state.customMascotGifUrl).toBeNull();
});
});
// Issue #4277 — second meeting mascot + per-mascot voices.
describe('secondary mascot id', () => {
it('defaults to null', () => {
const state = reducer(undefined, { type: '@@INIT' });
expect(state.secondaryMascotId).toBeNull();
expect(selectSecondaryMascotId({ mascot: state })).toBeNull();
});
it('setSecondaryMascotId trims and stores a valid id', () => {
const state = reducer(undefined, setSecondaryMascotId(' toshi '));
expect(state.secondaryMascotId).toBe('toshi');
});
it('null / whitespace / oversize input clears it', () => {
const set = reducer(undefined, setSecondaryMascotId('toshi'));
expect(reducer(set, setSecondaryMascotId(null)).secondaryMascotId).toBeNull();
expect(reducer(set, setSecondaryMascotId(' ')).secondaryMascotId).toBeNull();
const tooLong = 'x'.repeat(MAX_MASCOT_VOICE_ID_LEN + 1);
expect(reducer(set, setSecondaryMascotId(tooLong)).secondaryMascotId).toBeNull();
});
it('is cleared when a custom GIF avatar is set (mutually exclusive)', () => {
let state = reducer(undefined, setSecondaryMascotId('toshi'));
state = reducer(state, setCustomMascotGifUrl('https://example.com/avatar.gif'));
expect(state.secondaryMascotId).toBeNull();
});
it('resetUserScopedState clears it', () => {
let state = reducer(undefined, setSecondaryMascotId('toshi'));
state = reducer(state, resetUserScopedState());
expect(state.secondaryMascotId).toBeNull();
});
const rehydrate = (key: string, payload?: unknown) => ({ type: REHYDRATE, key, payload });
it('restores a valid persisted id; scrubs invalid; tolerates missing (older blobs)', () => {
expect(
reducer(undefined, rehydrate('mascot', { secondaryMascotId: 'toshi' })).secondaryMascotId
).toBe('toshi');
expect(
reducer(undefined, rehydrate('mascot', { secondaryMascotId: ' ' })).secondaryMascotId
).toBeNull();
expect(
reducer(undefined, rehydrate('mascot', { color: 'navy' })).secondaryMascotId
).toBeNull();
});
});
describe('per-mascot voices', () => {
it('defaults to an empty map', () => {
const state = reducer(undefined, { type: '@@INIT' });
expect(state.mascotVoices).toEqual({});
expect(selectMascotVoices({ mascot: state })).toEqual({});
expect(selectMascotVoiceFor('toshi')({ mascot: state })).toBeNull();
});
it('setMascotVoice records a trimmed mascotId → voiceId entry', () => {
const state = reducer(undefined, setMascotVoice({ mascotId: ' toshi ', voiceId: ' v-1 ' }));
expect(state.mascotVoices).toEqual({ toshi: 'v-1' });
expect(selectMascotVoiceFor('toshi')({ mascot: state })).toBe('v-1');
});
it('setMascotVoice with null / invalid voiceId removes the entry', () => {
const set = reducer(undefined, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-1' }));
expect(
reducer(set, setMascotVoice({ mascotId: 'toshi', voiceId: null })).mascotVoices
).toEqual({});
expect(
reducer(set, setMascotVoice({ mascotId: 'toshi', voiceId: ' ' })).mascotVoices
).toEqual({});
});
it('ignores an invalid mascotId key', () => {
const state = reducer(undefined, setMascotVoice({ mascotId: ' ', voiceId: 'v-1' }));
expect(state.mascotVoices).toEqual({});
});
it('caps new keys at MAX_MASCOT_VOICES but still updates existing ones', () => {
let state = reducer(undefined, { type: '@@INIT' });
for (let i = 0; i < MAX_MASCOT_VOICES; i += 1) {
state = reducer(state, setMascotVoice({ mascotId: `m-${i}`, voiceId: `v-${i}` }));
}
expect(Object.keys(state.mascotVoices)).toHaveLength(MAX_MASCOT_VOICES);
// A brand-new key over the cap is refused…
const overflow = reducer(state, setMascotVoice({ mascotId: 'm-extra', voiceId: 'v-x' }));
expect(overflow.mascotVoices['m-extra']).toBeUndefined();
// …but re-voicing an existing mascot is always allowed.
const updated = reducer(state, setMascotVoice({ mascotId: 'm-0', voiceId: 'v-new' }));
expect(updated.mascotVoices['m-0']).toBe('v-new');
});
it('resetUserScopedState clears the map', () => {
const dirty = reducer(undefined, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-1' }));
expect(reducer(dirty, resetUserScopedState()).mascotVoices).toEqual({});
});
const rehydrate = (key: string, payload?: unknown) => ({ type: REHYDRATE, key, payload });
it('REHYDRATE keeps only valid entries and tolerates a missing/garbage map', () => {
const restored = reducer(
undefined,
rehydrate('mascot', { mascotVoices: { toshi: 'v-1', bad: ' ', ' ': 'v-2', ok: 'v-3' } })
);
expect(restored.mascotVoices).toEqual({ toshi: 'v-1', ok: 'v-3' });
expect(reducer(undefined, rehydrate('mascot', { color: 'navy' })).mascotVoices).toEqual({});
expect(
reducer(undefined, rehydrate('mascot', { mascotVoices: 'not-an-object' })).mascotVoices
).toEqual({});
});
});
describe('dual-mascot resolution', () => {
it('selectDualMascotEnabled is false with one or duplicate mascots, true with two distinct', () => {
const one = reducer(undefined, setSelectedMascotId('tiny-mascot'));
expect(selectDualMascotEnabled({ mascot: one })).toBe(false);
const dup = reducer(one, setSecondaryMascotId('tiny-mascot'));
expect(selectDualMascotEnabled({ mascot: dup })).toBe(false);
const two = reducer(one, setSecondaryMascotId('toshi'));
expect(selectDualMascotEnabled({ mascot: two })).toBe(true);
});
it('selectMeetingMascotVoicePair returns null secondary when single', () => {
let state = reducer(undefined, setSelectedMascotId('tiny-mascot'));
state = reducer(state, setMascotVoiceId('v-primary'));
const pair = selectMeetingMascotVoicePair({ mascot: state });
expect(pair.secondary).toBeNull();
expect(pair.primary.mascotId).toBe('tiny-mascot');
expect(pair.primary.voiceId).toBe('v-primary');
});
it('selectMeetingMascotVoicePair resolves each slot to its per-mascot voice', () => {
let state = reducer(undefined, setSelectedMascotId('tiny-mascot'));
state = reducer(state, setSecondaryMascotId('toshi'));
state = reducer(state, setMascotVoice({ mascotId: 'tiny-mascot', voiceId: 'v-tiny' }));
state = reducer(state, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-toshi' }));
const pair = selectMeetingMascotVoicePair({ mascot: state });
expect(pair.primary).toEqual({ mascotId: 'tiny-mascot', voiceId: 'v-tiny' });
expect(pair.secondary).toEqual({ mascotId: 'toshi', voiceId: 'v-toshi' });
});
it('per-mascot voice falls back to the effective single voice when unset', () => {
let state = reducer(undefined, setSelectedMascotId('tiny-mascot'));
state = reducer(state, setSecondaryMascotId('toshi'));
state = reducer(state, setMascotVoiceId('v-effective'));
const pair = selectMeetingMascotVoicePair({ mascot: state });
// Neither mascot has an explicit override → both use the effective voice.
expect(pair.primary.voiceId).toBe('v-effective');
expect(pair.secondary?.voiceId).toBe('v-effective');
});
it('tolerates a legacy mascot state missing mascotVoices without throwing', () => {
// A pre-migration persisted blob or a partial preloadedState can omit
// `mascotVoices`; the meeting selectors must default it, not crash on
// `mascotVoices[selectedMascotId]`.
const legacy = {
mascot: { selectedMascotId: 'yellow', secondaryMascotId: null },
} as unknown as Parameters<typeof selectMeetingMascotVoicePair>[0];
expect(() => selectMeetingMascotVoicePair(legacy)).not.toThrow();
expect(selectMeetingMascotVoicePair(legacy).primary.mascotId).toBe('yellow');
expect(selectMascotVoiceFor('yellow')(legacy)).toBeNull();
expect(selectMascotVoices(legacy)).toEqual({});
});
});
});
+183 -1
View File
@@ -40,6 +40,17 @@ export const DEFAULT_MASCOT_VOICE_GENDER: MascotVoiceGender = 'male';
export const MAX_MASCOT_VOICE_ID_LEN = 128;
export const MAX_CUSTOM_MASCOT_GIF_URL_LEN = 2048;
/**
* Upper bound on how many per-mascot voice overrides we persist (issue
* #4277). A user only ever drives two mascots in a meeting, but they may
* try several before settling; the cap keeps the persisted map bounded
* against a runaway writer while comfortably covering real use. Once the
* cap is reached the reducer refuses NEW keys (an existing mascot can
* still be re-voiced); on rehydrate the first `MAX_MASCOT_VOICES` valid
* entries are kept and the rest dropped.
*/
export const MAX_MASCOT_VOICES = 16;
/**
* Loose shape check for a stored mascot voice id. Issue #1762 lets users
* paste a custom ElevenLabs voice id, so we cannot enumerate the valid
@@ -118,6 +129,21 @@ export interface MascotState {
* the persisted blob bounded.
*/
selectedMascotId: string | null;
/**
* Second mascot enabled for meetings (issue #4277). When set (and
* distinct from `selectedMascotId`) the meeting bot shows both mascots
* together and alternates who speaks each reply. `null` = single-mascot
* behavior, unchanged. Same validation/length cap as `selectedMascotId`.
*/
secondaryMascotId: string | null;
/**
* Per-mascot reply-voice overrides (issue #4277), keyed by manifest
* mascot id → ElevenLabs voice id. Lets each mascot in a two-mascot
* meeting speak in its own voice. Empty map = no per-mascot override,
* so every mascot falls back to `selectEffectiveMascotVoiceId` (the
* single-voice behavior). Bounded by `MAX_MASCOT_VOICES`.
*/
mascotVoices: Record<string, string>;
/**
* User-supplied animated avatar source. Kept as a plain validated
* string so the renderer can fall back to YellowMascot whenever the
@@ -134,11 +160,31 @@ const initialState: MascotState = {
voiceGender: DEFAULT_MASCOT_VOICE_GENDER,
voiceUseLocaleDefault: false,
selectedMascotId: null,
secondaryMascotId: null,
mascotVoices: {},
customMascotGifUrl: null,
customPrimaryColor: '#F7D145',
customSecondaryColor: '#B23C05',
};
/**
* Scrub a persisted / raw `mascotVoices` blob down to valid
* `mascotId → voiceId` entries under the size cap. Non-object inputs and
* any entry whose key or value fails `isMascotVoiceId` are dropped, so a
* corrupted localStorage blob can never poison the meeting TTS payload.
*/
function sanitizeMascotVoices(value: unknown): Record<string, string> {
if (value == null || typeof value !== 'object' || Array.isArray(value)) return {};
const out: Record<string, string> = {};
for (const [key, val] of Object.entries(value as Record<string, unknown>)) {
if (Object.keys(out).length >= MAX_MASCOT_VOICES) break;
if (isMascotVoiceId(key) && isMascotVoiceId(val)) {
out[key.trim()] = (val as string).trim();
}
}
return out;
}
function isMascotColor(value: unknown): value is MascotColor {
return (
typeof value === 'string' && (SUPPORTED_MASCOT_COLORS as readonly string[]).includes(value)
@@ -170,6 +216,49 @@ const mascotSlice = createSlice({
state.selectedMascotId = null;
}
},
/**
* Enable / clear the second meeting mascot (issue #4277). Trimmed;
* empty / oversize / null clears it (back to single-mascot). A custom
* GIF avatar and a second Rive mascot are mutually exclusive, so
* setting one clears the GIF override — mirroring `setSelectedMascotId`.
*/
setSecondaryMascotId(state, action: PayloadAction<string | null>) {
if (action.payload == null) {
state.secondaryMascotId = null;
return;
}
if (isMascotVoiceId(action.payload)) {
state.secondaryMascotId = action.payload.trim();
state.customMascotGifUrl = null;
} else {
state.secondaryMascotId = null;
}
},
/**
* Set or clear a per-mascot reply voice (issue #4277). A non-empty
* valid `voiceId` records `mascotId → voiceId`; a `null`/invalid
* `voiceId` removes the entry (that mascot falls back to the effective
* single voice). Both key and value are validated + trimmed so junk
* can't grow the persisted map. Over-cap writes are ignored.
*/
setMascotVoice(state, action: PayloadAction<{ mascotId: string; voiceId: string | null }>) {
const { mascotId, voiceId } = action.payload;
if (!isMascotVoiceId(mascotId)) return;
const key = mascotId.trim();
if (voiceId == null || !isMascotVoiceId(voiceId)) {
delete state.mascotVoices[key];
return;
}
// Only enforce the cap when introducing a NEW key — updating an
// existing mascot's voice must always be allowed.
if (
!(key in state.mascotVoices) &&
Object.keys(state.mascotVoices).length >= MAX_MASCOT_VOICES
) {
return;
}
state.mascotVoices[key] = voiceId.trim();
},
setCustomMascotGifUrl(state, action: PayloadAction<string | null>) {
if (action.payload == null) {
state.customMascotGifUrl = null;
@@ -178,6 +267,7 @@ const mascotSlice = createSlice({
if (isCustomMascotGifUrl(action.payload)) {
state.customMascotGifUrl = action.payload.trim();
state.selectedMascotId = null;
state.secondaryMascotId = null;
} else {
state.customMascotGifUrl = null;
}
@@ -231,6 +321,8 @@ const mascotSlice = createSlice({
voiceGender?: unknown;
voiceUseLocaleDefault?: unknown;
selectedMascotId?: unknown;
secondaryMascotId?: unknown;
mascotVoices?: unknown;
customMascotGifUrl?: unknown;
customPrimaryColor?: unknown;
customSecondaryColor?: unknown;
@@ -246,6 +338,17 @@ const mascotSlice = createSlice({
: isMascotVoiceId(restoredSelectedMascotId)
? (restoredSelectedMascotId as string).trim()
: null;
// Second mascot + per-mascot voices are absent in pre-#4277 blobs;
// the `null` / `{}` fallbacks match a fresh install and keep
// single-mascot users unchanged. Invalid values are scrubbed.
const restoredSecondaryMascotId = rehydrateAction.payload?.secondaryMascotId;
state.secondaryMascotId =
restoredSecondaryMascotId == null
? null
: isMascotVoiceId(restoredSecondaryMascotId)
? (restoredSecondaryMascotId as string).trim()
: null;
state.mascotVoices = sanitizeMascotVoices(rehydrateAction.payload?.mascotVoices);
const restoredCustomMascotGifUrl = rehydrateAction.payload?.customMascotGifUrl;
state.customMascotGifUrl =
restoredCustomMascotGifUrl == null
@@ -253,7 +356,12 @@ const mascotSlice = createSlice({
: isCustomMascotGifUrl(restoredCustomMascotGifUrl)
? (restoredCustomMascotGifUrl as string).trim()
: null;
if (state.customMascotGifUrl) state.selectedMascotId = null;
// A custom GIF avatar is mutually exclusive with Rive mascots —
// drop both mascot selections if a GIF override survived.
if (state.customMascotGifUrl) {
state.selectedMascotId = null;
state.secondaryMascotId = null;
}
// `voiceId` is optional in older persisted blobs (pre-#1762) — the
// `null` fallback is the intended default and matches a fresh
// install. Invalid values are scrubbed so a corrupted localStorage
@@ -289,6 +397,8 @@ export const {
setMascotVoiceGender,
setMascotVoiceUseLocaleDefault,
setSelectedMascotId,
setSecondaryMascotId,
setMascotVoice,
setCustomMascotGifUrl,
setCustomPrimaryColor,
setCustomSecondaryColor,
@@ -309,6 +419,32 @@ export const selectMascotVoiceUseLocaleDefault = (state: { mascot: MascotState }
export const selectSelectedMascotId = (state: { mascot: MascotState }): string | null =>
state.mascot.selectedMascotId;
export const selectSecondaryMascotId = (state: { mascot: MascotState }): string | null =>
state.mascot.secondaryMascotId;
export const selectMascotVoices = (state: { mascot: MascotState }): Record<string, string> =>
state.mascot.mascotVoices ?? {};
/**
* Explicit per-mascot voice override for `mascotId`, or `null` when none
* is set (caller falls back to the effective single voice). Curried so it
* reads like the other parameterised selectors at call sites.
*/
export const selectMascotVoiceFor =
(mascotId: string | null) =>
(state: { mascot: MascotState }): string | null =>
mascotId ? (state.mascot.mascotVoices?.[mascotId] ?? null) : null;
/**
* True when a distinct second mascot is enabled — the single gate the
* meeting render + join paths use to decide dual vs single behavior.
* Guards against the same mascot being picked twice.
*/
export const selectDualMascotEnabled = (state: { mascot: MascotState }): boolean => {
const { selectedMascotId, secondaryMascotId } = state.mascot;
return secondaryMascotId != null && secondaryMascotId !== selectedMascotId;
};
export const selectCustomMascotGifUrl = (state: { mascot: MascotState }): string | null =>
state.mascot.customMascotGifUrl;
@@ -354,5 +490,51 @@ export const selectEffectiveMascotVoiceId = (state: {
return MASCOT_VOICE_ID || ELEVENLABS_VOICE_PRESETS[0].id;
};
export interface MeetingMascotSlot {
/** Manifest mascot id, or `null` for the primary when the user is on
* the default (first-`ready`) mascot. */
mascotId: string | null;
/** Resolved voice id: the per-mascot override, else the effective
* single voice — never empty, so the join payload always carries one. */
voiceId: string;
}
export interface MeetingMascotVoicePair {
primary: MeetingMascotSlot;
secondary: MeetingMascotSlot | null;
}
/**
* Resolve the (up to two) mascots + voices a meeting join should use
* (issue #4277). Single source of truth for both join paths — the CEF
* `meet_call_open_window` sender and the backend `agent_meetings_join`
* sender — and for tests, so they can't drift.
*
* `secondary` is non-null only when a distinct second mascot is enabled
* (`selectDualMascotEnabled`). Each slot's voice is its per-mascot
* override, falling back to `selectEffectiveMascotVoiceId`; when the user
* hasn't set distinct voices both slots resolve to that same voice
* (harmless — alternation still works, it just sounds the same).
*/
export const selectMeetingMascotVoicePair = (state: {
mascot: MascotState;
locale?: { current: Locale };
}): MeetingMascotVoicePair => {
const effective = selectEffectiveMascotVoiceId(state);
const { selectedMascotId, secondaryMascotId } = state.mascot;
// Tolerate a partial / pre-migration mascot slice (e.g. a legacy persisted
// blob or a test's preloadedState) that predates `mascotVoices`.
const mascotVoices = state.mascot.mascotVoices ?? {};
const primary: MeetingMascotSlot = {
mascotId: selectedMascotId,
voiceId: (selectedMascotId && mascotVoices[selectedMascotId]) || effective,
};
const dualEnabled = secondaryMascotId != null && secondaryMascotId !== selectedMascotId;
const secondary: MeetingMascotSlot | null = dualEnabled
? { mascotId: secondaryMascotId, voiceId: mascotVoices[secondaryMascotId] || effective }
: null;
return { primary, secondary };
};
export { mascotSlice };
export default mascotSlice.reducer;
+4
View File
@@ -1203,6 +1203,10 @@ pub enum DomainEvent {
command_text: String,
recent_transcript: Vec<BackendMeetTurn>,
timestamp_ms: u64,
/// Dual-mascot name addressing (#4277 follow-up): slot (0 = primary,
/// 1 = secondary) whose mascot name was addressed, or `None` when no
/// specific mascot was named. Forwarded to `bot:speak` as `mascotSlot`.
mascot_slot: Option<u8>,
},
/// Core asked the backend bot to speak into the call (`bot:speak`).
/// Published for observability after the Socket.IO emit succeeds.
+4
View File
@@ -257,11 +257,13 @@ impl EventHandler for MeetingEventSubscriber {
command_text,
recent_transcript,
timestamp_ms,
mascot_slot,
} => {
tracing::info!(
correlation_id = ?correlation_id,
speaker = %speaker,
cmd_len = command_text.len(),
mascot_slot = ?mascot_slot,
"{LOG_PREFIX} in-call request received"
);
// The orchestrator turn can run for tens of seconds (tools,
@@ -271,6 +273,7 @@ impl EventHandler for MeetingEventSubscriber {
let command_text = command_text.clone();
let recent_transcript = recent_transcript.clone();
let timestamp_ms = *timestamp_ms;
let mascot_slot = *mascot_slot;
tokio::spawn(async move {
super::in_call::handle_in_call_request(
correlation_id,
@@ -278,6 +281,7 @@ impl EventHandler for MeetingEventSubscriber {
command_text,
recent_transcript,
timestamp_ms,
mascot_slot,
)
.await;
});
+76 -14
View File
@@ -161,6 +161,7 @@ pub async fn handle_in_call_request(
command_text: String,
recent_transcript: Vec<BackendMeetTurn>,
timestamp_ms: u64,
mascot_slot: Option<u8>,
) {
let command = command_text.trim();
if command.is_empty() {
@@ -201,12 +202,21 @@ pub async fn handle_in_call_request(
return;
}
// Name-addressed routing (#4277 follow-up): `mascot_slot` is decided by the
// backend's wake matcher (it sees the un-stripped caption and knows which
// mascot name was said) and pins this whole turn — the speculative ack, the
// streamed chunks, and the final reply — to that slot so the named mascot's
// voice + face own the answer. `None` (no name / single mascot / follow-up)
// leaves the backend's mechanical alternation untouched, i.e. today's
// behavior.
tracing::info!(
correlation_id = ?correlation_id,
speaker = %speaker,
cmd_len = command.len(),
transcript_turns = recent_transcript.len(),
timestamp_ms = timestamp_ms,
mascot_slot = ?mascot_slot,
"{LOG_PREFIX} dispatching in-call command to orchestrator"
);
@@ -220,7 +230,7 @@ pub async fn handle_in_call_request(
let ack_task = tokio::spawn(async move {
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(ACK_AFTER_SECS)) => {
if let Err(e) = emit_bot_filler(ACK_PHRASE, ack_cid.as_deref()).await {
if let Err(e) = emit_bot_filler(ACK_PHRASE, ack_cid.as_deref(), mascot_slot).await {
tracing::debug!("[agent_meetings::in_call] ack emit failed: {e}");
}
}
@@ -235,6 +245,7 @@ pub async fn handle_in_call_request(
&recent_transcript,
streaming,
ack_cancel.clone(),
mascot_slot,
)
.await;
@@ -253,7 +264,15 @@ pub async fn handle_in_call_request(
// In streaming mode the reply was already spoken sentence-by-
// sentence during the turn; only the buffered path emits here.
if !streaming {
if let Err(e) = emit_bot_speak(&text, correlation_id.as_deref()).await {
if let Err(e) = emit_bot_speak_inner(
&text,
correlation_id.as_deref(),
None,
"reply",
mascot_slot,
)
.await
{
tracing::warn!("{LOG_PREFIX} bot:speak emit failed: {e}");
}
}
@@ -264,7 +283,18 @@ pub async fn handle_in_call_request(
}
Err(e) => {
tracing::warn!("{LOG_PREFIX} in-call turn failed: {e}");
if let Err(e2) = emit_bot_speak(FAILURE_PHRASE, correlation_id.as_deref()).await {
// Speak the failure reply from the same mascot that was addressed
// (name-addressed routing, #4277 follow-up) — a terminal reply must
// carry `mascot_slot`, matching the success path above.
if let Err(e2) = emit_bot_speak_inner(
FAILURE_PHRASE,
correlation_id.as_deref(),
None,
"reply",
mascot_slot,
)
.await
{
tracing::debug!("{LOG_PREFIX} failure phrase emit failed: {e2}");
}
// The failure phrase promises a note in the thread — keep it.
@@ -363,8 +393,9 @@ pub(super) async fn speak_approval_prompt(action_summary: &str, correlation_id:
or — Hey Tiny, deny — to cancel."
);
// Filler, not a terminal reply: the bot is now waiting for a spoken
// decision, so the mascot should stay in its thinking cue.
if let Err(e) = emit_bot_filler(&prompt, correlation_id).await {
// decision, so the mascot should stay in its thinking cue. Slot-agnostic
// (an approval prompt isn't name-addressed) → default alternation.
if let Err(e) = emit_bot_filler(&prompt, correlation_id, None).await {
tracing::warn!("{LOG_PREFIX} approval prompt emit failed: {e}");
}
}
@@ -378,6 +409,7 @@ async fn run_orchestrator_turn(
recent_transcript: &[BackendMeetTurn],
streaming: bool,
ack_cancel: Arc<Notify>,
mascot_slot: Option<u8>,
) -> Result<String, String> {
let agent_lock = get_or_build_agent(correlation_id).await?;
let mut agent = agent_lock.lock().await;
@@ -408,7 +440,12 @@ async fn run_orchestrator_turn(
let (tx, rx) = mpsc::channel::<AgentProgress>(64);
agent.set_on_progress(Some(tx));
let cid_owned = correlation_id.map(String::from);
Some(tokio::spawn(stream_sentences(rx, cid_owned, ack_cancel)))
Some(tokio::spawn(stream_sentences(
rx,
cid_owned,
ack_cancel,
mascot_slot,
)))
} else {
None
};
@@ -487,6 +524,7 @@ async fn stream_sentences(
mut rx: mpsc::Receiver<AgentProgress>,
correlation_id: Option<String>,
ack_cancel: Arc<Notify>,
mascot_slot: Option<u8>,
) {
let mut buf = String::new();
let mut seq: u32 = 0;
@@ -501,6 +539,7 @@ async fn stream_sentences(
&mut seq,
&ack_cancel,
&mut spoke,
mascot_slot,
)
.await;
}
@@ -517,6 +556,7 @@ async fn stream_sentences(
&mut seq,
&ack_cancel,
&mut spoke,
mascot_slot,
)
.await;
}
@@ -528,6 +568,7 @@ async fn speak_stream_chunk(
seq: &mut u32,
ack_cancel: &Arc<Notify>,
spoke: &mut bool,
mascot_slot: Option<u8>,
) {
if text.is_empty() {
return;
@@ -536,7 +577,9 @@ async fn speak_stream_chunk(
*spoke = true;
ack_cancel.notify_one(); // first real audio — no need for the filler ack
}
if let Err(e) = emit_bot_speak_inner(text, correlation_id, Some(*seq), "reply").await {
if let Err(e) =
emit_bot_speak_inner(text, correlation_id, Some(*seq), "reply", mascot_slot).await
{
tracing::debug!("{LOG_PREFIX} streamed chunk emit failed: {e}");
}
*seq += 1;
@@ -799,26 +842,34 @@ async fn get_or_create_meeting_thread(
/// The backend's SpeakOrchestrator handles streaming TTS + the audio
/// politeness gate from there.
async fn emit_bot_speak(text: &str, correlation_id: Option<&str>) -> Result<(), String> {
emit_bot_speak_inner(text, correlation_id, None, "reply").await
emit_bot_speak_inner(text, correlation_id, None, "reply", None).await
}
/// Emit a non-terminal *filler* utterance (the speculative "On it" ack, an
/// approval prompt): tagged `kind="ack"` so the backend mascot returns to its
/// thinking cue afterwards instead of settling to idle — the real reply is
/// still on the way.
async fn emit_bot_filler(text: &str, correlation_id: Option<&str>) -> Result<(), String> {
emit_bot_speak_inner(text, correlation_id, None, "ack").await
/// still on the way. `mascot_slot` pins the filler to the name-addressed slot
/// so the ack comes from the same mascot that will answer.
async fn emit_bot_filler(
text: &str,
correlation_id: Option<&str>,
mascot_slot: Option<u8>,
) -> Result<(), String> {
emit_bot_speak_inner(text, correlation_id, None, "ack", mascot_slot).await
}
/// As [`emit_bot_speak`], plus an optional `seq` so the backend can order
/// the per-sentence chunks of a streamed reply, and a `kind` ("ack" filler vs
/// terminal "reply") so the backend can drive the mascot's post-speech pose
/// (both additive; older backends ignore them).
/// the per-sentence chunks of a streamed reply, a `kind` ("ack" filler vs
/// terminal "reply") so the backend can drive the mascot's post-speech pose,
/// and an optional `mascot_slot` (0|1) so a name-addressed turn is spoken by a
/// specific mascot instead of the backend's default alternation (all additive;
/// older backends ignore them).
async fn emit_bot_speak_inner(
text: &str,
correlation_id: Option<&str>,
seq: Option<u32>,
kind: &str,
mascot_slot: Option<u8>,
) -> Result<(), String> {
let mgr = global_socket_manager()
.ok_or_else(|| format!("{LOG_PREFIX} socket manager not initialized"))?;
@@ -835,11 +886,20 @@ async fn emit_bot_speak_inner(
map.insert("seq".to_string(), json!(s));
}
map.insert("kind".to_string(), json!(kind));
// Name-addressed routing (#4277 follow-up): pin this utterance to a
// specific mascot slot. Maps to `MeetingBotSpeakPayload.mascotSlot`,
// which the backend prefers over its alternation cursor.
if let Some(slot) = mascot_slot {
map.insert("mascotSlot".to_string(), json!(slot));
}
}
tracing::info!(
text_len = text.len(),
correlation_id = ?correlation_id,
kind = kind,
seq = ?seq,
mascot_slot = ?mascot_slot,
"{LOG_PREFIX} emitting bot:speak"
);
mgr.emit("bot:speak", payload)
@@ -1019,6 +1079,7 @@ mod tests {
" ".into(),
vec![],
0,
None,
)
.await;
}
@@ -1036,6 +1097,7 @@ mod tests {
"what time is it".into(),
vec![],
0,
None,
)
.await;
// No agent should have been built for the meeting.
+91
View File
@@ -761,6 +761,38 @@ fn build_join_payload(
}),
);
}
// Dual-mascot payload (issue #4277). Emitted only when the caller
// supplied a `mascots` array; the backend bot renders both slots,
// alternates the speaker per reply, and uses each slot's voice.
// Absent → the backend falls back to the single `mascotId` above,
// so one-mascot callers are unchanged.
if let Some(mascots) = &req.mascots {
let slots: Vec<Value> = mascots
.iter()
.map(|m| {
let mut slot = json!({ "mascotId": m.mascot_id });
if let Some(obj) = slot.as_object_mut() {
if let Some(name) = &m.name {
obj.insert("name".to_string(), json!(name));
}
if let Some(colors) = &m.rive_colors {
obj.insert(
"riveColors".to_string(),
json!({
"primaryColor": colors.primary_color,
"secondaryColor": colors.secondary_color,
}),
);
}
if let Some(vid) = &m.voice_id {
obj.insert("voiceId".to_string(), json!(vid));
}
}
slot
})
.collect();
map.insert("mascots".to_string(), json!(slots));
}
if let Some(respond_to) = &req.respond_to_participant {
map.insert("respondToParticipant".to_string(), json!(respond_to));
}
@@ -872,10 +904,23 @@ pub async fn handle_join(params: Map<String, Value>) -> Result<Value, String> {
return Err("[agent_meetings] socket not connected to backend".to_string());
}
// Name-addressing (#4277 follow-up) trace: the mascot ids + names forwarded
// to the backend. A `None` name on a slot means the backend can't route
// that mascot by name ("Hey Toshi").
let mascot_trace: Vec<(String, Option<String>)> = req
.mascots
.as_ref()
.map(|ms| {
ms.iter()
.map(|m| (m.mascot_id.clone(), m.name.clone()))
.collect()
})
.unwrap_or_default();
tracing::info!(
meet_url_host = %normalized_url.host_str().unwrap_or(""),
platform = %platform,
display_name_len = display_name.len(),
mascots = ?mascot_trace,
"[agent_meetings] emitting bot:join"
);
@@ -1783,6 +1828,8 @@ mod tests {
assert!(payload.get("systemPrompt").is_none());
assert!(payload.get("mascotId").is_none());
assert!(payload.get("riveColors").is_none());
// Single-mascot join carries no `mascots` array (issue #4277).
assert!(payload.get("mascots").is_none());
assert!(payload.get("respondToParticipant").is_none());
assert!(payload.get("wakePhrase").is_none());
}
@@ -1839,6 +1886,50 @@ mod tests {
assert_eq!(payload["riveColors"]["secondaryColor"], "#00ff00");
assert_eq!(payload["respondToParticipant"], "Bob");
assert_eq!(payload["wakePhrase"], "Hello bot");
// A single `mascot_id` (no `mascots` array) must NOT emit `mascots`
// — the legacy single-mascot wire shape is unchanged (issue #4277).
assert!(payload.get("mascots").is_none());
}
#[test]
fn build_join_payload_with_dual_mascots() {
// Two-mascot join (issue #4277): the `mascots` array is emitted with
// camelCase slot fields the backend expects; slot 0 = primary.
let req: BackendMeetJoinRequest = serde_json::from_value(json!({
"meet_url": "https://meet.google.com/abc-defg-hij",
"mascot_id": "tiny-mascot",
"mascots": [
{
"mascot_id": "tiny-mascot",
"name": "Tiny",
"voice_id": "voice-a",
"rive_colors": { "primary_color": "#111", "secondary_color": "#222" }
},
{ "mascot_id": "toshi", "voice_id": "voice-b" }
]
}))
.unwrap();
let payload = build_join_payload(
"https://meet.google.com/abc-defg-hij",
"OpenHuman",
"gmeet",
&req,
);
let mascots = payload["mascots"].as_array().expect("mascots array");
assert_eq!(mascots.len(), 2);
assert_eq!(mascots[0]["mascotId"], "tiny-mascot");
// Name-addressed routing (#4277 follow-up): the display name is
// forwarded so the bot can route "Hey Tiny …" to slot 0.
assert_eq!(mascots[0]["name"], "Tiny");
assert_eq!(mascots[0]["voiceId"], "voice-a");
assert_eq!(mascots[0]["riveColors"]["primaryColor"], "#111");
assert_eq!(mascots[0]["riveColors"]["secondaryColor"], "#222");
assert_eq!(mascots[1]["mascotId"], "toshi");
assert_eq!(mascots[1]["voiceId"], "voice-b");
// No name supplied for slot 1 → no `name` key.
assert!(mascots[1].get("name").is_none());
// No colors supplied for slot 1 → no `riveColors` key.
assert!(mascots[1].get("riveColors").is_none());
}
#[test]
+10
View File
@@ -126,6 +126,16 @@ fn schema_join() -> ControllerSchema {
comment: "Optional Rive mascot color overrides forwarded to the backend bot.",
required: false,
},
FieldSchema {
name: "mascots",
ty: TypeSchema::Json,
comment: "Optional dual-mascot config (issue #4277): array of up to 2 slots, each \
{ mascotId, name?, riveColors?, voiceId? }. When present the backend renders \
both mascots and alternates the speaker per reply. `name` (from the manifest) \
enables name-addressed routing (\"Hey Toshi …\" → that slot). Absent falls \
back to mascot_id.",
required: false,
},
FieldSchema {
name: "respond_to_participant",
ty: TypeSchema::String,
+35 -1
View File
@@ -90,6 +90,33 @@ pub struct RiveColors {
pub secondary_color: Option<String>,
}
/// One mascot slot for a multi-mascot backend meeting (issue #4277).
/// Slot 0 = primary speaker, slot 1 = secondary. The backend bot renders
/// both mascots side-by-side, alternates the speaking slot per reply, and
/// synthesizes each slot's replies with `voice_id` (falling back to its
/// configured default when absent).
///
/// Deserialize-only: the RPC input arrives snake_case (`mascot_id`,
/// `rive_colors`, `voice_id`); the outbound `bot:join` payload is built by
/// hand in `ops.rs` (camelCase), so this struct is never serialized.
#[derive(Debug, Clone, Deserialize)]
pub struct BackendMascotSlot {
/// Rive mascot id (e.g. "yellow", "toshi").
pub mascot_id: String,
/// Optional human-facing mascot name (e.g. "Toshi", "Tiny"), taken from the
/// manifest. Drives name-addressed routing (#4277 follow-up): a participant
/// who says "Hey Toshi …" is routed to this slot instead of the mechanical
/// alternation. Absent → that slot is not name-addressable.
#[serde(default)]
pub name: Option<String>,
/// Optional per-mascot color palette overrides.
#[serde(default)]
pub rive_colors: Option<RiveColors>,
/// Optional per-mascot ElevenLabs voice id.
#[serde(default)]
pub voice_id: Option<String>,
}
/// Inputs to `openhuman.agent_meetings_join`.
#[derive(Debug, Clone, Deserialize)]
pub struct BackendMeetJoinRequest {
@@ -105,11 +132,18 @@ pub struct BackendMeetJoinRequest {
#[serde(default)]
pub system_prompt: Option<String>,
/// Selects which Rive mascot appears in the meeting (e.g. "yellow", "blue").
/// Legacy single-mascot field; still honored when `mascots` is absent.
#[serde(default)]
pub mascot_id: Option<String>,
/// Optional Rive mascot color palette overrides.
/// Optional Rive mascot color palette overrides (single-mascot path).
#[serde(default)]
pub rive_colors: Option<RiveColors>,
/// Dual-mascot config (issue #4277). When present (2 slots) the backend
/// renders both mascots and alternates the speaker per reply, using each
/// slot's `voice_id`. Absent / single-element falls back to the legacy
/// single-mascot `mascot_id` + `rive_colors` behavior.
#[serde(default)]
pub mascots: Option<Vec<BackendMascotSlot>>,
/// Only respond to this participant's messages (empty/absent = respond to everyone).
#[serde(default)]
pub respond_to_participant: Option<String>,
+2 -2
View File
@@ -221,7 +221,7 @@ pub async fn run_grant_turn(request_id: &str, grantee: &str) -> Result<bool, Str
s.allow_speaker(grantee);
s.cancel_outbound();
});
let samples = match tts(&message).await {
let samples = match tts(&message, None).await {
Ok(samples) => samples,
Err(err) => {
log::warn!("[meet-agent] grant TTS failed request_id={request_id} err={err}");
@@ -301,7 +301,7 @@ pub async fn run_soft_deny_turn(
// Cancel any prior outbound so the refusal doesn't queue behind a
// half-drained reply from a previous turn.
let _ = registry().with_session(request_id, |s| s.cancel_outbound());
let samples = match tts(&message).await {
let samples = match tts(&message, None).await {
Ok(samples) => samples,
Err(err) => {
log::warn!("[meet-agent] soft-deny TTS failed request_id={request_id} err={err}");
+8 -1
View File
@@ -26,7 +26,11 @@ pub(super) async fn stt(samples: &[i16]) -> Result<String, String> {
// ─── Real TTS adapter ───────────────────────────────────────────────
pub(super) async fn tts(text: &str) -> Result<Vec<i16>, String> {
/// Synthesize `text` to PCM16LE @ 16 kHz. `voice_id` selects the
/// ElevenLabs voice for this utterance (the speaker-alternation slot's
/// voice); `None` lets the reply-speech backend pick its own default —
/// the exact prior behavior for single-mascot calls.
pub(super) async fn tts(text: &str, voice_id: Option<&str>) -> Result<Vec<i16>, String> {
use crate::openhuman::voice::reply_speech::{synthesize_reply, ReplySpeechOptions};
let config = crate::openhuman::config::ops::load_config_with_timeout().await?;
@@ -51,6 +55,9 @@ pub(super) async fn tts(text: &str) -> Result<Vec<i16>, String> {
output_format: Some("pcm_16000".to_string()),
model_id: Some(TTS_MODEL_ID.to_string()),
voice_settings: Some(voice_settings),
// Per-mascot voice for speaker alternation. `None` preserves the
// backend's default-voice pick (single-mascot behavior).
voice_id: voice_id.map(str::to_owned),
..Default::default()
};
let outcome = synthesize_reply(&config, text, &opts).await?;
+28 -4
View File
@@ -96,7 +96,13 @@ pub async fn run_turn(request_id: &str) -> Result<bool, String> {
let synthesized = if reply_text.trim().is_empty() {
Vec::new()
} else {
match tts(&reply_text).await {
// Pick this reply's speaker voice. For two-mascot calls this
// alternates voice + active slot once per spoken reply; a
// single-mascot call gets `None` (backend default voice) and the
// slot stays 0. Advanced only when we actually speak so declined
// turns don't rotate the speaker out of sync.
let voice_id = registry().with_session(request_id, |s| s.advance_speaker())?;
match tts(&reply_text, voice_id.as_deref()).await {
Ok(samples) => samples,
Err(err) => {
log::warn!("[meet-agent] TTS failed request_id={request_id} err={err}");
@@ -229,8 +235,20 @@ pub async fn run_caption_turn(request_id: &str) -> Result<bool, String> {
// can hear you" sounds redundant. The 50-char threshold is a
// rough proxy; real second-brain questions ("am I free Friday
// afternoon for a 30 min slot") are almost always longer.
if !was_bare_wake && prompt.chars().count() > PREROLL_SKIP_PROMPT_CHARS {
if let Ok(ack_pcm) = tts(PREROLL_ACK_PHRASE).await {
// One speaker per caption turn: the pre-roll ack and the final reply
// come from the SAME mascot, and the active slot advances exactly once
// per turn — but ONLY when the turn actually speaks. A turn that fires
// no ack and then declines must not rotate the speaker (mirrors
// `run_turn`, which advances only on a non-empty reply). `turn_voice`
// stays `None` for single-mascot calls (backend default voice, slot 0).
let will_ack = !was_bare_wake && prompt.chars().count() > PREROLL_SKIP_PROMPT_CHARS;
let mut turn_voice: Option<String> = None;
let mut speaker_advanced = false;
if will_ack {
turn_voice = registry().with_session(request_id, |s| s.advance_speaker())?;
speaker_advanced = true;
if let Ok(ack_pcm) = tts(PREROLL_ACK_PHRASE, turn_voice.as_deref()).await {
let _ = registry().with_session(request_id, |s| {
s.enqueue_outbound_pcm(&ack_pcm, false);
});
@@ -279,7 +297,13 @@ pub async fn run_caption_turn(request_id: &str) -> Result<bool, String> {
let synthesized = if reply_text.trim().is_empty() {
Vec::new()
} else {
match tts(&reply_text).await {
// If no pre-roll ack fired, this reply is the turn's first (and
// only) speech — advance the speaker now so a declined turn never
// rotated the slot, and a spoken turn rotates exactly once.
if !speaker_advanced {
turn_voice = registry().with_session(request_id, |s| s.advance_speaker())?;
}
match tts(&reply_text, turn_voice.as_deref()).await {
Ok(samples) => samples,
Err(err) => {
log::warn!(
+46 -4
View File
@@ -49,14 +49,22 @@ pub async fn handle_start_session(params: Map<String, Value>) -> Result<Value, S
registry().with_session(&req.request_id, |s| {
s.set_identities(&req.owner_display_name, &req.bot_display_name);
s.set_meet_url(&req.meet_url);
// Per-mascot voices for speaker alternation. Empty/absent =
// single-default-voice behavior (unchanged for one-mascot users).
s.set_voices(req.primary_voice_id.clone(), req.secondary_voice_id.clone());
})?;
let voice_count = [&req.primary_voice_id, &req.secondary_voice_id]
.into_iter()
.filter(|v| v.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false))
.count();
log::info!(
"{LOG_PREFIX} start_session request_id={} sample_rate_hz={} \
owner_chars={} bot_chars={}",
owner_chars={} bot_chars={} voice_count={}",
req.request_id,
req.sample_rate_hz,
req.owner_display_name.chars().count(),
req.bot_display_name.chars().count()
req.bot_display_name.chars().count(),
voice_count
);
RpcOutcome::new(
@@ -178,11 +186,11 @@ pub async fn handle_poll_speech(params: Map<String, Value>) -> Result<Value, Str
let req: PollSpeechRequest = serde_json::from_value(Value::Object(params))
.map_err(|e| format!("{LOG_PREFIX} invalid poll_speech params: {e}"))?;
let (pcm_base64, utterance_done, flush_pending) =
let (pcm_base64, utterance_done, flush_pending, active_mascot_slot) =
registry().with_session(&req.request_id, |s| {
let (b64, done) = s.poll_outbound();
let flush = s.take_flush_pending();
(b64, done, flush)
(b64, done, flush, s.active_slot())
})?;
RpcOutcome::new(
@@ -191,6 +199,11 @@ pub async fn handle_poll_speech(params: Map<String, Value>) -> Result<Value, Str
"pcm_base64": pcm_base64,
"utterance_done": utterance_done,
"flush_pending": flush_pending,
// Which mascot (0 = primary, 1 = secondary) is speaking the
// current outbound audio. The shell forwards this on the
// speaking-state edge so the frontend lip-syncs the right
// mascot and reacts the other. Always 0 for single-mascot.
"active_mascot_slot": active_mascot_slot,
}),
vec![],
)
@@ -334,6 +347,35 @@ mod tests {
assert_eq!(out.get("turn_count"), Some(&json!(0)));
}
#[tokio::test]
async fn poll_speech_reports_active_mascot_slot() {
// poll_speech surfaces `active_mascot_slot` (the mascot currently
// speaking, read from session state via `active_slot()`) on every
// reply so the shell can drive per-mascot lip-sync. This exercises
// that emit on an idle session — no LLM/brain turn needed, since
// `active_slot()` reads session state directly and defaults to 0.
let mut start = Map::new();
start.insert("request_id".into(), json!("rpc-active-slot"));
start.insert("sample_rate_hz".into(), json!(16_000));
start.insert("primary_voice_id".into(), json!("voice-primary"));
start.insert("secondary_voice_id".into(), json!("voice-secondary"));
handle_start_session(start).await.unwrap();
let mut poll = Map::new();
poll.insert("request_id".into(), json!("rpc-active-slot"));
let out = handle_poll_speech(poll).await.unwrap();
assert_eq!(out.get("ok"), Some(&json!(true)));
assert_eq!(
out.get("active_mascot_slot"),
Some(&json!(0)),
"idle session starts on the primary mascot (slot 0)"
);
let mut stop = Map::new();
stop.insert("request_id".into(), json!("rpc-active-slot"));
handle_stop_session(stop).await.unwrap();
}
#[tokio::test]
#[ignore = "flaky on CI; see PR #2588 follow-up"]
async fn push_then_poll_returns_audio_after_brain_turn() {
+126
View File
@@ -193,6 +193,20 @@ pub struct MeetAgentSession {
/// Resets on `stop_session` (the registry drops the whole
/// session). Empty by default — grants are opt-in per call.
allowlist: HashSet<String>,
/// Per-mascot TTS voice ids in speaker-slot order (slot 0 = primary,
/// slot 1 = secondary). 0, 1, or 2 entries. Empty preserves the old
/// single-default-voice behavior (the reply-speech backend picks its
/// own default). Set once at `start_session` from the shell payload.
voices: Vec<String>,
/// Index into `voices` of the mascot speaking the *current* turn.
/// Advanced once per assistant reply by [`Self::advance_speaker`] so
/// two-mascot calls alternate which voice speaks (and which mascot
/// the frontend lip-syncs). Meaningless when `voices.len() <= 1`.
active_voice_ix: usize,
/// False until the first [`Self::advance_speaker`] of a two-mascot call.
/// Keeps the first reply on the primary (slot 0) — so an idle session also
/// reports slot 0 — while every subsequent reply rotates the speaker.
speaker_primed: bool,
}
impl MeetAgentSession {
@@ -228,6 +242,9 @@ impl MeetAgentSession {
pending_unauthorized_speaker: None,
pending_unauthorized_at_ms: 0,
allowlist: HashSet::new(),
voices: Vec::new(),
active_voice_ix: 0,
speaker_primed: false,
}
}
@@ -305,6 +322,64 @@ impl MeetAgentSession {
self.bot_display_name = bot_display_name.trim().to_string();
}
/// Configure the per-mascot TTS voices for speaker alternation.
/// `primary` is slot 0, `secondary` is slot 1. Empty / whitespace
/// ids are dropped, so:
/// * both present → two-voice call, speaker alternates per reply;
/// * one present → that single voice speaks every turn;
/// * none present → `advance_speaker` yields `None` and the
/// reply-speech backend keeps picking its own default voice
/// (exact previous behavior).
/// The active slot starts at 0 so an idle session reports the primary
/// mascot; [`Self::advance_speaker`] keeps the first reply there and
/// rotates thereafter.
pub fn set_voices(&mut self, primary: Option<String>, secondary: Option<String>) {
let mut voices = Vec::new();
for id in [primary, secondary].into_iter().flatten() {
let trimmed = id.trim();
if !trimmed.is_empty() {
voices.push(trimmed.to_string());
}
}
self.active_voice_ix = 0;
self.speaker_primed = false;
self.voices = voices;
}
/// Pick the voice for the turn that is about to start and rotate the
/// active speaker for two-mascot calls. Returns `None` when no voices
/// were configured (single default-voice behavior). Call exactly once
/// per assistant reply, before synthesizing, so a multi-chunk reply
/// keeps one voice for its whole duration.
pub fn advance_speaker(&mut self) -> Option<String> {
match self.voices.len() {
0 => None,
1 => Some(self.voices[0].clone()),
n => {
// First reply stays on the primary (slot 0); each later reply
// rotates. This keeps an idle session on slot 0 too.
if self.speaker_primed {
self.active_voice_ix = (self.active_voice_ix + 1) % n;
} else {
self.speaker_primed = true;
}
Some(self.voices[self.active_voice_ix].clone())
}
}
}
/// Slot (0 = primary, 1 = secondary) of the mascot speaking the
/// current turn. The shell forwards this on the `meet-video:speaking-state`
/// edge so the frontend lip-syncs the right mascot and reacts the
/// other. Defaults to 0 (single-mascot rendering ignores it).
pub fn active_slot(&self) -> u8 {
if self.voices.is_empty() {
0
} else {
(self.active_voice_ix % self.voices.len()) as u8
}
}
/// Read accessor used by audit logging. Empty when set_identities
/// has not been called for this session.
pub fn owner_display_name(&self) -> &str {
@@ -950,6 +1025,57 @@ mod tests {
.unwrap();
}
// ─── Speaker alternation (issue #4277) ──────────────────────────
#[test]
fn set_voices_empty_disables_alternation() {
let mut s = MeetAgentSession::new("p".into(), 16_000);
s.set_voices(None, None);
// No voices → brain keeps the backend default voice, slot pinned 0.
assert_eq!(s.advance_speaker(), None);
assert_eq!(s.active_slot(), 0);
}
#[test]
fn single_voice_speaks_every_turn_on_slot_zero() {
let mut s = MeetAgentSession::new("p".into(), 16_000);
s.set_voices(Some("voice-a".into()), None);
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
assert_eq!(s.active_slot(), 0);
// A single voice never rotates.
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
assert_eq!(s.active_slot(), 0);
}
#[test]
fn two_voices_alternate_speaker_and_slot_per_turn() {
let mut s = MeetAgentSession::new("p".into(), 16_000);
s.set_voices(Some("voice-a".into()), Some("voice-b".into()));
// First reply = primary (slot 0)…
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
assert_eq!(s.active_slot(), 0);
// …second = secondary (slot 1)…
assert_eq!(s.advance_speaker().as_deref(), Some("voice-b"));
assert_eq!(s.active_slot(), 1);
// …third wraps back to primary.
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
assert_eq!(s.active_slot(), 0);
}
#[test]
fn set_voices_filters_blank_and_trims() {
let mut s = MeetAgentSession::new("p".into(), 16_000);
// Blank primary collapses to a single (secondary) voice on slot 0…
s.set_voices(Some(" ".into()), Some(" voice-b ".into()));
assert_eq!(s.advance_speaker().as_deref(), Some("voice-b"));
assert_eq!(s.advance_speaker().as_deref(), Some("voice-b"));
assert_eq!(s.active_slot(), 0);
// …and two real ids are trimmed before use.
s.set_voices(Some(" voice-a ".into()), Some(" voice-b ".into()));
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
assert_eq!(s.advance_speaker().as_deref(), Some("voice-b"));
}
/// Build a session pre-configured for the wake-word tests: Alice
/// is the call owner, "OpenHuman" is the bot's Meet tile. Every
/// wake-path test goes through this helper so the owner gate
+13
View File
@@ -45,6 +45,19 @@ pub struct StartSessionRequest {
/// updated to forward the URL still parse the payload.
#[serde(default)]
pub meet_url: String,
/// ElevenLabs voice id for the PRIMARY mascot (speaker slot 0). When
/// two mascots are enabled the session alternates the speaking voice
/// once per assistant reply (see [`super::session::MeetAgentSession::advance_speaker`]).
/// `None`/empty preserves the previous single-default-voice behavior
/// (the reply-speech backend picks its own default voice). Defaulted
/// so older shells / smoke tests still parse the payload.
#[serde(default)]
pub primary_voice_id: Option<String>,
/// ElevenLabs voice id for the SECONDARY mascot (speaker slot 1).
/// Absent when the user has only one mascot enabled — in that case no
/// alternation happens and slot 0 speaks every turn.
#[serde(default)]
pub secondary_voice_id: Option<String>,
}
fn default_sample_rate() -> u32 {
+11 -2
View File
@@ -390,10 +390,18 @@ pub(super) fn handle_sio_event(
.get("timestampMs")
.and_then(|v| v.as_u64())
.unwrap_or(0);
// Dual-mascot name addressing (#4277 follow-up): which slot the
// backend's wake matcher decided was addressed (0|1), if any.
let mascot_slot = data
.get("mascotSlot")
.and_then(|v| v.as_u64())
.filter(|s| *s <= 1)
.map(|s| s as u8);
log::info!(
"[socket] bot:in_call_request speaker={} cmd_len={}",
"[socket] bot:in_call_request speaker={} cmd_len={} mascot_slot={:?}",
speaker,
command_text.len()
command_text.len(),
mascot_slot
);
publish_global(DomainEvent::BackendMeetInCallRequest {
correlation_id,
@@ -401,6 +409,7 @@ pub(super) fn handle_sio_event(
command_text,
recent_transcript,
timestamp_ms,
mascot_slot,
});
}
"bot:error" => {