mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
This commit is contained in:
@@ -3678,6 +3678,10 @@ pub fn run() {
|
||||
// wake gate will fail-closed (no wakes fire) which
|
||||
// is the safe posture for an automated harness.
|
||||
owner_display_name: String::new(),
|
||||
// No mascot config on the dev-auto path → single
|
||||
// default voice (issue #4277).
|
||||
primary_voice_id: None,
|
||||
secondary_voice_id: None,
|
||||
};
|
||||
match meet_call::meet_call_open_window(app_handle.clone(), state, args)
|
||||
.await
|
||||
|
||||
@@ -89,13 +89,19 @@ pub async fn start<R: Runtime>(
|
||||
meet_url: String,
|
||||
owner_display_name: String,
|
||||
bot_display_name: String,
|
||||
primary_voice_id: Option<String>,
|
||||
secondary_voice_id: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
log::info!(
|
||||
"[meet-audio] start request_id={request_id} url_prefix={} \
|
||||
owner_chars={} bot_chars={}",
|
||||
owner_chars={} bot_chars={} voices={}",
|
||||
truncate_for_log(&meet_url, 64),
|
||||
owner_display_name.chars().count(),
|
||||
bot_display_name.chars().count()
|
||||
bot_display_name.chars().count(),
|
||||
[&primary_voice_id, &secondary_voice_id]
|
||||
.into_iter()
|
||||
.filter(|v| v.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false))
|
||||
.count()
|
||||
);
|
||||
|
||||
if let Some(state) = app.try_state::<MeetAudioState>() {
|
||||
@@ -125,6 +131,11 @@ pub async fn start<R: Runtime>(
|
||||
// <code>" in the history list. The URL the shell built
|
||||
// the CEF window with is the canonical value.
|
||||
"meet_url": meet_url,
|
||||
// Per-mascot TTS voices for speaker alternation (issue #4277).
|
||||
// Null for single-mascot calls → core keeps the backend
|
||||
// default voice (unchanged behavior).
|
||||
"primary_voice_id": primary_voice_id,
|
||||
"secondary_voice_id": secondary_voice_id,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -98,10 +98,10 @@ pub fn start<R: Runtime>(
|
||||
break;
|
||||
}
|
||||
_ = tick.tick() => {
|
||||
let had_pcm = match poll_and_feed(&request_id_for_task, &mut cdp, &session_id).await {
|
||||
Ok(had) => {
|
||||
let (had_pcm, active_slot) = match poll_and_feed(&request_id_for_task, &mut cdp, &session_id).await {
|
||||
Ok((had, slot)) => {
|
||||
feed_errors = 0;
|
||||
had
|
||||
(had, Some(slot))
|
||||
}
|
||||
Err(err) => {
|
||||
feed_errors += 1;
|
||||
@@ -118,11 +118,12 @@ pub fn start<R: Runtime>(
|
||||
// A failed tick is *not* evidence the bot
|
||||
// stopped speaking — leave the hangover to
|
||||
// expire naturally so transient CDP errors
|
||||
// don't flicker the mascot's mouth shut.
|
||||
false
|
||||
// don't flicker the mascot's mouth shut. No
|
||||
// fresh slot data → keep the last-known slot.
|
||||
(false, None)
|
||||
}
|
||||
};
|
||||
speaking_state.tick(had_pcm, &app, &request_id_for_task);
|
||||
speaking_state.tick(had_pcm, active_slot, &app, &request_id_for_task);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -148,6 +149,14 @@ struct SpeakingTracker {
|
||||
/// every tick that carries PCM; the state flips back to `false`
|
||||
/// only once `now > hangover_until` AND a tick with no PCM lands.
|
||||
hangover_until: Option<Instant>,
|
||||
/// Which mascot (0 = primary, 1 = secondary) is speaking the current
|
||||
/// outbound audio, as reported by `meet_agent_poll_speech`. For
|
||||
/// two-mascot calls the brain alternates this per reply; the frontend
|
||||
/// lip-syncs this slot and reacts the other. Emitted on the
|
||||
/// speaking-state edge AND whenever the slot changes mid-speech (so a
|
||||
/// back-to-back reply from the other mascot still switches lip-sync
|
||||
/// even if the hangover bridged the gap). 0 for single-mascot calls.
|
||||
active_slot: u8,
|
||||
}
|
||||
|
||||
impl SpeakingTracker {
|
||||
@@ -155,6 +164,7 @@ impl SpeakingTracker {
|
||||
Self {
|
||||
reported: false,
|
||||
hangover_until: None,
|
||||
active_slot: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,12 +172,21 @@ impl SpeakingTracker {
|
||||
/// is whether `poll_and_feed` saw a non-empty `pcm_base64` for
|
||||
/// this tick. Emits the Tauri event only when the reported
|
||||
/// state actually flips.
|
||||
fn tick<R: Runtime>(&mut self, had_pcm: bool, app: &AppHandle<R>, request_id: &str) {
|
||||
/// `active_slot` is the speaking mascot slot for this tick, or `None`
|
||||
/// when the tick had no fresh poll data (e.g. a CDP error) — in that
|
||||
/// case the last-known slot is retained.
|
||||
fn tick<R: Runtime>(
|
||||
&mut self,
|
||||
had_pcm: bool,
|
||||
active_slot: Option<u8>,
|
||||
app: &AppHandle<R>,
|
||||
request_id: &str,
|
||||
) {
|
||||
if had_pcm {
|
||||
// Extend the hangover. If we were idle, flip up to
|
||||
// speaking — the user hears audio starting now.
|
||||
self.hangover_until = Some(Instant::now() + SPEAKING_HANGOVER);
|
||||
self.set_reported(true, app, request_id);
|
||||
self.update(true, active_slot, app, request_id);
|
||||
return;
|
||||
}
|
||||
// No PCM this tick. If the hangover hasn't expired, stay in
|
||||
@@ -182,7 +201,7 @@ impl SpeakingTracker {
|
||||
self.hangover_until = None;
|
||||
}
|
||||
// Hangover expired or never armed → bot is genuinely idle.
|
||||
self.set_reported(false, app, request_id);
|
||||
self.update(false, active_slot, app, request_id);
|
||||
}
|
||||
|
||||
/// Force the reported state to `false` and emit an event if that's
|
||||
@@ -190,30 +209,71 @@ impl SpeakingTracker {
|
||||
/// can't get stuck mid-talk.
|
||||
fn force_off<R: Runtime>(&mut self, app: &AppHandle<R>, request_id: &str) {
|
||||
self.hangover_until = None;
|
||||
self.set_reported(false, app, request_id);
|
||||
self.update(false, None, app, request_id);
|
||||
}
|
||||
|
||||
fn set_reported<R: Runtime>(&mut self, next: bool, app: &AppHandle<R>, request_id: &str) {
|
||||
if self.reported == next {
|
||||
/// Update reported speaking state + active slot, emitting the Tauri
|
||||
/// event when either the speaking flag flips OR (while speaking) the
|
||||
/// active mascot slot changes. `slot = None` keeps the last-known
|
||||
/// slot (used when a tick carried no fresh poll data).
|
||||
fn update<R: Runtime>(
|
||||
&mut self,
|
||||
next: bool,
|
||||
slot: Option<u8>,
|
||||
app: &AppHandle<R>,
|
||||
request_id: &str,
|
||||
) {
|
||||
let (should_emit, resolved_slot) =
|
||||
next_speaking_state(self.reported, self.active_slot, next, slot);
|
||||
self.reported = next;
|
||||
self.active_slot = resolved_slot;
|
||||
if !should_emit {
|
||||
return;
|
||||
}
|
||||
self.reported = next;
|
||||
let payload = serde_json::json!({
|
||||
"requestId": request_id,
|
||||
"speaking": next,
|
||||
// Which mascot is speaking this audio (0 = primary, 1 =
|
||||
// secondary). Frontend lip-syncs this slot, reacts the other.
|
||||
"activeMascotSlot": self.active_slot,
|
||||
});
|
||||
if let Err(err) = app.emit(SPEAKING_STATE_EVENT, payload) {
|
||||
// Best-effort: a missing renderer (closed window mid-tick)
|
||||
// is the common case and not worth raising the log level.
|
||||
log::debug!(
|
||||
"[meet-audio] speaking-state emit failed request_id={request_id} speaking={next} err={err}"
|
||||
"[meet-audio] speaking-state emit failed request_id={request_id} speaking={next} slot={} err={err}",
|
||||
self.active_slot
|
||||
);
|
||||
} else {
|
||||
log::debug!("[meet-audio] speaking-state -> {next} request_id={request_id}");
|
||||
log::debug!(
|
||||
"[meet-audio] speaking-state -> {next} slot={} request_id={request_id}",
|
||||
self.active_slot
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure decision for [`SpeakingTracker::update`]: given the previously
|
||||
/// reported speaking flag + slot and the incoming `next`/`slot`, compute
|
||||
/// whether the Tauri event should be emitted and which slot to resolve to.
|
||||
///
|
||||
/// Extracted so the emit logic can be unit-tested without a live
|
||||
/// `AppHandle`. `slot = None` retains the previous slot (a tick with no
|
||||
/// fresh poll data). We emit when the speaking flag flips OR (while
|
||||
/// speaking) the active slot changes — the latter switches lip-sync on a
|
||||
/// back-to-back reply from the other mascot even if the hangover bridged
|
||||
/// the gap.
|
||||
fn next_speaking_state(
|
||||
prev_reported: bool,
|
||||
prev_slot: u8,
|
||||
next: bool,
|
||||
slot: Option<u8>,
|
||||
) -> (bool, u8) {
|
||||
let resolved = slot.unwrap_or(prev_slot);
|
||||
let should_emit = prev_reported != next || (next && prev_slot != resolved);
|
||||
(should_emit, resolved)
|
||||
}
|
||||
|
||||
/// No-op pump used when bridge install failed at session start. Keeps
|
||||
/// the rest of the session lifecycle uniform — `MeetAudioSession` can
|
||||
/// still hold a `SpeakPump` regardless of speak-path readiness.
|
||||
@@ -231,7 +291,7 @@ async fn poll_and_feed(
|
||||
request_id: &str,
|
||||
cdp: &mut CdpConn,
|
||||
session_id: &str,
|
||||
) -> Result<bool, String> {
|
||||
) -> Result<(bool, u8), String> {
|
||||
let v = super::rpc_call(
|
||||
"openhuman.meet_agent_poll_speech",
|
||||
serde_json::json!({ "request_id": request_id }),
|
||||
@@ -249,6 +309,12 @@ async fn poll_and_feed(
|
||||
.get("flush_pending")
|
||||
.and_then(|x| x.as_bool())
|
||||
.unwrap_or(false);
|
||||
// Which mascot slot is speaking this audio (0 = primary, 1 =
|
||||
// secondary). Absent on older cores / single-mascot → 0.
|
||||
let active_slot = v
|
||||
.get("active_mascot_slot")
|
||||
.and_then(|x| x.as_u64())
|
||||
.unwrap_or(0) as u8;
|
||||
|
||||
// Barge-in: brain set flush_pending when it cancelled the previous
|
||||
// outbound. Stop in-flight playback inside the JS bridge BEFORE we
|
||||
@@ -278,10 +344,45 @@ async fn poll_and_feed(
|
||||
bytes.len()
|
||||
);
|
||||
inject::feed_pcm_chunk(cdp, session_id, pcm_b64).await?;
|
||||
return Ok(true);
|
||||
return Ok((true, active_slot));
|
||||
}
|
||||
if utterance_done {
|
||||
log::info!("[meet-audio] speak pump utterance complete request_id={request_id}");
|
||||
}
|
||||
Ok(false)
|
||||
Ok((false, active_slot))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn speaking_edge_and_slot_change_emit_logic() {
|
||||
// idle -> speaking: a real edge, emits.
|
||||
let (emit, slot) = next_speaking_state(false, 0, true, Some(0));
|
||||
assert!(emit, "idle -> speaking must emit");
|
||||
assert_eq!(slot, 0);
|
||||
|
||||
// speaking -> speaking, same slot: no edge, no slot change, no emit.
|
||||
let (emit, slot) = next_speaking_state(true, 0, true, Some(0));
|
||||
assert!(!emit, "speaking -> speaking same slot must not emit");
|
||||
assert_eq!(slot, 0);
|
||||
|
||||
// speaking(slot 0) -> speaking(slot 1): mid-speech mascot switch emits
|
||||
// and resolves to the new slot so lip-sync moves to the other mascot.
|
||||
let (emit, slot) = next_speaking_state(true, 0, true, Some(1));
|
||||
assert!(emit, "speaking slot change must emit");
|
||||
assert_eq!(slot, 1, "resolved slot must be the new slot 1");
|
||||
|
||||
// slot = None retains the previous slot; with the same reported flag
|
||||
// that is neither an edge nor a slot change, so no emit.
|
||||
let (emit, slot) = next_speaking_state(true, 1, true, None);
|
||||
assert!(!emit, "no fresh slot + same reported must not emit");
|
||||
assert_eq!(slot, 1, "None retains the previous slot");
|
||||
|
||||
// speaking -> idle: a real edge, emits.
|
||||
let (emit, slot) = next_speaking_state(true, 1, false, None);
|
||||
assert!(emit, "speaking -> idle must emit");
|
||||
assert_eq!(slot, 1, "None retains the previous slot on the way down");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +93,15 @@ pub struct OpenWindowArgs {
|
||||
/// fails closed in core (no wakes fire).
|
||||
#[serde(default)]
|
||||
pub owner_display_name: String,
|
||||
/// ElevenLabs voice for the primary mascot (issue #4277). When the
|
||||
/// user has two mascots enabled the core alternates the speaking
|
||||
/// voice per reply. Absent/empty → core keeps its default voice.
|
||||
#[serde(default)]
|
||||
pub primary_voice_id: Option<String>,
|
||||
/// ElevenLabs voice for the secondary mascot. Absent when only one
|
||||
/// mascot is enabled.
|
||||
#[serde(default)]
|
||||
pub secondary_voice_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Open a dedicated top-level CEF webview window pointed at the Meet URL.
|
||||
@@ -262,6 +271,8 @@ pub async fn meet_call_open_window<R: Runtime>(
|
||||
let url_for_audio = parsed.to_string();
|
||||
let bot_for_audio = args.display_name.clone();
|
||||
let owner_for_audio = args.owner_display_name.clone();
|
||||
let primary_voice_for_audio = args.primary_voice_id.clone();
|
||||
let secondary_voice_for_audio = args.secondary_voice_id.clone();
|
||||
tauri::async_runtime::spawn(async move {
|
||||
if let Err(err) = crate::meet_audio::start(
|
||||
app_for_audio,
|
||||
@@ -269,6 +280,8 @@ pub async fn meet_call_open_window<R: Runtime>(
|
||||
url_for_audio,
|
||||
owner_for_audio,
|
||||
bot_for_audio,
|
||||
primary_voice_for_audio,
|
||||
secondary_voice_for_audio,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -551,6 +564,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_window_args_deserializes_voice_ids() {
|
||||
use serde_json::json;
|
||||
|
||||
// Both voice ids present → parse into Some(...). Proves the
|
||||
// #4277 per-mascot voice fields ride through the command payload.
|
||||
let args: OpenWindowArgs = serde_json::from_value(json!({
|
||||
"request_id": "r",
|
||||
"meet_url": "u",
|
||||
"display_name": "d",
|
||||
"primary_voice_id": "va",
|
||||
"secondary_voice_id": "vb",
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(args.primary_voice_id.as_deref(), Some("va"));
|
||||
assert_eq!(args.secondary_voice_id.as_deref(), Some("vb"));
|
||||
|
||||
// Both omitted → #[serde(default)] yields None (single-mascot
|
||||
// callers and older shells that don't forward voices still parse).
|
||||
let args: OpenWindowArgs = serde_json::from_value(json!({
|
||||
"request_id": "r",
|
||||
"meet_url": "u",
|
||||
"display_name": "d",
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(args.primary_voice_id, None);
|
||||
assert_eq!(args.secondary_voice_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meet_call_state_default_is_empty() {
|
||||
let state = MeetCallState::default();
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
import debug from 'debug';
|
||||
import { type RefObject, useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { useMascotManifest } from '../../features/human/Mascot/manifest/useMascotManifest';
|
||||
import { useComposioIntegrations } from '../../lib/composio/hooks';
|
||||
import { useT } from '../../lib/i18n/I18nContext';
|
||||
import {
|
||||
@@ -25,12 +26,15 @@ import { useAppDispatch, useAppSelector } from '../../store/hooks';
|
||||
import {
|
||||
selectCustomPrimaryColor,
|
||||
selectCustomSecondaryColor,
|
||||
selectDualMascotEnabled,
|
||||
selectMascotColor,
|
||||
selectMeetingMascotVoicePair,
|
||||
selectSelectedMascotId,
|
||||
} from '../../store/mascotSlice';
|
||||
import { selectPersonaDescription, selectPersonaDisplayName } from '../../store/personaSlice';
|
||||
import Button from '../ui/Button';
|
||||
import {
|
||||
buildMeetingMascots,
|
||||
platformLabel,
|
||||
platformUrlPlaceholder,
|
||||
resolveMeetingBotMascotId,
|
||||
@@ -76,6 +80,15 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
|
||||
const mascotColor = useAppSelector(selectMascotColor);
|
||||
const customPrimaryColor = useAppSelector(selectCustomPrimaryColor);
|
||||
const customSecondaryColor = useAppSelector(selectCustomSecondaryColor);
|
||||
// Dual-mascot config (issue #4277): when a distinct second mascot is enabled
|
||||
// we send both slots (each with its own voice) so the backend bot renders
|
||||
// two mascots and alternates who speaks. Single-mascot keeps the legacy
|
||||
// `mascotId` path below untouched.
|
||||
const dualMascotEnabled = useAppSelector(selectDualMascotEnabled);
|
||||
const mascotVoicePair = useAppSelector(selectMeetingMascotVoicePair);
|
||||
// Manifest drives name-addressed routing (#4277 follow-up): each dual slot is
|
||||
// tagged with its display name so "Hey Toshi …" routes to that mascot.
|
||||
const { manifest } = useMascotManifest();
|
||||
|
||||
// ── Meet slice ───────────────────────────────────────────────────────────
|
||||
const meetStatus = useAppSelector(selectBackendMeetStatus);
|
||||
@@ -129,6 +142,17 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
|
||||
mascotColor === 'custom'
|
||||
? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor }
|
||||
: undefined;
|
||||
// Two-mascot slots for the backend bot (issue #4277) — built via the shared
|
||||
// helper so this live-join path and the UpcomingTable scheduled-join path stay
|
||||
// behaviorally identical.
|
||||
const mascots = buildMeetingMascots({
|
||||
dualMascotEnabled,
|
||||
mascotVoicePair,
|
||||
manifest,
|
||||
mascotId,
|
||||
riveColors,
|
||||
agentName,
|
||||
});
|
||||
const wakePhrase = listenOnly ? undefined : `Hey ${agentName}`;
|
||||
|
||||
const handleSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
|
||||
@@ -143,6 +167,14 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
|
||||
!listenOnly,
|
||||
meetingId
|
||||
);
|
||||
// Name-addressing (#4277 follow-up) trace: the exact mascot ids + names sent
|
||||
// to the backend. If `mascots` is undefined or a slot's `name` is empty,
|
||||
// name addressing ("Hey Toshi") can't work — the backend needs both names.
|
||||
log(
|
||||
'[composer] join mascots=%o wakePhrase=%s',
|
||||
mascots?.map(m => ({ mascotId: m.mascotId, name: m.name })),
|
||||
wakePhrase
|
||||
);
|
||||
try {
|
||||
// Await the RPC BEFORE dispatching setBackendMeetJoining so that a
|
||||
// synchronous rejection (bad URL, auth failure) can be shown inline
|
||||
@@ -159,6 +191,8 @@ export function MeetComposer({ onToast, hasSubmittedRef }: MeetComposerProps) {
|
||||
systemPrompt,
|
||||
mascotId,
|
||||
riveColors,
|
||||
// Dual-mascot slots (issue #4277); undefined for single-mascot calls.
|
||||
mascots,
|
||||
correlationId: meetingId,
|
||||
respondToParticipant: displayedRespondTo.trim() || undefined,
|
||||
wakePhrase,
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
import debug from 'debug';
|
||||
import { useState } from 'react';
|
||||
|
||||
import { useMascotManifest } from '../../features/human/Mascot/manifest/useMascotManifest';
|
||||
import { useT } from '../../lib/i18n/I18nContext';
|
||||
import {
|
||||
joinMeetViaBackendBot,
|
||||
@@ -26,13 +27,21 @@ import { useAppSelector } from '../../store/hooks';
|
||||
import {
|
||||
selectCustomPrimaryColor,
|
||||
selectCustomSecondaryColor,
|
||||
selectDualMascotEnabled,
|
||||
selectMascotColor,
|
||||
selectMeetingMascotVoicePair,
|
||||
selectSelectedMascotId,
|
||||
} from '../../store/mascotSlice';
|
||||
import { selectPersonaDescription, selectPersonaDisplayName } from '../../store/personaSlice';
|
||||
import Button from '../ui/Button';
|
||||
import { type JoinPolicy, JoinPolicyToggle } from './JoinPolicyToggle';
|
||||
import { inferPlatformFromUrl, platformLabel, platformLogoUrl } from './meetingUtils';
|
||||
import {
|
||||
buildMeetingMascots,
|
||||
inferPlatformFromUrl,
|
||||
platformLabel,
|
||||
platformLogoUrl,
|
||||
resolveMeetingBotMascotId,
|
||||
} from './meetingUtils';
|
||||
import { useUpcomingMeetings } from './useUpcomingMeetings';
|
||||
|
||||
const log = debug('meetings:upcoming-table');
|
||||
@@ -359,6 +368,14 @@ export function UpcomingTable({
|
||||
const mascotColor = useAppSelector(selectMascotColor);
|
||||
const customPrimaryColor = useAppSelector(selectCustomPrimaryColor);
|
||||
const customSecondaryColor = useAppSelector(selectCustomSecondaryColor);
|
||||
// Dual-mascot config (issue #4277) — see MeetComposer for the rationale;
|
||||
// both join sites resolve the two-mascot slots the same way so a "Join now"
|
||||
// and a manual join behave identically.
|
||||
const dualMascotEnabled = useAppSelector(selectDualMascotEnabled);
|
||||
const mascotVoicePair = useAppSelector(selectMeetingMascotVoicePair);
|
||||
// Manifest drives name-addressed routing (#4277 follow-up): tag each dual slot
|
||||
// with its display name so "Hey Toshi …" routes to that mascot.
|
||||
const { manifest } = useMascotManifest();
|
||||
|
||||
// Live in-call state — lets a row detect that its meeting is already joined
|
||||
// and suppress the "Join now" button. correlationId is a fresh per-join UUID
|
||||
@@ -371,12 +388,29 @@ export function UpcomingTable({
|
||||
return Boolean(backendMeetUrl && m.meet_url && backendMeetUrl === m.meet_url);
|
||||
};
|
||||
|
||||
// Resolve bot join params the same way MeetComposer does.
|
||||
const mascotId = selectedMascotId ?? (mascotColor === 'custom' ? undefined : mascotColor);
|
||||
// Resolve bot join params the same way MeetComposer does — via
|
||||
// resolveMeetingBotMascotId, so a manifest-only id the backend bot doesn't
|
||||
// recognize is dropped here too (a raw `selectedMascotId` fallback would let
|
||||
// it through and diverge from the MeetComposer join).
|
||||
const mascotId = resolveMeetingBotMascotId(selectedMascotId, mascotColor);
|
||||
const riveColors =
|
||||
mascotColor === 'custom'
|
||||
? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor }
|
||||
: undefined;
|
||||
// Bot join name — hoisted to component-body scope because both the `mascots`
|
||||
// array here and the wake phrase inside handleJoin need it.
|
||||
const agentName = personaDisplayName.trim() || 'Tiny';
|
||||
// Two-mascot slots (issue #4277) — built via the shared helper so this
|
||||
// scheduled-join path and the MeetComposer live-join path stay behaviorally
|
||||
// identical.
|
||||
const mascots = buildMeetingMascots({
|
||||
dualMascotEnabled,
|
||||
mascotVoicePair,
|
||||
manifest,
|
||||
mascotId,
|
||||
riveColors,
|
||||
agentName,
|
||||
});
|
||||
|
||||
const handleJoin = async (meeting: UpcomingMeeting) => {
|
||||
if (!meeting.meet_url) return;
|
||||
@@ -388,9 +422,8 @@ export function UpcomingTable({
|
||||
const anchor = replyDisplayName.trim();
|
||||
// Reply mode gates the bot behind a wake phrase so it only reacts when
|
||||
// addressed ("Hey Alex, …"), never to every caption from the anchor —
|
||||
// mirroring MeetComposer. The bot joins as `agentName`, so the phrase must
|
||||
// match it. Listen-only joins (no anchor) send no wake phrase.
|
||||
const agentName = personaDisplayName.trim() || 'Tiny';
|
||||
// mirroring MeetComposer. The bot joins as `agentName` (hoisted above), so
|
||||
// the phrase must match it. Listen-only joins (no anchor) send no wake phrase.
|
||||
const wakePhrase = anchor ? `Hey ${agentName}` : undefined;
|
||||
// Mint a fresh correlation id per join. It becomes the call record's
|
||||
// `request_id` (recent-calls list key + per-call detail filename), so it
|
||||
@@ -408,6 +441,13 @@ export function UpcomingTable({
|
||||
correlationId
|
||||
);
|
||||
setJoiningId(meeting.calendar_event_id);
|
||||
// Name-addressing (#4277 follow-up) trace: mascot ids + names sent to the
|
||||
// backend. Empty `name` on a slot ⇒ name addressing can't route to it.
|
||||
log(
|
||||
'[upcoming] join mascots=%o wakePhrase=%s',
|
||||
mascots?.map(m => ({ mascotId: m.mascotId, name: m.name })),
|
||||
wakePhrase
|
||||
);
|
||||
try {
|
||||
await joinMeetViaBackendBot({
|
||||
meetUrl: meeting.meet_url,
|
||||
@@ -415,6 +455,8 @@ export function UpcomingTable({
|
||||
agentName,
|
||||
systemPrompt: personaDescription || undefined,
|
||||
mascotId: mascotId || undefined,
|
||||
// Dual-mascot slots (issue #4277); undefined for single-mascot calls.
|
||||
mascots,
|
||||
respondToParticipant: anchor || undefined,
|
||||
wakePhrase,
|
||||
listenOnly: !anchor,
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import type { MascotManifest } from '../../../features/human/Mascot/manifest/types';
|
||||
import type { ComposioConnection } from '../../../lib/composio/types';
|
||||
import {
|
||||
buildMeetingMascots,
|
||||
deriveDisplayNameFromEmail,
|
||||
inferPlatformFromUrl,
|
||||
MEETING_PLATFORMS,
|
||||
@@ -257,8 +259,16 @@ describe('resolveMeetingBotMascotId', () => {
|
||||
expect(resolveMeetingBotMascotId('navy', 'yellow')).toBe('navy');
|
||||
});
|
||||
|
||||
it('falls back to the legacy mascot color for a manifest-only mascot id', () => {
|
||||
expect(resolveMeetingBotMascotId('river-guide', 'yellow')).toBe('yellow');
|
||||
it('keeps the "toshi" manifest mascot id the backend now ships as an asset', () => {
|
||||
expect(resolveMeetingBotMascotId('toshi', 'yellow')).toBe('toshi');
|
||||
});
|
||||
|
||||
it('keeps the "tiny-mascot" manifest mascot id the backend now ships as an asset', () => {
|
||||
expect(resolveMeetingBotMascotId('tiny-mascot', 'navy')).toBe('tiny-mascot');
|
||||
});
|
||||
|
||||
it('falls back to the legacy mascot color for a manifest-only mascot id the backend still lacks', () => {
|
||||
expect(resolveMeetingBotMascotId('jarvis', 'yellow')).toBe('yellow');
|
||||
});
|
||||
|
||||
it('uses the mascot color when no mascot id is selected', () => {
|
||||
@@ -269,3 +279,121 @@ describe('resolveMeetingBotMascotId', () => {
|
||||
expect(resolveMeetingBotMascotId('river-guide', 'custom')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// buildMeetingMascots
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('buildMeetingMascots', () => {
|
||||
// findMascot only reads `.id`/`.name`, so a partial manifest is enough.
|
||||
const manifest = {
|
||||
mascots: [
|
||||
{ id: 'toshi', name: 'Toshi' },
|
||||
{ id: 'tiny-mascot', name: 'Tiny' },
|
||||
],
|
||||
} as unknown as MascotManifest;
|
||||
|
||||
const pair = {
|
||||
primary: { mascotId: 'toshi', voiceId: 'voice-a' },
|
||||
secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
|
||||
};
|
||||
|
||||
it('returns undefined when dual mode is off', () => {
|
||||
expect(
|
||||
buildMeetingMascots({
|
||||
dualMascotEnabled: false,
|
||||
mascotVoicePair: pair,
|
||||
manifest,
|
||||
mascotId: 'toshi',
|
||||
agentName: 'Tiny',
|
||||
})
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined when no secondary is configured', () => {
|
||||
expect(
|
||||
buildMeetingMascots({
|
||||
dualMascotEnabled: true,
|
||||
mascotVoicePair: { primary: { mascotId: 'toshi', voiceId: 'voice-a' } },
|
||||
manifest,
|
||||
mascotId: 'toshi',
|
||||
agentName: 'Tiny',
|
||||
})
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined when the primary slot id cannot be resolved', () => {
|
||||
// primary has no explicit mascot AND the resolved single-path id is undefined.
|
||||
expect(
|
||||
buildMeetingMascots({
|
||||
dualMascotEnabled: true,
|
||||
mascotVoicePair: {
|
||||
primary: { voiceId: 'voice-a' },
|
||||
secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
|
||||
},
|
||||
manifest,
|
||||
mascotId: undefined,
|
||||
agentName: 'Tiny',
|
||||
})
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('falls back to the resolved single-path mascotId for the primary slot', () => {
|
||||
const mascots = buildMeetingMascots({
|
||||
dualMascotEnabled: true,
|
||||
mascotVoicePair: {
|
||||
primary: { voiceId: 'voice-a' },
|
||||
secondary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
|
||||
},
|
||||
manifest,
|
||||
mascotId: 'toshi',
|
||||
agentName: 'Tiny',
|
||||
});
|
||||
expect(mascots?.[0].mascotId).toBe('toshi');
|
||||
});
|
||||
|
||||
it('builds two slots with manifest names and per-slot voices', () => {
|
||||
const riveColors = { primaryColor: '#111', secondaryColor: '#222' };
|
||||
const mascots = buildMeetingMascots({
|
||||
dualMascotEnabled: true,
|
||||
mascotVoicePair: pair,
|
||||
manifest,
|
||||
mascotId: 'toshi',
|
||||
riveColors,
|
||||
agentName: 'Tiny',
|
||||
});
|
||||
expect(mascots).toEqual([
|
||||
{ mascotId: 'toshi', name: 'Toshi', voiceId: 'voice-a', riveColors },
|
||||
{ mascotId: 'tiny-mascot', name: 'Tiny', voiceId: 'voice-b', riveColors },
|
||||
]);
|
||||
});
|
||||
|
||||
it('resolves names by slot so either mascot order works', () => {
|
||||
const mascots = buildMeetingMascots({
|
||||
dualMascotEnabled: true,
|
||||
mascotVoicePair: {
|
||||
primary: { mascotId: 'tiny-mascot', voiceId: 'voice-b' },
|
||||
secondary: { mascotId: 'toshi', voiceId: 'voice-a' },
|
||||
},
|
||||
manifest,
|
||||
mascotId: 'tiny-mascot',
|
||||
agentName: 'Tiny',
|
||||
});
|
||||
expect(mascots?.map(m => m.name)).toEqual(['Tiny', 'Toshi']);
|
||||
});
|
||||
|
||||
it('falls back to agentName for the primary when its manifest entry is missing', () => {
|
||||
const mascots = buildMeetingMascots({
|
||||
dualMascotEnabled: true,
|
||||
mascotVoicePair: {
|
||||
primary: { mascotId: 'river-guide', voiceId: 'voice-a' },
|
||||
secondary: { mascotId: 'toshi', voiceId: 'voice-a' },
|
||||
},
|
||||
manifest,
|
||||
mascotId: 'river-guide',
|
||||
agentName: 'Persona',
|
||||
});
|
||||
expect(mascots?.[0].name).toBe('Persona');
|
||||
expect(mascots?.[1].name).toBe('Toshi');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,17 +5,31 @@
|
||||
* previously embedded inside MeetingBotsCard so they can be unit-tested in
|
||||
* isolation and shared across the split composer components.
|
||||
*/
|
||||
import { findMascot } from '../../features/human/Mascot/manifest/manifestService';
|
||||
import type { MascotManifest } from '../../features/human/Mascot/manifest/types';
|
||||
import type { MascotColor } from '../../features/human/Mascot/mascotPalette';
|
||||
import type { ComposioConnection } from '../../lib/composio/types';
|
||||
import type { MeetingPlatform } from '../../services/meetCallService';
|
||||
import type { BackendMeetJoinInput, MeetingPlatform } from '../../services/meetCallService';
|
||||
import { composioLogoUrl } from '../composio/toolkitMeta';
|
||||
|
||||
/**
|
||||
* Mascot ids the meeting-bot backend recognizes. Newer manifest-only mascot
|
||||
* ids (e.g. "river-guide") aren't supported there, so the bot falls back to the
|
||||
* legacy mascot color for them.
|
||||
*
|
||||
* "toshi" and "tiny-mascot" are the tinyhumansai/mascots manifest ids that the
|
||||
* Recall meeting-bot backend now ships as real mascot assets, so they pass
|
||||
* through unchanged instead of being collapsed to a color.
|
||||
*/
|
||||
const MEETING_BOT_MASCOT_IDS = new Set(['yellow', 'blue', 'burgundy', 'black', 'navy']);
|
||||
const MEETING_BOT_MASCOT_IDS = new Set([
|
||||
'yellow',
|
||||
'blue',
|
||||
'burgundy',
|
||||
'black',
|
||||
'navy',
|
||||
'toshi',
|
||||
'tiny-mascot',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Resolve the mascot id to send to the meeting bot: the selected mascot id when
|
||||
@@ -31,6 +45,63 @@ export function resolveMeetingBotMascotId(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Voice/mascot pair the composer selects for the two speaking slots. */
|
||||
type MeetingMascotVoicePair = {
|
||||
primary: { mascotId?: string | null; voiceId?: string };
|
||||
secondary?: { mascotId?: string | null; voiceId?: string } | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Build the backend bot's dual-mascot `mascots` array (issue #4277), shared by
|
||||
* both join sites — MeetComposer (live join) and UpcomingTable (scheduled join)
|
||||
* — so the two paths stay behaviorally identical. They had already begun to
|
||||
* drift, which is exactly the class of bug centralising this prevents.
|
||||
*
|
||||
* Slot 0 (primary) reuses the resolved single-path `mascotId` when the voice
|
||||
* pair's primary carries no explicit mascot; slot 1 is the secondary. Slot names
|
||||
* come from the manifest so name-addressed routing works in either order ("Hey
|
||||
* Toshi" → whichever slot Toshi occupies); the primary falls back to `agentName`
|
||||
* only when its manifest entry is unavailable. Per-mascot colors are out of
|
||||
* scope — both slots reuse the single `riveColors`.
|
||||
*
|
||||
* Returns `undefined` (→ the single-mascot `mascotId` join) unless dual mode is
|
||||
* on, a secondary is configured, AND both slot ids resolve to a concrete value:
|
||||
* a blank slot-0 id would make the backend drop it and render the secondary
|
||||
* alone, mismatching the on-camera primary.
|
||||
*/
|
||||
export function buildMeetingMascots(input: {
|
||||
dualMascotEnabled: boolean;
|
||||
mascotVoicePair: MeetingMascotVoicePair;
|
||||
manifest: MascotManifest | null;
|
||||
mascotId: string | undefined;
|
||||
riveColors?: { primaryColor?: string; secondaryColor?: string };
|
||||
agentName: string;
|
||||
}): BackendMeetJoinInput['mascots'] {
|
||||
const { dualMascotEnabled, mascotVoicePair, manifest, mascotId, riveColors, agentName } = input;
|
||||
const primarySlotId = mascotVoicePair.primary.mascotId ?? mascotId;
|
||||
const secondarySlotId = mascotVoicePair.secondary?.mascotId ?? undefined;
|
||||
if (!dualMascotEnabled || !mascotVoicePair.secondary || !primarySlotId || !secondarySlotId) {
|
||||
return undefined;
|
||||
}
|
||||
const primaryName =
|
||||
(manifest ? findMascot(manifest, primarySlotId)?.name : undefined) ?? agentName;
|
||||
const secondaryName = manifest ? findMascot(manifest, secondarySlotId)?.name : undefined;
|
||||
return [
|
||||
{
|
||||
mascotId: primarySlotId,
|
||||
name: primaryName,
|
||||
voiceId: mascotVoicePair.primary.voiceId,
|
||||
riveColors,
|
||||
},
|
||||
{
|
||||
mascotId: secondarySlotId,
|
||||
name: secondaryName,
|
||||
voiceId: mascotVoicePair.secondary.voiceId,
|
||||
riveColors,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Platform registry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
selectMascotVoiceGender,
|
||||
selectMascotVoiceId,
|
||||
selectMascotVoiceUseLocaleDefault,
|
||||
selectSecondaryMascotId,
|
||||
selectSelectedMascotId,
|
||||
setCustomMascotGifUrl,
|
||||
setCustomPrimaryColor,
|
||||
@@ -30,6 +31,7 @@ import {
|
||||
setMascotVoiceGender,
|
||||
setMascotVoiceId,
|
||||
setMascotVoiceUseLocaleDefault,
|
||||
setSecondaryMascotId,
|
||||
setSelectedMascotId,
|
||||
SUPPORTED_MASCOT_COLORS,
|
||||
} from '../../../store/mascotSlice';
|
||||
@@ -41,6 +43,7 @@ import {
|
||||
ELEVENLABS_VOICE_PRESETS,
|
||||
isCuratedVoicePreset,
|
||||
} from './elevenlabsVoicePresets';
|
||||
import PerMascotVoiceRow from './PerMascotVoiceRow';
|
||||
|
||||
interface ColorOption {
|
||||
id: MascotColor;
|
||||
@@ -70,6 +73,7 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
const customPrimary = useAppSelector(selectCustomPrimaryColor);
|
||||
const customSecondary = useAppSelector(selectCustomSecondaryColor);
|
||||
const selectedMascotId = useAppSelector(selectSelectedMascotId);
|
||||
const secondaryMascotId = useAppSelector(selectSecondaryMascotId);
|
||||
const customMascotGifUrl = useAppSelector(selectCustomMascotGifUrl);
|
||||
const storedVoiceId = useAppSelector(selectMascotVoiceId);
|
||||
const voiceGender = useAppSelector(selectMascotVoiceGender);
|
||||
@@ -126,6 +130,21 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
// null ("default") case has to clear it here so the stage falls back to
|
||||
// the default manifest mascot rather than the GIF.
|
||||
if (id == null) dispatch(setCustomMascotGifUrl(null));
|
||||
// A newly-picked primary that collides with the current secondary would
|
||||
// leave both slots pointing at the same mascot; clear the secondary so
|
||||
// the duo never duplicates. The reducer's `selectDualMascotEnabled`
|
||||
// guard already treats a collision as single-mascot, but clearing here
|
||||
// keeps the picker's rendered state honest.
|
||||
if (id != null && id === secondaryMascotId) dispatch(setSecondaryMascotId(null));
|
||||
};
|
||||
|
||||
// ── Second-mascot picker (issue #4277) ───────────────────────────
|
||||
// Enable / clear the meeting duo's second mascot. `null` (the "None"
|
||||
// option) drops back to single-mascot. Picking the primary's id is
|
||||
// disabled in the dropdown, so this only ever dispatches a distinct id
|
||||
// or null.
|
||||
const handleSelectSecondaryMascot = (id: string | null) => {
|
||||
dispatch(setSecondaryMascotId(id));
|
||||
};
|
||||
|
||||
const onSaveCustomGif = () => {
|
||||
@@ -683,6 +702,82 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
{t('settings.mascot.characterDesc')}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* ── Meeting duo: second mascot + per-mascot voices (issue #4277) ─
|
||||
Only meaningful for manifest mascots — a custom GIF avatar is a
|
||||
single-figure path, so the whole block hides while one is set. */}
|
||||
{manifest && manifest.mascots.length > 0 && !customMascotGifUrl && (
|
||||
<div>
|
||||
<h3 className="text-xs font-semibold uppercase tracking-wider text-content-faint mb-2 px-1">
|
||||
{t('settings.mascot.secondaryHeading')}
|
||||
</h3>
|
||||
|
||||
{/* Second-mascot picker — bespoke label + select combo mirroring
|
||||
the voice preset dropdown. The primary's id is disabled so the
|
||||
duo can never duplicate a single mascot. */}
|
||||
<div className="bg-surface rounded-xl border border-line p-4 space-y-1">
|
||||
<label className="block space-y-1">
|
||||
<span className="sr-only">{t('settings.mascot.secondaryHeading')}</span>
|
||||
<SettingsSelect
|
||||
aria-label={t('settings.mascot.secondaryHeading')}
|
||||
data-testid="mascot-secondary-select"
|
||||
value={secondaryMascotId ?? '__none__'}
|
||||
onChange={e =>
|
||||
handleSelectSecondaryMascot(e.target.value === '__none__' ? null : e.target.value)
|
||||
}
|
||||
className="w-full">
|
||||
<option value="__none__">{t('settings.mascot.secondaryNone')}</option>
|
||||
{manifest.mascots.map(mascot => (
|
||||
<option
|
||||
key={mascot.id}
|
||||
value={mascot.id}
|
||||
// Skip the primary's id — it already speaks as the first
|
||||
// mascot, so offering it as the second is a no-op the
|
||||
// reducer would reject anyway. When the primary is still the
|
||||
// default (selectedMascotId is null), the effective primary
|
||||
// is the resolved default entry (activeEntry), so disable
|
||||
// that too — otherwise the same mascot could be picked for
|
||||
// both slots and the meeting would render two identical ones.
|
||||
disabled={mascot.id === (selectedMascotId ?? activeEntry?.id)}>
|
||||
{mascot.name}
|
||||
</option>
|
||||
))}
|
||||
</SettingsSelect>
|
||||
</label>
|
||||
</div>
|
||||
<p className="text-xs text-content-muted leading-relaxed px-1 mt-2">
|
||||
{t('settings.mascot.secondaryDesc')}
|
||||
</p>
|
||||
|
||||
{/* Per-mascot voices — a row per mascot whose voice is actually
|
||||
addressable in `mascotVoices` (keyed by a concrete manifest
|
||||
id). The join path (`selectMeetingMascotVoicePair`) resolves
|
||||
the primary slot's voice from `mascotVoices[selectedMascotId]`,
|
||||
so the primary row only appears once a specific primary mascot
|
||||
is pinned; on the default mascot the effective single voice
|
||||
(governed by the Voice section above) is what plays. Each row
|
||||
writes its own `mascotVoices` entry and owns a guarded preview. */}
|
||||
{secondaryMascotId != null && secondaryMascotId !== selectedMascotId && (
|
||||
<div className="mt-3 space-y-3">
|
||||
<h4 className="text-[11px] font-medium uppercase tracking-wide text-content-muted px-1">
|
||||
{t('settings.mascot.perMascotVoiceHeading')}
|
||||
</h4>
|
||||
{selectedMascotId != null && (
|
||||
<PerMascotVoiceRow
|
||||
mascotId={selectedMascotId}
|
||||
label={t('settings.mascot.primaryVoiceLabel')}
|
||||
testIdPrefix="mascot-voice-primary"
|
||||
/>
|
||||
)}
|
||||
<PerMascotVoiceRow
|
||||
mascotId={secondaryMascotId}
|
||||
label={t('settings.mascot.secondaryVoiceLabel')}
|
||||
testIdPrefix="mascot-voice-secondary"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { synthesizeSpeech } from '../../../features/human/voice/ttsClient';
|
||||
import { useT } from '../../../lib/i18n/I18nContext';
|
||||
import { useAppDispatch, useAppSelector } from '../../../store/hooks';
|
||||
import {
|
||||
selectEffectiveMascotVoiceId,
|
||||
selectMascotVoiceFor,
|
||||
selectMascotVoiceGender,
|
||||
setMascotVoice,
|
||||
} from '../../../store/mascotSlice';
|
||||
import Button from '../../ui/Button';
|
||||
import { SettingsSelect, SettingsTextField } from '../controls';
|
||||
import { ELEVENLABS_VOICE_PRESETS, isCuratedVoicePreset } from './elevenlabsVoicePresets';
|
||||
|
||||
interface PerMascotVoiceRowProps {
|
||||
/** Manifest mascot id this row controls the voice for. Writes land in
|
||||
* `mascotVoices[mascotId]` via `setMascotVoice`. */
|
||||
mascotId: string;
|
||||
/** Human-readable heading for the row (e.g. the mascot's name), already
|
||||
* localized by the caller. */
|
||||
label: string;
|
||||
/** Stable test hook so both the primary and secondary rows expose
|
||||
* distinct `data-testid`s (`mascot-voice-{primary,secondary}-*`). */
|
||||
testIdPrefix: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-mascot reply-voice control (issue #4277). A trimmed sibling of the
|
||||
* primary voice section in `MascotPanel`: it reuses the same preset
|
||||
* dropdown + custom-paste + guarded `synthesizeSpeech` preview, but writes
|
||||
* to `mascotVoices[mascotId]` via `setMascotVoice` instead of the single
|
||||
* `voiceId`. The current value is the per-mascot override
|
||||
* (`selectMascotVoiceFor`) falling back to the effective single voice
|
||||
* (`selectEffectiveMascotVoiceId`), so a mascot with no override sounds
|
||||
* exactly like the single-voice behaviour until the user picks one.
|
||||
*
|
||||
* Extracted from `MascotPanel` to keep that file within the ~500-line
|
||||
* budget while the per-mascot map (primary + secondary) doubles the voice
|
||||
* UI. Each instance owns its own preview-abort guard (`previewRequestIdRef`)
|
||||
* so the two rows never share an in-flight preview.
|
||||
*/
|
||||
const PerMascotVoiceRow = ({ mascotId, label, testIdPrefix }: PerMascotVoiceRowProps) => {
|
||||
const { t } = useT();
|
||||
const dispatch = useAppDispatch();
|
||||
// Per-mascot override, or the effective single voice when unset — the
|
||||
// same resolution the meeting join path uses, so the picker shows what
|
||||
// the mascot will actually speak with.
|
||||
const overrideVoiceId = useAppSelector(selectMascotVoiceFor(mascotId));
|
||||
const effectiveVoiceId = useAppSelector(selectEffectiveMascotVoiceId);
|
||||
const currentVoiceId = overrideVoiceId ?? effectiveVoiceId;
|
||||
// Reuse the global gender bucket to filter the preset dropdown — the
|
||||
// per-mascot control only overrides the voice id, not the gender filter.
|
||||
const voiceGender = useAppSelector(selectMascotVoiceGender);
|
||||
|
||||
// Paste-mode is sticky for the same reason as the primary control: a
|
||||
// curated preset id and a mid-paste custom id both leave the stored
|
||||
// value looking like a known id, so we can't derive the mode from it.
|
||||
const [voiceDraft, setVoiceDraft] = useState<string>(overrideVoiceId ?? '');
|
||||
const [voicePasteMode, setVoicePasteMode] = useState<boolean>(false);
|
||||
const [isPreviewingVoice, setIsPreviewingVoice] = useState(false);
|
||||
const [voicePreviewError, setVoicePreviewError] = useState<string | null>(null);
|
||||
const previewAudioRef = useRef<HTMLAudioElement | null>(null);
|
||||
// Monotonically-bumped preview-request id, mirroring the primary
|
||||
// control: unmount + each new preview both increment it so an in-flight
|
||||
// `synthesizeSpeech(...)` whose resolve loses the race bails before it
|
||||
// touches refs / state.
|
||||
const previewRequestIdRef = useRef(0);
|
||||
|
||||
// Stop any in-flight preview audio on unmount and invalidate a pending
|
||||
// `synthesizeSpeech(...)` so a late resolve can't start audio for a row
|
||||
// the user has already navigated away from.
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
previewRequestIdRef.current += 1;
|
||||
if (previewAudioRef.current) {
|
||||
previewAudioRef.current.pause();
|
||||
previewAudioRef.current.src = '';
|
||||
previewAudioRef.current = null;
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Presets the dropdown should expose: always include the current voice
|
||||
// (so the controlled select never points at an absent option) plus the
|
||||
// active gender bucket and any '*' fallback voices.
|
||||
const visiblePresets = ELEVENLABS_VOICE_PRESETS.filter(
|
||||
p => p.id === currentVoiceId || p.gender === voiceGender || p.locales.includes('*')
|
||||
);
|
||||
|
||||
// A custom (non-curated) override keeps the paste editor open so the
|
||||
// stored id stays visible; the effective-voice fallback is never treated
|
||||
// as "custom" because the mascot has no explicit override yet.
|
||||
const isCustomVoice =
|
||||
voicePasteMode || (overrideVoiceId != null && !isCuratedVoicePreset(overrideVoiceId));
|
||||
|
||||
const onPresetChange = (next: string) => {
|
||||
if (next === '__custom__') {
|
||||
setVoicePasteMode(true);
|
||||
setVoiceDraft(overrideVoiceId ?? '');
|
||||
return;
|
||||
}
|
||||
setVoicePasteMode(false);
|
||||
setVoicePreviewError(null);
|
||||
setVoiceDraft(next);
|
||||
dispatch(setMascotVoice({ mascotId, voiceId: next }));
|
||||
};
|
||||
|
||||
const onSavePaste = () => {
|
||||
setVoicePreviewError(null);
|
||||
const trimmed = voiceDraft.trim();
|
||||
setVoiceDraft(trimmed);
|
||||
dispatch(setMascotVoice({ mascotId, voiceId: trimmed.length > 0 ? trimmed : null }));
|
||||
};
|
||||
|
||||
const onVoiceReset = () => {
|
||||
setVoicePreviewError(null);
|
||||
setVoicePasteMode(false);
|
||||
setVoiceDraft('');
|
||||
dispatch(setMascotVoice({ mascotId, voiceId: null }));
|
||||
};
|
||||
|
||||
const onVoicePreview = async () => {
|
||||
// Same abort guard as the primary control: reserve a fresh id, and let
|
||||
// a stale resolve detect that a newer preview (or unmount) superseded
|
||||
// it before it mutates state or plays audio.
|
||||
const requestId = ++previewRequestIdRef.current;
|
||||
setIsPreviewingVoice(true);
|
||||
setVoicePreviewError(null);
|
||||
if (previewAudioRef.current) {
|
||||
previewAudioRef.current.pause();
|
||||
previewAudioRef.current.src = '';
|
||||
previewAudioRef.current = null;
|
||||
}
|
||||
try {
|
||||
const tts = await synthesizeSpeech(t('settings.mascot.voice.previewText'), {
|
||||
voiceId: currentVoiceId,
|
||||
});
|
||||
if (previewRequestIdRef.current !== requestId) return;
|
||||
const src = `data:${tts.audio_mime || 'audio/mpeg'};base64,${tts.audio_base64}`;
|
||||
const audio = new window.Audio(src);
|
||||
previewAudioRef.current = audio;
|
||||
await audio.play();
|
||||
} catch (err) {
|
||||
if (previewRequestIdRef.current !== requestId) return;
|
||||
const message = err instanceof Error ? err.message : t('settings.mascot.voice.previewError');
|
||||
setVoicePreviewError(message);
|
||||
} finally {
|
||||
if (previewRequestIdRef.current === requestId) setIsPreviewingVoice(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="bg-surface rounded-xl border border-line p-4 space-y-3">
|
||||
<span className="text-xs font-medium text-content-muted dark:text-content-secondary">
|
||||
{label}
|
||||
</span>
|
||||
|
||||
{/* Preset dropdown — mirrors the primary control's label + select combo */}
|
||||
<label className="block space-y-1">
|
||||
<span className="sr-only">{t('settings.mascot.voice.presetHeading')}</span>
|
||||
<SettingsSelect
|
||||
aria-label={`${label} — ${t('settings.mascot.voice.presetHeading')}`}
|
||||
data-testid={`${testIdPrefix}-select`}
|
||||
value={isCustomVoice ? '__custom__' : currentVoiceId}
|
||||
onChange={e => onPresetChange(e.target.value)}
|
||||
className="w-full">
|
||||
{visiblePresets.map(v => (
|
||||
<option key={v.id} value={v.id}>
|
||||
{v.label}
|
||||
</option>
|
||||
))}
|
||||
<option value="__custom__">{t('settings.mascot.voice.customOption')}</option>
|
||||
</SettingsSelect>
|
||||
</label>
|
||||
|
||||
{isCustomVoice && (
|
||||
<label className="block space-y-1">
|
||||
<span className="text-xs font-medium text-content-muted dark:text-content-secondary">
|
||||
{t('settings.mascot.voice.customHeading')}
|
||||
</span>
|
||||
<div className="flex gap-2">
|
||||
<SettingsTextField
|
||||
aria-label={`${label} — ${t('settings.mascot.voice.customHeading')}`}
|
||||
data-testid={`${testIdPrefix}-input`}
|
||||
value={voiceDraft}
|
||||
placeholder={t('settings.mascot.voice.customPlaceholder')}
|
||||
onChange={e => setVoiceDraft(e.target.value)}
|
||||
className="flex-1"
|
||||
/>
|
||||
<Button
|
||||
type="button"
|
||||
variant="primary"
|
||||
size="xs"
|
||||
data-testid={`${testIdPrefix}-save-paste`}
|
||||
onClick={onSavePaste}
|
||||
disabled={voiceDraft.trim() === (overrideVoiceId ?? '').trim()}>
|
||||
{t('common.save')}
|
||||
</Button>
|
||||
</div>
|
||||
</label>
|
||||
)}
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Button
|
||||
type="button"
|
||||
variant="primary"
|
||||
size="xs"
|
||||
data-testid={`${testIdPrefix}-preview`}
|
||||
onClick={() => void onVoicePreview()}
|
||||
disabled={isPreviewingVoice}
|
||||
className="bg-emerald-600 hover:bg-emerald-700 dark:bg-emerald-600 dark:hover:bg-emerald-500">
|
||||
{isPreviewingVoice
|
||||
? t('settings.mascot.voice.previewing')
|
||||
: t('settings.mascot.voice.preview')}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="secondary"
|
||||
size="xs"
|
||||
data-testid={`${testIdPrefix}-reset`}
|
||||
onClick={onVoiceReset}
|
||||
disabled={overrideVoiceId == null}>
|
||||
{t('settings.mascot.voice.reset')}
|
||||
</Button>
|
||||
<span
|
||||
data-testid={`${testIdPrefix}-current`}
|
||||
className="ml-1 text-[11px] text-content-muted truncate max-w-[18rem]"
|
||||
title={currentVoiceId}>
|
||||
{t('settings.mascot.voice.current')}: <code className="font-mono">{currentVoiceId}</code>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{voicePreviewError && (
|
||||
<div
|
||||
data-testid={`${testIdPrefix}-preview-error`}
|
||||
className="rounded-md border border-amber-200 dark:border-amber-500/30 bg-amber-50 dark:bg-amber-500/10 p-3 text-xs text-amber-800 dark:text-amber-200">
|
||||
{t('settings.mascot.voice.previewError')}: {voicePreviewError}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default PerMascotVoiceRow;
|
||||
@@ -10,6 +10,7 @@ import mascotReducer, {
|
||||
setCustomMascotGifUrl,
|
||||
setMascotColor,
|
||||
setMascotVoiceId,
|
||||
setSecondaryMascotId,
|
||||
setSelectedMascotId,
|
||||
} from '../../../../store/mascotSlice';
|
||||
import MascotPanel from '../MascotPanel';
|
||||
@@ -362,3 +363,90 @@ describe('MascotPanel — voice picker custom voice input (line 525)', () => {
|
||||
expect(store.getState().mascot.voiceId).toBe('new-voice-id');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Dual mascots + per-mascot voice (issue #4277) ────────────────────────────
|
||||
describe('MascotPanel — meeting duo (second mascot)', () => {
|
||||
const yellow = manifestEntry('yellow', 'Yellow');
|
||||
const toshi = manifestEntry('toshi', 'Toshi');
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
useMascotManifestMock.mockReturnValue(manifestResult([yellow, toshi]));
|
||||
mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0));
|
||||
});
|
||||
|
||||
it('dispatches setSecondaryMascotId when a second mascot is picked', () => {
|
||||
const { store } = renderPanel();
|
||||
fireEvent.change(screen.getByTestId('mascot-secondary-select'), { target: { value: 'toshi' } });
|
||||
expect(store.getState().mascot.secondaryMascotId).toBe('toshi');
|
||||
});
|
||||
|
||||
it('clears the second mascot when None is selected', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setSecondaryMascotId('toshi'));
|
||||
renderPanel(store);
|
||||
fireEvent.change(screen.getByTestId('mascot-secondary-select'), {
|
||||
target: { value: '__none__' },
|
||||
});
|
||||
expect(store.getState().mascot.secondaryMascotId).toBeNull();
|
||||
});
|
||||
|
||||
it('disables the primary mascot as a second-mascot option', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setSelectedMascotId('yellow'));
|
||||
renderPanel(store);
|
||||
const primaryOption = screen
|
||||
.getByTestId('mascot-secondary-select')
|
||||
.querySelector('option[value="yellow"]') as HTMLOptionElement | null;
|
||||
expect(primaryOption).not.toBeNull();
|
||||
expect(primaryOption).toBeDisabled();
|
||||
});
|
||||
|
||||
it('hides the duo picker when a custom GIF avatar is set', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setCustomMascotGifUrl('https://example.com/avatar.gif'));
|
||||
renderPanel(store);
|
||||
expect(screen.queryByTestId('mascot-secondary-select')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('dispatches setMascotVoice for the second mascot when its voice changes', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setSelectedMascotId('yellow'));
|
||||
store.dispatch(setSecondaryMascotId('toshi'));
|
||||
renderPanel(store);
|
||||
|
||||
// The secondary voice row renders once a distinct duo mascot is set.
|
||||
const select = screen.getByTestId('mascot-voice-secondary-select');
|
||||
fireEvent.change(select, { target: { value: 'pNInz6obpgDQGcFmaJgB' } });
|
||||
|
||||
expect(store.getState().mascot.mascotVoices.toshi).toBe('pNInz6obpgDQGcFmaJgB');
|
||||
});
|
||||
|
||||
it('dispatches setMascotVoice for the primary mascot when its voice changes', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setSelectedMascotId('yellow'));
|
||||
store.dispatch(setSecondaryMascotId('toshi'));
|
||||
renderPanel(store);
|
||||
|
||||
const select = screen.getByTestId('mascot-voice-primary-select');
|
||||
fireEvent.change(select, { target: { value: 'pNInz6obpgDQGcFmaJgB' } });
|
||||
|
||||
expect(store.getState().mascot.mascotVoices.yellow).toBe('pNInz6obpgDQGcFmaJgB');
|
||||
});
|
||||
|
||||
it('clears a per-mascot voice via the row reset button', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setSelectedMascotId('yellow'));
|
||||
store.dispatch(setSecondaryMascotId('toshi'));
|
||||
renderPanel(store);
|
||||
|
||||
// Seed an override, then reset it.
|
||||
fireEvent.change(screen.getByTestId('mascot-voice-secondary-select'), {
|
||||
target: { value: 'pNInz6obpgDQGcFmaJgB' },
|
||||
});
|
||||
expect(store.getState().mascot.mascotVoices.toshi).toBe('pNInz6obpgDQGcFmaJgB');
|
||||
|
||||
fireEvent.click(screen.getByTestId('mascot-voice-secondary-reset'));
|
||||
expect(store.getState().mascot.mascotVoices.toshi).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
import { configureStore } from '@reduxjs/toolkit';
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
||||
import { Provider } from 'react-redux';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import mascotReducer, { setMascotVoice } from '../../../../store/mascotSlice';
|
||||
import PerMascotVoiceRow from '../PerMascotVoiceRow';
|
||||
|
||||
const { mockSynthesizeSpeech } = vi.hoisted(() => ({ mockSynthesizeSpeech: vi.fn() }));
|
||||
|
||||
vi.mock('../../../../features/human/voice/ttsClient', () => ({
|
||||
synthesizeSpeech: (...args: unknown[]) => mockSynthesizeSpeech(...args),
|
||||
}));
|
||||
|
||||
const TEST_ID = 'mascot-voice-primary';
|
||||
const MASCOT_ID = 'yellow';
|
||||
|
||||
function buildStore() {
|
||||
return configureStore({ reducer: { mascot: mascotReducer } });
|
||||
}
|
||||
|
||||
function renderRow(store = buildStore()) {
|
||||
return {
|
||||
store,
|
||||
...render(
|
||||
<Provider store={store}>
|
||||
<PerMascotVoiceRow mascotId={MASCOT_ID} label="Yellow" testIdPrefix={TEST_ID} />
|
||||
</Provider>
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolve helper to control the timing of the mocked synthesizeSpeech. */
|
||||
function deferredTts() {
|
||||
let resolve!: (v: { audio_mime: string; audio_base64: string }) => void;
|
||||
let reject!: (err: unknown) => void;
|
||||
const promise = new Promise<{ audio_mime: string; audio_base64: string }>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
describe('PerMascotVoiceRow', () => {
|
||||
let playSpy: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
playSpy = vi.fn().mockResolvedValue(undefined);
|
||||
// Stub window.Audio so `new window.Audio(src)` records the src and exposes
|
||||
// a spy-able play()/pause() without touching the real audio pipeline.
|
||||
vi.stubGlobal(
|
||||
'Audio',
|
||||
class {
|
||||
src: string;
|
||||
constructor(src?: string) {
|
||||
this.src = src ?? '';
|
||||
}
|
||||
play = playSpy;
|
||||
pause = vi.fn();
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('reveals the paste input + Save when the custom option is selected and saves a new id', () => {
|
||||
const { store } = renderRow();
|
||||
|
||||
// No custom input until the __custom__ option is chosen.
|
||||
expect(screen.queryByTestId(`${TEST_ID}-input`)).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByTestId(`${TEST_ID}-select`), { target: { value: '__custom__' } });
|
||||
|
||||
const input = screen.getByTestId(`${TEST_ID}-input`);
|
||||
expect(input).toBeInTheDocument();
|
||||
|
||||
const saveBtn = screen.getByTestId(`${TEST_ID}-save-paste`);
|
||||
// Empty draft still equals the (empty) stored override → disabled.
|
||||
expect(saveBtn).toBeDisabled();
|
||||
|
||||
// A whitespace-padded id enables Save and is trimmed on dispatch.
|
||||
fireEvent.change(input, { target: { value: ' my-custom-voice ' } });
|
||||
expect(saveBtn).not.toBeDisabled();
|
||||
fireEvent.click(saveBtn);
|
||||
|
||||
expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBe('my-custom-voice');
|
||||
});
|
||||
|
||||
it('plays a data:audio preview and toggles the previewing label when synthesizeSpeech resolves', async () => {
|
||||
const d = deferredTts();
|
||||
mockSynthesizeSpeech.mockReturnValue(d.promise);
|
||||
renderRow();
|
||||
|
||||
const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
|
||||
fireEvent.click(previewBtn);
|
||||
|
||||
// Previewing state toggles on immediately (button disabled + label swap).
|
||||
await waitFor(() => expect(previewBtn).toBeDisabled());
|
||||
expect(previewBtn).toHaveTextContent('Previewing…');
|
||||
|
||||
d.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
|
||||
|
||||
await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(1));
|
||||
// A data:audio URI was constructed from the mime + base64 payload.
|
||||
expect(playSpy.mock.instances[0].src).toBe('data:audio/mpeg;base64,QUJD');
|
||||
|
||||
// Previewing resets once the preview finishes.
|
||||
await waitFor(() => expect(previewBtn).not.toBeDisabled());
|
||||
expect(previewBtn).toHaveTextContent('Preview voice');
|
||||
});
|
||||
|
||||
it('renders the preview-error text and resets previewing when synthesizeSpeech rejects', async () => {
|
||||
const d = deferredTts();
|
||||
mockSynthesizeSpeech.mockReturnValue(d.promise);
|
||||
renderRow();
|
||||
|
||||
const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
|
||||
fireEvent.click(previewBtn);
|
||||
await waitFor(() => expect(previewBtn).toBeDisabled());
|
||||
|
||||
d.reject(new Error('tts exploded'));
|
||||
|
||||
const errorBox = await screen.findByTestId(`${TEST_ID}-preview-error`);
|
||||
expect(errorBox).toHaveTextContent('tts exploded');
|
||||
expect(playSpy).not.toHaveBeenCalled();
|
||||
|
||||
// finally branch clears the previewing state.
|
||||
await waitFor(() => expect(previewBtn).not.toBeDisabled());
|
||||
expect(previewBtn).toHaveTextContent('Preview voice');
|
||||
});
|
||||
|
||||
it('is a no-op when a preview resolves after the row has unmounted (previewRequestIdRef guard)', async () => {
|
||||
const d = deferredTts();
|
||||
mockSynthesizeSpeech.mockReturnValue(d.promise);
|
||||
const { unmount } = renderRow();
|
||||
|
||||
fireEvent.click(screen.getByTestId(`${TEST_ID}-preview`));
|
||||
await waitFor(() => expect(mockSynthesizeSpeech).toHaveBeenCalledTimes(1));
|
||||
|
||||
// Unmount bumps previewRequestIdRef, so the late resolve loses the race.
|
||||
unmount();
|
||||
d.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
|
||||
|
||||
// Give the resolved promise a microtask/macrotask to flush.
|
||||
await new Promise(r => setTimeout(r, 0));
|
||||
expect(playSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('exposes the current voice id set via setMascotVoice', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setMascotVoice({ mascotId: MASCOT_ID, voiceId: 'pNInz6obpgDQGcFmaJgB' }));
|
||||
renderRow(store);
|
||||
expect(screen.getByTestId(`${TEST_ID}-current`)).toHaveTextContent('pNInz6obpgDQGcFmaJgB');
|
||||
});
|
||||
|
||||
it('dispatches setMascotVoice when a curated preset is picked from the dropdown', () => {
|
||||
const { store } = renderRow();
|
||||
fireEvent.change(screen.getByTestId(`${TEST_ID}-select`), {
|
||||
target: { value: 'pNInz6obpgDQGcFmaJgB' },
|
||||
});
|
||||
expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBe('pNInz6obpgDQGcFmaJgB');
|
||||
});
|
||||
|
||||
it('clears the per-mascot override via the reset button', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setMascotVoice({ mascotId: MASCOT_ID, voiceId: 'pNInz6obpgDQGcFmaJgB' }));
|
||||
renderRow(store);
|
||||
|
||||
const resetBtn = screen.getByTestId(`${TEST_ID}-reset`);
|
||||
expect(resetBtn).not.toBeDisabled();
|
||||
fireEvent.click(resetBtn);
|
||||
|
||||
expect(store.getState().mascot.mascotVoices[MASCOT_ID]).toBeUndefined();
|
||||
});
|
||||
|
||||
it('stops the prior preview audio when a second preview starts', async () => {
|
||||
const first = deferredTts();
|
||||
const second = deferredTts();
|
||||
mockSynthesizeSpeech.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
|
||||
renderRow();
|
||||
|
||||
const previewBtn = screen.getByTestId(`${TEST_ID}-preview`);
|
||||
|
||||
// First preview resolves and plays, seeding previewAudioRef.
|
||||
fireEvent.click(previewBtn);
|
||||
first.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'QUJD' });
|
||||
await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(1));
|
||||
const firstAudio = playSpy.mock.instances[0];
|
||||
await waitFor(() => expect(previewBtn).not.toBeDisabled());
|
||||
|
||||
// Second preview should pause + clear the prior audio before synthesizing.
|
||||
fireEvent.click(previewBtn);
|
||||
await waitFor(() => expect(mockSynthesizeSpeech).toHaveBeenCalledTimes(2));
|
||||
expect(firstAudio.pause).toHaveBeenCalled();
|
||||
expect(firstAudio.src).toBe('');
|
||||
|
||||
second.resolve({ audio_mime: 'audio/mpeg', audio_base64: 'WFla' });
|
||||
await waitFor(() => expect(playSpy).toHaveBeenCalledTimes(2));
|
||||
});
|
||||
});
|
||||
@@ -1,62 +1,69 @@
|
||||
import { listen, type UnlistenFn } from '@tauri-apps/api/event';
|
||||
import { type FC, useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { type FC, useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useSelector } from 'react-redux';
|
||||
|
||||
import { RiveMascot } from '../human/Mascot';
|
||||
import {
|
||||
selectCustomPrimaryColor,
|
||||
selectCustomSecondaryColor,
|
||||
selectMascotColor,
|
||||
selectSecondaryMascotId,
|
||||
} from '../../store/mascotSlice';
|
||||
import {
|
||||
getMascotPalette,
|
||||
hexToArgbInt,
|
||||
ManifestRiveMascot,
|
||||
type MascotFace,
|
||||
type MascotManifestEntry,
|
||||
RiveMascot,
|
||||
} from '../human/Mascot';
|
||||
import { findMascot } from '../human/Mascot/manifest/manifestService';
|
||||
import { useMascotManifest } from '../human/Mascot/manifest/useMascotManifest';
|
||||
import {
|
||||
drawMascotInCell,
|
||||
FRAME_H,
|
||||
FRAME_H_DUAL,
|
||||
FRAME_W,
|
||||
FRAME_W_DUAL,
|
||||
MASCOT_INSET,
|
||||
sampleCanvasPixels,
|
||||
} from './mascotFrameCompositor';
|
||||
import { type ActiveMascotSlot, type MeetingPhase, useMeetingMascots } from './useMeetingMascots';
|
||||
|
||||
const PRODUCER_FPS = 24;
|
||||
const FRAME_W = 320;
|
||||
const FRAME_H = 240;
|
||||
const JPEG_QUALITY = 0.7;
|
||||
|
||||
/**
|
||||
* How long the mascot(s) wave hello on join before settling into the live
|
||||
* `active` face state (ms). Matches the greeting the participants see so the
|
||||
* first frames read as "the bot is saying hi" rather than a cold stare.
|
||||
*/
|
||||
const GREETING_MS = 2500;
|
||||
/**
|
||||
* Teardown grace after `bus-stopped` (ms): keep the session mounted — WS +
|
||||
* worker alive — long enough for the goodbye wave to stream before the frame
|
||||
* pipeline is torn down. Without this the last thing the call sees is a hard
|
||||
* cut mid-pose instead of a wave.
|
||||
*/
|
||||
const SIGNOFF_MS = 1500;
|
||||
|
||||
interface BusSession {
|
||||
requestId: string;
|
||||
port: number;
|
||||
}
|
||||
|
||||
export function sampleCanvasPixels(
|
||||
ctx: OffscreenCanvasRenderingContext2D,
|
||||
width: number,
|
||||
height: number
|
||||
) {
|
||||
const cols = 7;
|
||||
const rows = 5;
|
||||
let sum = 0;
|
||||
let min = 255;
|
||||
let max = 0;
|
||||
let count = 0;
|
||||
let dark = 0;
|
||||
let bright = 0;
|
||||
|
||||
try {
|
||||
for (let y = 0; y < rows; y++) {
|
||||
for (let x = 0; x < cols; x++) {
|
||||
const px = Math.max(0, Math.min(width - 1, Math.floor(((x + 0.5) * width) / cols)));
|
||||
const py = Math.max(0, Math.min(height - 1, Math.floor(((y + 0.5) * height) / rows)));
|
||||
const [r, g, b] = ctx.getImageData(px, py, 1, 1).data;
|
||||
const luma = Math.round(r * 0.299 + g * 0.587 + b * 0.114);
|
||||
sum += luma;
|
||||
min = Math.min(min, luma);
|
||||
max = Math.max(max, luma);
|
||||
if (luma < 8) dark++;
|
||||
if (luma > 32) bright++;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return {
|
||||
avgLuma: Math.round(sum / Math.max(1, count)),
|
||||
minLuma: min,
|
||||
maxLuma: max,
|
||||
darkSamples: dark,
|
||||
brightSamples: bright,
|
||||
sampleCount: count,
|
||||
};
|
||||
} catch (err) {
|
||||
return { error: String(err instanceof Error ? err.message : err) };
|
||||
}
|
||||
}
|
||||
// Re-export from the compositor for back-compat with existing importers
|
||||
// (the producer test imports `sampleCanvasPixels` from here). The
|
||||
// implementation moved to mascotFrameCompositor.ts (issue #4277).
|
||||
export { sampleCanvasPixels };
|
||||
|
||||
export const MascotFrameProducer: FC = () => {
|
||||
const [session, setSession] = useState<BusSession | null>(null);
|
||||
// Meeting lifecycle phase, owned here so it survives across the brief
|
||||
// sign-off grace where `session` is still set but the bus has stopped.
|
||||
const [phase, setPhase] = useState<MeetingPhase>('greeting');
|
||||
// Set when `bus-stopped` fires; the session is cleared SIGNOFF_MS later so
|
||||
// the goodbye wave can stream. A ref so the timers don't re-arm on render.
|
||||
const signoffTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let unlistenStarted: UnlistenFn | undefined;
|
||||
@@ -67,6 +74,11 @@ export const MascotFrameProducer: FC = () => {
|
||||
const payload = event.payload;
|
||||
if (!payload || !payload.port) return;
|
||||
console.log('[meet-video-producer] bus-started', payload);
|
||||
if (signoffTimerRef.current) {
|
||||
clearTimeout(signoffTimerRef.current);
|
||||
signoffTimerRef.current = null;
|
||||
}
|
||||
setPhase('greeting');
|
||||
setSession(payload);
|
||||
})
|
||||
.then(stop => {
|
||||
@@ -77,7 +89,15 @@ export const MascotFrameProducer: FC = () => {
|
||||
|
||||
listen<{ requestId?: string; request_id?: string }>('meet-video:bus-stopped', event => {
|
||||
console.log('[meet-video-producer] bus-stopped', event.payload);
|
||||
setSession(null);
|
||||
// Enter the goodbye wave and keep the pipeline alive for SIGNOFF_MS so
|
||||
// the wave frames actually reach the call before we clear the session.
|
||||
setPhase('signoff');
|
||||
if (signoffTimerRef.current) clearTimeout(signoffTimerRef.current);
|
||||
signoffTimerRef.current = setTimeout(() => {
|
||||
console.log('[meet-video-producer] sign-off grace elapsed, clearing session');
|
||||
signoffTimerRef.current = null;
|
||||
setSession(null);
|
||||
}, SIGNOFF_MS);
|
||||
})
|
||||
.then(stop => {
|
||||
if (cancelled) stop();
|
||||
@@ -89,69 +109,178 @@ export const MascotFrameProducer: FC = () => {
|
||||
cancelled = true;
|
||||
if (unlistenStarted) unlistenStarted();
|
||||
if (unlistenStopped) unlistenStopped();
|
||||
if (signoffTimerRef.current) {
|
||||
clearTimeout(signoffTimerRef.current);
|
||||
signoffTimerRef.current = null;
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Advance greeting → active after GREETING_MS, per active session. Bound to
|
||||
// requestId so a fresh session restarts the greeting.
|
||||
useEffect(() => {
|
||||
if (!session || phase !== 'greeting') return;
|
||||
const id = setTimeout(() => {
|
||||
console.log('[meet-video-producer] greeting elapsed → active', session.requestId);
|
||||
setPhase('active');
|
||||
}, GREETING_MS);
|
||||
return () => clearTimeout(id);
|
||||
}, [session, phase]);
|
||||
|
||||
if (!session) return null;
|
||||
return <ProducerSession key={session.requestId} session={session} />;
|
||||
return <ProducerSession key={session.requestId} session={session} phase={phase} />;
|
||||
};
|
||||
|
||||
const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
|
||||
const ProducerSession: FC<{ session: BusSession; phase: MeetingPhase }> = ({ session, phase }) => {
|
||||
const hostRef = useRef<HTMLDivElement>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const wsReadyRef = useRef(false);
|
||||
const stoppedRef = useRef(false);
|
||||
const inflightRef = useRef(false);
|
||||
const lastDiagAtRef = useRef(0);
|
||||
const isSpeakingRef = useRef(false);
|
||||
// True while the bot is actively producing PCM into the Meet call.
|
||||
// Drives the mascot face so the mouth animates in time with the audio
|
||||
// participants hear. Source of truth is the Rust speak_pump (edge-detected
|
||||
// from the RPC poll loop). Same requestId guards against stale events from
|
||||
// a previous session bleeding into this session's mascot state.
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
// Which mascot slot is speaking the current audio (issue #4277). Single-
|
||||
// mascot calls always report slot 0. Read off the speaking-state event.
|
||||
const [activeMascotSlot, setActiveMascotSlot] = useState<ActiveMascotSlot>(0);
|
||||
|
||||
// Per-slot render state (which mascot + which face) for this tick.
|
||||
const render = useMeetingMascots({ speaking: isSpeaking, activeMascotSlot, phase });
|
||||
const { dualEnabled } = render;
|
||||
|
||||
// Resolve the manifest entries for both slots. The primary follows the
|
||||
// user's selection (same resolution the Human page uses); the secondary is
|
||||
// looked up by its explicit id so the frame honors both selections and no
|
||||
// longer always shows the bundled default (fixes the pre-existing single-
|
||||
// mascot bug where the meeting camera ignored `selectedMascotId`).
|
||||
const { manifest, entry: primaryEntry } = useMascotManifest();
|
||||
const secondaryMascotId = useSelector(selectSecondaryMascotId);
|
||||
const secondaryEntry: MascotManifestEntry | null =
|
||||
dualEnabled && manifest ? (findMascot(manifest, secondaryMascotId) ?? null) : null;
|
||||
|
||||
// Mascot body colors, mirroring HumanPage so the meeting mascot matches the
|
||||
// one on the Human stage. Per-mascot colors are out of scope (#4277) — both
|
||||
// slots share the single selected color.
|
||||
const mascotColor = useSelector(selectMascotColor);
|
||||
const customPrimary = useSelector(selectCustomPrimaryColor);
|
||||
const customSecondary = useSelector(selectCustomSecondaryColor);
|
||||
const palette = getMascotPalette(mascotColor);
|
||||
const primaryColor = useMemo(
|
||||
() => hexToArgbInt(mascotColor === 'custom' ? customPrimary : palette.bodyFill),
|
||||
[mascotColor, customPrimary, palette]
|
||||
);
|
||||
const secondaryColor = useMemo(
|
||||
() => hexToArgbInt(mascotColor === 'custom' ? customSecondary : palette.neckShadowColor),
|
||||
[mascotColor, customSecondary, palette]
|
||||
);
|
||||
|
||||
const isSpeakingRef = useRef(isSpeaking);
|
||||
useEffect(() => {
|
||||
isSpeakingRef.current = isSpeaking;
|
||||
}, [isSpeaking]);
|
||||
|
||||
// `dualEnabled` and `activeMascotSlot` are read inside captureFrame via refs
|
||||
// so a speaker switch (activeMascotSlot 0↔1) or a dual-mode toggle does NOT
|
||||
// change captureFrame's identity — otherwise the WS/worker effect below
|
||||
// (which depends on captureFrame) would tear down and rebuild the socket +
|
||||
// frame worker on every alternation. captureFrame stays keyed to the session
|
||||
// only, matching the original single-mascot behavior.
|
||||
const dualEnabledRef = useRef(dualEnabled);
|
||||
useEffect(() => {
|
||||
dualEnabledRef.current = dualEnabled;
|
||||
}, [dualEnabled]);
|
||||
const activeMascotSlotRef = useRef(activeMascotSlot);
|
||||
useEffect(() => {
|
||||
activeMascotSlotRef.current = activeMascotSlot;
|
||||
}, [activeMascotSlot]);
|
||||
|
||||
const captureFrame = useCallback(async () => {
|
||||
if (stoppedRef.current || !wsReadyRef.current || inflightRef.current) return;
|
||||
const host = hostRef.current;
|
||||
if (!host) return;
|
||||
const canvas = host.querySelector('canvas');
|
||||
if (!canvas) return;
|
||||
const dualEnabledNow = dualEnabledRef.current;
|
||||
// Look the source canvases up by slot so a face change (which re-renders
|
||||
// the mascot) never changes which canvas we sample.
|
||||
const primaryCanvas = host.querySelector<HTMLCanvasElement>(
|
||||
'[data-mascot-slot="primary"] canvas'
|
||||
);
|
||||
if (!primaryCanvas) return;
|
||||
const secondaryCanvas = dualEnabledNow
|
||||
? host.querySelector<HTMLCanvasElement>('[data-mascot-slot="secondary"] canvas')
|
||||
: null;
|
||||
|
||||
const frameW = dualEnabledNow ? FRAME_W_DUAL : FRAME_W;
|
||||
// The dual frame is taller (16:9) than the single frame so the fake-camera
|
||||
// bridge's cover-scale fills the 1280×720 canvas without cropping — see
|
||||
// FRAME_H_DUAL in mascotFrameCompositor.ts.
|
||||
const frameH = dualEnabledNow ? FRAME_H_DUAL : FRAME_H;
|
||||
|
||||
inflightRef.current = true;
|
||||
try {
|
||||
const offscreen = new OffscreenCanvas(FRAME_W, FRAME_H);
|
||||
const offscreen = new OffscreenCanvas(frameW, frameH);
|
||||
const ctx = offscreen.getContext('2d');
|
||||
if (!ctx) return;
|
||||
|
||||
const grad = ctx.createRadialGradient(
|
||||
FRAME_W / 2,
|
||||
FRAME_H / 2,
|
||||
frameW / 2,
|
||||
frameH / 2,
|
||||
0,
|
||||
FRAME_W / 2,
|
||||
FRAME_H / 2,
|
||||
Math.max(FRAME_W, FRAME_H) * 0.7
|
||||
frameW / 2,
|
||||
frameH / 2,
|
||||
Math.max(frameW, frameH) * 0.7
|
||||
);
|
||||
grad.addColorStop(0, '#FBF3D9');
|
||||
grad.addColorStop(1, '#EFE3B8');
|
||||
ctx.fillStyle = grad;
|
||||
ctx.fillRect(0, 0, FRAME_W, FRAME_H);
|
||||
ctx.fillRect(0, 0, frameW, frameH);
|
||||
|
||||
const inset = 0.06;
|
||||
const fitW = FRAME_W * (1 - 2 * inset);
|
||||
const fitH = FRAME_H * (1 - 2 * inset);
|
||||
const scale = Math.min(fitW / canvas.width, fitH / canvas.height);
|
||||
const dw = canvas.width * scale;
|
||||
const dh = canvas.height * scale;
|
||||
const dx = (FRAME_W - dw) / 2;
|
||||
const dy = (FRAME_H - dh) / 2;
|
||||
ctx.drawImage(canvas, dx, dy, dw, dh);
|
||||
if (dualEnabledNow && secondaryCanvas) {
|
||||
// Two half-cells: [0..half] primary, [half..frameW] secondary.
|
||||
const half = frameW / 2;
|
||||
drawMascotInCell(
|
||||
ctx,
|
||||
primaryCanvas,
|
||||
0,
|
||||
0,
|
||||
half,
|
||||
frameH,
|
||||
MASCOT_INSET,
|
||||
primaryCanvas.width,
|
||||
primaryCanvas.height
|
||||
);
|
||||
drawMascotInCell(
|
||||
ctx,
|
||||
secondaryCanvas,
|
||||
half,
|
||||
0,
|
||||
half,
|
||||
frameH,
|
||||
MASCOT_INSET,
|
||||
secondaryCanvas.width,
|
||||
secondaryCanvas.height
|
||||
);
|
||||
} else {
|
||||
// Single-cell draw. This also covers the dual-but-secondary-not-yet-
|
||||
// mounted tick (a 2.2MB mascot still decoding): rather than emit a
|
||||
// black half we draw the primary across the whole frame for this tick.
|
||||
drawMascotInCell(
|
||||
ctx,
|
||||
primaryCanvas,
|
||||
0,
|
||||
0,
|
||||
frameW,
|
||||
frameH,
|
||||
MASCOT_INSET,
|
||||
primaryCanvas.width,
|
||||
primaryCanvas.height
|
||||
);
|
||||
}
|
||||
|
||||
const probe = sampleCanvasPixels(ctx, FRAME_W, FRAME_H);
|
||||
const probe = sampleCanvasPixels(ctx, frameW, frameH);
|
||||
const blob = await offscreen.convertToBlob({ type: 'image/jpeg', quality: JPEG_QUALITY });
|
||||
const buffer = await blob.arrayBuffer();
|
||||
const ws = wsRef.current;
|
||||
@@ -163,12 +292,17 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
|
||||
JSON.stringify({
|
||||
kind: 'producer-pixel-probe',
|
||||
requestId: session.requestId,
|
||||
canvasWidth: canvas.width,
|
||||
canvasHeight: canvas.height,
|
||||
frameWidth: FRAME_W,
|
||||
frameHeight: FRAME_H,
|
||||
canvasWidth: primaryCanvas.width,
|
||||
canvasHeight: primaryCanvas.height,
|
||||
frameWidth: frameW,
|
||||
frameHeight: frameH,
|
||||
jpegBytes: blob.size,
|
||||
isSpeaking: isSpeakingRef.current,
|
||||
// Dual-mascot diagnostics (issue #4277): whether we drew two
|
||||
// cells this tick, and which slot the audio is on.
|
||||
dualEnabled: dualEnabledNow,
|
||||
secondaryMounted: dualEnabledNow ? Boolean(secondaryCanvas) : undefined,
|
||||
activeMascotSlot: activeMascotSlotRef.current,
|
||||
probe,
|
||||
})
|
||||
);
|
||||
@@ -241,13 +375,19 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
|
||||
// session — a remount tears it down with the rest of the pipeline.
|
||||
let unlistenSpeaking: UnlistenFn | undefined;
|
||||
let speakingListenerCancelled = false;
|
||||
listen<{ requestId?: string; speaking?: boolean }>('meet-video:speaking-state', event => {
|
||||
const payload = event.payload;
|
||||
if (!payload) return;
|
||||
// Ignore events from a different session during teardown / restart.
|
||||
if (payload.requestId && payload.requestId !== session.requestId) return;
|
||||
setIsSpeaking(!!payload.speaking);
|
||||
})
|
||||
listen<{ requestId?: string; speaking?: boolean; activeMascotSlot?: number }>(
|
||||
'meet-video:speaking-state',
|
||||
event => {
|
||||
const payload = event.payload;
|
||||
if (!payload) return;
|
||||
// Ignore events from a different session during teardown / restart.
|
||||
if (payload.requestId && payload.requestId !== session.requestId) return;
|
||||
setIsSpeaking(!!payload.speaking);
|
||||
// `activeMascotSlot` names which mascot is speaking this audio (0|1);
|
||||
// default to slot 0 for single-mascot / older core builds that omit it.
|
||||
setActiveMascotSlot(payload.activeMascotSlot === 1 ? 1 : 0);
|
||||
}
|
||||
)
|
||||
.then(stop => {
|
||||
if (speakingListenerCancelled) stop();
|
||||
else unlistenSpeaking = stop;
|
||||
@@ -289,14 +429,70 @@ const ProducerSession: FC<{ session: BusSession }> = ({ session }) => {
|
||||
position: 'fixed',
|
||||
left: '-99999px',
|
||||
top: 0,
|
||||
width: FRAME_H,
|
||||
width: dualEnabled ? FRAME_H * 2 : FRAME_H,
|
||||
height: FRAME_H,
|
||||
pointerEvents: 'none',
|
||||
opacity: 0,
|
||||
}}>
|
||||
<RiveMascot face={isSpeaking ? 'speaking' : 'idle'} size={FRAME_H} />
|
||||
{/* Slot 0 (primary). Stable key per mascot id so a face change updates in
|
||||
place instead of remounting the Rive/WebGL context. */}
|
||||
<div data-mascot-slot="primary" style={{ width: FRAME_H, height: FRAME_H }}>
|
||||
<MascotStage
|
||||
entry={primaryEntry}
|
||||
face={render.primary.face}
|
||||
primaryColor={primaryColor}
|
||||
secondaryColor={secondaryColor}
|
||||
/>
|
||||
</div>
|
||||
{/* Slot 1 (secondary) — only mounted when a distinct second mascot is
|
||||
enabled. Its 2.2MB asset may still be decoding for the first frames;
|
||||
captureFrame falls back to a single-cell draw until its canvas exists. */}
|
||||
{dualEnabled && render.secondary && (
|
||||
<div data-mascot-slot="secondary" style={{ width: FRAME_H, height: FRAME_H }}>
|
||||
<MascotStage
|
||||
entry={secondaryEntry}
|
||||
face={render.secondary.face}
|
||||
primaryColor={primaryColor}
|
||||
secondaryColor={secondaryColor}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Render one mascot slot. Prefers the manifest mascot (honoring the user's
|
||||
* selection) and falls back to the bundled `RiveMascot` while the manifest is
|
||||
* still resolving so the frame never blanks. Keyed by mascot id upstream so a
|
||||
* *selection* change remounts, while a *face* change updates in place.
|
||||
*/
|
||||
const MascotStage: FC<{
|
||||
entry: MascotManifestEntry | null;
|
||||
face: MascotFace;
|
||||
primaryColor: number;
|
||||
secondaryColor: number;
|
||||
}> = ({ entry, face, primaryColor, secondaryColor }) => {
|
||||
if (entry) {
|
||||
return (
|
||||
<ManifestRiveMascot
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
face={face}
|
||||
size={FRAME_H}
|
||||
primaryColor={primaryColor}
|
||||
secondaryColor={secondaryColor}
|
||||
/>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<RiveMascot
|
||||
face={face}
|
||||
size={FRAME_H}
|
||||
primaryColor={primaryColor}
|
||||
secondaryColor={secondaryColor}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export default MascotFrameProducer;
|
||||
|
||||
@@ -1,17 +1,204 @@
|
||||
import { cleanup } from '@testing-library/react';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { listen } from '@tauri-apps/api/event';
|
||||
import { act, cleanup, waitFor } from '@testing-library/react';
|
||||
import { createElement } from 'react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { renderWithProviders } from '../../../test/test-utils';
|
||||
import { MascotFrameProducer, sampleCanvasPixels } from '../MascotFrameProducer';
|
||||
|
||||
// @tauri-apps/api/event is already mocked in setup.ts (listen → vi.fn())
|
||||
// @tauri-apps/api/event is mocked in setup.ts (listen → vi.fn()). Here we
|
||||
// override it per-test to capture the event handlers so we can drive a fake
|
||||
// `meet-video:bus-started` and assert how many mascot hosts mount.
|
||||
type Listener = (event: { payload: unknown }) => void;
|
||||
|
||||
// The producer renders ManifestRiveMascot / RiveMascot (WebGL). Mock both leaf
|
||||
// renderers to a plain <canvas> host so the frame's slot structure is
|
||||
// assertable without the Rive runtime. Each records the face it was asked to
|
||||
// render via a data attribute.
|
||||
vi.mock('../../human/Mascot', async () => {
|
||||
const actual = await vi.importActual<typeof import('../../human/Mascot')>('../../human/Mascot');
|
||||
const stub = (props: { face?: string }) =>
|
||||
createElement('canvas', { 'data-face': props.face, width: 200, height: 200 });
|
||||
return { ...actual, ManifestRiveMascot: stub, RiveMascot: stub };
|
||||
});
|
||||
|
||||
// Keep the manifest resolution deterministic + synchronous — the host tree
|
||||
// mounts regardless (MascotStage falls back to RiveMascot when entry is null),
|
||||
// so we just avoid a real network fetch.
|
||||
vi.mock('../../human/Mascot/manifest/useMascotManifest', () => ({
|
||||
useMascotManifest: () => ({ manifest: null, entry: null, loading: false, error: null }),
|
||||
}));
|
||||
|
||||
vi.mock('../../human/Mascot/manifest/manifestService', () => ({ findMascot: () => null }));
|
||||
|
||||
// Timings the component uses (kept in sync with MascotFrameProducer.tsx).
|
||||
const GREETING_MS = 2500;
|
||||
const SIGNOFF_MS = 1500;
|
||||
|
||||
/**
|
||||
* A fake `Worker` whose `postMessage({cmd:'start'})` records itself so the test
|
||||
* can fire a single tick on demand (invoking the producer's `onmessage`, which
|
||||
* calls `captureFrame`). Real timers/intervals are avoided — the test drives
|
||||
* ticks explicitly so the capture path runs deterministically under fake
|
||||
* timers.
|
||||
*/
|
||||
const workers: FakeWorker[] = [];
|
||||
class FakeWorker {
|
||||
onmessage: ((e: { data: unknown }) => void) | null = null;
|
||||
started = false;
|
||||
terminated = false;
|
||||
constructor() {
|
||||
workers.push(this);
|
||||
}
|
||||
postMessage(msg: { cmd?: string }) {
|
||||
if (msg?.cmd === 'start') this.started = true;
|
||||
else if (msg?.cmd === 'stop') this.started = false;
|
||||
}
|
||||
terminate() {
|
||||
this.terminated = true;
|
||||
}
|
||||
/** Deliver one 'tick' to the producer, as the interval would. */
|
||||
tick() {
|
||||
this.onmessage?.({ data: 'tick' });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A fake `WebSocket` that opens synchronously (so `wsReadyRef` flips true) and
|
||||
* records every `send()` so the capture path's binary frame + JSON probe are
|
||||
* assertable.
|
||||
*/
|
||||
const sockets: FakeWebSocket[] = [];
|
||||
class FakeWebSocket {
|
||||
static OPEN = 1;
|
||||
readyState = FakeWebSocket.OPEN;
|
||||
binaryType = 'arraybuffer';
|
||||
onopen: (() => void) | null = null;
|
||||
onclose: (() => void) | null = null;
|
||||
onerror: ((e: unknown) => void) | null = null;
|
||||
sent: unknown[] = [];
|
||||
closed = false;
|
||||
constructor() {
|
||||
sockets.push(this);
|
||||
// Fire onopen on the next microtask so the effect that assigns
|
||||
// `ws.onopen = ...` has run before we invoke it.
|
||||
queueMicrotask(() => this.onopen?.());
|
||||
}
|
||||
send(data: unknown) {
|
||||
this.sent.push(data);
|
||||
}
|
||||
close() {
|
||||
this.closed = true;
|
||||
this.onclose?.();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A fake 2D context + OffscreenCanvas that satisfies the capture path:
|
||||
* gradient fill, per-cell drawImage, a pixel probe read, and a JPEG blob whose
|
||||
* `arrayBuffer()` resolves so the buffer is `send()`-able.
|
||||
*/
|
||||
class FakeOffscreenCanvas {
|
||||
drawImageCalls = 0;
|
||||
constructor(
|
||||
public width: number,
|
||||
public height: number
|
||||
) {
|
||||
offscreens.push(this);
|
||||
}
|
||||
getContext() {
|
||||
const self = this;
|
||||
return {
|
||||
createRadialGradient: () => ({ addColorStop() {} }),
|
||||
fillStyle: '' as unknown,
|
||||
fillRect() {},
|
||||
drawImage() {
|
||||
self.drawImageCalls++;
|
||||
},
|
||||
getImageData: () => ({ data: [128, 128, 128, 255] }),
|
||||
};
|
||||
}
|
||||
convertToBlob() {
|
||||
return Promise.resolve({ size: 1234, arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)) });
|
||||
}
|
||||
}
|
||||
const offscreens: FakeOffscreenCanvas[] = [];
|
||||
|
||||
/** Install the browser globals the ProducerSession effect touches. */
|
||||
function installBrowserStubs() {
|
||||
workers.length = 0;
|
||||
sockets.length = 0;
|
||||
offscreens.length = 0;
|
||||
vi.stubGlobal('Worker', FakeWorker);
|
||||
vi.stubGlobal('WebSocket', FakeWebSocket);
|
||||
vi.stubGlobal('OffscreenCanvas', FakeOffscreenCanvas);
|
||||
if (!('createObjectURL' in URL)) {
|
||||
(URL as unknown as { createObjectURL: () => string }).createObjectURL = () => 'blob:x';
|
||||
}
|
||||
if (!('revokeObjectURL' in URL)) {
|
||||
(URL as unknown as { revokeObjectURL: () => void }).revokeObjectURL = () => {};
|
||||
}
|
||||
// jsdom does not implement HTMLMediaElement.play(); the silent keep-alive
|
||||
// audio the producer creates calls `.play().catch(...)`, so give it a
|
||||
// resolving stub.
|
||||
vi.spyOn(HTMLMediaElement.prototype, 'play').mockResolvedValue(undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire the mocked `listen` so each event name captures its handler. Returns a
|
||||
* fn to fire a fake payload for a given event.
|
||||
*/
|
||||
function captureListeners() {
|
||||
const handlers = new Map<string, Listener>();
|
||||
vi.mocked(listen).mockImplementation((event: string, handler: unknown) => {
|
||||
handlers.set(event, handler as Listener);
|
||||
return Promise.resolve(vi.fn());
|
||||
});
|
||||
return {
|
||||
fire(event: string, payload: unknown) {
|
||||
const h = handlers.get(event);
|
||||
if (!h) throw new Error(`no listener registered for ${event}`);
|
||||
h({ payload });
|
||||
},
|
||||
has(event: string) {
|
||||
return handlers.has(event);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const SINGLE_MASCOT_STATE = {
|
||||
mascot: {
|
||||
color: 'yellow',
|
||||
voiceId: null,
|
||||
voiceGender: 'male',
|
||||
voiceUseLocaleDefault: false,
|
||||
selectedMascotId: 'tiny-mascot',
|
||||
secondaryMascotId: null,
|
||||
mascotVoices: {},
|
||||
customMascotGifUrl: null,
|
||||
customPrimaryColor: '#F7D145',
|
||||
customSecondaryColor: '#B23C05',
|
||||
},
|
||||
};
|
||||
|
||||
const DUAL_MASCOT_STATE = { mascot: { ...SINGLE_MASCOT_STATE.mascot, secondaryMascotId: 'toshi' } };
|
||||
|
||||
describe('MascotFrameProducer', () => {
|
||||
afterEach(() => cleanup());
|
||||
beforeEach(() => {
|
||||
installBrowserStubs();
|
||||
});
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
// Restore the setup.ts default so other files that rely on the shared
|
||||
// `listen` mock (resolving to an unlisten fn) are unaffected.
|
||||
vi.mocked(listen).mockReset();
|
||||
vi.mocked(listen).mockResolvedValue(vi.fn());
|
||||
});
|
||||
|
||||
it('renders nothing when no bus session is active', () => {
|
||||
const { container } = renderWithProviders(<MascotFrameProducer />);
|
||||
// Component returns null until a meet-video:bus-started Tauri event fires
|
||||
expect(container.firstChild).toBeNull();
|
||||
});
|
||||
|
||||
@@ -21,44 +208,332 @@ describe('MascotFrameProducer', () => {
|
||||
unmount();
|
||||
}).not.toThrow();
|
||||
});
|
||||
|
||||
it('renders ONE mascot host for a single mascot', async () => {
|
||||
const bus = captureListeners();
|
||||
const { container } = renderWithProviders(<MascotFrameProducer />, {
|
||||
preloadedState: SINGLE_MASCOT_STATE,
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'r1', port: 55555 });
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(container.querySelectorAll('[data-mascot-slot]').length).toBe(1);
|
||||
});
|
||||
expect(container.querySelector('[data-mascot-slot="primary"]')).not.toBeNull();
|
||||
expect(container.querySelector('[data-mascot-slot="secondary"]')).toBeNull();
|
||||
});
|
||||
|
||||
it('renders TWO mascot hosts for two distinct mascots', async () => {
|
||||
const bus = captureListeners();
|
||||
const { container } = renderWithProviders(<MascotFrameProducer />, {
|
||||
preloadedState: DUAL_MASCOT_STATE,
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'r2', port: 55556 });
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(container.querySelectorAll('[data-mascot-slot]').length).toBe(2);
|
||||
});
|
||||
expect(container.querySelector('[data-mascot-slot="primary"]')).not.toBeNull();
|
||||
expect(container.querySelector('[data-mascot-slot="secondary"]')).not.toBeNull();
|
||||
});
|
||||
|
||||
it('ignores a bus-started payload with no port', async () => {
|
||||
const bus = captureListeners();
|
||||
const { container } = renderWithProviders(<MascotFrameProducer />, {
|
||||
preloadedState: SINGLE_MASCOT_STATE,
|
||||
});
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'r0' });
|
||||
});
|
||||
// No port → guard returns early, session never set, nothing renders.
|
||||
expect(container.querySelector('[data-mascot-slot]')).toBeNull();
|
||||
});
|
||||
|
||||
it('waves during greeting then transitions to active after GREETING_MS', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const bus = captureListeners();
|
||||
const { container } = renderWithProviders(<MascotFrameProducer />, {
|
||||
preloadedState: DUAL_MASCOT_STATE,
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'g1', port: 4100 });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
});
|
||||
|
||||
// Greeting phase: both mascots wave.
|
||||
const facesDuringGreeting = Array.from(
|
||||
container.querySelectorAll('[data-mascot-slot] canvas')
|
||||
).map(c => c.getAttribute('data-face'));
|
||||
expect(facesDuringGreeting).toEqual(['waving', 'waving']);
|
||||
|
||||
// Advance past the greeting window → active phase. With no speaking
|
||||
// event yet, the active-slot mascot rests (listening) and the other
|
||||
// shows thinking — no longer both waving.
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(GREETING_MS + 10);
|
||||
});
|
||||
|
||||
const facesAfter = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(
|
||||
c => c.getAttribute('data-face')
|
||||
);
|
||||
expect(facesAfter).not.toEqual(['waving', 'waving']);
|
||||
expect(facesAfter).toContain('listening');
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('holds the sign-off wave for SIGNOFF_MS then clears the session', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const bus = captureListeners();
|
||||
const { container } = renderWithProviders(<MascotFrameProducer />, {
|
||||
preloadedState: DUAL_MASCOT_STATE,
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 's1', port: 4200 });
|
||||
await vi.advanceTimersByTimeAsync(GREETING_MS + 10);
|
||||
});
|
||||
// Session mounted (active phase).
|
||||
expect(container.querySelector('[data-mascot-slot]')).not.toBeNull();
|
||||
|
||||
// bus-stopped → signoff phase; both mascots wave goodbye and the
|
||||
// session stays mounted through the grace window.
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-stopped', { requestId: 's1' });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
});
|
||||
const signoffFaces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(
|
||||
c => c.getAttribute('data-face')
|
||||
);
|
||||
expect(signoffFaces).toEqual(['waving', 'waving']);
|
||||
// Still mounted just before the grace elapses.
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(SIGNOFF_MS - 50);
|
||||
});
|
||||
expect(container.querySelector('[data-mascot-slot]')).not.toBeNull();
|
||||
|
||||
// Grace elapsed → session cleared → producer renders nothing.
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
});
|
||||
expect(container.querySelector('[data-mascot-slot]')).toBeNull();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('re-arms a fresh greeting when bus-started fires during the sign-off grace', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const bus = captureListeners();
|
||||
const { container } = renderWithProviders(<MascotFrameProducer />, {
|
||||
preloadedState: DUAL_MASCOT_STATE,
|
||||
});
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'a1', port: 4300 });
|
||||
await vi.advanceTimersByTimeAsync(GREETING_MS + 10);
|
||||
// Stop → enters signoff + arms the clear timer.
|
||||
bus.fire('meet-video:bus-stopped', { requestId: 'a1' });
|
||||
await vi.advanceTimersByTimeAsync(SIGNOFF_MS - 200);
|
||||
// A new session starts before the clear fires → clears the signoff
|
||||
// timer and restarts greeting.
|
||||
bus.fire('meet-video:bus-started', { requestId: 'a2', port: 4301 });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
});
|
||||
const faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
|
||||
c.getAttribute('data-face')
|
||||
);
|
||||
expect(faces).toEqual(['waving', 'waving']);
|
||||
|
||||
// The old signoff clear must NOT fire now; session stays mounted.
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(SIGNOFF_MS);
|
||||
});
|
||||
expect(container.querySelector('[data-mascot-slot]')).not.toBeNull();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('updates speaking state only for a matching requestId (gate)', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const bus = captureListeners();
|
||||
const { container } = renderWithProviders(<MascotFrameProducer />, {
|
||||
preloadedState: DUAL_MASCOT_STATE,
|
||||
});
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'sp1', port: 4400 });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
});
|
||||
// Reach active phase so the face reflects speaking/slot rather than
|
||||
// the greeting wave.
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(GREETING_MS + 10);
|
||||
});
|
||||
expect(
|
||||
Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
|
||||
c.getAttribute('data-face')
|
||||
)
|
||||
).not.toEqual(['waving', 'waving']);
|
||||
|
||||
// Non-matching requestId is ignored: slot 1 speaking with the wrong id
|
||||
// must NOT flip any mascot to the speaking face.
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:speaking-state', {
|
||||
requestId: 'STALE',
|
||||
speaking: true,
|
||||
activeMascotSlot: 1,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
});
|
||||
let faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
|
||||
c.getAttribute('data-face')
|
||||
);
|
||||
expect(faces).not.toContain('speaking');
|
||||
|
||||
// Matching requestId with slot 1 speaking → the secondary slot animates
|
||||
// (speaking), primary shows thinking.
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:speaking-state', {
|
||||
requestId: 'sp1',
|
||||
speaking: true,
|
||||
activeMascotSlot: 1,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
});
|
||||
faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
|
||||
c.getAttribute('data-face')
|
||||
);
|
||||
expect(faces).toEqual(['thinking', 'speaking']);
|
||||
|
||||
// Slot 0 speaking → primary animates, secondary shows thinking. Also
|
||||
// exercises the `activeMascotSlot === 1 ? 1 : 0` default-to-0 branch.
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:speaking-state', {
|
||||
requestId: 'sp1',
|
||||
speaking: true,
|
||||
activeMascotSlot: 0,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
});
|
||||
faces = Array.from(container.querySelectorAll('[data-mascot-slot] canvas')).map(c =>
|
||||
c.getAttribute('data-face')
|
||||
);
|
||||
expect(faces).toEqual(['speaking', 'thinking']);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('ignores a speaking-state event with no payload', async () => {
|
||||
const bus = captureListeners();
|
||||
renderWithProviders(<MascotFrameProducer />, { preloadedState: SINGLE_MASCOT_STATE });
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'np1', port: 4500 });
|
||||
});
|
||||
await waitFor(() => expect(bus.has('meet-video:speaking-state')).toBe(true));
|
||||
expect(() =>
|
||||
act(() => {
|
||||
bus.fire('meet-video:speaking-state', null);
|
||||
})
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('captures and sends a single-mascot frame over the websocket on a worker tick', async () => {
|
||||
const bus = captureListeners();
|
||||
renderWithProviders(<MascotFrameProducer />, { preloadedState: SINGLE_MASCOT_STATE });
|
||||
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'cap1', port: 4600 });
|
||||
});
|
||||
// Wait for the WS/worker effect to wire up (worker registered, socket
|
||||
// open).
|
||||
await waitFor(() => expect(workers.length).toBeGreaterThan(0));
|
||||
await act(async () => {
|
||||
// let the queued microtask fire ws.onopen
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
workers[workers.length - 1].tick();
|
||||
// captureFrame is async (blob → arrayBuffer); flush its microtasks.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
const ws = sockets[sockets.length - 1];
|
||||
// A binary ArrayBuffer frame was sent.
|
||||
const binary = ws.sent.filter(m => m instanceof ArrayBuffer);
|
||||
expect(binary.length).toBeGreaterThan(0);
|
||||
// The diagnostic JSON probe was also sent, and reports single-mascot.
|
||||
const jsonMsgs = ws.sent
|
||||
.filter((m): m is string => typeof m === 'string')
|
||||
.map(m => JSON.parse(m));
|
||||
expect(jsonMsgs.some(p => p.kind === 'producer-pixel-probe' && p.dualEnabled === false)).toBe(
|
||||
true
|
||||
);
|
||||
// Single-cell draw → drawMascotInCell called once.
|
||||
expect(offscreens[offscreens.length - 1].drawImageCalls).toBe(1);
|
||||
});
|
||||
|
||||
it('captures and sends a dual-mascot frame (two-cell composite) on a worker tick', async () => {
|
||||
const bus = captureListeners();
|
||||
renderWithProviders(<MascotFrameProducer />, { preloadedState: DUAL_MASCOT_STATE });
|
||||
|
||||
await act(async () => {
|
||||
bus.fire('meet-video:bus-started', { requestId: 'cap2', port: 4700 });
|
||||
});
|
||||
await waitFor(() => expect(workers.length).toBeGreaterThan(0));
|
||||
// Ensure both mascot slots are mounted so the dual (two-cell) branch runs.
|
||||
await waitFor(() =>
|
||||
expect(document.querySelectorAll('[data-mascot-slot="secondary"] canvas').length).toBe(1)
|
||||
);
|
||||
await act(async () => {
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
workers[workers.length - 1].tick();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
const ws = sockets[sockets.length - 1];
|
||||
const jsonMsgs = ws.sent
|
||||
.filter((m): m is string => typeof m === 'string')
|
||||
.map(m => JSON.parse(m));
|
||||
expect(
|
||||
jsonMsgs.some(
|
||||
p => p.kind === 'producer-pixel-probe' && p.dualEnabled === true && p.secondaryMounted
|
||||
)
|
||||
).toBe(true);
|
||||
// Two-cell draw → drawMascotInCell called twice (primary + secondary).
|
||||
expect(offscreens[offscreens.length - 1].drawImageCalls).toBe(2);
|
||||
expect(ws.sent.filter(m => m instanceof ArrayBuffer).length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sampleCanvasPixels', () => {
|
||||
it('returns pixel stats for a canvas with mid-range luma', () => {
|
||||
// luma = 0.299*128 + 0.587*128 + 0.114*128 ≈ 128
|
||||
// sampleCanvasPixels is still exported from the producer (re-exported from the
|
||||
// compositor for back-compat); a light smoke check keeps that surface covered
|
||||
// here. Full assertions live in mascotFrameCompositor.test.ts.
|
||||
describe('sampleCanvasPixels (re-export)', () => {
|
||||
it('is re-exported and returns pixel stats', () => {
|
||||
const mockCtx = {
|
||||
getImageData: vi.fn().mockReturnValue({ data: [128, 128, 128, 255] }),
|
||||
} as unknown as OffscreenCanvasRenderingContext2D;
|
||||
|
||||
const result = sampleCanvasPixels(mockCtx, 320, 240);
|
||||
expect(result).toMatchObject({
|
||||
avgLuma: 128,
|
||||
minLuma: 128,
|
||||
maxLuma: 128,
|
||||
darkSamples: 0,
|
||||
brightSamples: 35, // all 35 samples have luma > 32
|
||||
sampleCount: 35, // 7 cols × 5 rows
|
||||
});
|
||||
});
|
||||
|
||||
it('counts dark samples correctly for near-black pixels', () => {
|
||||
// luma ≈ 0.299*4 + 0.587*4 + 0.114*4 ≈ 4 → dark (< 8), not bright (> 32)
|
||||
const mockCtx = {
|
||||
getImageData: vi.fn().mockReturnValue({ data: [4, 4, 4, 255] }),
|
||||
} as unknown as OffscreenCanvasRenderingContext2D;
|
||||
|
||||
const result = sampleCanvasPixels(mockCtx, 320, 240);
|
||||
expect(result).toMatchObject({ darkSamples: 35, brightSamples: 0 });
|
||||
});
|
||||
|
||||
it('returns an error object when getImageData throws', () => {
|
||||
const mockCtx = {
|
||||
getImageData: vi.fn().mockImplementation(() => {
|
||||
throw new Error('canvas tainted');
|
||||
}),
|
||||
} as unknown as OffscreenCanvasRenderingContext2D;
|
||||
|
||||
const result = sampleCanvasPixels(mockCtx, 320, 240);
|
||||
expect(result).toMatchObject({ error: 'canvas tainted' });
|
||||
expect(sampleCanvasPixels(mockCtx, 320, 240)).toMatchObject({ avgLuma: 128, sampleCount: 35 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import {
|
||||
drawMascotInCell,
|
||||
FRAME_H,
|
||||
FRAME_H_DUAL,
|
||||
FRAME_W,
|
||||
FRAME_W_DUAL,
|
||||
MASCOT_INSET,
|
||||
sampleCanvasPixels,
|
||||
} from '../mascotFrameCompositor';
|
||||
|
||||
/** A drawImage-only ctx stub that records the last destination rect. */
|
||||
function makeCtx() {
|
||||
const drawImage =
|
||||
vi.fn<(image: CanvasImageSource, dx: number, dy: number, dw: number, dh: number) => void>();
|
||||
return { drawImage };
|
||||
}
|
||||
|
||||
describe('mascotFrameCompositor geometry constants', () => {
|
||||
it('exposes the locked single/dual frame geometry', () => {
|
||||
expect(FRAME_W).toBe(320);
|
||||
expect(FRAME_W_DUAL).toBe(480);
|
||||
expect(FRAME_H).toBe(240);
|
||||
expect(FRAME_H_DUAL).toBe(270);
|
||||
expect(MASCOT_INSET).toBeCloseTo(0.06);
|
||||
});
|
||||
|
||||
it('keeps the dual frame at 16:9 so the camera bridge cover-scale never crops', () => {
|
||||
// The Tauri camera bridge cover-scales the frame onto a 1280×720 (16:9)
|
||||
// canvas; any non-16:9 frame loses its overflowing axis. Locking the dual
|
||||
// frame to 16:9 keeps both mascots' outer edges intact.
|
||||
expect(FRAME_W_DUAL / FRAME_H_DUAL).toBeCloseTo(16 / 9);
|
||||
});
|
||||
});
|
||||
|
||||
describe('drawMascotInCell — containment (AC#6)', () => {
|
||||
it('contain-scales a TALL source and centers it (fits height, letterboxed x)', () => {
|
||||
const ctx = makeCtx();
|
||||
// Cell = full single frame; source is taller than wide.
|
||||
const srcW = 100;
|
||||
const srcH = 400;
|
||||
const rect = drawMascotInCell(
|
||||
ctx,
|
||||
{} as CanvasImageSource,
|
||||
0,
|
||||
0,
|
||||
FRAME_W,
|
||||
FRAME_H,
|
||||
MASCOT_INSET,
|
||||
srcW,
|
||||
srcH
|
||||
);
|
||||
|
||||
const fitW = FRAME_W * (1 - 2 * MASCOT_INSET); // 281.6
|
||||
const fitH = FRAME_H * (1 - 2 * MASCOT_INSET); // 211.2
|
||||
const scale = Math.min(fitW / srcW, fitH / srcH); // height-bound → fitH/400
|
||||
const dw = srcW * scale;
|
||||
const dh = srcH * scale;
|
||||
const dx = (FRAME_W - dw) / 2;
|
||||
const dy = (FRAME_H - dh) / 2;
|
||||
|
||||
// Height-bound: dh must equal the padded fit height, and never exceed the cell.
|
||||
expect(dh).toBeCloseTo(fitH);
|
||||
expect(rect.dw).toBeCloseTo(dw);
|
||||
expect(rect.dh).toBeCloseTo(dh);
|
||||
expect(rect.dx).toBeCloseTo(dx);
|
||||
expect(rect.dy).toBeCloseTo(dy);
|
||||
// No clipping: the drawn rect stays fully inside the cell bounds.
|
||||
expect(rect.dx).toBeGreaterThanOrEqual(0);
|
||||
expect(rect.dy).toBeGreaterThanOrEqual(0);
|
||||
expect(rect.dx + rect.dw).toBeLessThanOrEqual(FRAME_W + 1e-6);
|
||||
expect(rect.dy + rect.dh).toBeLessThanOrEqual(FRAME_H + 1e-6);
|
||||
expect(ctx.drawImage).toHaveBeenCalledWith(expect.anything(), dx, dy, dw, dh);
|
||||
});
|
||||
|
||||
it('contain-scales a WIDE source and centers it (fits width, letterboxed y)', () => {
|
||||
const ctx = makeCtx();
|
||||
const srcW = 400;
|
||||
const srcH = 100;
|
||||
const rect = drawMascotInCell(
|
||||
ctx,
|
||||
{} as CanvasImageSource,
|
||||
0,
|
||||
0,
|
||||
FRAME_W,
|
||||
FRAME_H,
|
||||
MASCOT_INSET,
|
||||
srcW,
|
||||
srcH
|
||||
);
|
||||
|
||||
const fitW = FRAME_W * (1 - 2 * MASCOT_INSET);
|
||||
const fitH = FRAME_H * (1 - 2 * MASCOT_INSET);
|
||||
const scale = Math.min(fitW / srcW, fitH / srcH); // width-bound → fitW/400
|
||||
const dw = srcW * scale;
|
||||
const dh = srcH * scale;
|
||||
|
||||
// Width-bound: dw must equal the padded fit width.
|
||||
expect(dw).toBeCloseTo(fitW);
|
||||
expect(rect.dw).toBeCloseTo(dw);
|
||||
expect(rect.dh).toBeCloseTo(dh);
|
||||
// Still fully contained.
|
||||
expect(rect.dx).toBeGreaterThanOrEqual(0);
|
||||
expect(rect.dx + rect.dw).toBeLessThanOrEqual(FRAME_W + 1e-6);
|
||||
expect(rect.dy + rect.dh).toBeLessThanOrEqual(FRAME_H + 1e-6);
|
||||
});
|
||||
|
||||
it('draws into each half-cell in dual mode without crossing the divider', () => {
|
||||
const half = FRAME_W_DUAL / 2; // 240
|
||||
// Square source so scale is symmetric and easy to reason about.
|
||||
const srcW = 200;
|
||||
const srcH = 200;
|
||||
|
||||
const leftCtx = makeCtx();
|
||||
const left = drawMascotInCell(
|
||||
leftCtx,
|
||||
{} as CanvasImageSource,
|
||||
0,
|
||||
0,
|
||||
half,
|
||||
FRAME_H,
|
||||
MASCOT_INSET,
|
||||
srcW,
|
||||
srcH
|
||||
);
|
||||
const rightCtx = makeCtx();
|
||||
const right = drawMascotInCell(
|
||||
rightCtx,
|
||||
{} as CanvasImageSource,
|
||||
half,
|
||||
0,
|
||||
half,
|
||||
FRAME_H,
|
||||
MASCOT_INSET,
|
||||
srcW,
|
||||
srcH
|
||||
);
|
||||
|
||||
// Left cell stays entirely left of the divider.
|
||||
expect(left.dx).toBeGreaterThanOrEqual(0);
|
||||
expect(left.dx + left.dw).toBeLessThanOrEqual(half + 1e-6);
|
||||
// Right cell stays entirely right of the divider.
|
||||
expect(right.dx).toBeGreaterThanOrEqual(half - 1e-6);
|
||||
expect(right.dx + right.dw).toBeLessThanOrEqual(FRAME_W_DUAL + 1e-6);
|
||||
// Both cells share the same size (identical source + cell dims).
|
||||
expect(right.dw).toBeCloseTo(left.dw);
|
||||
expect(right.dh).toBeCloseTo(left.dh);
|
||||
// The right cell is offset by exactly `half` from the left one.
|
||||
expect(right.dx - left.dx).toBeCloseTo(half);
|
||||
});
|
||||
|
||||
it('never divides by zero for a not-yet-laid-out (0×0) source', () => {
|
||||
const ctx = makeCtx();
|
||||
const rect = drawMascotInCell(
|
||||
ctx,
|
||||
{} as CanvasImageSource,
|
||||
0,
|
||||
0,
|
||||
FRAME_W,
|
||||
FRAME_H,
|
||||
MASCOT_INSET,
|
||||
0,
|
||||
0
|
||||
);
|
||||
// With the 1px guard the rect is finite and contained, not NaN.
|
||||
expect(Number.isFinite(rect.dw)).toBe(true);
|
||||
expect(Number.isFinite(rect.dh)).toBe(true);
|
||||
expect(rect.dx).toBeGreaterThanOrEqual(0);
|
||||
expect(rect.dy).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
// Moved from MascotFrameProducer.test.tsx (issue #4277) — sampleCanvasPixels
|
||||
// now lives in the compositor module.
|
||||
describe('sampleCanvasPixels', () => {
|
||||
it('returns pixel stats for a canvas with mid-range luma', () => {
|
||||
// luma = 0.299*128 + 0.587*128 + 0.114*128 ≈ 128
|
||||
const mockCtx = {
|
||||
getImageData: vi.fn().mockReturnValue({ data: [128, 128, 128, 255] }),
|
||||
} as unknown as OffscreenCanvasRenderingContext2D;
|
||||
|
||||
const result = sampleCanvasPixels(mockCtx, 320, 240);
|
||||
expect(result).toMatchObject({
|
||||
avgLuma: 128,
|
||||
minLuma: 128,
|
||||
maxLuma: 128,
|
||||
darkSamples: 0,
|
||||
brightSamples: 35, // all 35 samples have luma > 32
|
||||
sampleCount: 35, // 7 cols × 5 rows
|
||||
});
|
||||
});
|
||||
|
||||
it('counts dark samples correctly for near-black pixels', () => {
|
||||
// luma ≈ 0.299*4 + 0.587*4 + 0.114*4 ≈ 4 → dark (< 8), not bright (> 32)
|
||||
const mockCtx = {
|
||||
getImageData: vi.fn().mockReturnValue({ data: [4, 4, 4, 255] }),
|
||||
} as unknown as OffscreenCanvasRenderingContext2D;
|
||||
|
||||
const result = sampleCanvasPixels(mockCtx, 320, 240);
|
||||
expect(result).toMatchObject({ darkSamples: 35, brightSamples: 0 });
|
||||
});
|
||||
|
||||
it('returns an error object when getImageData throws', () => {
|
||||
const mockCtx = {
|
||||
getImageData: vi.fn().mockImplementation(() => {
|
||||
throw new Error('canvas tainted');
|
||||
}),
|
||||
} as unknown as OffscreenCanvasRenderingContext2D;
|
||||
|
||||
const result = sampleCanvasPixels(mockCtx, 320, 240);
|
||||
expect(result).toMatchObject({ error: 'canvas tainted' });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
import { renderHook } from '@testing-library/react';
|
||||
import { createElement, type PropsWithChildren } from 'react';
|
||||
import { Provider } from 'react-redux';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { createTestStore } from '../../../test/test-utils';
|
||||
import {
|
||||
type MeetingPhase,
|
||||
useMeetingMascots,
|
||||
type UseMeetingMascotsInput,
|
||||
} from '../useMeetingMascots';
|
||||
|
||||
/**
|
||||
* Seed the mascot slice via preloadedState. `secondaryMascotId` distinct from
|
||||
* `selectedMascotId` is what flips `selectDualMascotEnabled` on.
|
||||
*/
|
||||
function makeWrapper(mascot: Record<string, unknown>) {
|
||||
const store = createTestStore({
|
||||
mascot: {
|
||||
color: 'yellow',
|
||||
voiceId: null,
|
||||
voiceGender: 'male',
|
||||
voiceUseLocaleDefault: false,
|
||||
selectedMascotId: null,
|
||||
secondaryMascotId: null,
|
||||
mascotVoices: {},
|
||||
customMascotGifUrl: null,
|
||||
customPrimaryColor: '#F7D145',
|
||||
customSecondaryColor: '#B23C05',
|
||||
...mascot,
|
||||
},
|
||||
});
|
||||
return function Wrapper({ children }: PropsWithChildren) {
|
||||
return createElement(Provider, { store, children });
|
||||
};
|
||||
}
|
||||
|
||||
function run(mascot: Record<string, unknown>, input: UseMeetingMascotsInput) {
|
||||
const { result } = renderHook(() => useMeetingMascots(input), { wrapper: makeWrapper(mascot) });
|
||||
return result.current;
|
||||
}
|
||||
|
||||
const SINGLE = { selectedMascotId: 'tiny-mascot', secondaryMascotId: null };
|
||||
const DUAL = { selectedMascotId: 'tiny-mascot', secondaryMascotId: 'toshi' };
|
||||
|
||||
describe('useMeetingMascots — dualEnabled gating', () => {
|
||||
it('is single when no secondary mascot is set', () => {
|
||||
const state = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase: 'active' });
|
||||
expect(state.dualEnabled).toBe(false);
|
||||
expect(state.secondary).toBeNull();
|
||||
expect(state.primary.mascotId).toBe('tiny-mascot');
|
||||
});
|
||||
|
||||
it('is single when the secondary equals the primary (same mascot picked twice)', () => {
|
||||
const state = run(
|
||||
{ selectedMascotId: 'tiny-mascot', secondaryMascotId: 'tiny-mascot' },
|
||||
{ speaking: false, activeMascotSlot: 0, phase: 'active' }
|
||||
);
|
||||
expect(state.dualEnabled).toBe(false);
|
||||
expect(state.secondary).toBeNull();
|
||||
});
|
||||
|
||||
it('is dual when a distinct secondary mascot is set', () => {
|
||||
const state = run(DUAL, { speaking: false, activeMascotSlot: 0, phase: 'active' });
|
||||
expect(state.dualEnabled).toBe(true);
|
||||
expect(state.primary.mascotId).toBe('tiny-mascot');
|
||||
expect(state.secondary?.mascotId).toBe('toshi');
|
||||
});
|
||||
});
|
||||
|
||||
describe('useMeetingMascots — single-mascot face (preserves original behavior)', () => {
|
||||
it('primary follows speaking → speaking, else idle; secondary null', () => {
|
||||
const speaking = run(SINGLE, { speaking: true, activeMascotSlot: 0, phase: 'active' });
|
||||
expect(speaking.primary.face).toBe('speaking');
|
||||
expect(speaking.secondary).toBeNull();
|
||||
|
||||
const silent = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase: 'active' });
|
||||
expect(silent.primary.face).toBe('idle');
|
||||
});
|
||||
|
||||
it('single-mascot ignores phase for the face (no greeting/signoff wave)', () => {
|
||||
// The single path deliberately keeps the legacy speaking/idle mapping.
|
||||
for (const phase of ['greeting', 'active', 'signoff'] as MeetingPhase[]) {
|
||||
const state = run(SINGLE, { speaking: false, activeMascotSlot: 0, phase });
|
||||
expect(state.primary.face).toBe('idle');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('useMeetingMascots — dual face table', () => {
|
||||
it('greeting → both slots wave, regardless of speaking/activeSlot', () => {
|
||||
for (const activeMascotSlot of [0, 1] as const) {
|
||||
for (const speaking of [false, true]) {
|
||||
const state = run(DUAL, { speaking, activeMascotSlot, phase: 'greeting' });
|
||||
expect(state.primary.face).toBe('waving');
|
||||
expect(state.secondary?.face).toBe('waving');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('signoff → both slots wave, regardless of speaking/activeSlot', () => {
|
||||
for (const activeMascotSlot of [0, 1] as const) {
|
||||
for (const speaking of [false, true]) {
|
||||
const state = run(DUAL, { speaking, activeMascotSlot, phase: 'signoff' });
|
||||
expect(state.primary.face).toBe('waving');
|
||||
expect(state.secondary?.face).toBe('waving');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe('active phase — activeSlot × speaking', () => {
|
||||
it('slot 0 active + speaking → primary speaking, secondary thinking', () => {
|
||||
const s = run(DUAL, { speaking: true, activeMascotSlot: 0, phase: 'active' });
|
||||
expect(s.primary.face).toBe('speaking');
|
||||
expect(s.secondary?.face).toBe('thinking');
|
||||
});
|
||||
|
||||
it('slot 0 active + not speaking → primary listening, secondary thinking', () => {
|
||||
const s = run(DUAL, { speaking: false, activeMascotSlot: 0, phase: 'active' });
|
||||
expect(s.primary.face).toBe('listening');
|
||||
expect(s.secondary?.face).toBe('thinking');
|
||||
});
|
||||
|
||||
it('slot 1 active + speaking → secondary speaking, primary thinking', () => {
|
||||
const s = run(DUAL, { speaking: true, activeMascotSlot: 1, phase: 'active' });
|
||||
expect(s.secondary?.face).toBe('speaking');
|
||||
expect(s.primary.face).toBe('thinking');
|
||||
});
|
||||
|
||||
it('slot 1 active + not speaking → secondary listening, primary thinking', () => {
|
||||
const s = run(DUAL, { speaking: false, activeMascotSlot: 1, phase: 'active' });
|
||||
expect(s.secondary?.face).toBe('listening');
|
||||
expect(s.primary.face).toBe('thinking');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* Pure canvas-geometry helpers for the Meet video producer (issue #4277).
|
||||
*
|
||||
* Kept free of any WebGL / DOM-mount dependency so the containment math is
|
||||
* cheap to unit-test in isolation. The producer (`MascotFrameProducer.tsx`)
|
||||
* owns the OffscreenCanvas + WebSocket pipeline; this module owns only the
|
||||
* "where does each mascot get drawn" arithmetic, so the no-clip guarantee
|
||||
* (AC#6) is proven by the compositor tests rather than a WebGL render.
|
||||
*/
|
||||
|
||||
/** Frame width for a single mascot (unchanged from the original producer). */
|
||||
export const FRAME_W = 320;
|
||||
/** Frame height for a single mascot (unchanged from the original producer). */
|
||||
export const FRAME_H = 240;
|
||||
/**
|
||||
* Frame dimensions when two mascots share the frame side-by-side (issue #4277).
|
||||
* Wider than the single frame so each mascot keeps roughly the single-frame
|
||||
* cell width instead of being squeezed to half.
|
||||
*
|
||||
* 480×270 is deliberately 16:9 — the same aspect as the fake-camera capture
|
||||
* canvas (1280×720). The Tauri camera bridge cover-scales the received frame
|
||||
* onto that canvas (`scale = Math.max(W/bw, H/bh)` in `camera_bridge.js`), so a
|
||||
* frame whose aspect differs from 16:9 gets its overflowing axis cropped. A
|
||||
* 480×240 (2:1) dual frame would be scaled to 1440×720 and lose ~27 source px
|
||||
* off each side — clipping the two mascots' outer edges. Matching 16:9 makes the
|
||||
* cover-scale a pure fit with no crop.
|
||||
*/
|
||||
export const FRAME_W_DUAL = 480;
|
||||
export const FRAME_H_DUAL = 270;
|
||||
/**
|
||||
* Fraction of each cell reserved as padding on every side before the mascot
|
||||
* is scaled to fit. Matches the original single-mascot inset so the framing
|
||||
* is visually identical in the single path.
|
||||
*/
|
||||
export const MASCOT_INSET = 0.06;
|
||||
|
||||
/**
|
||||
* Draw `sourceCanvas` scaled to *contain* (never crop) inside the cell at
|
||||
* `(cellX, cellY)` of size `cellW × cellH`, centred, with `inset` padding on
|
||||
* every side.
|
||||
*
|
||||
* Contain-scaling (`min` of the two axis ratios) guarantees the mascot always
|
||||
* fits within the padded cell, so it can never be clipped by the cell edge
|
||||
* (AC#6) regardless of the source canvas aspect ratio. The mascot is centred
|
||||
* within the cell so any leftover space is split evenly.
|
||||
*
|
||||
* Returns the computed destination rect for the caller's diagnostics (and so
|
||||
* the geometry is directly assertable in tests) — the draw itself is the side
|
||||
* effect.
|
||||
*/
|
||||
export function drawMascotInCell(
|
||||
ctx: {
|
||||
drawImage: (image: CanvasImageSource, dx: number, dy: number, dw: number, dh: number) => void;
|
||||
},
|
||||
sourceCanvas: CanvasImageSource,
|
||||
cellX: number,
|
||||
cellY: number,
|
||||
cellW: number,
|
||||
cellH: number,
|
||||
inset: number,
|
||||
srcW: number,
|
||||
srcH: number
|
||||
): { dx: number; dy: number; dw: number; dh: number } {
|
||||
// Guard against a zero-sized source (a canvas that hasn't laid out yet):
|
||||
// scaling by it would produce NaN and a silent no-op draw.
|
||||
const safeSrcW = srcW > 0 ? srcW : 1;
|
||||
const safeSrcH = srcH > 0 ? srcH : 1;
|
||||
|
||||
const fitW = cellW * (1 - 2 * inset);
|
||||
const fitH = cellH * (1 - 2 * inset);
|
||||
const scale = Math.min(fitW / safeSrcW, fitH / safeSrcH);
|
||||
const dw = safeSrcW * scale;
|
||||
const dh = safeSrcH * scale;
|
||||
const dx = cellX + (cellW - dw) / 2;
|
||||
const dy = cellY + (cellH - dh) / 2;
|
||||
ctx.drawImage(sourceCanvas, dx, dy, dw, dh);
|
||||
return { dx, dy, dw, dh };
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a coarse 7×5 grid of luma values from a rendered frame. Used as a
|
||||
* cheap "is the mascot actually on the frame or is it black?" diagnostic that
|
||||
* the producer streams over the debug WebSocket every couple of seconds.
|
||||
*
|
||||
* Moved here from `MascotFrameProducer.tsx` (issue #4277) so it lives next to
|
||||
* the rest of the frame geometry; the producer re-exports it for back-compat
|
||||
* with existing importers.
|
||||
*/
|
||||
export function sampleCanvasPixels(
|
||||
ctx: OffscreenCanvasRenderingContext2D,
|
||||
width: number,
|
||||
height: number
|
||||
) {
|
||||
const cols = 7;
|
||||
const rows = 5;
|
||||
let sum = 0;
|
||||
let min = 255;
|
||||
let max = 0;
|
||||
let count = 0;
|
||||
let dark = 0;
|
||||
let bright = 0;
|
||||
|
||||
try {
|
||||
for (let y = 0; y < rows; y++) {
|
||||
for (let x = 0; x < cols; x++) {
|
||||
const px = Math.max(0, Math.min(width - 1, Math.floor(((x + 0.5) * width) / cols)));
|
||||
const py = Math.max(0, Math.min(height - 1, Math.floor(((y + 0.5) * height) / rows)));
|
||||
const [r, g, b] = ctx.getImageData(px, py, 1, 1).data;
|
||||
const luma = Math.round(r * 0.299 + g * 0.587 + b * 0.114);
|
||||
sum += luma;
|
||||
min = Math.min(min, luma);
|
||||
max = Math.max(max, luma);
|
||||
if (luma < 8) dark++;
|
||||
if (luma > 32) bright++;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return {
|
||||
avgLuma: Math.round(sum / Math.max(1, count)),
|
||||
minLuma: min,
|
||||
maxLuma: max,
|
||||
darkSamples: dark,
|
||||
brightSamples: bright,
|
||||
sampleCount: count,
|
||||
};
|
||||
} catch (err) {
|
||||
return { error: String(err instanceof Error ? err.message : err) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
/**
|
||||
* Resolve per-slot mascot render state for the Meet video frame (issue #4277).
|
||||
*
|
||||
* The producer (`MascotFrameProducer.tsx`) composites up to two mascots into
|
||||
* the outgoing camera frame. This hook is the single source of truth for
|
||||
* *which* mascot each slot shows and *what face* it wears, given the live
|
||||
* speaking-state event + the meeting phase. Kept as a thin, pure-ish selector
|
||||
* hook (redux in → render state out) so the face table is unit-testable via
|
||||
* `renderHook` + `preloadedState` without mounting any WebGL.
|
||||
*
|
||||
* Slot mapping (locked by the mascotSlice contract):
|
||||
* - slot 0 = primary = `selectedMascotId`
|
||||
* - slot 1 = secondary = `secondaryMascotId`
|
||||
* and `activeMascotSlot` on the speaking-state event names which slot is
|
||||
* currently speaking the audio participants hear.
|
||||
*/
|
||||
import debug from 'debug';
|
||||
|
||||
import { useAppSelector } from '../../store/hooks';
|
||||
import {
|
||||
selectDualMascotEnabled,
|
||||
selectMeetingMascotVoicePair,
|
||||
selectSecondaryMascotId,
|
||||
selectSelectedMascotId,
|
||||
} from '../../store/mascotSlice';
|
||||
import type { MascotFace } from '../human/Mascot';
|
||||
|
||||
const log = debug('meet:mascots');
|
||||
|
||||
/**
|
||||
* Coarse lifecycle phase of the on-camera mascot(s):
|
||||
* - `greeting` — just joined; both mascots wave hello.
|
||||
* - `active` — the call is live; faces track who is speaking.
|
||||
* - `signoff` — the session is tearing down; both wave goodbye.
|
||||
*/
|
||||
export type MeetingPhase = 'greeting' | 'active' | 'signoff';
|
||||
|
||||
/** Which slot is producing the audio the call currently hears. */
|
||||
export type ActiveMascotSlot = 0 | 1;
|
||||
|
||||
export interface MeetingMascotSlotRender {
|
||||
/** Manifest mascot id, or `null` for the primary on the default mascot. */
|
||||
mascotId: string | null;
|
||||
/** MascotFace to render this tick. */
|
||||
face: MascotFace;
|
||||
}
|
||||
|
||||
export interface MeetingMascotsRenderState {
|
||||
/** True when a distinct second mascot is enabled (drives dual composite). */
|
||||
dualEnabled: boolean;
|
||||
primary: MeetingMascotSlotRender;
|
||||
/** Non-null only when `dualEnabled`. */
|
||||
secondary: MeetingMascotSlotRender | null;
|
||||
}
|
||||
|
||||
export interface UseMeetingMascotsInput {
|
||||
/** Live from `meet-video:speaking-state` — is audio currently streaming. */
|
||||
speaking: boolean;
|
||||
/** Live from `meet-video:speaking-state` — which slot owns that audio. */
|
||||
activeMascotSlot: ActiveMascotSlot;
|
||||
/** Meeting lifecycle phase; drives the greeting / sign-off wave. */
|
||||
phase: MeetingPhase;
|
||||
}
|
||||
|
||||
/**
|
||||
* Face the *speaking* slot wears while the call is active: mouth-animating
|
||||
* `speaking` when audio is streaming, otherwise `listening` (attentive rest).
|
||||
* Both map to the `idle` body pose in `FACE_TO_POSE`, but `speaking` is what
|
||||
* drives the viseme mouth on the producer, so the distinction is load-bearing.
|
||||
*/
|
||||
function activeSpeakerFace(speaking: boolean): MascotFace {
|
||||
return speaking ? 'speaking' : 'listening';
|
||||
}
|
||||
|
||||
/**
|
||||
* The face for a single slot in the two-mascot layout.
|
||||
* - greeting / signoff → both wave (`waving` → `hand_wave` pose).
|
||||
* - active: the slot that owns the current audio follows
|
||||
* {@link activeSpeakerFace}; the other slot shows `thinking` — the only
|
||||
* asset-distinct "reacting / listening" body pose (verified against
|
||||
* `FACE_TO_POSE` in riveMaps: `listening` collapses to `idle`, so it would
|
||||
* be visually indistinguishable from the speaker's rest state; `thinking`
|
||||
* is the distinct pose that reads as "the other mascot is paying
|
||||
* attention").
|
||||
*/
|
||||
function dualSlotFace(slot: ActiveMascotSlot, input: UseMeetingMascotsInput): MascotFace {
|
||||
if (input.phase === 'greeting' || input.phase === 'signoff') return 'waving';
|
||||
if (slot === input.activeMascotSlot) return activeSpeakerFace(input.speaking);
|
||||
return 'thinking';
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the per-slot render state. Exposed as a standalone pure function so
|
||||
* the face table can be exercised directly (the hook is a thin redux wrapper
|
||||
* over it).
|
||||
*/
|
||||
export function computeMeetingMascotsRenderState(
|
||||
dualEnabled: boolean,
|
||||
primaryMascotId: string | null,
|
||||
secondaryMascotId: string | null,
|
||||
input: UseMeetingMascotsInput
|
||||
): MeetingMascotsRenderState {
|
||||
if (!dualEnabled) {
|
||||
// Single-mascot path — preserves the original producer behavior exactly:
|
||||
// primary follows speaking → speaking/idle, no secondary slot.
|
||||
return {
|
||||
dualEnabled: false,
|
||||
primary: { mascotId: primaryMascotId, face: input.speaking ? 'speaking' : 'idle' },
|
||||
secondary: null,
|
||||
};
|
||||
}
|
||||
return {
|
||||
dualEnabled: true,
|
||||
primary: { mascotId: primaryMascotId, face: dualSlotFace(0, input) },
|
||||
secondary: { mascotId: secondaryMascotId, face: dualSlotFace(1, input) },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook form: reads the mascot slice and folds in the live speaking-state +
|
||||
* phase to produce the render state the producer composites from.
|
||||
*/
|
||||
export function useMeetingMascots(input: UseMeetingMascotsInput): MeetingMascotsRenderState {
|
||||
const dualEnabled = useAppSelector(selectDualMascotEnabled);
|
||||
const primaryMascotId = useAppSelector(selectSelectedMascotId);
|
||||
const secondaryMascotId = useAppSelector(selectSecondaryMascotId);
|
||||
// Resolve the voice pair too so the primary slot's mascotId matches exactly
|
||||
// what the join payload sent — the pair is the single source of truth and
|
||||
// reading it here keeps the on-camera mascot and the spoken voice aligned.
|
||||
const pair = useAppSelector(selectMeetingMascotVoicePair);
|
||||
|
||||
const state = computeMeetingMascotsRenderState(
|
||||
dualEnabled,
|
||||
pair.primary.mascotId ?? primaryMascotId,
|
||||
pair.secondary?.mascotId ?? secondaryMascotId,
|
||||
input
|
||||
);
|
||||
|
||||
log(
|
||||
'render state dual=%s phase=%s speaking=%s activeSlot=%d primaryFace=%s secondaryFace=%s',
|
||||
state.dualEnabled,
|
||||
input.phase,
|
||||
input.speaking,
|
||||
input.activeMascotSlot,
|
||||
state.primary.face,
|
||||
state.secondary?.face ?? 'none'
|
||||
);
|
||||
|
||||
return state;
|
||||
}
|
||||
@@ -5267,6 +5267,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'اختر لون التميمة المستخدم في جميع أنحاء التطبيق',
|
||||
'settings.mascot.noCharacters': 'لا توجد شخصيات OpenHuman متاحة بعد',
|
||||
'settings.mascot.noColorVariants': 'لا توجد ألوان متاحة',
|
||||
'settings.mascot.secondaryHeading': 'ثنائي الاجتماع (تميمة ثانية)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'أضِف تميمة ثانية للاجتماعات. عند تعيين اثنتين تظهران معًا وتتناوبان في الحديث. اتركها بلا لتميمة واحدة.',
|
||||
'settings.mascot.secondaryNone': 'بلا (تميمة واحدة)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'أصوات لكل تميمة',
|
||||
'settings.mascot.primaryVoiceLabel': 'صوت التميمة الأولى',
|
||||
'settings.mascot.secondaryVoiceLabel': 'صوت التميمة الثانية',
|
||||
'settings.mascot.voice.current': 'الحالي',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'ابحث عن معرّفات الصوت في api.elevenlabs.io/v1/voices أو لوحة تحكم ElevenLabs الخاصة بك. يُخزَّن المعرّف فقط — يبقى مفتاح API الخاص بك على الخادم.',
|
||||
|
||||
@@ -5386,6 +5386,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'অ্যাপ জুড়ে ব্যবহৃত মাসকটের রঙ বেছে নিন',
|
||||
'settings.mascot.noCharacters': 'কোনো OpenHuman ক্যারেক্টার এখনও উপলব্ধ নেই',
|
||||
'settings.mascot.noColorVariants': 'কোনো রঙের ভেরিয়েন্ট নেই',
|
||||
'settings.mascot.secondaryHeading': 'মিটিং জুটি (দ্বিতীয় ম্যাসকট)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'মিটিংয়ের জন্য একটি দ্বিতীয় ম্যাসকট যোগ করুন। দুটি সেট করা থাকলে তারা একসাথে দেখা দেয় এবং পালা করে কথা বলে। একটি ম্যাসকটের জন্য কোনোটি নয় রাখুন।',
|
||||
'settings.mascot.secondaryNone': 'কোনোটি নয় (একটি ম্যাসকট)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'প্রতি ম্যাসকটের কণ্ঠস্বর',
|
||||
'settings.mascot.primaryVoiceLabel': 'প্রথম ম্যাসকটের কণ্ঠস্বর',
|
||||
'settings.mascot.secondaryVoiceLabel': 'দ্বিতীয় ম্যাসকটের কণ্ঠস্বর',
|
||||
'settings.mascot.voice.current': 'বর্তমান',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'ভয়েস আইডি খুঁজুন api.elevenlabs.io/v1/voices বা আপনার ElevenLabs ড্যাশবোর্ডে। শুধু আইডি সংরক্ষিত থাকে — আপনার API কী ব্যাকএন্ডে থাকে।',
|
||||
|
||||
@@ -5528,6 +5528,13 @@ const messages: TranslationMap = {
|
||||
'Wähle die Maskottchenfarbe aus, die in der gesamten App verwendet wird',
|
||||
'settings.mascot.noCharacters': 'Es sind noch keine OpenHuman Zeichen verfügbar',
|
||||
'settings.mascot.noColorVariants': 'Keine Farbvarianten',
|
||||
'settings.mascot.secondaryHeading': 'Meeting-Duo (zweites Maskottchen)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Füge ein zweites Maskottchen für Meetings hinzu. Sind zwei gesetzt, erscheinen sie gemeinsam und sprechen abwechselnd. Für ein einzelnes Maskottchen auf Keines lassen.',
|
||||
'settings.mascot.secondaryNone': 'Keines (einzelnes Maskottchen)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Stimmen pro Maskottchen',
|
||||
'settings.mascot.primaryVoiceLabel': 'Stimme des ersten Maskottchens',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Stimme des zweiten Maskottchens',
|
||||
'settings.mascot.voice.current': 'aktuell',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'Sprach-IDs findest du unter api.elevenlabs.io/v1/voices oder in deinem ElevenLabs-Dashboard. Es wird nur die ID gespeichert – dein API-Schlüssel bleibt im Backend.',
|
||||
|
||||
@@ -6069,6 +6069,13 @@ const en: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'Pick the mascot color used across the app',
|
||||
'settings.mascot.noCharacters': 'No OpenHuman characters are available yet',
|
||||
'settings.mascot.noColorVariants': 'No color variants',
|
||||
'settings.mascot.secondaryHeading': 'Meeting duo (second mascot)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Add a second mascot for meetings. When two are set they appear together and take turns speaking. Leave as None for a single mascot.',
|
||||
'settings.mascot.secondaryNone': 'None (single mascot)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Per-mascot voices',
|
||||
'settings.mascot.primaryVoiceLabel': 'First mascot voice',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Second mascot voice',
|
||||
'settings.mascot.voice.current': 'current',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'Find voice ids at api.elevenlabs.io/v1/voices or your ElevenLabs dashboard. Only the id is stored — your API key stays on the backend.',
|
||||
|
||||
@@ -5484,6 +5484,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'Elige el color de la mascota usado en toda la app',
|
||||
'settings.mascot.noCharacters': 'Aún no hay personajes de OpenHuman disponibles',
|
||||
'settings.mascot.noColorVariants': 'Sin variantes de color',
|
||||
'settings.mascot.secondaryHeading': 'Dúo de reunión (segunda mascota)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Añade una segunda mascota para las reuniones. Al fijar dos, aparecen juntas y hablan por turnos. Déjalo en Ninguna para una sola mascota.',
|
||||
'settings.mascot.secondaryNone': 'Ninguna (una sola mascota)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Voces por mascota',
|
||||
'settings.mascot.primaryVoiceLabel': 'Voz de la primera mascota',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Voz de la segunda mascota',
|
||||
'settings.mascot.voice.current': 'actual',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'Encuentra los ID de voz en api.elevenlabs.io/v1/voices o en tu panel de ElevenLabs. Solo se almacena el ID — tu clave de API permanece en el backend.',
|
||||
|
||||
@@ -5504,6 +5504,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': "Choisis la couleur de la mascotte utilisée dans toute l'application",
|
||||
'settings.mascot.noCharacters': "Aucun personnage OpenHuman n'est encore disponible",
|
||||
'settings.mascot.noColorVariants': 'Aucune variante de couleur',
|
||||
'settings.mascot.secondaryHeading': 'Duo de réunion (seconde mascotte)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Ajoute une seconde mascotte pour les réunions. Lorsque deux sont définies, elles apparaissent ensemble et parlent à tour de rôle. Laisse sur Aucune pour une seule mascotte.',
|
||||
'settings.mascot.secondaryNone': 'Aucune (une seule mascotte)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Voix par mascotte',
|
||||
'settings.mascot.primaryVoiceLabel': 'Voix de la première mascotte',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Voix de la seconde mascotte',
|
||||
'settings.mascot.voice.current': 'actuel',
|
||||
'settings.mascot.voice.customDesc':
|
||||
"Trouvez les identifiants vocaux sur api.elevenlabs.io/v1/voices ou dans votre tableau de bord ElevenLabs. Seul l'identifiant est stocké — votre clé API reste sur le backend.",
|
||||
|
||||
@@ -5386,6 +5386,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'ऐप में उपयोग होने वाला मास्कॉट रंग चुनें',
|
||||
'settings.mascot.noCharacters': 'अभी तक कोई OpenHuman कैरेक्टर उपलब्ध नहीं है',
|
||||
'settings.mascot.noColorVariants': 'कोई कलर वेरिएंट नहीं',
|
||||
'settings.mascot.secondaryHeading': 'मीटिंग जोड़ी (दूसरा मैस्कट)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'मीटिंग के लिए एक दूसरा मैस्कट जोड़ें। दो सेट होने पर वे साथ दिखते हैं और बारी-बारी से बोलते हैं। एक ही मैस्कट के लिए कोई नहीं पर छोड़ें।',
|
||||
'settings.mascot.secondaryNone': 'कोई नहीं (एकल मैस्कट)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'प्रति-मैस्कट आवाज़ें',
|
||||
'settings.mascot.primaryVoiceLabel': 'पहले मैस्कट की आवाज़',
|
||||
'settings.mascot.secondaryVoiceLabel': 'दूसरे मैस्कट की आवाज़',
|
||||
'settings.mascot.voice.current': 'वर्तमान',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'वॉइस आईडी api.elevenlabs.io/v1/voices या अपने ElevenLabs डैशबोर्ड पर खोजें। केवल आईडी संग्रहीत होती है — आपकी API कुंजी बैकएंड पर रहती है।',
|
||||
|
||||
@@ -5402,6 +5402,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'Pilih warna maskot yang digunakan di seluruh aplikasi',
|
||||
'settings.mascot.noCharacters': 'Belum ada karakter OpenHuman yang tersedia',
|
||||
'settings.mascot.noColorVariants': 'Tidak ada varian warna',
|
||||
'settings.mascot.secondaryHeading': 'Duo rapat (maskot kedua)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Tambahkan maskot kedua untuk rapat. Bila dua diatur, keduanya muncul bersama dan berbicara bergantian. Biarkan Tidak ada untuk satu maskot saja.',
|
||||
'settings.mascot.secondaryNone': 'Tidak ada (maskot tunggal)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Suara per maskot',
|
||||
'settings.mascot.primaryVoiceLabel': 'Suara maskot pertama',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Suara maskot kedua',
|
||||
'settings.mascot.voice.current': 'saat ini',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'Temukan ID suara di api.elevenlabs.io/v1/voices atau dasbor ElevenLabs Anda. Hanya ID yang disimpan — kunci API Anda tetap di backend.',
|
||||
|
||||
@@ -5473,6 +5473,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': "Scegli il colore della mascotte usato in tutta l'app",
|
||||
'settings.mascot.noCharacters': 'Nessun personaggio OpenHuman disponibile',
|
||||
'settings.mascot.noColorVariants': 'Nessuna variante di colore',
|
||||
'settings.mascot.secondaryHeading': 'Duo riunione (seconda mascotte)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Aggiungi una seconda mascotte per le riunioni. Quando ne imposti due, compaiono insieme e parlano a turno. Lascia su Nessuna per una sola mascotte.',
|
||||
'settings.mascot.secondaryNone': 'Nessuna (una sola mascotte)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Voci per mascotte',
|
||||
'settings.mascot.primaryVoiceLabel': 'Voce della prima mascotte',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Voce della seconda mascotte',
|
||||
'settings.mascot.voice.current': 'attuale',
|
||||
'settings.mascot.voice.customDesc':
|
||||
"Trova gli ID vocali su api.elevenlabs.io/v1/voices o nel tuo dashboard ElevenLabs. Viene salvato solo l'ID — la tua chiave API rimane sul backend.",
|
||||
|
||||
@@ -5324,6 +5324,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': '앱 전체에서 사용되는 마스코트 색상 선택',
|
||||
'settings.mascot.noCharacters': '아직 사용할 수 있는 OpenHuman 캐릭터가 없습니다',
|
||||
'settings.mascot.noColorVariants': '색상 변형 없음',
|
||||
'settings.mascot.secondaryHeading': '회의 듀오 (두 번째 마스코트)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'회의에 두 번째 마스코트를 추가하세요. 둘을 설정하면 함께 나타나 번갈아 말합니다. 마스코트를 하나만 쓰려면 없음으로 두세요.',
|
||||
'settings.mascot.secondaryNone': '없음 (단일 마스코트)',
|
||||
'settings.mascot.perMascotVoiceHeading': '마스코트별 음성',
|
||||
'settings.mascot.primaryVoiceLabel': '첫 번째 마스코트 음성',
|
||||
'settings.mascot.secondaryVoiceLabel': '두 번째 마스코트 음성',
|
||||
'settings.mascot.voice.current': '현재',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'api.elevenlabs.io/v1/voices 또는 ElevenLabs 대시보드에서 음성 ID를 찾으세요. ID만 저장되며 API 키는 백엔드에 유지됩니다.',
|
||||
|
||||
@@ -5464,6 +5464,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'Wybierz kolor maskotki używany w aplikacji',
|
||||
'settings.mascot.noCharacters': 'Nie ma jeszcze dostępnych postaci OpenHuman',
|
||||
'settings.mascot.noColorVariants': 'Brak wariantów kolorystycznych',
|
||||
'settings.mascot.secondaryHeading': 'Duet spotkania (druga maskotka)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Dodaj drugą maskotkę do spotkań. Gdy ustawisz dwie, pojawiają się razem i mówią na zmianę. Pozostaw Brak, aby korzystać z jednej maskotki.',
|
||||
'settings.mascot.secondaryNone': 'Brak (pojedyncza maskotka)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Głosy poszczególnych maskotek',
|
||||
'settings.mascot.primaryVoiceLabel': 'Głos pierwszej maskotki',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Głos drugiej maskotki',
|
||||
'settings.mascot.voice.current': 'bieżący',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'Znajdź ID głosów w api.elevenlabs.io/v1/voices lub w panelu ElevenLabs. Przechowywane jest tylko ID — Twój klucz API pozostaje na backendzie.',
|
||||
|
||||
@@ -5473,6 +5473,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'Escolha a cor do mascote usada em todo o app',
|
||||
'settings.mascot.noCharacters': 'Nenhum personagem do OpenHuman disponível ainda',
|
||||
'settings.mascot.noColorVariants': 'Sem variantes de cor',
|
||||
'settings.mascot.secondaryHeading': 'Dupla de reunião (segundo mascote)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Adicione um segundo mascote para as reuniões. Com dois definidos, eles aparecem juntos e falam em turnos. Deixe como Nenhum para um único mascote.',
|
||||
'settings.mascot.secondaryNone': 'Nenhum (mascote único)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Vozes por mascote',
|
||||
'settings.mascot.primaryVoiceLabel': 'Voz do primeiro mascote',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Voz do segundo mascote',
|
||||
'settings.mascot.voice.current': 'atual',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'Encontre IDs de voz em api.elevenlabs.io/v1/voices ou no seu painel da ElevenLabs. Apenas o ID é armazenado — sua chave de API permanece no backend.',
|
||||
|
||||
@@ -5439,6 +5439,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': 'Выберите цвет маскота, используемый во всем приложении',
|
||||
'settings.mascot.noCharacters': 'Персонажи OpenHuman пока недоступны',
|
||||
'settings.mascot.noColorVariants': 'Нет цветовых вариантов',
|
||||
'settings.mascot.secondaryHeading': 'Дуэт для встреч (второй маскот)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'Добавьте второго маскота для встреч. Когда заданы двое, они появляются вместе и говорят по очереди. Оставьте Нет для одного маскота.',
|
||||
'settings.mascot.secondaryNone': 'Нет (один маскот)',
|
||||
'settings.mascot.perMascotVoiceHeading': 'Голоса для каждого маскота',
|
||||
'settings.mascot.primaryVoiceLabel': 'Голос первого маскота',
|
||||
'settings.mascot.secondaryVoiceLabel': 'Голос второго маскота',
|
||||
'settings.mascot.voice.current': 'текущий',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'Идентификаторы голосов можно найти на api.elevenlabs.io/v1/voices или в вашей панели ElevenLabs. Сохраняется только идентификатор — ваш API-ключ остаётся на бэкенде.',
|
||||
|
||||
@@ -5103,6 +5103,13 @@ const messages: TranslationMap = {
|
||||
'settings.mascot.menuDesc': '选择应用内使用的吉祥物颜色',
|
||||
'settings.mascot.noCharacters': '暂无可用的 OpenHuman 角色',
|
||||
'settings.mascot.noColorVariants': '无颜色变体',
|
||||
'settings.mascot.secondaryHeading': '会议搭档(第二个吉祥物)',
|
||||
'settings.mascot.secondaryDesc':
|
||||
'为会议添加第二个吉祥物。设置两个后,它们会一起出现并轮流说话。若只用一个吉祥物,请保持为无。',
|
||||
'settings.mascot.secondaryNone': '无(单个吉祥物)',
|
||||
'settings.mascot.perMascotVoiceHeading': '每个吉祥物的语音',
|
||||
'settings.mascot.primaryVoiceLabel': '第一个吉祥物的语音',
|
||||
'settings.mascot.secondaryVoiceLabel': '第二个吉祥物的语音',
|
||||
'settings.mascot.voice.current': '当前',
|
||||
'settings.mascot.voice.customDesc':
|
||||
'在 api.elevenlabs.io/v1/voices 或您的 ElevenLabs 仪表板中查找语音 ID。仅存储 ID — 您的 API 密钥保留在后端。',
|
||||
|
||||
@@ -77,6 +77,35 @@ describe('joinMeetCall', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('forwards per-mascot voices to the shell when provided (issue #4277)', async () => {
|
||||
vi.mocked(callCoreRpc).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
request_id: 'req-2',
|
||||
meet_url: 'https://meet.google.com/abc-defg-hij',
|
||||
display_name: 'Agent Alice',
|
||||
} as never);
|
||||
vi.mocked(invoke).mockResolvedValueOnce('meet-call-req-2');
|
||||
|
||||
await joinMeetCall({
|
||||
meetUrl: 'https://meet.google.com/abc-defg-hij',
|
||||
displayName: 'Agent Alice',
|
||||
ownerDisplayName: 'Owner Bob',
|
||||
primaryVoiceId: ' voice-a ',
|
||||
secondaryVoiceId: 'voice-b',
|
||||
});
|
||||
|
||||
expect(invoke).toHaveBeenCalledWith('meet_call_open_window', {
|
||||
args: {
|
||||
request_id: 'req-2',
|
||||
meet_url: 'https://meet.google.com/abc-defg-hij',
|
||||
display_name: 'Agent Alice',
|
||||
owner_display_name: 'Owner Bob',
|
||||
primary_voice_id: 'voice-a',
|
||||
secondary_voice_id: 'voice-b',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('throws if core rejects the request', async () => {
|
||||
vi.mocked(callCoreRpc).mockResolvedValueOnce({ ok: false } as never);
|
||||
await expect(
|
||||
@@ -233,6 +262,60 @@ describe('joinMeetViaBackendBot', () => {
|
||||
expect(result).toEqual({ meetUrl: 'https://meet.google.com/abc-defg-hij', platform: 'gmeet' });
|
||||
});
|
||||
|
||||
it('omits mascots[] for a single-mascot join (unchanged wire shape)', async () => {
|
||||
vi.mocked(callCoreRpc).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
meet_url: 'https://meet.google.com/abc-defg-hij',
|
||||
platform: 'gmeet',
|
||||
} as never);
|
||||
|
||||
await joinMeetViaBackendBot({
|
||||
meetUrl: 'https://meet.google.com/abc-defg-hij',
|
||||
mascotId: 'yellow',
|
||||
});
|
||||
|
||||
// No `mascots` key on the params → the single `mascot_id` path is
|
||||
// byte-identical to before (undefined is dropped by the RPC boundary).
|
||||
const params = vi.mocked(callCoreRpc).mock.calls[0][0].params as Record<string, unknown>;
|
||||
expect(params.mascots).toBeUndefined();
|
||||
expect(params.mascot_id).toBe('yellow');
|
||||
});
|
||||
|
||||
it('maps dual mascots[] to snake_case slots and drops blank ids (issue #4277)', async () => {
|
||||
vi.mocked(callCoreRpc).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
meet_url: 'https://meet.google.com/abc-defg-hij',
|
||||
platform: 'gmeet',
|
||||
} as never);
|
||||
|
||||
await joinMeetViaBackendBot({
|
||||
meetUrl: 'https://meet.google.com/abc-defg-hij',
|
||||
mascots: [
|
||||
{
|
||||
mascotId: ' tiny-mascot ',
|
||||
name: ' Tiny ',
|
||||
voiceId: ' voice-a ',
|
||||
riveColors: { primaryColor: '#111', secondaryColor: '#222' },
|
||||
},
|
||||
{ mascotId: 'toshi', voiceId: 'voice-b' },
|
||||
{ mascotId: ' ', voiceId: 'ignored' },
|
||||
],
|
||||
});
|
||||
|
||||
const params = vi.mocked(callCoreRpc).mock.calls[0][0].params as Record<string, unknown>;
|
||||
expect(params.mascots).toEqual([
|
||||
{
|
||||
mascot_id: 'tiny-mascot',
|
||||
// Name-addressed routing (#4277 follow-up): trimmed + forwarded.
|
||||
name: 'Tiny',
|
||||
voice_id: 'voice-a',
|
||||
rive_colors: { primary_color: '#111', secondary_color: '#222' },
|
||||
},
|
||||
// Slot 1 supplies no name → `name` omitted (undefined).
|
||||
{ mascot_id: 'toshi', name: undefined, voice_id: 'voice-b', rive_colors: undefined },
|
||||
]);
|
||||
});
|
||||
|
||||
it('rejects an empty meeting link before contacting core', async () => {
|
||||
await expect(joinMeetViaBackendBot({ meetUrl: ' ' })).rejects.toThrow(/meeting link/i);
|
||||
expect(callCoreRpc).not.toHaveBeenCalled();
|
||||
|
||||
@@ -10,11 +10,32 @@
|
||||
// Splitting it this way keeps platform-specific window code in the shell
|
||||
// while the validation rules live (and are tested) in the core.
|
||||
import { invoke } from '@tauri-apps/api/core';
|
||||
import debug from 'debug';
|
||||
|
||||
import { isTauri } from '../utils/tauriCommands/common';
|
||||
import { apiClient } from './apiClient';
|
||||
import { callCoreRpc } from './coreRpcClient';
|
||||
|
||||
// Shares the sibling hook's namespace (`useMeetingMascots.ts`) so the whole
|
||||
// dual-mascot flow can be traced under one prefix.
|
||||
const log = debug('meet:mascots');
|
||||
|
||||
/**
|
||||
* Map optional Rive colors to the backend's snake_case wire shape, trimming
|
||||
* blanks and collapsing an all-empty pair to `undefined`. Shared by the
|
||||
* top-level and per-slot color payloads so the two can't drift.
|
||||
*/
|
||||
function mapRiveColors(colors?: {
|
||||
primaryColor?: string;
|
||||
secondaryColor?: string;
|
||||
}): { primary_color?: string; secondary_color?: string } | undefined {
|
||||
if (!colors) return undefined;
|
||||
const primary = colors.primaryColor?.trim() || undefined;
|
||||
const secondary = colors.secondaryColor?.trim() || undefined;
|
||||
if (!primary && !secondary) return undefined;
|
||||
return { primary_color: primary, secondary_color: secondary };
|
||||
}
|
||||
|
||||
export type MeetJoinCallInput = {
|
||||
meetUrl: string;
|
||||
/** Bot's display name in Meet's "Your name" prompt. */
|
||||
@@ -27,6 +48,14 @@ export type MeetJoinCallInput = {
|
||||
* (no wakes fire) which is the safe default during the rollout.
|
||||
*/
|
||||
ownerDisplayName?: string;
|
||||
/**
|
||||
* ElevenLabs voice id for the primary mascot (issue #4277). When two
|
||||
* mascots are enabled the core alternates the speaking voice per reply.
|
||||
* Omit for single-mascot calls (core keeps its default voice).
|
||||
*/
|
||||
primaryVoiceId?: string;
|
||||
/** Voice id for the secondary mascot; present only in two-mascot calls. */
|
||||
secondaryVoiceId?: string;
|
||||
};
|
||||
|
||||
export type MeetJoinCallResult = {
|
||||
@@ -86,6 +115,10 @@ export async function joinMeetCall(input: MeetJoinCallInput): Promise<MeetJoinCa
|
||||
// hand it to the wake-word gate. See feat/mascot-meet-flowA
|
||||
// Plan C — owner-only privacy lock.
|
||||
owner_display_name: ownerDisplayName,
|
||||
// Per-mascot voices for speaker alternation (issue #4277). Absent
|
||||
// → core keeps its single default voice (unchanged behavior).
|
||||
primary_voice_id: input.primaryVoiceId?.trim() || undefined,
|
||||
secondary_voice_id: input.secondaryVoiceId?.trim() || undefined,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
@@ -264,6 +297,25 @@ export type BackendMeetJoinInput = {
|
||||
systemPrompt?: string;
|
||||
mascotId?: string;
|
||||
riveColors?: { primaryColor?: string; secondaryColor?: string };
|
||||
/**
|
||||
* Dual-mascot config (issue #4277). Up to 2 slots; slot 0 = primary,
|
||||
* slot 1 = secondary. When two are present the backend bot renders both
|
||||
* mascots and alternates who speaks each reply (each with its own
|
||||
* `voiceId`). Omit / single element = legacy single-mascot behavior via
|
||||
* `mascotId`.
|
||||
*/
|
||||
mascots?: Array<{
|
||||
mascotId: string;
|
||||
/**
|
||||
* Human-facing mascot name (from the manifest, e.g. "Toshi"). Enables
|
||||
* name-addressed routing (#4277 follow-up): a participant who says
|
||||
* "Hey Toshi …" is answered by this slot instead of the mechanical
|
||||
* alternation. Omit → that slot is not name-addressable.
|
||||
*/
|
||||
name?: string;
|
||||
riveColors?: { primaryColor?: string; secondaryColor?: string };
|
||||
voiceId?: string;
|
||||
}>;
|
||||
/** Only respond to messages from this participant name (empty = respond to all). */
|
||||
respondToParticipant?: string;
|
||||
/** Wake phrase the participant must say before the bot responds (empty = no wake phrase). */
|
||||
@@ -291,6 +343,29 @@ export async function joinMeetViaBackendBot(
|
||||
const meetUrl = input.meetUrl.trim();
|
||||
if (!meetUrl) throw new Error('Please paste a meeting link.');
|
||||
|
||||
// Dual-mascot slots (issue #4277), mapped to the backend's snake_case wire
|
||||
// shape. Absent → backend falls back to `mascot_id`.
|
||||
const slots = input.mascots?.filter(m => m.mascotId?.trim());
|
||||
const mascots =
|
||||
slots && slots.length > 0
|
||||
? slots.map(m => ({
|
||||
mascot_id: m.mascotId.trim(),
|
||||
name: m.name?.trim() || undefined,
|
||||
voice_id: m.voiceId?.trim() || undefined,
|
||||
rive_colors: mapRiveColors(m.riveColors),
|
||||
}))
|
||||
: undefined;
|
||||
|
||||
// Flow/state metadata only — no participant names, voices, or the meet URL.
|
||||
log(
|
||||
'backend bot join corr=%s dual=%s slots=%d singleMascot=%s riveColors=%s',
|
||||
input.correlationId?.trim() || '-',
|
||||
Boolean(input.mascots?.length),
|
||||
mascots?.length ?? 0,
|
||||
Boolean(input.mascotId?.trim()),
|
||||
Boolean(mapRiveColors(input.riveColors))
|
||||
);
|
||||
|
||||
const result = await callCoreRpc<CoreBackendMeetJoinResponse>({
|
||||
method: 'openhuman.agent_meetings_join',
|
||||
params: {
|
||||
@@ -304,13 +379,8 @@ export async function joinMeetViaBackendBot(
|
||||
wake_phrase: input.wakePhrase?.trim() || undefined,
|
||||
correlation_id: input.correlationId?.trim() || undefined,
|
||||
listen_only: input.listenOnly ?? undefined,
|
||||
rive_colors: (() => {
|
||||
if (!input.riveColors) return undefined;
|
||||
const primary = input.riveColors.primaryColor?.trim() || undefined;
|
||||
const secondary = input.riveColors.secondaryColor?.trim() || undefined;
|
||||
if (!primary && !secondary) return undefined;
|
||||
return { primary_color: primary, secondary_color: secondary };
|
||||
})(),
|
||||
rive_colors: mapRiveColors(input.riveColors),
|
||||
mascots,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -6,13 +6,21 @@ import reducer, {
|
||||
isCustomMascotGifUrl,
|
||||
MAX_CUSTOM_MASCOT_GIF_URL_LEN,
|
||||
MAX_MASCOT_VOICE_ID_LEN,
|
||||
MAX_MASCOT_VOICES,
|
||||
selectCustomMascotGifUrl,
|
||||
selectDualMascotEnabled,
|
||||
selectMascotColor,
|
||||
selectMascotVoiceFor,
|
||||
selectMascotVoiceId,
|
||||
selectMascotVoices,
|
||||
selectMeetingMascotVoicePair,
|
||||
selectSecondaryMascotId,
|
||||
selectSelectedMascotId,
|
||||
setCustomMascotGifUrl,
|
||||
setMascotColor,
|
||||
setMascotVoice,
|
||||
setMascotVoiceId,
|
||||
setSecondaryMascotId,
|
||||
setSelectedMascotId,
|
||||
SUPPORTED_MASCOT_COLORS,
|
||||
} from '../mascotSlice';
|
||||
@@ -277,4 +285,168 @@ describe('mascotSlice', () => {
|
||||
expect(state.customMascotGifUrl).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// Issue #4277 — second meeting mascot + per-mascot voices.
|
||||
describe('secondary mascot id', () => {
|
||||
it('defaults to null', () => {
|
||||
const state = reducer(undefined, { type: '@@INIT' });
|
||||
expect(state.secondaryMascotId).toBeNull();
|
||||
expect(selectSecondaryMascotId({ mascot: state })).toBeNull();
|
||||
});
|
||||
|
||||
it('setSecondaryMascotId trims and stores a valid id', () => {
|
||||
const state = reducer(undefined, setSecondaryMascotId(' toshi '));
|
||||
expect(state.secondaryMascotId).toBe('toshi');
|
||||
});
|
||||
|
||||
it('null / whitespace / oversize input clears it', () => {
|
||||
const set = reducer(undefined, setSecondaryMascotId('toshi'));
|
||||
expect(reducer(set, setSecondaryMascotId(null)).secondaryMascotId).toBeNull();
|
||||
expect(reducer(set, setSecondaryMascotId(' ')).secondaryMascotId).toBeNull();
|
||||
const tooLong = 'x'.repeat(MAX_MASCOT_VOICE_ID_LEN + 1);
|
||||
expect(reducer(set, setSecondaryMascotId(tooLong)).secondaryMascotId).toBeNull();
|
||||
});
|
||||
|
||||
it('is cleared when a custom GIF avatar is set (mutually exclusive)', () => {
|
||||
let state = reducer(undefined, setSecondaryMascotId('toshi'));
|
||||
state = reducer(state, setCustomMascotGifUrl('https://example.com/avatar.gif'));
|
||||
expect(state.secondaryMascotId).toBeNull();
|
||||
});
|
||||
|
||||
it('resetUserScopedState clears it', () => {
|
||||
let state = reducer(undefined, setSecondaryMascotId('toshi'));
|
||||
state = reducer(state, resetUserScopedState());
|
||||
expect(state.secondaryMascotId).toBeNull();
|
||||
});
|
||||
|
||||
const rehydrate = (key: string, payload?: unknown) => ({ type: REHYDRATE, key, payload });
|
||||
|
||||
it('restores a valid persisted id; scrubs invalid; tolerates missing (older blobs)', () => {
|
||||
expect(
|
||||
reducer(undefined, rehydrate('mascot', { secondaryMascotId: 'toshi' })).secondaryMascotId
|
||||
).toBe('toshi');
|
||||
expect(
|
||||
reducer(undefined, rehydrate('mascot', { secondaryMascotId: ' ' })).secondaryMascotId
|
||||
).toBeNull();
|
||||
expect(
|
||||
reducer(undefined, rehydrate('mascot', { color: 'navy' })).secondaryMascotId
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('per-mascot voices', () => {
|
||||
it('defaults to an empty map', () => {
|
||||
const state = reducer(undefined, { type: '@@INIT' });
|
||||
expect(state.mascotVoices).toEqual({});
|
||||
expect(selectMascotVoices({ mascot: state })).toEqual({});
|
||||
expect(selectMascotVoiceFor('toshi')({ mascot: state })).toBeNull();
|
||||
});
|
||||
|
||||
it('setMascotVoice records a trimmed mascotId → voiceId entry', () => {
|
||||
const state = reducer(undefined, setMascotVoice({ mascotId: ' toshi ', voiceId: ' v-1 ' }));
|
||||
expect(state.mascotVoices).toEqual({ toshi: 'v-1' });
|
||||
expect(selectMascotVoiceFor('toshi')({ mascot: state })).toBe('v-1');
|
||||
});
|
||||
|
||||
it('setMascotVoice with null / invalid voiceId removes the entry', () => {
|
||||
const set = reducer(undefined, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-1' }));
|
||||
expect(
|
||||
reducer(set, setMascotVoice({ mascotId: 'toshi', voiceId: null })).mascotVoices
|
||||
).toEqual({});
|
||||
expect(
|
||||
reducer(set, setMascotVoice({ mascotId: 'toshi', voiceId: ' ' })).mascotVoices
|
||||
).toEqual({});
|
||||
});
|
||||
|
||||
it('ignores an invalid mascotId key', () => {
|
||||
const state = reducer(undefined, setMascotVoice({ mascotId: ' ', voiceId: 'v-1' }));
|
||||
expect(state.mascotVoices).toEqual({});
|
||||
});
|
||||
|
||||
it('caps new keys at MAX_MASCOT_VOICES but still updates existing ones', () => {
|
||||
let state = reducer(undefined, { type: '@@INIT' });
|
||||
for (let i = 0; i < MAX_MASCOT_VOICES; i += 1) {
|
||||
state = reducer(state, setMascotVoice({ mascotId: `m-${i}`, voiceId: `v-${i}` }));
|
||||
}
|
||||
expect(Object.keys(state.mascotVoices)).toHaveLength(MAX_MASCOT_VOICES);
|
||||
// A brand-new key over the cap is refused…
|
||||
const overflow = reducer(state, setMascotVoice({ mascotId: 'm-extra', voiceId: 'v-x' }));
|
||||
expect(overflow.mascotVoices['m-extra']).toBeUndefined();
|
||||
// …but re-voicing an existing mascot is always allowed.
|
||||
const updated = reducer(state, setMascotVoice({ mascotId: 'm-0', voiceId: 'v-new' }));
|
||||
expect(updated.mascotVoices['m-0']).toBe('v-new');
|
||||
});
|
||||
|
||||
it('resetUserScopedState clears the map', () => {
|
||||
const dirty = reducer(undefined, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-1' }));
|
||||
expect(reducer(dirty, resetUserScopedState()).mascotVoices).toEqual({});
|
||||
});
|
||||
|
||||
const rehydrate = (key: string, payload?: unknown) => ({ type: REHYDRATE, key, payload });
|
||||
|
||||
it('REHYDRATE keeps only valid entries and tolerates a missing/garbage map', () => {
|
||||
const restored = reducer(
|
||||
undefined,
|
||||
rehydrate('mascot', { mascotVoices: { toshi: 'v-1', bad: ' ', ' ': 'v-2', ok: 'v-3' } })
|
||||
);
|
||||
expect(restored.mascotVoices).toEqual({ toshi: 'v-1', ok: 'v-3' });
|
||||
expect(reducer(undefined, rehydrate('mascot', { color: 'navy' })).mascotVoices).toEqual({});
|
||||
expect(
|
||||
reducer(undefined, rehydrate('mascot', { mascotVoices: 'not-an-object' })).mascotVoices
|
||||
).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('dual-mascot resolution', () => {
|
||||
it('selectDualMascotEnabled is false with one or duplicate mascots, true with two distinct', () => {
|
||||
const one = reducer(undefined, setSelectedMascotId('tiny-mascot'));
|
||||
expect(selectDualMascotEnabled({ mascot: one })).toBe(false);
|
||||
const dup = reducer(one, setSecondaryMascotId('tiny-mascot'));
|
||||
expect(selectDualMascotEnabled({ mascot: dup })).toBe(false);
|
||||
const two = reducer(one, setSecondaryMascotId('toshi'));
|
||||
expect(selectDualMascotEnabled({ mascot: two })).toBe(true);
|
||||
});
|
||||
|
||||
it('selectMeetingMascotVoicePair returns null secondary when single', () => {
|
||||
let state = reducer(undefined, setSelectedMascotId('tiny-mascot'));
|
||||
state = reducer(state, setMascotVoiceId('v-primary'));
|
||||
const pair = selectMeetingMascotVoicePair({ mascot: state });
|
||||
expect(pair.secondary).toBeNull();
|
||||
expect(pair.primary.mascotId).toBe('tiny-mascot');
|
||||
expect(pair.primary.voiceId).toBe('v-primary');
|
||||
});
|
||||
|
||||
it('selectMeetingMascotVoicePair resolves each slot to its per-mascot voice', () => {
|
||||
let state = reducer(undefined, setSelectedMascotId('tiny-mascot'));
|
||||
state = reducer(state, setSecondaryMascotId('toshi'));
|
||||
state = reducer(state, setMascotVoice({ mascotId: 'tiny-mascot', voiceId: 'v-tiny' }));
|
||||
state = reducer(state, setMascotVoice({ mascotId: 'toshi', voiceId: 'v-toshi' }));
|
||||
const pair = selectMeetingMascotVoicePair({ mascot: state });
|
||||
expect(pair.primary).toEqual({ mascotId: 'tiny-mascot', voiceId: 'v-tiny' });
|
||||
expect(pair.secondary).toEqual({ mascotId: 'toshi', voiceId: 'v-toshi' });
|
||||
});
|
||||
|
||||
it('per-mascot voice falls back to the effective single voice when unset', () => {
|
||||
let state = reducer(undefined, setSelectedMascotId('tiny-mascot'));
|
||||
state = reducer(state, setSecondaryMascotId('toshi'));
|
||||
state = reducer(state, setMascotVoiceId('v-effective'));
|
||||
const pair = selectMeetingMascotVoicePair({ mascot: state });
|
||||
// Neither mascot has an explicit override → both use the effective voice.
|
||||
expect(pair.primary.voiceId).toBe('v-effective');
|
||||
expect(pair.secondary?.voiceId).toBe('v-effective');
|
||||
});
|
||||
|
||||
it('tolerates a legacy mascot state missing mascotVoices without throwing', () => {
|
||||
// A pre-migration persisted blob or a partial preloadedState can omit
|
||||
// `mascotVoices`; the meeting selectors must default it, not crash on
|
||||
// `mascotVoices[selectedMascotId]`.
|
||||
const legacy = {
|
||||
mascot: { selectedMascotId: 'yellow', secondaryMascotId: null },
|
||||
} as unknown as Parameters<typeof selectMeetingMascotVoicePair>[0];
|
||||
expect(() => selectMeetingMascotVoicePair(legacy)).not.toThrow();
|
||||
expect(selectMeetingMascotVoicePair(legacy).primary.mascotId).toBe('yellow');
|
||||
expect(selectMascotVoiceFor('yellow')(legacy)).toBeNull();
|
||||
expect(selectMascotVoices(legacy)).toEqual({});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -40,6 +40,17 @@ export const DEFAULT_MASCOT_VOICE_GENDER: MascotVoiceGender = 'male';
|
||||
export const MAX_MASCOT_VOICE_ID_LEN = 128;
|
||||
export const MAX_CUSTOM_MASCOT_GIF_URL_LEN = 2048;
|
||||
|
||||
/**
|
||||
* Upper bound on how many per-mascot voice overrides we persist (issue
|
||||
* #4277). A user only ever drives two mascots in a meeting, but they may
|
||||
* try several before settling; the cap keeps the persisted map bounded
|
||||
* against a runaway writer while comfortably covering real use. Once the
|
||||
* cap is reached the reducer refuses NEW keys (an existing mascot can
|
||||
* still be re-voiced); on rehydrate the first `MAX_MASCOT_VOICES` valid
|
||||
* entries are kept and the rest dropped.
|
||||
*/
|
||||
export const MAX_MASCOT_VOICES = 16;
|
||||
|
||||
/**
|
||||
* Loose shape check for a stored mascot voice id. Issue #1762 lets users
|
||||
* paste a custom ElevenLabs voice id, so we cannot enumerate the valid
|
||||
@@ -118,6 +129,21 @@ export interface MascotState {
|
||||
* the persisted blob bounded.
|
||||
*/
|
||||
selectedMascotId: string | null;
|
||||
/**
|
||||
* Second mascot enabled for meetings (issue #4277). When set (and
|
||||
* distinct from `selectedMascotId`) the meeting bot shows both mascots
|
||||
* together and alternates who speaks each reply. `null` = single-mascot
|
||||
* behavior, unchanged. Same validation/length cap as `selectedMascotId`.
|
||||
*/
|
||||
secondaryMascotId: string | null;
|
||||
/**
|
||||
* Per-mascot reply-voice overrides (issue #4277), keyed by manifest
|
||||
* mascot id → ElevenLabs voice id. Lets each mascot in a two-mascot
|
||||
* meeting speak in its own voice. Empty map = no per-mascot override,
|
||||
* so every mascot falls back to `selectEffectiveMascotVoiceId` (the
|
||||
* single-voice behavior). Bounded by `MAX_MASCOT_VOICES`.
|
||||
*/
|
||||
mascotVoices: Record<string, string>;
|
||||
/**
|
||||
* User-supplied animated avatar source. Kept as a plain validated
|
||||
* string so the renderer can fall back to YellowMascot whenever the
|
||||
@@ -134,11 +160,31 @@ const initialState: MascotState = {
|
||||
voiceGender: DEFAULT_MASCOT_VOICE_GENDER,
|
||||
voiceUseLocaleDefault: false,
|
||||
selectedMascotId: null,
|
||||
secondaryMascotId: null,
|
||||
mascotVoices: {},
|
||||
customMascotGifUrl: null,
|
||||
customPrimaryColor: '#F7D145',
|
||||
customSecondaryColor: '#B23C05',
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrub a persisted / raw `mascotVoices` blob down to valid
|
||||
* `mascotId → voiceId` entries under the size cap. Non-object inputs and
|
||||
* any entry whose key or value fails `isMascotVoiceId` are dropped, so a
|
||||
* corrupted localStorage blob can never poison the meeting TTS payload.
|
||||
*/
|
||||
function sanitizeMascotVoices(value: unknown): Record<string, string> {
|
||||
if (value == null || typeof value !== 'object' || Array.isArray(value)) return {};
|
||||
const out: Record<string, string> = {};
|
||||
for (const [key, val] of Object.entries(value as Record<string, unknown>)) {
|
||||
if (Object.keys(out).length >= MAX_MASCOT_VOICES) break;
|
||||
if (isMascotVoiceId(key) && isMascotVoiceId(val)) {
|
||||
out[key.trim()] = (val as string).trim();
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function isMascotColor(value: unknown): value is MascotColor {
|
||||
return (
|
||||
typeof value === 'string' && (SUPPORTED_MASCOT_COLORS as readonly string[]).includes(value)
|
||||
@@ -170,6 +216,49 @@ const mascotSlice = createSlice({
|
||||
state.selectedMascotId = null;
|
||||
}
|
||||
},
|
||||
/**
|
||||
* Enable / clear the second meeting mascot (issue #4277). Trimmed;
|
||||
* empty / oversize / null clears it (back to single-mascot). A custom
|
||||
* GIF avatar and a second Rive mascot are mutually exclusive, so
|
||||
* setting one clears the GIF override — mirroring `setSelectedMascotId`.
|
||||
*/
|
||||
setSecondaryMascotId(state, action: PayloadAction<string | null>) {
|
||||
if (action.payload == null) {
|
||||
state.secondaryMascotId = null;
|
||||
return;
|
||||
}
|
||||
if (isMascotVoiceId(action.payload)) {
|
||||
state.secondaryMascotId = action.payload.trim();
|
||||
state.customMascotGifUrl = null;
|
||||
} else {
|
||||
state.secondaryMascotId = null;
|
||||
}
|
||||
},
|
||||
/**
|
||||
* Set or clear a per-mascot reply voice (issue #4277). A non-empty
|
||||
* valid `voiceId` records `mascotId → voiceId`; a `null`/invalid
|
||||
* `voiceId` removes the entry (that mascot falls back to the effective
|
||||
* single voice). Both key and value are validated + trimmed so junk
|
||||
* can't grow the persisted map. Over-cap writes are ignored.
|
||||
*/
|
||||
setMascotVoice(state, action: PayloadAction<{ mascotId: string; voiceId: string | null }>) {
|
||||
const { mascotId, voiceId } = action.payload;
|
||||
if (!isMascotVoiceId(mascotId)) return;
|
||||
const key = mascotId.trim();
|
||||
if (voiceId == null || !isMascotVoiceId(voiceId)) {
|
||||
delete state.mascotVoices[key];
|
||||
return;
|
||||
}
|
||||
// Only enforce the cap when introducing a NEW key — updating an
|
||||
// existing mascot's voice must always be allowed.
|
||||
if (
|
||||
!(key in state.mascotVoices) &&
|
||||
Object.keys(state.mascotVoices).length >= MAX_MASCOT_VOICES
|
||||
) {
|
||||
return;
|
||||
}
|
||||
state.mascotVoices[key] = voiceId.trim();
|
||||
},
|
||||
setCustomMascotGifUrl(state, action: PayloadAction<string | null>) {
|
||||
if (action.payload == null) {
|
||||
state.customMascotGifUrl = null;
|
||||
@@ -178,6 +267,7 @@ const mascotSlice = createSlice({
|
||||
if (isCustomMascotGifUrl(action.payload)) {
|
||||
state.customMascotGifUrl = action.payload.trim();
|
||||
state.selectedMascotId = null;
|
||||
state.secondaryMascotId = null;
|
||||
} else {
|
||||
state.customMascotGifUrl = null;
|
||||
}
|
||||
@@ -231,6 +321,8 @@ const mascotSlice = createSlice({
|
||||
voiceGender?: unknown;
|
||||
voiceUseLocaleDefault?: unknown;
|
||||
selectedMascotId?: unknown;
|
||||
secondaryMascotId?: unknown;
|
||||
mascotVoices?: unknown;
|
||||
customMascotGifUrl?: unknown;
|
||||
customPrimaryColor?: unknown;
|
||||
customSecondaryColor?: unknown;
|
||||
@@ -246,6 +338,17 @@ const mascotSlice = createSlice({
|
||||
: isMascotVoiceId(restoredSelectedMascotId)
|
||||
? (restoredSelectedMascotId as string).trim()
|
||||
: null;
|
||||
// Second mascot + per-mascot voices are absent in pre-#4277 blobs;
|
||||
// the `null` / `{}` fallbacks match a fresh install and keep
|
||||
// single-mascot users unchanged. Invalid values are scrubbed.
|
||||
const restoredSecondaryMascotId = rehydrateAction.payload?.secondaryMascotId;
|
||||
state.secondaryMascotId =
|
||||
restoredSecondaryMascotId == null
|
||||
? null
|
||||
: isMascotVoiceId(restoredSecondaryMascotId)
|
||||
? (restoredSecondaryMascotId as string).trim()
|
||||
: null;
|
||||
state.mascotVoices = sanitizeMascotVoices(rehydrateAction.payload?.mascotVoices);
|
||||
const restoredCustomMascotGifUrl = rehydrateAction.payload?.customMascotGifUrl;
|
||||
state.customMascotGifUrl =
|
||||
restoredCustomMascotGifUrl == null
|
||||
@@ -253,7 +356,12 @@ const mascotSlice = createSlice({
|
||||
: isCustomMascotGifUrl(restoredCustomMascotGifUrl)
|
||||
? (restoredCustomMascotGifUrl as string).trim()
|
||||
: null;
|
||||
if (state.customMascotGifUrl) state.selectedMascotId = null;
|
||||
// A custom GIF avatar is mutually exclusive with Rive mascots —
|
||||
// drop both mascot selections if a GIF override survived.
|
||||
if (state.customMascotGifUrl) {
|
||||
state.selectedMascotId = null;
|
||||
state.secondaryMascotId = null;
|
||||
}
|
||||
// `voiceId` is optional in older persisted blobs (pre-#1762) — the
|
||||
// `null` fallback is the intended default and matches a fresh
|
||||
// install. Invalid values are scrubbed so a corrupted localStorage
|
||||
@@ -289,6 +397,8 @@ export const {
|
||||
setMascotVoiceGender,
|
||||
setMascotVoiceUseLocaleDefault,
|
||||
setSelectedMascotId,
|
||||
setSecondaryMascotId,
|
||||
setMascotVoice,
|
||||
setCustomMascotGifUrl,
|
||||
setCustomPrimaryColor,
|
||||
setCustomSecondaryColor,
|
||||
@@ -309,6 +419,32 @@ export const selectMascotVoiceUseLocaleDefault = (state: { mascot: MascotState }
|
||||
export const selectSelectedMascotId = (state: { mascot: MascotState }): string | null =>
|
||||
state.mascot.selectedMascotId;
|
||||
|
||||
export const selectSecondaryMascotId = (state: { mascot: MascotState }): string | null =>
|
||||
state.mascot.secondaryMascotId;
|
||||
|
||||
export const selectMascotVoices = (state: { mascot: MascotState }): Record<string, string> =>
|
||||
state.mascot.mascotVoices ?? {};
|
||||
|
||||
/**
|
||||
* Explicit per-mascot voice override for `mascotId`, or `null` when none
|
||||
* is set (caller falls back to the effective single voice). Curried so it
|
||||
* reads like the other parameterised selectors at call sites.
|
||||
*/
|
||||
export const selectMascotVoiceFor =
|
||||
(mascotId: string | null) =>
|
||||
(state: { mascot: MascotState }): string | null =>
|
||||
mascotId ? (state.mascot.mascotVoices?.[mascotId] ?? null) : null;
|
||||
|
||||
/**
|
||||
* True when a distinct second mascot is enabled — the single gate the
|
||||
* meeting render + join paths use to decide dual vs single behavior.
|
||||
* Guards against the same mascot being picked twice.
|
||||
*/
|
||||
export const selectDualMascotEnabled = (state: { mascot: MascotState }): boolean => {
|
||||
const { selectedMascotId, secondaryMascotId } = state.mascot;
|
||||
return secondaryMascotId != null && secondaryMascotId !== selectedMascotId;
|
||||
};
|
||||
|
||||
export const selectCustomMascotGifUrl = (state: { mascot: MascotState }): string | null =>
|
||||
state.mascot.customMascotGifUrl;
|
||||
|
||||
@@ -354,5 +490,51 @@ export const selectEffectiveMascotVoiceId = (state: {
|
||||
return MASCOT_VOICE_ID || ELEVENLABS_VOICE_PRESETS[0].id;
|
||||
};
|
||||
|
||||
export interface MeetingMascotSlot {
|
||||
/** Manifest mascot id, or `null` for the primary when the user is on
|
||||
* the default (first-`ready`) mascot. */
|
||||
mascotId: string | null;
|
||||
/** Resolved voice id: the per-mascot override, else the effective
|
||||
* single voice — never empty, so the join payload always carries one. */
|
||||
voiceId: string;
|
||||
}
|
||||
|
||||
export interface MeetingMascotVoicePair {
|
||||
primary: MeetingMascotSlot;
|
||||
secondary: MeetingMascotSlot | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the (up to two) mascots + voices a meeting join should use
|
||||
* (issue #4277). Single source of truth for both join paths — the CEF
|
||||
* `meet_call_open_window` sender and the backend `agent_meetings_join`
|
||||
* sender — and for tests, so they can't drift.
|
||||
*
|
||||
* `secondary` is non-null only when a distinct second mascot is enabled
|
||||
* (`selectDualMascotEnabled`). Each slot's voice is its per-mascot
|
||||
* override, falling back to `selectEffectiveMascotVoiceId`; when the user
|
||||
* hasn't set distinct voices both slots resolve to that same voice
|
||||
* (harmless — alternation still works, it just sounds the same).
|
||||
*/
|
||||
export const selectMeetingMascotVoicePair = (state: {
|
||||
mascot: MascotState;
|
||||
locale?: { current: Locale };
|
||||
}): MeetingMascotVoicePair => {
|
||||
const effective = selectEffectiveMascotVoiceId(state);
|
||||
const { selectedMascotId, secondaryMascotId } = state.mascot;
|
||||
// Tolerate a partial / pre-migration mascot slice (e.g. a legacy persisted
|
||||
// blob or a test's preloadedState) that predates `mascotVoices`.
|
||||
const mascotVoices = state.mascot.mascotVoices ?? {};
|
||||
const primary: MeetingMascotSlot = {
|
||||
mascotId: selectedMascotId,
|
||||
voiceId: (selectedMascotId && mascotVoices[selectedMascotId]) || effective,
|
||||
};
|
||||
const dualEnabled = secondaryMascotId != null && secondaryMascotId !== selectedMascotId;
|
||||
const secondary: MeetingMascotSlot | null = dualEnabled
|
||||
? { mascotId: secondaryMascotId, voiceId: mascotVoices[secondaryMascotId] || effective }
|
||||
: null;
|
||||
return { primary, secondary };
|
||||
};
|
||||
|
||||
export { mascotSlice };
|
||||
export default mascotSlice.reducer;
|
||||
|
||||
@@ -1203,6 +1203,10 @@ pub enum DomainEvent {
|
||||
command_text: String,
|
||||
recent_transcript: Vec<BackendMeetTurn>,
|
||||
timestamp_ms: u64,
|
||||
/// Dual-mascot name addressing (#4277 follow-up): slot (0 = primary,
|
||||
/// 1 = secondary) whose mascot name was addressed, or `None` when no
|
||||
/// specific mascot was named. Forwarded to `bot:speak` as `mascotSlot`.
|
||||
mascot_slot: Option<u8>,
|
||||
},
|
||||
/// Core asked the backend bot to speak into the call (`bot:speak`).
|
||||
/// Published for observability after the Socket.IO emit succeeds.
|
||||
|
||||
@@ -257,11 +257,13 @@ impl EventHandler for MeetingEventSubscriber {
|
||||
command_text,
|
||||
recent_transcript,
|
||||
timestamp_ms,
|
||||
mascot_slot,
|
||||
} => {
|
||||
tracing::info!(
|
||||
correlation_id = ?correlation_id,
|
||||
speaker = %speaker,
|
||||
cmd_len = command_text.len(),
|
||||
mascot_slot = ?mascot_slot,
|
||||
"{LOG_PREFIX} in-call request received"
|
||||
);
|
||||
// The orchestrator turn can run for tens of seconds (tools,
|
||||
@@ -271,6 +273,7 @@ impl EventHandler for MeetingEventSubscriber {
|
||||
let command_text = command_text.clone();
|
||||
let recent_transcript = recent_transcript.clone();
|
||||
let timestamp_ms = *timestamp_ms;
|
||||
let mascot_slot = *mascot_slot;
|
||||
tokio::spawn(async move {
|
||||
super::in_call::handle_in_call_request(
|
||||
correlation_id,
|
||||
@@ -278,6 +281,7 @@ impl EventHandler for MeetingEventSubscriber {
|
||||
command_text,
|
||||
recent_transcript,
|
||||
timestamp_ms,
|
||||
mascot_slot,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
@@ -161,6 +161,7 @@ pub async fn handle_in_call_request(
|
||||
command_text: String,
|
||||
recent_transcript: Vec<BackendMeetTurn>,
|
||||
timestamp_ms: u64,
|
||||
mascot_slot: Option<u8>,
|
||||
) {
|
||||
let command = command_text.trim();
|
||||
if command.is_empty() {
|
||||
@@ -201,12 +202,21 @@ pub async fn handle_in_call_request(
|
||||
return;
|
||||
}
|
||||
|
||||
// Name-addressed routing (#4277 follow-up): `mascot_slot` is decided by the
|
||||
// backend's wake matcher (it sees the un-stripped caption and knows which
|
||||
// mascot name was said) and pins this whole turn — the speculative ack, the
|
||||
// streamed chunks, and the final reply — to that slot so the named mascot's
|
||||
// voice + face own the answer. `None` (no name / single mascot / follow-up)
|
||||
// leaves the backend's mechanical alternation untouched, i.e. today's
|
||||
// behavior.
|
||||
|
||||
tracing::info!(
|
||||
correlation_id = ?correlation_id,
|
||||
speaker = %speaker,
|
||||
cmd_len = command.len(),
|
||||
transcript_turns = recent_transcript.len(),
|
||||
timestamp_ms = timestamp_ms,
|
||||
mascot_slot = ?mascot_slot,
|
||||
"{LOG_PREFIX} dispatching in-call command to orchestrator"
|
||||
);
|
||||
|
||||
@@ -220,7 +230,7 @@ pub async fn handle_in_call_request(
|
||||
let ack_task = tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(ACK_AFTER_SECS)) => {
|
||||
if let Err(e) = emit_bot_filler(ACK_PHRASE, ack_cid.as_deref()).await {
|
||||
if let Err(e) = emit_bot_filler(ACK_PHRASE, ack_cid.as_deref(), mascot_slot).await {
|
||||
tracing::debug!("[agent_meetings::in_call] ack emit failed: {e}");
|
||||
}
|
||||
}
|
||||
@@ -235,6 +245,7 @@ pub async fn handle_in_call_request(
|
||||
&recent_transcript,
|
||||
streaming,
|
||||
ack_cancel.clone(),
|
||||
mascot_slot,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -253,7 +264,15 @@ pub async fn handle_in_call_request(
|
||||
// In streaming mode the reply was already spoken sentence-by-
|
||||
// sentence during the turn; only the buffered path emits here.
|
||||
if !streaming {
|
||||
if let Err(e) = emit_bot_speak(&text, correlation_id.as_deref()).await {
|
||||
if let Err(e) = emit_bot_speak_inner(
|
||||
&text,
|
||||
correlation_id.as_deref(),
|
||||
None,
|
||||
"reply",
|
||||
mascot_slot,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("{LOG_PREFIX} bot:speak emit failed: {e}");
|
||||
}
|
||||
}
|
||||
@@ -264,7 +283,18 @@ pub async fn handle_in_call_request(
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("{LOG_PREFIX} in-call turn failed: {e}");
|
||||
if let Err(e2) = emit_bot_speak(FAILURE_PHRASE, correlation_id.as_deref()).await {
|
||||
// Speak the failure reply from the same mascot that was addressed
|
||||
// (name-addressed routing, #4277 follow-up) — a terminal reply must
|
||||
// carry `mascot_slot`, matching the success path above.
|
||||
if let Err(e2) = emit_bot_speak_inner(
|
||||
FAILURE_PHRASE,
|
||||
correlation_id.as_deref(),
|
||||
None,
|
||||
"reply",
|
||||
mascot_slot,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::debug!("{LOG_PREFIX} failure phrase emit failed: {e2}");
|
||||
}
|
||||
// The failure phrase promises a note in the thread — keep it.
|
||||
@@ -363,8 +393,9 @@ pub(super) async fn speak_approval_prompt(action_summary: &str, correlation_id:
|
||||
or — Hey Tiny, deny — to cancel."
|
||||
);
|
||||
// Filler, not a terminal reply: the bot is now waiting for a spoken
|
||||
// decision, so the mascot should stay in its thinking cue.
|
||||
if let Err(e) = emit_bot_filler(&prompt, correlation_id).await {
|
||||
// decision, so the mascot should stay in its thinking cue. Slot-agnostic
|
||||
// (an approval prompt isn't name-addressed) → default alternation.
|
||||
if let Err(e) = emit_bot_filler(&prompt, correlation_id, None).await {
|
||||
tracing::warn!("{LOG_PREFIX} approval prompt emit failed: {e}");
|
||||
}
|
||||
}
|
||||
@@ -378,6 +409,7 @@ async fn run_orchestrator_turn(
|
||||
recent_transcript: &[BackendMeetTurn],
|
||||
streaming: bool,
|
||||
ack_cancel: Arc<Notify>,
|
||||
mascot_slot: Option<u8>,
|
||||
) -> Result<String, String> {
|
||||
let agent_lock = get_or_build_agent(correlation_id).await?;
|
||||
let mut agent = agent_lock.lock().await;
|
||||
@@ -408,7 +440,12 @@ async fn run_orchestrator_turn(
|
||||
let (tx, rx) = mpsc::channel::<AgentProgress>(64);
|
||||
agent.set_on_progress(Some(tx));
|
||||
let cid_owned = correlation_id.map(String::from);
|
||||
Some(tokio::spawn(stream_sentences(rx, cid_owned, ack_cancel)))
|
||||
Some(tokio::spawn(stream_sentences(
|
||||
rx,
|
||||
cid_owned,
|
||||
ack_cancel,
|
||||
mascot_slot,
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -487,6 +524,7 @@ async fn stream_sentences(
|
||||
mut rx: mpsc::Receiver<AgentProgress>,
|
||||
correlation_id: Option<String>,
|
||||
ack_cancel: Arc<Notify>,
|
||||
mascot_slot: Option<u8>,
|
||||
) {
|
||||
let mut buf = String::new();
|
||||
let mut seq: u32 = 0;
|
||||
@@ -501,6 +539,7 @@ async fn stream_sentences(
|
||||
&mut seq,
|
||||
&ack_cancel,
|
||||
&mut spoke,
|
||||
mascot_slot,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -517,6 +556,7 @@ async fn stream_sentences(
|
||||
&mut seq,
|
||||
&ack_cancel,
|
||||
&mut spoke,
|
||||
mascot_slot,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -528,6 +568,7 @@ async fn speak_stream_chunk(
|
||||
seq: &mut u32,
|
||||
ack_cancel: &Arc<Notify>,
|
||||
spoke: &mut bool,
|
||||
mascot_slot: Option<u8>,
|
||||
) {
|
||||
if text.is_empty() {
|
||||
return;
|
||||
@@ -536,7 +577,9 @@ async fn speak_stream_chunk(
|
||||
*spoke = true;
|
||||
ack_cancel.notify_one(); // first real audio — no need for the filler ack
|
||||
}
|
||||
if let Err(e) = emit_bot_speak_inner(text, correlation_id, Some(*seq), "reply").await {
|
||||
if let Err(e) =
|
||||
emit_bot_speak_inner(text, correlation_id, Some(*seq), "reply", mascot_slot).await
|
||||
{
|
||||
tracing::debug!("{LOG_PREFIX} streamed chunk emit failed: {e}");
|
||||
}
|
||||
*seq += 1;
|
||||
@@ -799,26 +842,34 @@ async fn get_or_create_meeting_thread(
|
||||
/// The backend's SpeakOrchestrator handles streaming TTS + the audio
|
||||
/// politeness gate from there.
|
||||
async fn emit_bot_speak(text: &str, correlation_id: Option<&str>) -> Result<(), String> {
|
||||
emit_bot_speak_inner(text, correlation_id, None, "reply").await
|
||||
emit_bot_speak_inner(text, correlation_id, None, "reply", None).await
|
||||
}
|
||||
|
||||
/// Emit a non-terminal *filler* utterance (the speculative "On it" ack, an
|
||||
/// approval prompt): tagged `kind="ack"` so the backend mascot returns to its
|
||||
/// thinking cue afterwards instead of settling to idle — the real reply is
|
||||
/// still on the way.
|
||||
async fn emit_bot_filler(text: &str, correlation_id: Option<&str>) -> Result<(), String> {
|
||||
emit_bot_speak_inner(text, correlation_id, None, "ack").await
|
||||
/// still on the way. `mascot_slot` pins the filler to the name-addressed slot
|
||||
/// so the ack comes from the same mascot that will answer.
|
||||
async fn emit_bot_filler(
|
||||
text: &str,
|
||||
correlation_id: Option<&str>,
|
||||
mascot_slot: Option<u8>,
|
||||
) -> Result<(), String> {
|
||||
emit_bot_speak_inner(text, correlation_id, None, "ack", mascot_slot).await
|
||||
}
|
||||
|
||||
/// As [`emit_bot_speak`], plus an optional `seq` so the backend can order
|
||||
/// the per-sentence chunks of a streamed reply, and a `kind` ("ack" filler vs
|
||||
/// terminal "reply") so the backend can drive the mascot's post-speech pose
|
||||
/// (both additive; older backends ignore them).
|
||||
/// the per-sentence chunks of a streamed reply, a `kind` ("ack" filler vs
|
||||
/// terminal "reply") so the backend can drive the mascot's post-speech pose,
|
||||
/// and an optional `mascot_slot` (0|1) so a name-addressed turn is spoken by a
|
||||
/// specific mascot instead of the backend's default alternation (all additive;
|
||||
/// older backends ignore them).
|
||||
async fn emit_bot_speak_inner(
|
||||
text: &str,
|
||||
correlation_id: Option<&str>,
|
||||
seq: Option<u32>,
|
||||
kind: &str,
|
||||
mascot_slot: Option<u8>,
|
||||
) -> Result<(), String> {
|
||||
let mgr = global_socket_manager()
|
||||
.ok_or_else(|| format!("{LOG_PREFIX} socket manager not initialized"))?;
|
||||
@@ -835,11 +886,20 @@ async fn emit_bot_speak_inner(
|
||||
map.insert("seq".to_string(), json!(s));
|
||||
}
|
||||
map.insert("kind".to_string(), json!(kind));
|
||||
// Name-addressed routing (#4277 follow-up): pin this utterance to a
|
||||
// specific mascot slot. Maps to `MeetingBotSpeakPayload.mascotSlot`,
|
||||
// which the backend prefers over its alternation cursor.
|
||||
if let Some(slot) = mascot_slot {
|
||||
map.insert("mascotSlot".to_string(), json!(slot));
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
text_len = text.len(),
|
||||
correlation_id = ?correlation_id,
|
||||
kind = kind,
|
||||
seq = ?seq,
|
||||
mascot_slot = ?mascot_slot,
|
||||
"{LOG_PREFIX} emitting bot:speak"
|
||||
);
|
||||
mgr.emit("bot:speak", payload)
|
||||
@@ -1019,6 +1079,7 @@ mod tests {
|
||||
" ".into(),
|
||||
vec![],
|
||||
0,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -1036,6 +1097,7 @@ mod tests {
|
||||
"what time is it".into(),
|
||||
vec![],
|
||||
0,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
// No agent should have been built for the meeting.
|
||||
|
||||
@@ -761,6 +761,38 @@ fn build_join_payload(
|
||||
}),
|
||||
);
|
||||
}
|
||||
// Dual-mascot payload (issue #4277). Emitted only when the caller
|
||||
// supplied a `mascots` array; the backend bot renders both slots,
|
||||
// alternates the speaker per reply, and uses each slot's voice.
|
||||
// Absent → the backend falls back to the single `mascotId` above,
|
||||
// so one-mascot callers are unchanged.
|
||||
if let Some(mascots) = &req.mascots {
|
||||
let slots: Vec<Value> = mascots
|
||||
.iter()
|
||||
.map(|m| {
|
||||
let mut slot = json!({ "mascotId": m.mascot_id });
|
||||
if let Some(obj) = slot.as_object_mut() {
|
||||
if let Some(name) = &m.name {
|
||||
obj.insert("name".to_string(), json!(name));
|
||||
}
|
||||
if let Some(colors) = &m.rive_colors {
|
||||
obj.insert(
|
||||
"riveColors".to_string(),
|
||||
json!({
|
||||
"primaryColor": colors.primary_color,
|
||||
"secondaryColor": colors.secondary_color,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if let Some(vid) = &m.voice_id {
|
||||
obj.insert("voiceId".to_string(), json!(vid));
|
||||
}
|
||||
}
|
||||
slot
|
||||
})
|
||||
.collect();
|
||||
map.insert("mascots".to_string(), json!(slots));
|
||||
}
|
||||
if let Some(respond_to) = &req.respond_to_participant {
|
||||
map.insert("respondToParticipant".to_string(), json!(respond_to));
|
||||
}
|
||||
@@ -872,10 +904,23 @@ pub async fn handle_join(params: Map<String, Value>) -> Result<Value, String> {
|
||||
return Err("[agent_meetings] socket not connected to backend".to_string());
|
||||
}
|
||||
|
||||
// Name-addressing (#4277 follow-up) trace: the mascot ids + names forwarded
|
||||
// to the backend. A `None` name on a slot means the backend can't route
|
||||
// that mascot by name ("Hey Toshi").
|
||||
let mascot_trace: Vec<(String, Option<String>)> = req
|
||||
.mascots
|
||||
.as_ref()
|
||||
.map(|ms| {
|
||||
ms.iter()
|
||||
.map(|m| (m.mascot_id.clone(), m.name.clone()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
tracing::info!(
|
||||
meet_url_host = %normalized_url.host_str().unwrap_or(""),
|
||||
platform = %platform,
|
||||
display_name_len = display_name.len(),
|
||||
mascots = ?mascot_trace,
|
||||
"[agent_meetings] emitting bot:join"
|
||||
);
|
||||
|
||||
@@ -1783,6 +1828,8 @@ mod tests {
|
||||
assert!(payload.get("systemPrompt").is_none());
|
||||
assert!(payload.get("mascotId").is_none());
|
||||
assert!(payload.get("riveColors").is_none());
|
||||
// Single-mascot join carries no `mascots` array (issue #4277).
|
||||
assert!(payload.get("mascots").is_none());
|
||||
assert!(payload.get("respondToParticipant").is_none());
|
||||
assert!(payload.get("wakePhrase").is_none());
|
||||
}
|
||||
@@ -1839,6 +1886,50 @@ mod tests {
|
||||
assert_eq!(payload["riveColors"]["secondaryColor"], "#00ff00");
|
||||
assert_eq!(payload["respondToParticipant"], "Bob");
|
||||
assert_eq!(payload["wakePhrase"], "Hello bot");
|
||||
// A single `mascot_id` (no `mascots` array) must NOT emit `mascots`
|
||||
// — the legacy single-mascot wire shape is unchanged (issue #4277).
|
||||
assert!(payload.get("mascots").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_join_payload_with_dual_mascots() {
|
||||
// Two-mascot join (issue #4277): the `mascots` array is emitted with
|
||||
// camelCase slot fields the backend expects; slot 0 = primary.
|
||||
let req: BackendMeetJoinRequest = serde_json::from_value(json!({
|
||||
"meet_url": "https://meet.google.com/abc-defg-hij",
|
||||
"mascot_id": "tiny-mascot",
|
||||
"mascots": [
|
||||
{
|
||||
"mascot_id": "tiny-mascot",
|
||||
"name": "Tiny",
|
||||
"voice_id": "voice-a",
|
||||
"rive_colors": { "primary_color": "#111", "secondary_color": "#222" }
|
||||
},
|
||||
{ "mascot_id": "toshi", "voice_id": "voice-b" }
|
||||
]
|
||||
}))
|
||||
.unwrap();
|
||||
let payload = build_join_payload(
|
||||
"https://meet.google.com/abc-defg-hij",
|
||||
"OpenHuman",
|
||||
"gmeet",
|
||||
&req,
|
||||
);
|
||||
let mascots = payload["mascots"].as_array().expect("mascots array");
|
||||
assert_eq!(mascots.len(), 2);
|
||||
assert_eq!(mascots[0]["mascotId"], "tiny-mascot");
|
||||
// Name-addressed routing (#4277 follow-up): the display name is
|
||||
// forwarded so the bot can route "Hey Tiny …" to slot 0.
|
||||
assert_eq!(mascots[0]["name"], "Tiny");
|
||||
assert_eq!(mascots[0]["voiceId"], "voice-a");
|
||||
assert_eq!(mascots[0]["riveColors"]["primaryColor"], "#111");
|
||||
assert_eq!(mascots[0]["riveColors"]["secondaryColor"], "#222");
|
||||
assert_eq!(mascots[1]["mascotId"], "toshi");
|
||||
assert_eq!(mascots[1]["voiceId"], "voice-b");
|
||||
// No name supplied for slot 1 → no `name` key.
|
||||
assert!(mascots[1].get("name").is_none());
|
||||
// No colors supplied for slot 1 → no `riveColors` key.
|
||||
assert!(mascots[1].get("riveColors").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -126,6 +126,16 @@ fn schema_join() -> ControllerSchema {
|
||||
comment: "Optional Rive mascot color overrides forwarded to the backend bot.",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "mascots",
|
||||
ty: TypeSchema::Json,
|
||||
comment: "Optional dual-mascot config (issue #4277): array of up to 2 slots, each \
|
||||
{ mascotId, name?, riveColors?, voiceId? }. When present the backend renders \
|
||||
both mascots and alternates the speaker per reply. `name` (from the manifest) \
|
||||
enables name-addressed routing (\"Hey Toshi …\" → that slot). Absent falls \
|
||||
back to mascot_id.",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "respond_to_participant",
|
||||
ty: TypeSchema::String,
|
||||
|
||||
@@ -90,6 +90,33 @@ pub struct RiveColors {
|
||||
pub secondary_color: Option<String>,
|
||||
}
|
||||
|
||||
/// One mascot slot for a multi-mascot backend meeting (issue #4277).
|
||||
/// Slot 0 = primary speaker, slot 1 = secondary. The backend bot renders
|
||||
/// both mascots side-by-side, alternates the speaking slot per reply, and
|
||||
/// synthesizes each slot's replies with `voice_id` (falling back to its
|
||||
/// configured default when absent).
|
||||
///
|
||||
/// Deserialize-only: the RPC input arrives snake_case (`mascot_id`,
|
||||
/// `rive_colors`, `voice_id`); the outbound `bot:join` payload is built by
|
||||
/// hand in `ops.rs` (camelCase), so this struct is never serialized.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct BackendMascotSlot {
|
||||
/// Rive mascot id (e.g. "yellow", "toshi").
|
||||
pub mascot_id: String,
|
||||
/// Optional human-facing mascot name (e.g. "Toshi", "Tiny"), taken from the
|
||||
/// manifest. Drives name-addressed routing (#4277 follow-up): a participant
|
||||
/// who says "Hey Toshi …" is routed to this slot instead of the mechanical
|
||||
/// alternation. Absent → that slot is not name-addressable.
|
||||
#[serde(default)]
|
||||
pub name: Option<String>,
|
||||
/// Optional per-mascot color palette overrides.
|
||||
#[serde(default)]
|
||||
pub rive_colors: Option<RiveColors>,
|
||||
/// Optional per-mascot ElevenLabs voice id.
|
||||
#[serde(default)]
|
||||
pub voice_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Inputs to `openhuman.agent_meetings_join`.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct BackendMeetJoinRequest {
|
||||
@@ -105,11 +132,18 @@ pub struct BackendMeetJoinRequest {
|
||||
#[serde(default)]
|
||||
pub system_prompt: Option<String>,
|
||||
/// Selects which Rive mascot appears in the meeting (e.g. "yellow", "blue").
|
||||
/// Legacy single-mascot field; still honored when `mascots` is absent.
|
||||
#[serde(default)]
|
||||
pub mascot_id: Option<String>,
|
||||
/// Optional Rive mascot color palette overrides.
|
||||
/// Optional Rive mascot color palette overrides (single-mascot path).
|
||||
#[serde(default)]
|
||||
pub rive_colors: Option<RiveColors>,
|
||||
/// Dual-mascot config (issue #4277). When present (2 slots) the backend
|
||||
/// renders both mascots and alternates the speaker per reply, using each
|
||||
/// slot's `voice_id`. Absent / single-element falls back to the legacy
|
||||
/// single-mascot `mascot_id` + `rive_colors` behavior.
|
||||
#[serde(default)]
|
||||
pub mascots: Option<Vec<BackendMascotSlot>>,
|
||||
/// Only respond to this participant's messages (empty/absent = respond to everyone).
|
||||
#[serde(default)]
|
||||
pub respond_to_participant: Option<String>,
|
||||
|
||||
@@ -221,7 +221,7 @@ pub async fn run_grant_turn(request_id: &str, grantee: &str) -> Result<bool, Str
|
||||
s.allow_speaker(grantee);
|
||||
s.cancel_outbound();
|
||||
});
|
||||
let samples = match tts(&message).await {
|
||||
let samples = match tts(&message, None).await {
|
||||
Ok(samples) => samples,
|
||||
Err(err) => {
|
||||
log::warn!("[meet-agent] grant TTS failed request_id={request_id} err={err}");
|
||||
@@ -301,7 +301,7 @@ pub async fn run_soft_deny_turn(
|
||||
// Cancel any prior outbound so the refusal doesn't queue behind a
|
||||
// half-drained reply from a previous turn.
|
||||
let _ = registry().with_session(request_id, |s| s.cancel_outbound());
|
||||
let samples = match tts(&message).await {
|
||||
let samples = match tts(&message, None).await {
|
||||
Ok(samples) => samples,
|
||||
Err(err) => {
|
||||
log::warn!("[meet-agent] soft-deny TTS failed request_id={request_id} err={err}");
|
||||
|
||||
@@ -26,7 +26,11 @@ pub(super) async fn stt(samples: &[i16]) -> Result<String, String> {
|
||||
|
||||
// ─── Real TTS adapter ───────────────────────────────────────────────
|
||||
|
||||
pub(super) async fn tts(text: &str) -> Result<Vec<i16>, String> {
|
||||
/// Synthesize `text` to PCM16LE @ 16 kHz. `voice_id` selects the
|
||||
/// ElevenLabs voice for this utterance (the speaker-alternation slot's
|
||||
/// voice); `None` lets the reply-speech backend pick its own default —
|
||||
/// the exact prior behavior for single-mascot calls.
|
||||
pub(super) async fn tts(text: &str, voice_id: Option<&str>) -> Result<Vec<i16>, String> {
|
||||
use crate::openhuman::voice::reply_speech::{synthesize_reply, ReplySpeechOptions};
|
||||
|
||||
let config = crate::openhuman::config::ops::load_config_with_timeout().await?;
|
||||
@@ -51,6 +55,9 @@ pub(super) async fn tts(text: &str) -> Result<Vec<i16>, String> {
|
||||
output_format: Some("pcm_16000".to_string()),
|
||||
model_id: Some(TTS_MODEL_ID.to_string()),
|
||||
voice_settings: Some(voice_settings),
|
||||
// Per-mascot voice for speaker alternation. `None` preserves the
|
||||
// backend's default-voice pick (single-mascot behavior).
|
||||
voice_id: voice_id.map(str::to_owned),
|
||||
..Default::default()
|
||||
};
|
||||
let outcome = synthesize_reply(&config, text, &opts).await?;
|
||||
|
||||
@@ -96,7 +96,13 @@ pub async fn run_turn(request_id: &str) -> Result<bool, String> {
|
||||
let synthesized = if reply_text.trim().is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
match tts(&reply_text).await {
|
||||
// Pick this reply's speaker voice. For two-mascot calls this
|
||||
// alternates voice + active slot once per spoken reply; a
|
||||
// single-mascot call gets `None` (backend default voice) and the
|
||||
// slot stays 0. Advanced only when we actually speak so declined
|
||||
// turns don't rotate the speaker out of sync.
|
||||
let voice_id = registry().with_session(request_id, |s| s.advance_speaker())?;
|
||||
match tts(&reply_text, voice_id.as_deref()).await {
|
||||
Ok(samples) => samples,
|
||||
Err(err) => {
|
||||
log::warn!("[meet-agent] TTS failed request_id={request_id} err={err}");
|
||||
@@ -229,8 +235,20 @@ pub async fn run_caption_turn(request_id: &str) -> Result<bool, String> {
|
||||
// can hear you" sounds redundant. The 50-char threshold is a
|
||||
// rough proxy; real second-brain questions ("am I free Friday
|
||||
// afternoon for a 30 min slot") are almost always longer.
|
||||
if !was_bare_wake && prompt.chars().count() > PREROLL_SKIP_PROMPT_CHARS {
|
||||
if let Ok(ack_pcm) = tts(PREROLL_ACK_PHRASE).await {
|
||||
// One speaker per caption turn: the pre-roll ack and the final reply
|
||||
// come from the SAME mascot, and the active slot advances exactly once
|
||||
// per turn — but ONLY when the turn actually speaks. A turn that fires
|
||||
// no ack and then declines must not rotate the speaker (mirrors
|
||||
// `run_turn`, which advances only on a non-empty reply). `turn_voice`
|
||||
// stays `None` for single-mascot calls (backend default voice, slot 0).
|
||||
let will_ack = !was_bare_wake && prompt.chars().count() > PREROLL_SKIP_PROMPT_CHARS;
|
||||
let mut turn_voice: Option<String> = None;
|
||||
let mut speaker_advanced = false;
|
||||
|
||||
if will_ack {
|
||||
turn_voice = registry().with_session(request_id, |s| s.advance_speaker())?;
|
||||
speaker_advanced = true;
|
||||
if let Ok(ack_pcm) = tts(PREROLL_ACK_PHRASE, turn_voice.as_deref()).await {
|
||||
let _ = registry().with_session(request_id, |s| {
|
||||
s.enqueue_outbound_pcm(&ack_pcm, false);
|
||||
});
|
||||
@@ -279,7 +297,13 @@ pub async fn run_caption_turn(request_id: &str) -> Result<bool, String> {
|
||||
let synthesized = if reply_text.trim().is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
match tts(&reply_text).await {
|
||||
// If no pre-roll ack fired, this reply is the turn's first (and
|
||||
// only) speech — advance the speaker now so a declined turn never
|
||||
// rotated the slot, and a spoken turn rotates exactly once.
|
||||
if !speaker_advanced {
|
||||
turn_voice = registry().with_session(request_id, |s| s.advance_speaker())?;
|
||||
}
|
||||
match tts(&reply_text, turn_voice.as_deref()).await {
|
||||
Ok(samples) => samples,
|
||||
Err(err) => {
|
||||
log::warn!(
|
||||
|
||||
@@ -49,14 +49,22 @@ pub async fn handle_start_session(params: Map<String, Value>) -> Result<Value, S
|
||||
registry().with_session(&req.request_id, |s| {
|
||||
s.set_identities(&req.owner_display_name, &req.bot_display_name);
|
||||
s.set_meet_url(&req.meet_url);
|
||||
// Per-mascot voices for speaker alternation. Empty/absent =
|
||||
// single-default-voice behavior (unchanged for one-mascot users).
|
||||
s.set_voices(req.primary_voice_id.clone(), req.secondary_voice_id.clone());
|
||||
})?;
|
||||
let voice_count = [&req.primary_voice_id, &req.secondary_voice_id]
|
||||
.into_iter()
|
||||
.filter(|v| v.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false))
|
||||
.count();
|
||||
log::info!(
|
||||
"{LOG_PREFIX} start_session request_id={} sample_rate_hz={} \
|
||||
owner_chars={} bot_chars={}",
|
||||
owner_chars={} bot_chars={} voice_count={}",
|
||||
req.request_id,
|
||||
req.sample_rate_hz,
|
||||
req.owner_display_name.chars().count(),
|
||||
req.bot_display_name.chars().count()
|
||||
req.bot_display_name.chars().count(),
|
||||
voice_count
|
||||
);
|
||||
|
||||
RpcOutcome::new(
|
||||
@@ -178,11 +186,11 @@ pub async fn handle_poll_speech(params: Map<String, Value>) -> Result<Value, Str
|
||||
let req: PollSpeechRequest = serde_json::from_value(Value::Object(params))
|
||||
.map_err(|e| format!("{LOG_PREFIX} invalid poll_speech params: {e}"))?;
|
||||
|
||||
let (pcm_base64, utterance_done, flush_pending) =
|
||||
let (pcm_base64, utterance_done, flush_pending, active_mascot_slot) =
|
||||
registry().with_session(&req.request_id, |s| {
|
||||
let (b64, done) = s.poll_outbound();
|
||||
let flush = s.take_flush_pending();
|
||||
(b64, done, flush)
|
||||
(b64, done, flush, s.active_slot())
|
||||
})?;
|
||||
|
||||
RpcOutcome::new(
|
||||
@@ -191,6 +199,11 @@ pub async fn handle_poll_speech(params: Map<String, Value>) -> Result<Value, Str
|
||||
"pcm_base64": pcm_base64,
|
||||
"utterance_done": utterance_done,
|
||||
"flush_pending": flush_pending,
|
||||
// Which mascot (0 = primary, 1 = secondary) is speaking the
|
||||
// current outbound audio. The shell forwards this on the
|
||||
// speaking-state edge so the frontend lip-syncs the right
|
||||
// mascot and reacts the other. Always 0 for single-mascot.
|
||||
"active_mascot_slot": active_mascot_slot,
|
||||
}),
|
||||
vec![],
|
||||
)
|
||||
@@ -334,6 +347,35 @@ mod tests {
|
||||
assert_eq!(out.get("turn_count"), Some(&json!(0)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn poll_speech_reports_active_mascot_slot() {
|
||||
// poll_speech surfaces `active_mascot_slot` (the mascot currently
|
||||
// speaking, read from session state via `active_slot()`) on every
|
||||
// reply so the shell can drive per-mascot lip-sync. This exercises
|
||||
// that emit on an idle session — no LLM/brain turn needed, since
|
||||
// `active_slot()` reads session state directly and defaults to 0.
|
||||
let mut start = Map::new();
|
||||
start.insert("request_id".into(), json!("rpc-active-slot"));
|
||||
start.insert("sample_rate_hz".into(), json!(16_000));
|
||||
start.insert("primary_voice_id".into(), json!("voice-primary"));
|
||||
start.insert("secondary_voice_id".into(), json!("voice-secondary"));
|
||||
handle_start_session(start).await.unwrap();
|
||||
|
||||
let mut poll = Map::new();
|
||||
poll.insert("request_id".into(), json!("rpc-active-slot"));
|
||||
let out = handle_poll_speech(poll).await.unwrap();
|
||||
assert_eq!(out.get("ok"), Some(&json!(true)));
|
||||
assert_eq!(
|
||||
out.get("active_mascot_slot"),
|
||||
Some(&json!(0)),
|
||||
"idle session starts on the primary mascot (slot 0)"
|
||||
);
|
||||
|
||||
let mut stop = Map::new();
|
||||
stop.insert("request_id".into(), json!("rpc-active-slot"));
|
||||
handle_stop_session(stop).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "flaky on CI; see PR #2588 follow-up"]
|
||||
async fn push_then_poll_returns_audio_after_brain_turn() {
|
||||
|
||||
@@ -193,6 +193,20 @@ pub struct MeetAgentSession {
|
||||
/// Resets on `stop_session` (the registry drops the whole
|
||||
/// session). Empty by default — grants are opt-in per call.
|
||||
allowlist: HashSet<String>,
|
||||
/// Per-mascot TTS voice ids in speaker-slot order (slot 0 = primary,
|
||||
/// slot 1 = secondary). 0, 1, or 2 entries. Empty preserves the old
|
||||
/// single-default-voice behavior (the reply-speech backend picks its
|
||||
/// own default). Set once at `start_session` from the shell payload.
|
||||
voices: Vec<String>,
|
||||
/// Index into `voices` of the mascot speaking the *current* turn.
|
||||
/// Advanced once per assistant reply by [`Self::advance_speaker`] so
|
||||
/// two-mascot calls alternate which voice speaks (and which mascot
|
||||
/// the frontend lip-syncs). Meaningless when `voices.len() <= 1`.
|
||||
active_voice_ix: usize,
|
||||
/// False until the first [`Self::advance_speaker`] of a two-mascot call.
|
||||
/// Keeps the first reply on the primary (slot 0) — so an idle session also
|
||||
/// reports slot 0 — while every subsequent reply rotates the speaker.
|
||||
speaker_primed: bool,
|
||||
}
|
||||
|
||||
impl MeetAgentSession {
|
||||
@@ -228,6 +242,9 @@ impl MeetAgentSession {
|
||||
pending_unauthorized_speaker: None,
|
||||
pending_unauthorized_at_ms: 0,
|
||||
allowlist: HashSet::new(),
|
||||
voices: Vec::new(),
|
||||
active_voice_ix: 0,
|
||||
speaker_primed: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -305,6 +322,64 @@ impl MeetAgentSession {
|
||||
self.bot_display_name = bot_display_name.trim().to_string();
|
||||
}
|
||||
|
||||
/// Configure the per-mascot TTS voices for speaker alternation.
|
||||
/// `primary` is slot 0, `secondary` is slot 1. Empty / whitespace
|
||||
/// ids are dropped, so:
|
||||
/// * both present → two-voice call, speaker alternates per reply;
|
||||
/// * one present → that single voice speaks every turn;
|
||||
/// * none present → `advance_speaker` yields `None` and the
|
||||
/// reply-speech backend keeps picking its own default voice
|
||||
/// (exact previous behavior).
|
||||
/// The active slot starts at 0 so an idle session reports the primary
|
||||
/// mascot; [`Self::advance_speaker`] keeps the first reply there and
|
||||
/// rotates thereafter.
|
||||
pub fn set_voices(&mut self, primary: Option<String>, secondary: Option<String>) {
|
||||
let mut voices = Vec::new();
|
||||
for id in [primary, secondary].into_iter().flatten() {
|
||||
let trimmed = id.trim();
|
||||
if !trimmed.is_empty() {
|
||||
voices.push(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
self.active_voice_ix = 0;
|
||||
self.speaker_primed = false;
|
||||
self.voices = voices;
|
||||
}
|
||||
|
||||
/// Pick the voice for the turn that is about to start and rotate the
|
||||
/// active speaker for two-mascot calls. Returns `None` when no voices
|
||||
/// were configured (single default-voice behavior). Call exactly once
|
||||
/// per assistant reply, before synthesizing, so a multi-chunk reply
|
||||
/// keeps one voice for its whole duration.
|
||||
pub fn advance_speaker(&mut self) -> Option<String> {
|
||||
match self.voices.len() {
|
||||
0 => None,
|
||||
1 => Some(self.voices[0].clone()),
|
||||
n => {
|
||||
// First reply stays on the primary (slot 0); each later reply
|
||||
// rotates. This keeps an idle session on slot 0 too.
|
||||
if self.speaker_primed {
|
||||
self.active_voice_ix = (self.active_voice_ix + 1) % n;
|
||||
} else {
|
||||
self.speaker_primed = true;
|
||||
}
|
||||
Some(self.voices[self.active_voice_ix].clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Slot (0 = primary, 1 = secondary) of the mascot speaking the
|
||||
/// current turn. The shell forwards this on the `meet-video:speaking-state`
|
||||
/// edge so the frontend lip-syncs the right mascot and reacts the
|
||||
/// other. Defaults to 0 (single-mascot rendering ignores it).
|
||||
pub fn active_slot(&self) -> u8 {
|
||||
if self.voices.is_empty() {
|
||||
0
|
||||
} else {
|
||||
(self.active_voice_ix % self.voices.len()) as u8
|
||||
}
|
||||
}
|
||||
|
||||
/// Read accessor used by audit logging. Empty when set_identities
|
||||
/// has not been called for this session.
|
||||
pub fn owner_display_name(&self) -> &str {
|
||||
@@ -950,6 +1025,57 @@ mod tests {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// ─── Speaker alternation (issue #4277) ──────────────────────────
|
||||
|
||||
#[test]
|
||||
fn set_voices_empty_disables_alternation() {
|
||||
let mut s = MeetAgentSession::new("p".into(), 16_000);
|
||||
s.set_voices(None, None);
|
||||
// No voices → brain keeps the backend default voice, slot pinned 0.
|
||||
assert_eq!(s.advance_speaker(), None);
|
||||
assert_eq!(s.active_slot(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_voice_speaks_every_turn_on_slot_zero() {
|
||||
let mut s = MeetAgentSession::new("p".into(), 16_000);
|
||||
s.set_voices(Some("voice-a".into()), None);
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
|
||||
assert_eq!(s.active_slot(), 0);
|
||||
// A single voice never rotates.
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
|
||||
assert_eq!(s.active_slot(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_voices_alternate_speaker_and_slot_per_turn() {
|
||||
let mut s = MeetAgentSession::new("p".into(), 16_000);
|
||||
s.set_voices(Some("voice-a".into()), Some("voice-b".into()));
|
||||
// First reply = primary (slot 0)…
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
|
||||
assert_eq!(s.active_slot(), 0);
|
||||
// …second = secondary (slot 1)…
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-b"));
|
||||
assert_eq!(s.active_slot(), 1);
|
||||
// …third wraps back to primary.
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
|
||||
assert_eq!(s.active_slot(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_voices_filters_blank_and_trims() {
|
||||
let mut s = MeetAgentSession::new("p".into(), 16_000);
|
||||
// Blank primary collapses to a single (secondary) voice on slot 0…
|
||||
s.set_voices(Some(" ".into()), Some(" voice-b ".into()));
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-b"));
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-b"));
|
||||
assert_eq!(s.active_slot(), 0);
|
||||
// …and two real ids are trimmed before use.
|
||||
s.set_voices(Some(" voice-a ".into()), Some(" voice-b ".into()));
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-a"));
|
||||
assert_eq!(s.advance_speaker().as_deref(), Some("voice-b"));
|
||||
}
|
||||
|
||||
/// Build a session pre-configured for the wake-word tests: Alice
|
||||
/// is the call owner, "OpenHuman" is the bot's Meet tile. Every
|
||||
/// wake-path test goes through this helper so the owner gate
|
||||
|
||||
@@ -45,6 +45,19 @@ pub struct StartSessionRequest {
|
||||
/// updated to forward the URL still parse the payload.
|
||||
#[serde(default)]
|
||||
pub meet_url: String,
|
||||
/// ElevenLabs voice id for the PRIMARY mascot (speaker slot 0). When
|
||||
/// two mascots are enabled the session alternates the speaking voice
|
||||
/// once per assistant reply (see [`super::session::MeetAgentSession::advance_speaker`]).
|
||||
/// `None`/empty preserves the previous single-default-voice behavior
|
||||
/// (the reply-speech backend picks its own default voice). Defaulted
|
||||
/// so older shells / smoke tests still parse the payload.
|
||||
#[serde(default)]
|
||||
pub primary_voice_id: Option<String>,
|
||||
/// ElevenLabs voice id for the SECONDARY mascot (speaker slot 1).
|
||||
/// Absent when the user has only one mascot enabled — in that case no
|
||||
/// alternation happens and slot 0 speaks every turn.
|
||||
#[serde(default)]
|
||||
pub secondary_voice_id: Option<String>,
|
||||
}
|
||||
|
||||
fn default_sample_rate() -> u32 {
|
||||
|
||||
@@ -390,10 +390,18 @@ pub(super) fn handle_sio_event(
|
||||
.get("timestampMs")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
// Dual-mascot name addressing (#4277 follow-up): which slot the
|
||||
// backend's wake matcher decided was addressed (0|1), if any.
|
||||
let mascot_slot = data
|
||||
.get("mascotSlot")
|
||||
.and_then(|v| v.as_u64())
|
||||
.filter(|s| *s <= 1)
|
||||
.map(|s| s as u8);
|
||||
log::info!(
|
||||
"[socket] bot:in_call_request speaker={} cmd_len={}",
|
||||
"[socket] bot:in_call_request speaker={} cmd_len={} mascot_slot={:?}",
|
||||
speaker,
|
||||
command_text.len()
|
||||
command_text.len(),
|
||||
mascot_slot
|
||||
);
|
||||
publish_global(DomainEvent::BackendMeetInCallRequest {
|
||||
correlation_id,
|
||||
@@ -401,6 +409,7 @@ pub(super) fn handle_sio_event(
|
||||
command_text,
|
||||
recent_transcript,
|
||||
timestamp_ms,
|
||||
mascot_slot,
|
||||
});
|
||||
}
|
||||
"bot:error" => {
|
||||
|
||||
Reference in New Issue
Block a user