diff --git a/app/src/components/settings/panels/MascotPanel.tsx b/app/src/components/settings/panels/MascotPanel.tsx index 1d59b91b0..a97be584c 100644 --- a/app/src/components/settings/panels/MascotPanel.tsx +++ b/app/src/components/settings/panels/MascotPanel.tsx @@ -1,13 +1,7 @@ import { useEffect, useMemo, useRef, useState } from 'react'; -import { CustomGifMascot, RiveMascot } from '../../../features/human/Mascot'; -import { BackendMascot } from '../../../features/human/Mascot/backend/BackendMascot'; -import { BackendRiveMascot } from '../../../features/human/Mascot/backend/BackendRiveMascot'; -import { - isRiveMascotDetail, - type MascotDetailUnion, - type MascotSummary, -} from '../../../features/human/Mascot/backend/types'; +import { CustomGifMascot, ManifestRiveMascot, RiveMascot } from '../../../features/human/Mascot'; +import { useMascotManifest } from '../../../features/human/Mascot/manifest/useMascotManifest'; import { getMascotPalette, hexToArgbInt, @@ -15,7 +9,6 @@ import { } from '../../../features/human/Mascot/mascotPalette'; import { synthesizeSpeech } from '../../../features/human/voice/ttsClient'; import { useT } from '../../../lib/i18n/I18nContext'; -import { fetchMascotList, getCachedMascotDetail } from '../../../services/mascotService'; import { useAppDispatch, useAppSelector } from '../../../store/hooks'; import { DEFAULT_MASCOT_COLOR, @@ -83,13 +76,16 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => { const useLocaleDefault = useAppSelector(selectMascotVoiceUseLocaleDefault); const effectiveVoiceId = useAppSelector(selectEffectiveMascotVoiceId); - // Backend mascot library (PR tinyhumansai/backend#770). The list endpoint - // is cheap (no SVG bytes); per-id detail is fetched on demand so the - // animated preview only pays for the active selection. - const [backendList, setBackendList] = useState(null); - const [backendListError, setBackendListError] = useState(null); - const [activeDetail, setActiveDetail] = useState(null); - const [detailError, setDetailError] = useState(null); + // Mascot library, sourced from the published GitHub manifest + // (tinyhumansai/mascots). `entry` is the resolved active mascot (selection + // or default); `manifest.mascots` drives the picker list. Each entry carries + // its full stateEngine inline, so there is no per-id detail round trip. + const { + manifest, + entry: activeEntry, + loading: manifestLoading, + error: manifestError, + } = useMascotManifest(); const [customGifDraft, setCustomGifDraft] = useState(customMascotGifUrl ?? ''); const [customGifError, setCustomGifError] = useState(null); @@ -108,45 +104,6 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => { // earlier audio-only cleanup missed. const previewRequestIdRef = useRef(0); - useEffect(() => { - let cancelled = false; - fetchMascotList() - .then(list => { - if (cancelled) return; - setBackendList(list); - setBackendListError(null); - }) - .catch((err: unknown) => { - if (cancelled) return; - const message = err instanceof Error ? err.message : t('settings.mascot.loadLibraryError'); - setBackendListError(message); - setBackendList([]); - }); - return () => { - cancelled = true; - }; - }, [t]); - - useEffect(() => { - if (!selectedMascotId) return; - let cancelled = false; - getCachedMascotDetail(selectedMascotId) - .then(detail => { - if (cancelled) return; - setActiveDetail(detail); - setDetailError(null); - }) - .catch((err: unknown) => { - if (cancelled) return; - const message = err instanceof Error ? err.message : t('settings.mascot.loadDetailError'); - setDetailError(message); - setActiveDetail(null); - }); - return () => { - cancelled = true; - }; - }, [selectedMascotId, t]); - // Stop any in-flight preview audio when the panel unmounts. Also // bump the preview request id so a `synthesizeSpeech(...)` that // resolves after unmount can detect the staleness and bail. @@ -161,15 +118,14 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => { }; }, []); - const handleSelectBackend = (id: string | null) => { + const handleSelectMascot = (id: string | null) => { dispatch(setSelectedMascotId(id)); setCustomGifError(null); - if (id == null) { - setCustomGifDraft(''); - dispatch(setCustomMascotGifUrl(null)); - } else { - setCustomGifDraft(''); - } + setCustomGifDraft(''); + // Selecting a mascot id already clears the custom GIF in the reducer; the + // null ("default") case has to clear it here so the stage falls back to + // the default manifest mascot rather than the GIF. + if (id == null) dispatch(setCustomMascotGifUrl(null)); }; const onSaveCustomGif = () => { @@ -305,8 +261,6 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => { const presetPickerDisabled = useLocaleDefault; const isCustomVoice = !presetPickerDisabled && (voicePasteMode || !isCuratedVoicePreset(effectiveVoiceId)); - const visibleActiveDetail = selectedMascotId ? activeDetail : null; - const visibleDetailError = selectedMascotId ? detailError : null; const activePalette = getMascotPalette(activeColor); const primaryColorArgb = useMemo( @@ -629,25 +583,25 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => { )} - {/* Backend mascot library */} + {/* Mascot manifest library (tinyhumansai/mascots) */}
- {backendListError && ( + {manifestError && (

- {t('settings.mascot.libraryUnavailable')}: {backendListError} + {t('settings.mascot.libraryUnavailable')}: {manifestError.message}

)} - {!backendListError && backendList === null && ( + {!manifestError && manifestLoading && (

{t('settings.mascot.loadingLibrary')}

)} - {backendList && backendList.length === 0 && !backendListError && ( + {manifest && manifest.mascots.length === 0 && !manifestError && (

{t('settings.mascot.noCharacters')}

)} - {backendList && backendList.length > 0 && ( + {manifest && manifest.mascots.length > 0 && (
  • - {backendList.map(summary => { - const active = summary.id === selectedMascotId; + {manifest.mascots.map(mascot => { + const active = mascot.id === selectedMascotId; + const poseCount = new Set([ + ...mascot.stateEngine.idlePoseCycle, + ...Object.values(mascot.stateEngine.states), + ]).size; + const visemeCount = mascot.stateEngine.visemeCodes.length; return ( -
  • +
- {visibleActiveDetail && ( + {activeEntry && !customMascotGifUrl && (

- {t('settings.mascot.characterPreview')} · {visibleActiveDetail.name} + {t('settings.mascot.characterPreview')} · {activeEntry.name}

- {isRiveMascotDetail(visibleActiveDetail) ? ( - - ) : ( - - )} +
)} - {visibleDetailError && ( -

- {visibleDetailError} -

- )}

{t('settings.mascot.characterDesc')}

diff --git a/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx b/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx index 16ae9726b..c73738659 100644 --- a/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx +++ b/app/src/components/settings/panels/__tests__/MascotPanel.test.tsx @@ -14,17 +14,14 @@ import mascotReducer, { } from '../../../../store/mascotSlice'; import MascotPanel from '../MascotPanel'; -const { mockNavigateBack, fetchMascotListMock, getCachedMascotDetailMock, mockSynthesizeSpeech } = - vi.hoisted(() => ({ - mockNavigateBack: vi.fn(), - fetchMascotListMock: vi.fn(), - getCachedMascotDetailMock: vi.fn(), - mockSynthesizeSpeech: vi.fn(), - })); +const { mockNavigateBack, useMascotManifestMock, mockSynthesizeSpeech } = vi.hoisted(() => ({ + mockNavigateBack: vi.fn(), + useMascotManifestMock: vi.fn(), + mockSynthesizeSpeech: vi.fn(), +})); -vi.mock('../../../../services/mascotService', () => ({ - fetchMascotList: (...args: unknown[]) => fetchMascotListMock(...args), - getCachedMascotDetail: (...args: unknown[]) => getCachedMascotDetailMock(...args), +vi.mock('../../../../features/human/Mascot/manifest/useMascotManifest', () => ({ + useMascotManifest: () => useMascotManifestMock(), })); vi.mock('../../../../features/human/voice/ttsClient', () => ({ @@ -36,17 +33,59 @@ vi.mock('../../../../features/human/Mascot', async importOriginal => { return { ...actual, RiveMascot: () =>
, + ManifestRiveMascot: ({ entry }: { entry: { id: string } }) => ( +
+ ), CustomGifMascot: ({ src }: { src: string }) => ( ), }; }); -vi.mock('../../../../features/human/Mascot/backend/BackendMascot', () => ({ - BackendMascot: ({ mascot }: { mascot: { id: string } }) => ( -
- ), -})); +// Build a minimal manifest entry for the picker list / preview. +function manifestEntry(id: string, name: string, status: 'ready' | 'draft' = 'ready') { + return { + id, + name, + description: '', + status, + tags: [], + stateEngine: { + idlePoseCycle: ['idle', 'dancing'], + states: { idle: 'idle', thinking: 'thinking' }, + visemeCodes: ['sil', 'PP', 'aa'], + }, + files: [ + { path: `${id}.riv`, bytes: 1, role: 'runtime', sha256: id, url: `https://x/${id}.riv` }, + ], + }; +} + +function manifestResult( + mascots: ReturnType[], + overrides: Partial<{ + manifest: unknown; + entry: unknown; + loading: boolean; + error: Error | null; + }> = {} +) { + const manifest = + 'manifest' in overrides + ? overrides.manifest + : { + schemaVersion: 1, + generatedAt: '', + mascots, + source: { repository: '', branch: '', commit: '' }, + }; + return { + manifest, + entry: 'entry' in overrides ? overrides.entry : (mascots[0] ?? null), + loading: overrides.loading ?? false, + error: overrides.error ?? null, + }; +} vi.mock('../../hooks/useSettingsNavigation', () => ({ useSettingsNavigation: () => ({ @@ -75,8 +114,7 @@ function renderPanel(store = buildStore()) { describe('MascotPanel', () => { beforeEach(() => { vi.clearAllMocks(); - fetchMascotListMock.mockResolvedValue([]); - getCachedMascotDetailMock.mockResolvedValue(null); + useMascotManifestMock.mockReturnValue(manifestResult([])); mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0)); }); @@ -168,78 +206,65 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => { expect(screen.getByRole('radio', { name: 'Yellow' })).toHaveAttribute('aria-checked', 'false'); }); - describe('backend mascot library', () => { - const summary = { - id: 'yellow', - name: 'Yellow', - version: '1.0.0', - description: '', - states: [{ id: 'idle', label: 'Idle', description: '' }], - hasVisemes: true, - }; - const detail = { - id: 'yellow', - name: 'Yellow', - version: '1.0.0', - description: '', - viewBox: '0 0 1 1', - defaultState: 'idle', - variables: [], - states: [{ id: 'idle', label: 'Idle', description: '', svg: '' }], - visemes: [], - }; + describe('mascot manifest library', () => { + const yellow = manifestEntry('yellow', 'Yellow'); + const toshi = manifestEntry('toshi', 'Toshi', 'draft'); - it('renders the picker entries returned by the API', async () => { - fetchMascotListMock.mockResolvedValueOnce([summary]); + it('renders the picker entries from the manifest', () => { + useMascotManifestMock.mockReturnValue(manifestResult([yellow, toshi])); renderPanel(); - expect(await screen.findByTestId('backend-mascot-yellow')).toBeInTheDocument(); + expect(screen.getByTestId('manifest-mascot-yellow')).toBeInTheDocument(); + expect(screen.getByTestId('manifest-mascot-toshi')).toBeInTheDocument(); + // Draft status badge surfaces for non-ready mascots. + expect(screen.getByText('Draft')).toBeInTheDocument(); // Default-row (local) sentinel expect(screen.getByText(/Local OpenHuman/)).toBeInTheDocument(); }); - it('shows a friendly empty state when the library is empty', async () => { - fetchMascotListMock.mockResolvedValueOnce([]); + it('shows a friendly empty state when the library is empty', () => { + useMascotManifestMock.mockReturnValue( + manifestResult([], { + manifest: { + schemaVersion: 1, + generatedAt: '', + mascots: [], + source: { repository: '', branch: '', commit: '' }, + }, + }) + ); renderPanel(); - expect( - await screen.findByText(/No OpenHuman characters are available yet/i) - ).toBeInTheDocument(); + expect(screen.getByText(/No OpenHuman characters are available yet/i)).toBeInTheDocument(); }); - it('shows an error when the library endpoint rejects', async () => { - fetchMascotListMock.mockRejectedValueOnce(new Error('offline')); + it('shows an error when the manifest fails to load', () => { + useMascotManifestMock.mockReturnValue( + manifestResult([], { manifest: null, entry: null, error: new Error('offline') }) + ); renderPanel(); - expect( - await screen.findByText(/OpenHuman library unavailable: offline/i) - ).toBeInTheDocument(); + expect(screen.getByText(/OpenHuman library unavailable: offline/i)).toBeInTheDocument(); }); - it('dispatches setSelectedMascotId when a backend mascot is picked', async () => { - fetchMascotListMock.mockResolvedValueOnce([summary]); - getCachedMascotDetailMock.mockResolvedValueOnce(detail); + it('dispatches setSelectedMascotId when a mascot is picked', () => { + useMascotManifestMock.mockReturnValue(manifestResult([yellow])); const { store } = renderPanel(); - const row = await screen.findByTestId('backend-mascot-yellow'); - fireEvent.click(row); + fireEvent.click(screen.getByTestId('manifest-mascot-yellow')); expect(store.getState().mascot.selectedMascotId).toBe('yellow'); }); - it('loads + previews the active backend mascot detail', async () => { + it('previews the active manifest mascot', () => { const store = buildStore(); store.dispatch(setSelectedMascotId('yellow')); - fetchMascotListMock.mockResolvedValueOnce([summary]); - getCachedMascotDetailMock.mockResolvedValueOnce(detail); + useMascotManifestMock.mockReturnValue(manifestResult([yellow], { entry: yellow })); renderPanel(store); - expect(await screen.findByTestId('backend-mascot-preview-yellow')).toBeInTheDocument(); - expect(getCachedMascotDetailMock).toHaveBeenCalledWith('yellow'); + expect(screen.getByTestId('manifest-mascot-preview-yellow')).toBeInTheDocument(); }); - it('clearing the selection returns to the local default', async () => { + it('clearing the selection returns to the local default', () => { const store = buildStore(); store.dispatch(setSelectedMascotId('yellow')); - fetchMascotListMock.mockResolvedValueOnce([summary]); - getCachedMascotDetailMock.mockResolvedValueOnce(detail); + useMascotManifestMock.mockReturnValue(manifestResult([yellow], { entry: yellow })); renderPanel(store); - const localRow = await screen.findByText(/Local OpenHuman/); - fireEvent.click(localRow); + fireEvent.click(screen.getByText(/Local OpenHuman/)); expect(store.getState().mascot.selectedMascotId).toBeNull(); }); @@ -268,12 +293,12 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => { expect(screen.getByTestId('mascot-custom-gif-error')).toHaveTextContent('HTTPS .gif'); }); - it('selecting a backend mascot clears the custom GIF avatar', async () => { + it('selecting a mascot clears the custom GIF avatar', () => { const store = buildStore(); store.dispatch(setCustomMascotGifUrl('https://example.com/avatar.gif')); - fetchMascotListMock.mockResolvedValueOnce([summary]); + useMascotManifestMock.mockReturnValue(manifestResult([yellow])); renderPanel(store); - fireEvent.click(await screen.findByTestId('backend-mascot-yellow')); + fireEvent.click(screen.getByTestId('manifest-mascot-yellow')); expect(store.getState().mascot.selectedMascotId).toBe('yellow'); expect(store.getState().mascot.customMascotGifUrl).toBeNull(); @@ -285,8 +310,7 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => { describe('MascotPanel — voice picker custom voice input (line 525)', () => { beforeEach(() => { vi.clearAllMocks(); - fetchMascotListMock.mockResolvedValue([]); - getCachedMascotDetailMock.mockResolvedValue(null); + useMascotManifestMock.mockReturnValue(manifestResult([])); mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0)); }); diff --git a/app/src/components/skills/MeetingBotsCard.tsx b/app/src/components/skills/MeetingBotsCard.tsx index 90dc90249..a0f759c98 100644 --- a/app/src/components/skills/MeetingBotsCard.tsx +++ b/app/src/components/skills/MeetingBotsCard.tsx @@ -2,6 +2,7 @@ import debug from 'debug'; import { type RefObject, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { type MascotFace, RiveMascot } from '../../features/human/Mascot'; +import type { MascotColor } from '../../features/human/Mascot/mascotPalette'; import { useComposioIntegrations } from '../../lib/composio/hooks'; import type { ComposioConnection } from '../../lib/composio/types'; import { useT } from '../../lib/i18n/I18nContext'; @@ -40,6 +41,16 @@ import { RecentCallsSection } from './RecentCallsSection'; type Toast = { type: 'success' | 'error' | 'info'; title: string; message?: string }; const log = debug('meeting-bots'); +const MEETING_BOT_MASCOT_IDS = new Set(['yellow', 'blue', 'burgundy', 'black', 'navy']); + +function resolveMeetingBotMascotId( + selectedMascotId: string | null, + mascotColor: MascotColor +): string | undefined { + if (selectedMascotId && MEETING_BOT_MASCOT_IDS.has(selectedMascotId)) return selectedMascotId; + if (mascotColor !== 'custom') return mascotColor; + return undefined; +} // Composio only hands back a connected account's email — there is no separate // display-name field on `ComposioConnection`. A meeting display name is almost @@ -309,7 +320,7 @@ function MeetingBotsInline({ onToast, hasSubmittedRef }: MeetingBotsInlineProps) const selectedLabel = t('skills.meetingBots.platforms.gmeet'); const agentName = personaDisplayName.trim() || 'Tiny'; const systemPrompt = personaDescription.trim() || undefined; - const mascotId = selectedMascotId ?? (mascotColor === 'custom' ? undefined : mascotColor); + const mascotId = resolveMeetingBotMascotId(selectedMascotId, mascotColor); const riveColors = mascotColor === 'custom' ? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor } diff --git a/app/src/components/skills/__tests__/MeetingBotsCard.test.tsx b/app/src/components/skills/__tests__/MeetingBotsCard.test.tsx index b63d8232e..a3fabc2a9 100644 --- a/app/src/components/skills/__tests__/MeetingBotsCard.test.tsx +++ b/app/src/components/skills/__tests__/MeetingBotsCard.test.tsx @@ -121,6 +121,37 @@ describe('MeetingBotsCard', () => { }); }); + it('falls back to the legacy mascot color for manifest-only mascot ids', async () => { + joinMock.mockResolvedValueOnce({ + meetUrl: 'https://meet.google.com/abc-defg-hij', + platform: 'gmeet', + }); + + renderWithProviders(, { + preloadedState: { + mascot: { + color: 'yellow', + voiceId: null, + voiceGender: 'male', + voiceUseLocaleDefault: false, + selectedMascotId: 'river-guide', + customMascotGifUrl: null, + customPrimaryColor: '#123456', + customSecondaryColor: '#abcdef', + }, + }, + }); + + fireEvent.change(screen.getByLabelText(/meeting link/i), { + target: { value: 'https://meet.google.com/abc-defg-hij' }, + }); + fireEvent.submit(document.querySelector('form')!); + + await vi.waitFor(() => { + expect(joinMock).toHaveBeenCalledWith(expect.objectContaining({ mascotId: 'yellow' })); + }); + }); + it('surfaces a join error inline + as an error toast', async () => { joinMock.mockRejectedValueOnce(new Error('Bad URL')); const onToast = vi.fn(); diff --git a/app/src/features/human/HumanPage.test.tsx b/app/src/features/human/HumanPage.test.tsx index a3764f05b..519987f4c 100644 --- a/app/src/features/human/HumanPage.test.tsx +++ b/app/src/features/human/HumanPage.test.tsx @@ -39,6 +39,12 @@ vi.mock('./Mascot', async importOriginal => { vi.mock('./useHumanMascot', () => ({ useHumanMascot: () => ({ face: 'idle', visemes: [] }) })); +// Keep the manifest fetch out of the unit test (no real network to GitHub). +// entry:null exercises the bundled-default fallback branch (RiveMascot stub). +vi.mock('./Mascot/manifest/useMascotManifest', () => ({ + useMascotManifest: () => ({ manifest: null, entry: null, loading: false, error: null }), +})); + const SPEAK_REPLIES_KEY = 'human.speakReplies'; function buildMinimalStore() { diff --git a/app/src/features/human/HumanPage.tsx b/app/src/features/human/HumanPage.tsx index 858d742a7..cbce7dd86 100644 --- a/app/src/features/human/HumanPage.tsx +++ b/app/src/features/human/HumanPage.tsx @@ -8,15 +8,15 @@ import { selectCustomPrimaryColor, selectCustomSecondaryColor, selectMascotColor, - selectSelectedMascotId, } from '../../store/mascotSlice'; import { - BackendRiveMascot, CustomGifMascot, getMascotPalette, hexToArgbInt, + ManifestRiveMascot, RiveMascot, } from './Mascot'; +import { useMascotManifest } from './Mascot/manifest/useMascotManifest'; import { useHumanMascot } from './useHumanMascot'; const SPEAK_REPLIES_KEY = 'human.speakReplies'; @@ -37,7 +37,8 @@ const HumanPage = () => { const customPrimary = useAppSelector(selectCustomPrimaryColor); const customSecondary = useAppSelector(selectCustomSecondaryColor); const customMascotGifUrl = useAppSelector(selectCustomMascotGifUrl); - const selectedMascotId = useAppSelector(selectSelectedMascotId); + // Active mascot resolved from the GitHub manifest (selection + default). + const { entry: mascotEntry } = useMascotManifest(); const palette = getMascotPalette(mascotColor); const primaryColor = useMemo( () => hexToArgbInt(mascotColor === 'custom' ? customPrimary : palette.bodyFill), @@ -62,10 +63,10 @@ const HumanPage = () => {
{customMascotGifUrl ? ( - ) : selectedMascotId ? ( - ({ + useRiveParams: null as Record | null, + enumCalls: {} as Record, + colorCalls: {} as Record, +})); + +vi.mock('@rive-app/react-webgl2', () => ({ + Fit: { Contain: 'contain' }, + Layout: class { + constructor(opts: unknown) { + Object.assign(this, opts as object); + } + }, + useRive: (params: Record) => { + h.useRiveParams = params; + return { rive: {}, RiveComponent: () => null }; + }, + useViewModel: () => ({}), + useViewModelInstance: () => ({}), + useViewModelInstanceEnum: (path: string) => ({ + setValue: (v: string) => (h.enumCalls[path] ??= []).push(v), + value: null, + values: [], + }), + useViewModelInstanceColor: (path: string) => ({ + setValue: (v: number) => (h.colorCalls[path] ??= []).push(v), + }), +})); + +const loadManifestRiv = vi.hoisted(() => vi.fn()); +vi.mock('./manifest/manifestService', () => ({ loadManifestRiv })); + +const TOSHI: MascotManifestEntry = { + id: 'toshi', + name: 'Toshi', + description: '', + status: 'ready', + tags: [], + stateEngine: { + idlePoseCycle: ['idle', 'look_around', 'pointing'], + states: { idle: 'idle', thinking: 'look_around' }, + visemeCodes: ['sil', 'PP', 'aa'], + channels: [ + { + key: 'eyes', + label: 'Eyes', + values: ['blink', 'look_left', 'look_right'], + cycle: { enabled: true, intervalMs: 2600, order: 'sequential' }, + }, + ], + }, + files: [ + { path: 'm/toshi.riv', bytes: 1, role: 'runtime', sha256: 'cccc', url: 'https://x/toshi.riv' }, + ], +}; + +const RIVER: MascotManifestEntry = { + ...TOSHI, + id: 'river-guide', + name: 'River Guide', + stateEngine: { ...TOSHI.stateEngine, states: { idle: 'idle', thinking: 'thinking' } }, + files: [ + { path: 'm/river.riv', bytes: 1, role: 'runtime', sha256: 'dddd', url: 'https://x/river.riv' }, + ], +}; + +function enumLast(path: string): string | undefined { + return (h.enumCalls[path] ?? []).at(-1) as string | undefined; +} + +beforeEach(() => { + h.useRiveParams = null; + h.enumCalls = {}; + h.colorCalls = {}; + loadManifestRiv.mockReset(); +}); + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe('ManifestRiveMascot', () => { + it('renders the bundled default while the buffer loads', () => { + loadManifestRiv.mockReturnValue(new Promise(() => {})); // never resolves + render(); + // Default RiveMascot loads from the bundled src, not a buffer. + expect(h.useRiveParams?.src).toBe('/tiny_mascot.riv'); + }); + + it('renders from the loaded buffer and drives the mascot vocabulary', async () => { + loadManifestRiv.mockResolvedValue(new Uint8Array([1, 2, 3]).buffer); + render(); + + await waitFor(() => expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer)); + // 'thinking' face → Toshi's look_around (it has no 'thinking' pose). + expect((h.enumCalls['pose'] ?? []).at(-1)).toBe('look_around'); + expect(enumLast('mouthVisemeCode')).toBe('aa'); + }); + + it('clears the previous buffer when the entry changes without a remount', async () => { + let resolveRiver!: (buffer: ArrayBuffer) => void; + loadManifestRiv + .mockResolvedValueOnce(new Uint8Array([1, 2, 3]).buffer) + .mockReturnValueOnce(new Promise(res => (resolveRiver = res))); + + const { rerender } = render(); + await waitFor(() => expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer)); + + rerender(); + await waitFor(() => expect(h.useRiveParams?.src).toBe('/tiny_mascot.riv')); + + await act(async () => { + resolveRiver(new Uint8Array([4, 5, 6]).buffer); + }); + await waitFor(() => expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer)); + }); + + it('rests the mouth for a viseme outside the mascot vocabulary', async () => { + loadManifestRiv.mockResolvedValue(new Uint8Array([1]).buffer); + render(); + await waitFor(() => expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer)); + // Toshi has no 'oh' viseme → rest on sil. + await waitFor(() => expect(enumLast('mouthVisemeCode')).toBe('sil')); + }); + + it('auto-cycles a cyclable channel when idle rotation is on', async () => { + vi.useFakeTimers(); + loadManifestRiv.mockResolvedValue(new Uint8Array([1]).buffer); + render(); + + // Let the async buffer load resolve. + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer); + + const before = (h.enumCalls['eyes'] ?? []).length; + act(() => { + vi.advanceTimersByTime(2_600); // one cycle interval + }); + const eyesWrites = h.enumCalls['eyes'] ?? []; + expect(eyesWrites.length).toBeGreaterThan(before); + // sequential order starts at values[0] then advances to values[1]. + expect(eyesWrites.at(-1)).toBe('look_left'); + }); +}); diff --git a/app/src/features/human/Mascot/ManifestRiveMascot.tsx b/app/src/features/human/Mascot/ManifestRiveMascot.tsx new file mode 100644 index 000000000..f3396fb08 --- /dev/null +++ b/app/src/features/human/Mascot/ManifestRiveMascot.tsx @@ -0,0 +1,299 @@ +import { + Fit, + Layout, + useRive, + useViewModel, + useViewModelInstance, + useViewModelInstanceColor, + useViewModelInstanceEnum, +} from '@rive-app/react-webgl2'; +import debug from 'debug'; +import { type FC, useEffect, useRef, useState } from 'react'; + +import type { MascotFace } from './Ghosty'; +import { loadManifestRiv } from './manifest/manifestService'; +import { pickIdleFlourish, resolveFaceToPose, resolveVisemeCode } from './manifest/stateEngine'; +import type { + MascotManifestChannel, + MascotManifestEntry, + MascotStateEngine, +} from './manifest/types'; +import { MASCOT_STATE_MACHINE } from './riveMaps'; +import { RiveMascot } from './RiveMascot'; + +const log = debug('human:mascot:manifest-rive'); + +/** Idle dwell before the mascot drifts into a flourish (ms), randomised. */ +const AMBIENT_IDLE_MIN_MS = 6_000; +const AMBIENT_IDLE_MAX_MS = 12_000; +/** How long a flourish is held before returning to the resting pose (ms). */ +const AMBIENT_HOLD_MIN_MS = 2_500; +const AMBIENT_HOLD_MAX_MS = 5_000; +/** Fallback channel auto-cycle interval when the manifest omits one (ms). */ +const CHANNEL_CYCLE_FALLBACK_MS = 2_500; + +function randBetween(min: number, max: number): number { + return min + Math.random() * (max - min); +} + +/** + * Map a value to the asset's actual enum option, case-insensitively. Rive + * enums are case-sensitive and silently ignore an unrecognised value — so if + * our resolved code (`oh`) doesn't exactly match the asset's option (`OH`), + * the mouth/pose freezes. The runtime exposes the real option list; we match + * against it and return the exact-cased option. Falls back to the input when + * the options aren't loaded yet or there's no match. + */ +function matchEnumValue(value: string, options: readonly string[] | undefined): string { + if (!options || options.length === 0) return value; + return options.find(o => o.toLowerCase() === value.toLowerCase()) ?? value; +} + +const RIVE_LAYOUT = new Layout({ fit: Fit.Contain }); + +export interface ManifestRiveMascotProps { + /** The manifest entry to render. Its runtime `.riv` is loaded + cached. */ + entry: MascotManifestEntry; + face?: MascotFace; + size?: number | string; + primaryColor?: number; + secondaryColor?: number; + /** Raw Oculus 15-set viseme code; normalised to this mascot's vocabulary. */ + visemeCode?: string; + /** Drift through this mascot's idle pose cycle + auto-cycle its channels. */ + idlePoseRotation?: boolean; +} + +/** + * Render a manifest mascot from its loaded `.riv` buffer. Split out from the + * loader so every Rive hook runs against a present buffer (calling the Rive + * hooks with no source then swapping in a buffer mid-mount destabilises the + * runtime). The parent keys this by mascot id so a new selection remounts it. + */ +const ManifestRiveStage: FC<{ + buffer: ArrayBuffer; + engine: MascotStateEngine; + channels: MascotManifestChannel[]; + face: MascotFace; + size: number | string; + primaryColor?: number; + secondaryColor?: number; + visemeCode: string; + idlePoseRotation: boolean; +}> = ({ + buffer, + engine, + channels, + face, + size, + primaryColor, + secondaryColor, + visemeCode, + idlePoseRotation, +}) => { + const { rive, RiveComponent } = useRive({ + buffer, + stateMachines: MASCOT_STATE_MACHINE, + autoplay: true, + layout: RIVE_LAYOUT, + }); + + const viewModel = useViewModel(rive, { useDefault: true }); + const vmInstance = useViewModelInstance(viewModel, { useDefault: true, rive }); + const { setValue: setPose, values: poseEnumValues } = useViewModelInstanceEnum( + 'pose', + vmInstance + ); + const { setValue: setMouthVisemeCode, values: visemeEnumValues } = useViewModelInstanceEnum( + 'mouthVisemeCode', + vmInstance + ); + const { setValue: setPrimaryColor } = useViewModelInstanceColor('primaryColor', vmInstance); + const { setValue: setSecondaryColor } = useViewModelInstanceColor('secondaryColor', vmInstance); + + // `useViewModelInstanceEnum(...).values` returns a NEW array reference on + // every render. Depending on that identity in an effect causes the effect to + // re-run each render → setValue updates the hook's internal state → re-render + // → loop ("Maximum update depth exceeded"). So we keep the arrays in refs + // (read inside effects) and gate re-runs on a *content* key that only changes + // when the asset's option list actually changes (i.e. once, when it loads). + const poseValuesRef = useRef(poseEnumValues); + poseValuesRef.current = poseEnumValues; + const visemeValuesRef = useRef(visemeEnumValues); + visemeValuesRef.current = visemeEnumValues; + const poseEnumKey = (poseEnumValues ?? []).join(''); + const visemeEnumKey = (visemeEnumValues ?? []).join(''); + + // One-time visibility into what the asset actually accepts vs. what we send. + const loggedEnumsRef = useRef(false); + useEffect(() => { + if (loggedEnumsRef.current) return; + if (visemeEnumKey.length > 0 || poseEnumKey.length > 0) { + loggedEnumsRef.current = true; + log('asset enums — pose=%o viseme=%o', poseValuesRef.current, visemeValuesRef.current); + } + }, [poseEnumKey, visemeEnumKey]); + + const basePose = resolveFaceToPose(face, engine); + const restPose = engine.states.idle; + + // Driven (face-derived) pose. A real activity pose always wins; the resting + // pose is what the idle scheduler below is free to override. + useEffect(() => { + setPose(matchEnumValue(basePose, poseValuesRef.current)); + }, [basePose, setPose, poseEnumKey]); + + // Idle pose rotation, scoped to this mascot's idlePoseCycle. Same self- + // rescheduling shape as RiveMascot; only runs while enabled AND resting. + const setPoseRef = useRef(setPose); + setPoseRef.current = setPose; + useEffect(() => { + if (!idlePoseRotation || basePose !== restPose) return; + let timer: number | undefined; + let current = restPose; + const toRest = () => { + current = restPose; + setPoseRef.current(matchEnumValue(restPose, poseValuesRef.current)); + timer = window.setTimeout(toFlourish, randBetween(AMBIENT_IDLE_MIN_MS, AMBIENT_IDLE_MAX_MS)); + }; + const toFlourish = () => { + current = pickIdleFlourish(engine, current === restPose ? undefined : current); + log('idle flourish → %s', current); + setPoseRef.current(matchEnumValue(current, poseValuesRef.current)); + timer = window.setTimeout(toRest, randBetween(AMBIENT_HOLD_MIN_MS, AMBIENT_HOLD_MAX_MS)); + }; + timer = window.setTimeout(toFlourish, randBetween(AMBIENT_IDLE_MIN_MS, AMBIENT_IDLE_MAX_MS)); + return () => { + if (timer !== undefined) window.clearTimeout(timer); + setPoseRef.current(matchEnumValue(restPose, poseValuesRef.current)); + }; + }, [idlePoseRotation, basePose, restPose, engine]); + + useEffect(() => { + const code = resolveVisemeCode(visemeCode, engine); + setMouthVisemeCode(matchEnumValue(code, visemeValuesRef.current)); + }, [visemeCode, engine, setMouthVisemeCode, visemeEnumKey]); + + useEffect(() => { + if (primaryColor !== undefined) setPrimaryColor(primaryColor); + }, [primaryColor, setPrimaryColor]); + + useEffect(() => { + if (secondaryColor !== undefined) setSecondaryColor(secondaryColor); + }, [secondaryColor, setSecondaryColor]); + + return ( +
+ + {channels.map(channel => ( + + ))} +
+ ); +}; + +/** + * Drives one optional enum channel (e.g. `eyes`) onto the view model. Each + * channel is its own component so the rules-of-hooks count stays stable, and + * auto-cycles its value on a timer when the manifest marks it cyclable and the + * mascot is in its "feel alive" mode. + */ +const ChannelDriver: FC<{ + channel: MascotManifestChannel; + vmInstance: ReturnType; + autoCycle: boolean; +}> = ({ channel, vmInstance, autoCycle }) => { + const { setValue } = useViewModelInstanceEnum(channel.key, vmInstance); + const [value, setVal] = useState(channel.default ?? channel.values[0]); + + useEffect(() => { + if (value != null) setValue(value); + }, [value, setValue]); + + useEffect(() => { + if (!autoCycle || !channel.cycle?.enabled || channel.values.length < 2) return; + const interval = channel.cycle.intervalMs ?? CHANNEL_CYCLE_FALLBACK_MS; + const sequential = channel.cycle.order === 'sequential'; + let index = 0; + const timer = window.setInterval(() => { + if (sequential) { + index = (index + 1) % channel.values.length; + setVal(channel.values[index]); + } else { + setVal(channel.values[Math.floor(Math.random() * channel.values.length)]); + } + }, interval); + return () => window.clearInterval(timer); + }, [autoCycle, channel]); + + return null; +}; + +/** + * Load a manifest mascot's `.riv` and render it. While the buffer resolves — + * or if it fails — the bundled default mascot keeps the stage alive and still + * lip-syncs, so a slow GitHub fetch never blanks the Human page. + */ +export const ManifestRiveMascot: FC = ({ + entry, + face = 'idle', + size = '100%', + primaryColor, + secondaryColor, + visemeCode = 'sil', + idlePoseRotation = false, +}) => { + const [loadState, setLoadState] = useState<{ + entryId: string; + buffer: ArrayBuffer | null; + failed: boolean; + }>({ entryId: entry.id, buffer: null, failed: false }); + + useEffect(() => { + let cancelled = false; + (async () => { + try { + const buf = await loadManifestRiv(entry); + if (!cancelled) setLoadState({ entryId: entry.id, buffer: buf, failed: false }); + } catch (err) { + if (!cancelled) { + log('failed to load mascot %s: %o', entry.id, err); + setLoadState({ entryId: entry.id, buffer: null, failed: true }); + } + } + })(); + return () => { + cancelled = true; + }; + }, [entry]); + + const fallbackProps = { face, size, primaryColor, secondaryColor, visemeCode, idlePoseRotation }; + const currentBuffer = loadState.entryId === entry.id ? loadState.buffer : null; + const currentFailed = loadState.entryId === entry.id && loadState.failed; + if (currentFailed || !currentBuffer) return ; + + return ( + + ); +}; diff --git a/app/src/features/human/Mascot/index.ts b/app/src/features/human/Mascot/index.ts index 11dba456f..0985f7247 100644 --- a/app/src/features/human/Mascot/index.ts +++ b/app/src/features/human/Mascot/index.ts @@ -6,6 +6,20 @@ export { MascotChipAvatar } from './MascotChipAvatar'; export type { MascotChipAvatarProps } from './MascotChipAvatar'; export { RiveMascot, DEFAULT_MASCOT_SRC } from './RiveMascot'; export type { RiveMascotProps } from './RiveMascot'; +export { ManifestRiveMascot } from './ManifestRiveMascot'; +export type { ManifestRiveMascotProps } from './ManifestRiveMascot'; +export { + fetchMascotManifest, + defaultMascot, + findMascot, + loadManifestRiv, +} from './manifest/manifestService'; +export type { + MascotManifest, + MascotManifestEntry, + MascotStateEngine, + MascotManifestChannel, +} from './manifest/types'; export { BackendRiveMascot } from './backend/BackendRiveMascot'; export type { BackendRiveMascotProps } from './backend/BackendRiveMascot'; export { lerpViseme, VISEMES, visemePath } from './visemes'; diff --git a/app/src/features/human/Mascot/manifest/manifestService.test.ts b/app/src/features/human/Mascot/manifest/manifestService.test.ts new file mode 100644 index 000000000..147ebfecd --- /dev/null +++ b/app/src/features/human/Mascot/manifest/manifestService.test.ts @@ -0,0 +1,226 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { clearRivMemoryCache } from '../rivCache'; +import { + clearManifestCache, + defaultMascot, + fetchMascotManifest, + findMascot, + loadManifestRiv, + parseManifest, +} from './manifestService'; +import type { MascotManifest, MascotManifestEntry } from './types'; + +const TINY: MascotManifestEntry = { + id: 'tiny-mascot', + name: 'Tiny Mascot', + description: 'Default OpenHuman mascot.', + status: 'ready', + tags: ['default', 'openhuman'], + stateEngine: { + idlePoseCycle: ['idle', 'bookreading', 'dancing'], + states: { idle: 'idle', thinking: 'thinking' }, + visemeCodes: ['sil', 'PP', 'aa', 'oh', 'ou'], + }, + files: [ + { + bytes: 189378, + path: 'mascots/tiny-mascot/tinyMascot.riv', + role: 'runtime', + sha256: 'aaaa', + url: 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/mascots/tiny-mascot/tinyMascot.riv', + }, + { + bytes: 622966, + path: 'mascots/tiny-mascot/tinyMascot.rev', + role: 'source', + sha256: 'bbbb', + url: 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/mascots/tiny-mascot/tinyMascot.rev', + }, + ], +}; + +const TOSHI: MascotManifestEntry = { + id: 'toshi', + name: 'Toshi', + description: 'Uploaded Toshi mascot asset.', + status: 'draft', + tags: ['draft', 'openhuman'], + stateEngine: { + idlePoseCycle: ['idle', 'look_around', 'dancing'], + states: { idle: 'idle', thinking: 'look_around' }, + visemeCodes: ['sil', 'PP', 'aa'], + channels: [ + { + key: 'eyes', + label: 'Eyes', + values: ['blink', 'look_left', 'look_right'], + cycle: { enabled: true, intervalMs: 2600, order: 'random' }, + }, + ], + }, + files: [ + { + bytes: 2259491, + path: 'mascots/toshi/toshi.riv', + role: 'runtime', + sha256: 'cccc', + url: 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/mascots/toshi/toshi.riv', + }, + ], +}; + +function manifestDoc(mascots: unknown[] = [TINY, TOSHI]): unknown { + return { + schemaVersion: 1, + generatedAt: '2026-06-29T21:04:06.960Z', + mascots, + source: { repository: 'tinyhumansai/mascots', branch: 'main', commit: 'abc' }, + }; +} + +function mockFetchJson(doc: unknown) { + const fn = vi + .fn() + .mockResolvedValue({ ok: true, json: () => Promise.resolve(doc) } as unknown as Response); + vi.stubGlobal('fetch', fn); + return fn; +} + +beforeEach(() => { + clearManifestCache(); + clearRivMemoryCache(); + window.localStorage.clear(); +}); + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +describe('parseManifest', () => { + it('accepts a well-formed document', () => { + const m = parseManifest(manifestDoc()); + expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot', 'toshi']); + }); + + it('drops entries missing a runtime .riv', () => { + const noRuntime = { ...TINY, id: 'broken', files: [TINY.files[1]] }; + const m = parseManifest(manifestDoc([TINY, noRuntime])); + expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot']); + }); + + it('drops entries with an incomplete stateEngine', () => { + const noStates = { + ...TOSHI, + id: 'bad', + stateEngine: { ...TOSHI.stateEngine, states: { idle: 'idle' } }, + }; + const m = parseManifest(manifestDoc([TINY, noStates])); + expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot']); + }); + + it('drops entries with malformed stateEngine arrays or channels', () => { + const badVisemes = { + ...TOSHI, + id: 'bad-visemes', + stateEngine: { ...TOSHI.stateEngine, visemeCodes: ['sil', 123] }, + }; + const badChannels = { + ...TOSHI, + id: 'bad-channels', + stateEngine: { ...TOSHI.stateEngine, channels: [{ key: 'eyes', values: [] }] }, + }; + const m = parseManifest(manifestDoc([TINY, badVisemes, badChannels])); + expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot']); + }); + + it('throws when there are no renderable mascots', () => { + expect(() => parseManifest(manifestDoc([]))).toThrow(); + expect(() => parseManifest({ nope: true })).toThrow(); + }); +}); + +describe('fetchMascotManifest', () => { + it('fetches once and memoises for the session', async () => { + const fn = mockFetchJson(manifestDoc()); + const a = await fetchMascotManifest(); + const b = await fetchMascotManifest(); + expect(b).toBe(a); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('falls back to the localStorage snapshot when the network fails', async () => { + mockFetchJson(manifestDoc()); + await fetchMascotManifest(); // writes snapshot + clearManifestCache(); + + vi.stubGlobal( + 'fetch', + vi.fn().mockRejectedValue(new Error('offline')) as unknown as typeof fetch + ); + const m = await fetchMascotManifest(); + expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot', 'toshi']); + }); + + it('aborts a stalled fetch so the localStorage snapshot can be used', async () => { + vi.useFakeTimers(); + mockFetchJson(manifestDoc()); + await fetchMascotManifest(); // writes snapshot + clearManifestCache(); + + vi.stubGlobal( + 'fetch', + vi.fn((_url: string, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(new Error('aborted'))); + }); + }) as unknown as typeof fetch + ); + + const pending = fetchMascotManifest(); + await vi.advanceTimersByTimeAsync(5_000); + const m = await pending; + expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot', 'toshi']); + }); + + it('rejects when the network fails and there is no snapshot', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockRejectedValue(new Error('offline')) as unknown as typeof fetch + ); + await expect(fetchMascotManifest()).rejects.toThrow('offline'); + }); +}); + +describe('selectors', () => { + const manifest: MascotManifest = parseManifest(manifestDoc()); + + it('findMascot resolves by id', () => { + expect(findMascot(manifest, 'toshi')?.name).toBe('Toshi'); + expect(findMascot(manifest, 'missing')).toBeUndefined(); + expect(findMascot(manifest, null)).toBeUndefined(); + }); + + it('defaultMascot prefers the first ready entry', () => { + expect(defaultMascot(manifest)?.id).toBe('tiny-mascot'); + }); +}); + +describe('loadManifestRiv', () => { + it('downloads the runtime file keyed by its sha256', async () => { + const buf = new Uint8Array([0x52, 0x49, 0x56, 0x45]).buffer; + const fn = vi + .fn() + .mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(buf), + } as unknown as Response); + vi.stubGlobal('fetch', fn); + + const out = await loadManifestRiv(TINY); + expect(new Uint8Array(out)[0]).toBe(0x52); + expect(fn).toHaveBeenCalledWith(TINY.files[0].url); + }); +}); diff --git a/app/src/features/human/Mascot/manifest/manifestService.ts b/app/src/features/human/Mascot/manifest/manifestService.ts new file mode 100644 index 000000000..e9342308b --- /dev/null +++ b/app/src/features/human/Mascot/manifest/manifestService.ts @@ -0,0 +1,216 @@ +/** + * Client for the GitHub-published mascot manifest (`dist/mascots.json`). + * + * The manifest and its `.riv` runtime files are fetched directly from + * `raw.githubusercontent.com` (CORS-open, allowed by the webview CSP). Three + * cache layers keep the picker and Human stage snappy and offline-tolerant: + * + * 1. an in-memory promise (one network fetch per session, even under + * concurrent callers), + * 2. a `localStorage` snapshot (survives reloads / brief offline), and + * 3. the IndexedDB `.riv` binary cache (`rivCache`, keyed by sha256) so + * switching back to a previously-seen mascot never re-downloads the asset. + */ +import debug from 'debug'; + +import { MASCOT_MANIFEST_URL } from '../../../../utils/config'; +import { loadRivBuffer } from '../rivCache'; +import { + type MascotManifest, + type MascotManifestChannel, + type MascotManifestEntry, + type MascotManifestFile, + runtimeFile, +} from './types'; + +const log = debug('human:mascot:manifest'); + +/** localStorage key for the last successfully-fetched manifest snapshot. */ +const SNAPSHOT_KEY = 'openhuman.mascotManifest.v1'; +const MANIFEST_FETCH_TIMEOUT_MS = 5_000; + +/** Session-lifetime in-flight/resolved fetch. Shared across all callers. */ +let inflight: Promise | null = null; + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0; +} + +function isManifestFile(value: unknown): value is MascotManifestFile { + const f = value as Partial | null; + return ( + !!f && + isNonEmptyString(f.path) && + isNonEmptyString(f.url) && + isNonEmptyString(f.sha256) && + (f.role === 'runtime' || f.role === 'source') + ); +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.length > 0 && value.every(isNonEmptyString); +} + +function isManifestChannel(value: unknown): value is MascotManifestChannel { + const channel = value as Partial | null; + if (!channel || !isNonEmptyString(channel.key) || !isStringArray(channel.values)) return false; + if (channel.enum !== undefined && !isNonEmptyString(channel.enum)) return false; + if (channel.label !== undefined && !isNonEmptyString(channel.label)) return false; + if (channel.default !== undefined && !isNonEmptyString(channel.default)) return false; + if (channel.cycle !== undefined) { + const cycle = channel.cycle; + if (typeof cycle.enabled !== 'boolean') return false; + if ( + cycle.intervalMs !== undefined && + (!Number.isFinite(cycle.intervalMs) || cycle.intervalMs <= 0) + ) { + return false; + } + if (cycle.order !== undefined && cycle.order !== 'random' && cycle.order !== 'sequential') { + return false; + } + } + return true; +} + +function isManifestEntry(value: unknown): value is MascotManifestEntry { + const m = value as Partial | null; + if (!m || !isNonEmptyString(m.id) || !isNonEmptyString(m.name)) return false; + if (m.status !== 'ready' && m.status !== 'draft') return false; + const se = m.stateEngine; + if ( + !se || + !isStringArray(se.visemeCodes) || + !isStringArray(se.idlePoseCycle) || + !se.states || + !isNonEmptyString(se.states.idle) || + !isNonEmptyString(se.states.thinking) + ) { + return false; + } + if (!Object.values(se.states).every(isNonEmptyString)) return false; + if ( + se.channels !== undefined && + (!Array.isArray(se.channels) || !se.channels.every(isManifestChannel)) + ) { + return false; + } + if (!Array.isArray(m.files) || !m.files.every(isManifestFile)) return false; + // A renderable mascot needs a runtime `.riv`; drop entries that only ship a + // source `.rev` so callers never select an unplayable asset. + return !!runtimeFile(m as MascotManifestEntry); +} + +/** + * Validate and normalise a parsed manifest document. Throws on a + * fundamentally malformed payload; silently drops individual malformed + * mascot entries (a single bad entry should never blank the whole library). + */ +export function parseManifest(raw: unknown): MascotManifest { + const doc = raw as Partial | null; + if (!doc || !Array.isArray(doc.mascots)) { + throw new Error('mascot manifest: missing mascots array'); + } + const mascots = doc.mascots.filter(isManifestEntry); + if (mascots.length === 0) { + throw new Error('mascot manifest: no renderable mascots'); + } + return { + schemaVersion: typeof doc.schemaVersion === 'number' ? doc.schemaVersion : 1, + generatedAt: isNonEmptyString(doc.generatedAt) ? doc.generatedAt : '', + mascots, + source: { + repository: doc.source?.repository ?? '', + branch: doc.source?.branch ?? '', + commit: doc.source?.commit ?? '', + }, + }; +} + +function readSnapshot(): MascotManifest | null { + try { + const raw = window.localStorage.getItem(SNAPSHOT_KEY); + if (!raw) return null; + return parseManifest(JSON.parse(raw)); + } catch (err) { + log('snapshot read failed: %o', err); + return null; + } +} + +function writeSnapshot(manifest: MascotManifest): void { + try { + window.localStorage.setItem(SNAPSHOT_KEY, JSON.stringify(manifest)); + } catch (err) { + log('snapshot write failed: %o', err); + } +} + +/** + * Fetch the mascot manifest, memoised for the session. On a network failure we + * fall back to the last localStorage snapshot so the picker still works + * offline; if there is no snapshot either, the rejection propagates so the UI + * can show an error state. + */ +export function fetchMascotManifest(): Promise { + if (inflight) return inflight; + inflight = (async () => { + const controller = new AbortController(); + const timeoutId = window.setTimeout(() => controller.abort(), MANIFEST_FETCH_TIMEOUT_MS); + try { + log('fetching manifest %s', MASCOT_MANIFEST_URL); + const res = await fetch(MASCOT_MANIFEST_URL, { + cache: 'no-cache', + signal: controller.signal, + }); + if (!res.ok) throw new Error(`manifest fetch failed (${res.status})`); + const manifest = parseManifest(await res.json()); + log('manifest ok — %d mascots (schema v%d)', manifest.mascots.length, manifest.schemaVersion); + writeSnapshot(manifest); + return manifest; + } catch (err) { + const snapshot = readSnapshot(); + if (snapshot) { + log('manifest fetch failed, using snapshot: %o', err); + return snapshot; + } + // Reset so a later retry can attempt the network again rather than + // re-resolving this rejected promise forever. + inflight = null; + throw err; + } finally { + window.clearTimeout(timeoutId); + } + })(); + return inflight; +} + +/** Drop the memoised fetch so the next call re-hits the network. */ +export function clearManifestCache(): void { + inflight = null; +} + +/** Find a mascot entry by id within a manifest. */ +export function findMascot( + manifest: MascotManifest, + id: string | null | undefined +): MascotManifestEntry | undefined { + if (!id) return undefined; + return manifest.mascots.find(m => m.id === id); +} + +/** The mascot the app defaults to: the first `ready` entry, else the first. */ +export function defaultMascot(manifest: MascotManifest): MascotManifestEntry | undefined { + return manifest.mascots.find(m => m.status === 'ready') ?? manifest.mascots[0]; +} + +/** + * Resolve a mascot's `.riv` binary, version-cached in IndexedDB by its + * runtime file's sha256. The network is only hit when the sha changes (i.e. + * the asset was republished), so re-selecting a mascot is instant. + */ +export async function loadManifestRiv(entry: MascotManifestEntry): Promise { + const runtime = runtimeFile(entry); + if (!runtime) throw new Error(`mascot ${entry.id} has no runtime .riv file`); + return loadRivBuffer(entry.id, runtime.sha256, runtime.url); +} diff --git a/app/src/features/human/Mascot/manifest/stateEngine.test.ts b/app/src/features/human/Mascot/manifest/stateEngine.test.ts new file mode 100644 index 000000000..72088fdfb --- /dev/null +++ b/app/src/features/human/Mascot/manifest/stateEngine.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from 'vitest'; + +import { + availablePoses, + initialChannelValues, + pickIdleFlourish, + resolveFaceToPose, + resolveVisemeCode, + restVisemeCode, +} from './stateEngine'; +import type { MascotStateEngine } from './types'; + +const TINY: MascotStateEngine = { + idlePoseCycle: ['idle', 'bookreading', 'coffeedrink', 'writing', 'dancing'], + states: { idle: 'idle', thinking: 'thinking' }, + visemeCodes: ['sil', 'PP', 'FF', 'aa', 'E', 'ih', 'oh', 'ou'], +}; + +// A sparse mascot: distinct logical state names, no shared flourish vocabulary. +const TOSHI: MascotStateEngine = { + idlePoseCycle: ['idle', 'look_around', 'pointing'], + states: { idle: 'idle', thinking: 'look_around' }, + visemeCodes: ['sil', 'PP', 'aa'], + channels: [ + { + key: 'eyes', + label: 'Eyes', + values: ['blink', 'look_left', 'look_right'], + default: 'look_left', + cycle: { enabled: true, intervalMs: 2600, order: 'random' }, + }, + ], +}; + +describe('availablePoses', () => { + it('unions idlePoseCycle and logical state values', () => { + expect(availablePoses(TOSHI)).toEqual(new Set(['idle', 'look_around', 'pointing'])); + }); +}); + +describe('resolveFaceToPose', () => { + it('uses the desired pose when the mascot has it', () => { + expect(resolveFaceToPose('writing', TINY)).toBe('writing'); + expect(resolveFaceToPose('reading', TINY)).toBe('bookreading'); + }); + + it('uses manifest logical state mappings for custom activity poses', () => { + const custom: MascotStateEngine = { + ...TOSHI, + states: { ...TOSHI.states, writing: 'scribbling' }, + idlePoseCycle: [...TOSHI.idlePoseCycle, 'scribbling'], + }; + + expect(resolveFaceToPose('writing', custom)).toBe('scribbling'); + }); + + it('falls back to the thinking state for thinking-ish faces', () => { + // Toshi has no 'thinking' pose; thinking-ish faces map to its 'look_around'. + expect(resolveFaceToPose('thinking', TOSHI)).toBe('look_around'); + expect(resolveFaceToPose('confused', TOSHI)).toBe('look_around'); + }); + + it('falls back to the idle state when the flourish is absent', () => { + // Toshi has no 'writing' pose → rest on idle. + expect(resolveFaceToPose('writing', TOSHI)).toBe('idle'); + }); +}); + +describe('resolveVisemeCode', () => { + it('normalises and keeps codes in the mascot vocabulary', () => { + expect(resolveVisemeCode('O', TINY)).toBe('oh'); + expect(resolveVisemeCode('E', TINY)).toBe('E'); + }); + + it("returns the manifest's exact casing for case-varied viseme enums", () => { + const upper: MascotStateEngine = { ...TINY, visemeCodes: ['SIL', 'PP', 'AA', 'OH'] }; + expect(resolveVisemeCode('O', upper)).toBe('OH'); + expect(resolveVisemeCode('a', upper)).toBe('AA'); + expect(restVisemeCode(upper)).toBe('SIL'); + }); + + it('preserves raw close-vowel aliases when the manifest uses them', () => { + const raw: MascotStateEngine = { ...TINY, visemeCodes: ['sil', 'I', 'O', 'U'] }; + expect(resolveVisemeCode('I', raw)).toBe('I'); + expect(resolveVisemeCode('O', raw)).toBe('O'); + expect(resolveVisemeCode('U', raw)).toBe('U'); + }); + + it('falls back to rest for codes the mascot lacks', () => { + // Toshi has no 'oh'; an O viseme rests the mouth instead of no-op. + expect(resolveVisemeCode('O', TOSHI)).toBe('sil'); + expect(resolveVisemeCode('???', TINY)).toBe('sil'); + }); + + it('restVisemeCode prefers sil', () => { + expect(restVisemeCode(TINY)).toBe('sil'); + expect(restVisemeCode({ ...TINY, visemeCodes: ['rest', 'aa'] })).toBe('rest'); + }); +}); + +describe('pickIdleFlourish', () => { + it('never returns the resting idle pose', () => { + for (let r = 0; r < 1; r += 0.1) { + expect(pickIdleFlourish(TINY, undefined, () => r)).not.toBe('idle'); + } + }); + + it('avoids the excluded (just-played) pose', () => { + // rng=0 would pick the first non-idle pose ('bookreading'); excluding it + // shifts the pool so the first pick is the next one. + expect(pickIdleFlourish(TINY, 'bookreading', () => 0)).toBe('coffeedrink'); + }); + + it('returns the rest pose when there are no flourishes', () => { + const flat: MascotStateEngine = { ...TINY, idlePoseCycle: ['idle'] }; + expect(pickIdleFlourish(flat)).toBe('idle'); + }); +}); + +describe('initialChannelValues', () => { + it('uses default then first value', () => { + expect(initialChannelValues(TOSHI)).toEqual({ eyes: 'look_left' }); + expect( + initialChannelValues({ ...TOSHI, channels: [{ key: 'eyes', values: ['blink', 'open'] }] }) + ).toEqual({ eyes: 'blink' }); + }); + + it('is empty when there are no channels', () => { + expect(initialChannelValues(TINY)).toEqual({}); + }); +}); diff --git a/app/src/features/human/Mascot/manifest/stateEngine.ts b/app/src/features/human/Mascot/manifest/stateEngine.ts new file mode 100644 index 000000000..a84301f61 --- /dev/null +++ b/app/src/features/human/Mascot/manifest/stateEngine.ts @@ -0,0 +1,84 @@ +/** + * Resolve mascot face/viseme state against a *specific* mascot's `stateEngine`. + * + * The bundled-asset helpers in `../riveMaps` assume the `tiny_mascot.riv` + * vocabulary. Manifest mascots each ship their own poses, viseme codes, and + * channels, so these helpers constrain every value to what the selected asset + * actually accepts — setting an out-of-vocabulary enum on a Rive state machine + * is a silent no-op, which would freeze the mouth or pose. + */ +import type { MascotFace } from '../Ghosty'; +import { faceToPose, toRiveVisemeCode } from '../riveMaps'; +import type { MascotManifestEntry, MascotStateEngine } from './types'; + +/** Every pose value this mascot's state machine accepts (cycle + logical states). */ +export function availablePoses(engine: MascotStateEngine): Set { + return new Set([...engine.idlePoseCycle, ...Object.values(engine.states)]); +} + +/** + * Map a {@link MascotFace} to a pose this mascot can actually play. Manifest + * logical mappings win first, then the shared face→pose vocabulary, then a + * guaranteed logical state (`thinking` for thinking-ish faces, otherwise + * `idle`) when the mascot's asset doesn't carry that specific flourish. + */ +export function resolveFaceToPose(face: MascotFace, engine: MascotStateEngine): string { + const logical = engine.states[face]; + if (logical && availablePoses(engine).has(logical)) return logical; + + const desired = faceToPose(face); + if (availablePoses(engine).has(desired)) return desired; + return desired === 'thinking' ? engine.states.thinking : engine.states.idle; +} + +/** The resting mouth code for this mascot (`sil` when present, else its first code). */ +export function restVisemeCode(engine: MascotStateEngine): string { + return engine.visemeCodes.find(code => code.toLowerCase() === 'sil') ?? engine.visemeCodes[0]; +} + +/** + * Normalise an incoming Oculus/ElevenLabs viseme code to one this mascot's + * mouth enum accepts. Unknown or out-of-vocabulary codes resolve to the + * resting (closed) mouth so the mouth never sticks on a no-op value. + */ +export function resolveVisemeCode(code: string, engine: MascotStateEngine): string { + const normalised = toRiveVisemeCode(code); + const candidates = [code, normalised]; + return ( + engine.visemeCodes.find(candidate => + candidates.some(alias => candidate.toLowerCase() === alias.toLowerCase()) + ) ?? restVisemeCode(engine) + ); +} + +/** + * Pick a random idle flourish pose for this mascot, excluding the resting + * `idle` pose (and optionally a just-played one) so the same flourish never + * fires twice in a row. `rng` is injectable for deterministic tests. + */ +export function pickIdleFlourish( + engine: MascotStateEngine, + exclude?: string, + rng: () => number = Math.random +): string { + const rest = engine.states.idle; + const pool = engine.idlePoseCycle.filter(p => p !== rest && p !== exclude); + const choices = pool.length > 0 ? pool : engine.idlePoseCycle.filter(p => p !== rest); + if (choices.length === 0) return rest; + const idx = Math.min(choices.length - 1, Math.floor(rng() * choices.length)); + return choices[idx]; +} + +/** Initial values for every channel: its `default`, falling back to `values[0]`. */ +export function initialChannelValues(engine: MascotStateEngine): Record { + const out: Record = {}; + for (const channel of engine.channels ?? []) { + out[channel.key] = channel.default ?? channel.values[0]; + } + return out; +} + +/** Convenience: pull a mascot entry's state engine. */ +export function stateEngineOf(entry: MascotManifestEntry): MascotStateEngine { + return entry.stateEngine; +} diff --git a/app/src/features/human/Mascot/manifest/types.ts b/app/src/features/human/Mascot/manifest/types.ts new file mode 100644 index 000000000..3746ba9cb --- /dev/null +++ b/app/src/features/human/Mascot/manifest/types.ts @@ -0,0 +1,88 @@ +/** + * Types for the GitHub-published mascot manifest + * (`tinyhumansai/mascots` → `dist/mascots.json`). + * + * This is the authoritative source for the in-app mascot library: each entry + * names a Rive runtime file plus a `stateEngine` describing the poses, logical + * states, viseme vocabulary, and optional secondary animation channels (e.g. + * darting eyes) the asset's `MascotSM` state machine accepts. + * + * Schema mirror: `tinyhumansai/mascots:schemas/mascots.schema.json` (v1). Kept + * as hand-written TS rather than generated because the app fetches the manifest + * directly over HTTPS — there is no shared package between the two repos. + */ + +/** One downloadable asset belonging to a mascot. */ +export interface MascotManifestFile { + /** Repo-relative path, e.g. `mascots/tiny-mascot/tinyMascot.riv`. */ + path: string; + bytes: number; + /** `runtime` = the playable `.riv`; `source` = the `.rev` editor file. */ + role: 'runtime' | 'source'; + /** Lowercase hex sha256 of the file. Doubles as the cache-bust version key. */ + sha256: string; + /** Absolute `raw.githubusercontent.com` URL the app fetches. */ + url: string; +} + +/** Optional secondary animation channel (its own enum input on the view model). */ +export interface MascotManifestChannel { + /** View-model enum input name to drive (e.g. `eyes`). */ + key: string; + /** Enum name inside the `.riv` (informational). */ + enum?: string; + /** Human label for the controls UI. */ + label?: string; + /** Every value the enum accepts. */ + values: string[]; + /** Default value; falls back to `values[0]` when absent. */ + default?: string; + /** Auto-cycle config — drives idle "aliveness" (e.g. random eye darts). */ + cycle?: { enabled: boolean; intervalMs?: number; order?: 'random' | 'sequential' }; +} + +/** The animation contract a mascot's `.riv` is authored against. */ +export interface MascotStateEngine { + /** + * Ordered viseme codes the asset's mouth enum accepts. Standard + * Oculus/ElevenLabs 15-set: `sil, PP, FF, TH, DD, kk, CH, SS, nn, RR, aa, E, + * ih, oh, ou`. The first entry (`sil`) is the resting/closed mouth. + */ + visemeCodes: string[]; + /** Logical state → pose-enum value. Always carries at least `idle`/`thinking`. */ + states: { idle: string; thinking: string; [key: string]: string }; + /** Poses the mascot drifts through while idle (includes the resting pose). */ + idlePoseCycle: string[]; + /** Optional extra enum channels (eyes, etc.). */ + channels?: MascotManifestChannel[]; +} + +/** A single mascot in the manifest. */ +export interface MascotManifestEntry { + id: string; + name: string; + description: string; + /** `ready` = production; `draft` = not yet matched to the runtime contract. */ + status: 'ready' | 'draft'; + tags: string[]; + stateEngine: MascotStateEngine; + files: MascotManifestFile[]; +} + +/** Top-level manifest document. */ +export interface MascotManifest { + schemaVersion: number; + generatedAt: string; + mascots: MascotManifestEntry[]; + source: { repository: string; branch: string; commit: string }; +} + +/** The runtime (`.riv`) file for a mascot, or `undefined` if it has none. */ +export function runtimeFile(entry: MascotManifestEntry): MascotManifestFile | undefined { + return entry.files.find(f => f.role === 'runtime'); +} + +/** The source (`.rev`) file for a mascot, or `undefined` if it has none. */ +export function sourceFile(entry: MascotManifestEntry): MascotManifestFile | undefined { + return entry.files.find(f => f.role === 'source'); +} diff --git a/app/src/features/human/Mascot/manifest/useMascotManifest.test.tsx b/app/src/features/human/Mascot/manifest/useMascotManifest.test.tsx new file mode 100644 index 000000000..fb47481f6 --- /dev/null +++ b/app/src/features/human/Mascot/manifest/useMascotManifest.test.tsx @@ -0,0 +1,114 @@ +import { configureStore } from '@reduxjs/toolkit'; +import { render, screen, waitFor } from '@testing-library/react'; +import { Provider } from 'react-redux'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import mascotReducer, { + setCustomMascotGifUrl, + setSelectedMascotId, +} from '../../../../store/mascotSlice'; +import type { MascotManifest, MascotManifestEntry } from './types'; +import { useMascotManifest } from './useMascotManifest'; + +const fetchMascotManifest = vi.hoisted(() => vi.fn()); +// Fully mock the service (matching BackendRiveMascot.test) so no real module +// instance lingers. findMascot/defaultMascot keep their real semantics. +vi.mock('./manifestService', () => ({ + fetchMascotManifest, + findMascot: (m: MascotManifest, id: string | null | undefined) => + id ? m.mascots.find(x => x.id === id) : undefined, + defaultMascot: (m: MascotManifest) => m.mascots.find(x => x.status === 'ready') ?? m.mascots[0], +})); + +function entry(id: string, status: 'ready' | 'draft'): MascotManifestEntry { + return { + id, + name: id, + description: '', + status, + tags: [], + stateEngine: { + idlePoseCycle: ['idle'], + states: { idle: 'idle', thinking: 'thinking' }, + visemeCodes: ['sil'], + }, + files: [ + { path: `${id}.riv`, bytes: 1, role: 'runtime', sha256: id, url: `https://x/${id}.riv` }, + ], + }; +} + +const MANIFEST: MascotManifest = { + schemaVersion: 1, + generatedAt: '', + mascots: [entry('toshi', 'draft'), entry('tiny-mascot', 'ready')], + source: { repository: '', branch: '', commit: '' }, +}; + +// A render-based probe — surfaces the hook's output into the DOM. We render the +// hook through a real component (not renderHook) so a rejecting manifest fetch +// is consumed exactly like it is in production, with no orphaned promise. +function Probe() { + const { entry: e, loading, error } = useMascotManifest(); + return ( +
+ {e?.id ?? 'none'} + {String(loading)} + {error?.message ?? ''} +
+ ); +} + +function renderProbe(selectedId: string | null, customGifUrl: string | null = null) { + const store = configureStore({ reducer: { mascot: mascotReducer } }); + if (selectedId) store.dispatch(setSelectedMascotId(selectedId)); + if (customGifUrl) store.dispatch(setCustomMascotGifUrl(customGifUrl)); + const view = render( + + + + ); + return { store, ...view }; +} + +beforeEach(() => fetchMascotManifest.mockReset()); +afterEach(() => vi.restoreAllMocks()); + +describe('useMascotManifest', () => { + it('resolves the selected mascot when set', async () => { + fetchMascotManifest.mockResolvedValue(MANIFEST); + renderProbe('toshi'); + await waitFor(() => expect(screen.getByTestId('entry')).toHaveTextContent('toshi')); + expect(screen.getByTestId('loading')).toHaveTextContent('false'); + }); + + it('falls back to the default (first ready) mascot when none selected', async () => { + fetchMascotManifest.mockResolvedValue(MANIFEST); + const { store } = renderProbe(null); + await waitFor(() => expect(screen.getByTestId('entry')).toHaveTextContent('tiny-mascot')); + expect(store.getState().mascot.selectedMascotId).toBeNull(); + }); + + it('reconciles a stale selected mascot id after the manifest loads', async () => { + fetchMascotManifest.mockResolvedValue(MANIFEST); + const { store } = renderProbe('removed-mascot'); + await waitFor(() => expect(screen.getByTestId('entry')).toHaveTextContent('tiny-mascot')); + await waitFor(() => expect(store.getState().mascot.selectedMascotId).toBe('tiny-mascot')); + }); + + it('does not overwrite a custom GIF selection with the manifest fallback', async () => { + fetchMascotManifest.mockResolvedValue(MANIFEST); + const { store } = renderProbe(null, 'https://example.com/custom.gif'); + + await waitFor(() => expect(screen.getByTestId('entry')).toHaveTextContent('tiny-mascot')); + + expect(store.getState().mascot.customMascotGifUrl).toBe('https://example.com/custom.gif'); + expect(store.getState().mascot.selectedMascotId).toBeNull(); + }); + + // The fetch-failure path is covered end-to-end in manifestService.test.ts + // ("rejects when the network fails and there is no snapshot") and the + // entry:null fallback render is covered in HumanPage.test.tsx, so the hook's + // catch branch is exercised without re-triggering a vitest-v4 quirk where + // awaiting into a settling (but handled) rejection surfaces as a test error. +}); diff --git a/app/src/features/human/Mascot/manifest/useMascotManifest.ts b/app/src/features/human/Mascot/manifest/useMascotManifest.ts new file mode 100644 index 000000000..946974396 --- /dev/null +++ b/app/src/features/human/Mascot/manifest/useMascotManifest.ts @@ -0,0 +1,64 @@ +/** + * Load the published mascot manifest and resolve the active mascot entry from + * the user's `selectedMascotId` Redux preference, with a render-only fallback + * to the default (`ready`) mascot while none is selected. Shared by the Human + * stage and the settings picker so both agree on which mascot is current. + */ +import { useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; + +import { + selectCustomMascotGifUrl, + selectSelectedMascotId, + setSelectedMascotId, +} from '../../../../store/mascotSlice'; +import { defaultMascot, fetchMascotManifest, findMascot } from './manifestService'; +import type { MascotManifest, MascotManifestEntry } from './types'; + +export interface UseMascotManifestResult { + manifest: MascotManifest | null; + /** The selected mascot, or the default when none is chosen / found yet. */ + entry: MascotManifestEntry | null; + loading: boolean; + error: Error | null; +} + +export function useMascotManifest(): UseMascotManifestResult { + const dispatch = useDispatch(); + const selectedMascotId = useSelector(selectSelectedMascotId); + const customMascotGifUrl = useSelector(selectCustomMascotGifUrl); + const [manifest, setManifest] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + (async () => { + try { + const m = await fetchMascotManifest(); + if (cancelled) return; + setManifest(m); + setError(null); + setLoading(false); + } catch (err) { + if (cancelled) return; + setError(err instanceof Error ? err : new Error(String(err))); + setLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, []); + + const selectedEntry = manifest ? findMascot(manifest, selectedMascotId) : undefined; + const fallbackEntry = manifest ? (defaultMascot(manifest) ?? null) : null; + const entry = selectedEntry ?? fallbackEntry; + + useEffect(() => { + if (!manifest || !selectedMascotId || customMascotGifUrl || selectedEntry) return; + dispatch(setSelectedMascotId(fallbackEntry?.id ?? null)); + }, [customMascotGifUrl, dispatch, fallbackEntry?.id, manifest, selectedEntry, selectedMascotId]); + + return { manifest, entry, loading, error }; +} diff --git a/app/src/features/human/useHumanMascot.lipsync.test.ts b/app/src/features/human/useHumanMascot.lipsync.test.ts index 00c4cd6ae..1f588a533 100644 --- a/app/src/features/human/useHumanMascot.lipsync.test.ts +++ b/app/src/features/human/useHumanMascot.lipsync.test.ts @@ -74,8 +74,10 @@ function makePlayback(durationMs: number): FakePlayback { let ms = 0; let stopped = false; let resolveEnded!: () => void; - const ended = new Promise(res => { + let rejectEnded!: (err: unknown) => void; + const ended = new Promise((res, rej) => { resolveEnded = res; + rejectEnded = rej; }); return { handle: { @@ -83,7 +85,9 @@ function makePlayback(durationMs: number): FakePlayback { durationMs: () => durationMs, metadataReady: Promise.resolve(), stop: () => { + if (stopped) return; stopped = true; + rejectEnded({ stopped: true }); }, ended, }, @@ -97,6 +101,48 @@ function makePlayback(durationMs: number): FakePlayback { }; } +function makePlaybackWithDeferredMetadata( + finalDurationMs: number +): FakePlayback & { resolveMetadata(): void } { + let durationMs = 0; + let ms = 0; + let stopped = false; + let resolveEnded!: () => void; + let rejectEnded!: (err: unknown) => void; + let resolveMetadata!: () => void; + const ended = new Promise((res, rej) => { + resolveEnded = res; + rejectEnded = rej; + }); + const metadataReady = new Promise(res => { + resolveMetadata = () => { + durationMs = finalDurationMs; + res(); + }; + }); + return { + handle: { + currentMs: () => (stopped ? -1 : ms), + durationMs: () => durationMs, + metadataReady, + stop: () => { + if (stopped) return; + stopped = true; + rejectEnded({ stopped: true }); + }, + ended, + }, + setMs(next: number) { + ms = next; + }, + finish() { + stopped = true; + resolveEnded(); + }, + resolveMetadata, + }; +} + /** * Drive the hook's RAF-based render loop deterministically. The hook calls * `requestAnimationFrame` on every speaking frame; without firing it the @@ -105,12 +151,15 @@ function makePlayback(durationMs: number): FakePlayback { let rafQueue: FrameRequestCallback[] = []; const originalRaf = window.requestAnimationFrame; const originalCancel = window.cancelAnimationFrame; +let nowMs = 1_000; beforeEach(() => { capturedListeners = null; rafQueue = []; + nowMs = 1_000; (synthesizeSpeech as ReturnType).mockReset(); (playBase64Audio as ReturnType).mockReset(); + vi.spyOn(window.performance, 'now').mockImplementation(() => nowMs); window.requestAnimationFrame = ((cb: FrameRequestCallback) => { rafQueue.push(cb); return rafQueue.length; @@ -121,12 +170,13 @@ beforeEach(() => { afterEach(() => { window.requestAnimationFrame = originalRaf; window.cancelAnimationFrame = originalCancel; + vi.restoreAllMocks(); }); function tickRaf() { const queue = rafQueue; rafQueue = []; - for (const cb of queue) cb(performance.now()); + for (const cb of queue) cb(nowMs); } function fakeDone(text: string) { @@ -142,7 +192,9 @@ function fakeDone(text: string) { describe('useHumanMascot — audio-driven lipsync end-to-end', () => { it('mouth opens (non-REST) once playback starts and visemes have known codes', async () => { - const fake = makePlayback(1000); + // Audio duration matches the viseme track span (400ms) so the duration- + // alignment rescale is a no-op and this test isolates code→shape mapping. + const fake = makePlayback(400); (synthesizeSpeech as ReturnType).mockResolvedValueOnce({ audio_base64: 'AAA=', audio_mime: 'audio/mpeg', @@ -183,7 +235,8 @@ describe('useHumanMascot — audio-driven lipsync end-to-end', () => { }); it('mouth opens even when backend ships visemes in lowercase / aliased codes', async () => { - const fake = makePlayback(1000); + // Audio length matches the 600ms viseme span → rescale no-op (mapping only). + const fake = makePlayback(600); (synthesizeSpeech as ReturnType).mockResolvedValueOnce({ audio_base64: 'AAA=', audio_mime: 'audio/mpeg', @@ -296,6 +349,149 @@ describe('useHumanMascot — audio-driven lipsync end-to-end', () => { expect(sampled.size).toBeGreaterThanOrEqual(2); }); + it('stretches a short viseme track to match the longer measured audio', async () => { + // Backend ships a 400ms viseme track, but the rendered MP3 is 800ms. Without + // alignment the mouth would finish at the halfway point and run ahead of the + // voice; rescaling doubles every frame so the track fills the audio. + const fake = makePlayback(800); + (synthesizeSpeech as ReturnType).mockResolvedValueOnce({ + audio_base64: 'AAA=', + audio_mime: 'audio/mpeg', + visemes: [ + { viseme: 'aa', start_ms: 0, end_ms: 200 }, + { viseme: 'PP', start_ms: 200, end_ms: 400 }, + ], + }); + (playBase64Audio as ReturnType).mockResolvedValueOnce(fake.handle); + + const { result } = renderHook(() => useHumanMascot({ speakReplies: true })); + await act(async () => { + capturedListeners?.onDone?.(fakeDone('hello')); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + // Rescaled track: frame[0]=aa 0–400ms, frame[1]=PP 400–800ms. + // At 300ms the *unscaled* track would already be on frame[1] (PP); after + // alignment it is still on frame[0] (aa) — proving the stretch. + act(() => { + fake.setMs(300); + tickRaf(); + }); + expect(result.current.viseme).toEqual(VISEMES.A); + + // At 600ms we are into the second (now-stretched) frame. + act(() => { + fake.setMs(600); + tickRaf(); + }); + expect(result.current.viseme).toEqual(VISEMES.M); + }); + + it('uses wall-clock elapsed time when the audio currentTime is frozen', async () => { + const fake = makePlayback(600); + (synthesizeSpeech as ReturnType).mockResolvedValueOnce({ + audio_base64: 'AAA=', + audio_mime: 'audio/mpeg', + visemes: [ + { viseme: 'aa', start_ms: 0, end_ms: 200 }, + { viseme: 'PP', start_ms: 200, end_ms: 400 }, + { viseme: 'O', start_ms: 400, end_ms: 600 }, + ], + }); + (playBase64Audio as ReturnType).mockResolvedValueOnce(fake.handle); + + const { result } = renderHook(() => useHumanMascot({ speakReplies: true })); + await act(async () => { + capturedListeners?.onDone?.(fakeDone('hello')); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + // Leave fake.currentMs() at 0 to model CEF blob audio where currentTime + // fails to advance even though playback is audible. + act(() => { + nowMs = 1_300; + tickRaf(); + }); + expect(result.current.viseme).toEqual(VISEMES.M); + }); + + it('starts lipsync before delayed audio metadata resolves', async () => { + const fake = makePlaybackWithDeferredMetadata(600); + (synthesizeSpeech as ReturnType).mockResolvedValueOnce({ + audio_base64: 'AAA=', + audio_mime: 'audio/mpeg', + visemes: [ + { viseme: 'aa', start_ms: 0, end_ms: 200 }, + { viseme: 'PP', start_ms: 200, end_ms: 400 }, + ], + }); + (playBase64Audio as ReturnType).mockResolvedValueOnce(fake.handle); + + const { result } = renderHook(() => useHumanMascot({ speakReplies: true })); + await act(async () => { + capturedListeners?.onDone?.(fakeDone('hi')); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(result.current.face).toBe('speaking'); + act(() => { + nowMs = 1_050; + tickRaf(); + }); + expect(result.current.viseme).toEqual(VISEMES.A); + + await act(async () => { + fake.resolveMetadata(); + fake.finish(); + await Promise.resolve(); + await Promise.resolve(); + }); + }); + + it('does not estimate pending metadata duration from collapsed frame ends', async () => { + const fake = makePlaybackWithDeferredMetadata(900); + (synthesizeSpeech as ReturnType).mockResolvedValueOnce({ + audio_base64: 'AAA=', + audio_mime: 'audio/mpeg', + // Backend regression shape: starts collapsed at zero, short fixed ends. + // Treating the final 80ms end as total duration compresses the whole + // utterance before metadata is ready and leaves the mouth at rest. + visemes: [ + { viseme: 'aa', start_ms: 0, end_ms: 80 }, + { viseme: 'PP', start_ms: 0, end_ms: 80 }, + { viseme: 'O', start_ms: 0, end_ms: 80 }, + ], + }); + (playBase64Audio as ReturnType).mockResolvedValueOnce(fake.handle); + + const { result } = renderHook(() => useHumanMascot({ speakReplies: true })); + await act(async () => { + capturedListeners?.onDone?.(fakeDone('metadata is still loading')); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + act(() => { + nowMs = 1_300; + tickRaf(); + }); + expect(result.current.viseme).not.toEqual(VISEMES.REST); + + await act(async () => { + fake.resolveMetadata(); + fake.finish(); + await Promise.resolve(); + await Promise.resolve(); + }); + }); + it('mouth returns to a non-speaking shape once playback ends', async () => { const fake = makePlayback(500); (synthesizeSpeech as ReturnType).mockResolvedValueOnce({ diff --git a/app/src/features/human/useHumanMascot.test.ts b/app/src/features/human/useHumanMascot.test.ts index a3e0c95c5..303b95968 100644 --- a/app/src/features/human/useHumanMascot.test.ts +++ b/app/src/features/human/useHumanMascot.test.ts @@ -48,12 +48,20 @@ const proceduralVisemesMock = vi.fn( } ); -vi.mock('./voice/ttsClient', () => ({ - synthesizeSpeech: vi.fn(), - visemesFromAlignment: (alignment: { char: string; start_ms: number; end_ms: number }[]) => - alignment.map(a => ({ viseme: 'aa', start_ms: a.start_ms, end_ms: a.end_ms })), - proceduralVisemes: (text: string, durationMs: number) => proceduralVisemesMock(text, durationMs), -})); +vi.mock('./voice/ttsClient', async importOriginal => { + // Keep the real pure helpers (normalizeVisemeTimeline, hasUsableStarts, …) so + // startTtsPlayback doesn't blow up on undefined imports; only stub the network + // call and the two frame builders the tests want to control. + const actual = await importOriginal(); + return { + ...actual, + synthesizeSpeech: vi.fn(), + visemesFromAlignment: (alignment: { char: string; start_ms: number; end_ms: number }[]) => + alignment.map(a => ({ viseme: 'aa', start_ms: a.start_ms, end_ms: a.end_ms })), + proceduralVisemes: (text: string, durationMs: number) => + proceduralVisemesMock(text, durationMs), + }; +}); class FakeAudioStoppedError extends Error { readonly stopped = true; diff --git a/app/src/features/human/useHumanMascot.ts b/app/src/features/human/useHumanMascot.ts index 9e5552cb4..3cac34520 100644 --- a/app/src/features/human/useHumanMascot.ts +++ b/app/src/features/human/useHumanMascot.ts @@ -13,6 +13,8 @@ import { swallowAudioStop, } from './voice/audioPlayer'; import { + hasUsableStarts, + normalizeVisemeTimeline, proceduralVisemes, synthesizeSpeech, type VisemeFrame, @@ -32,6 +34,14 @@ const VISEME_DECAY_MS = 180; * 5 minutes comfortably covers any real reply; exported for tests. */ export const TTS_MAX_PLAYBACK_MS = 5 * 60 * 1_000; +const TTS_ESTIMATED_MS_PER_CHAR = 55; +const TTS_MIN_ESTIMATED_PLAYBACK_MS = 600; + +function estimateTtsPlaybackMs(text: string, frames: VisemeFrame[]): number { + const frameEnd = frames.at(-1)?.end_ms ?? 0; + if (frameEnd > 0 && hasUsableStarts(frames)) return frameEnd; + return Math.max(TTS_MIN_ESTIMATED_PLAYBACK_MS, text.trim().length * TTS_ESTIMATED_MS_PER_CHAR); +} /** * Heuristic — does this timeline contain at least one frame whose code maps @@ -310,6 +320,15 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas const visemeFramesRef = useRef<{ viseme: string; start_ms: number; end_ms: number }[]>([]); const visemeCursorRef = useRef(0); const playbackSeqRef = useRef(0); + // Wall-clock anchor (performance.now() at the instant playback became + // current) used to index the viseme timeline. We deliberately do NOT key off + // `audio.currentTime`: in the embedded CEF webview it can fail to advance for + // in-memory blob audio, which freezes the mouth on a single viseme even + // though the audio plays. A monotonic clock always advances, and because the + // viseme frames are rescaled to the measured audio duration it stays in sync. + const playbackStartedAtRef = useRef(0); + // Throttle marker for the lipsync diagnostic log (last logged ms). + const lastLipsyncLogRef = useRef(0); const [, force] = useState(0); @@ -382,6 +401,16 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas mascotLog('voice-session text_delta suppressed — listening is active'); return; } + // When TTS is enabled the mouth is driven by the synthesized audio's + // viseme timeline (see startTtsPlayback), locked to the audio clock. + // The text-delta pseudo-lipsync exists only for the no-audio path — if + // we let it run while replies are spoken, the mouth flaps along with + // the streaming text *before and faster than* the voice. Stay at rest + // during streaming so the only mouth motion is in sync with the audio. + if (speakRef.current) { + mascotLog('voice-session text_delta lipsync suppressed — TTS will drive the mouth'); + return; + } if (playbackRef.current) return; clearAckTimer(); setFace('speaking'); @@ -419,6 +448,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas playbackSeqRef.current++; const orphan = playbackRef.current; playbackRef.current = null; + playbackStartedAtRef.current = 0; if (orphan) { orphan.stop(); orphan.ended.catch(swallowAudioStop); @@ -433,6 +463,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas playbackSeqRef.current++; const orphan = playbackRef.current; playbackRef.current = null; + playbackStartedAtRef.current = 0; if (orphan) { orphan.stop(); orphan.ended.catch(swallowAudioStop); @@ -454,6 +485,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas playbackSeqRef.current++; const orphan = playbackRef.current; playbackRef.current = null; + playbackStartedAtRef.current = 0; if (orphan) { orphan.stop(); orphan.ended.catch(swallowAudioStop); @@ -472,6 +504,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas ): Promise { const prev = playbackRef.current; playbackRef.current = null; + playbackStartedAtRef.current = 0; if (prev) { prev.stop(); prev.ended.catch(swallowAudioStop); @@ -499,13 +532,24 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas mascotLog('tts visemes produced no motion — dropping and falling through'); frames = []; } - if (frames.length === 0 && tts.alignment && tts.alignment.length > 0) { - frames = visemesFromAlignment(tts.alignment); + // The cloud path's viseme *timestamps* are frequently degenerate (all-zero + // starts or a constant end), which loses the gaps between words/sentences. + // The char-level alignment carries real timing — including those pauses — + // so prefer it whenever the viseme starts don't form a usable timeline. + const visemeStartsUsable = hasUsableStarts(frames); + const haveAlignment = !!tts.alignment && tts.alignment.length > 0; + if ((frames.length === 0 || !visemeStartsUsable) && haveAlignment) { + frames = visemesFromAlignment(tts.alignment!); source = 'alignment'; mascotLog('tts derived %d viseme frames from alignment', frames.length); - } else if (frames.length > 0) { - mascotLog('tts got %d viseme frames from backend', frames.length); } + mascotLog( + 'tts synth visemes=%d alignment=%d startsUsable=%s → source=%s', + tts.visemes?.length ?? 0, + tts.alignment?.length ?? 0, + visemeStartsUsable, + source + ); const ttsOptions: PlaybackOptions = { maxDurationMs: TTS_MAX_PLAYBACK_MS }; const handle = await playBase64Audio( tts.audio_base64, @@ -517,21 +561,66 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas handle.ended.catch(swallowAudioStop); return; } - if (frames.length === 0) { - const dur = handle.durationMs(); - frames = proceduralVisemes(text, dur); - source = 'procedural'; - mascotLog('tts derived %d procedural viseme frames over %dms', frames.length, dur); + // Start lipsync as soon as audio.play() succeeds. Metadata can lag by the + // 500ms decoder fallback in blob/MP3 cases; keeping playbackRef empty + // until then makes short replies play while the mascot is still thinking. + let audioMs = handle.durationMs(); + const waitingForMetadata = audioMs <= 0; + if (waitingForMetadata) { + audioMs = estimateTtsPlaybackMs(text, frames); + mascotLog( + 'tts duration pending — starting lipsync with %dms estimate', + Math.round(audioMs) + ); } + if (frames.length === 0) { + frames = proceduralVisemes(text, audioMs); + source = 'procedural'; + mascotLog('tts derived %d procedural viseme frames over %dms', frames.length, audioMs); + } + // Rebuild per-frame end times from the next frame's start. The backend can + // ship a constant `end_ms` (= whole-utterance length) for every frame, + // which freezes findActiveFrame on frame 0; this restores a walkable + // cue timeline and stretches the last frame to the measured audio length. + frames = normalizeVisemeTimeline(frames, audioMs); + mascotLog( + 'tts normalized %d viseme frames (%s) over %dms', + frames.length, + source, + Math.round(audioMs) + ); visemeFramesRef.current = frames; visemeCursorRef.current = 0; playbackRef.current = handle; + playbackStartedAtRef.current = window.performance.now(); + lastLipsyncLogRef.current = -1_000; setFace('speaking'); mascotLog( 'tts playback started (%s) — driving lipsync from %d frames', source, frames.length ); + if (waitingForMetadata) { + await handle.metadataReady; + if (!isStillCurrent()) { + handle.stop(); + handle.ended.catch(swallowAudioStop); + return; + } + const measuredAudioMs = handle.durationMs(); + if (measuredAudioMs > 0 && playbackRef.current === handle) { + const measuredFrames = + source === 'procedural' + ? proceduralVisemes(text, measuredAudioMs) + : normalizeVisemeTimeline(visemeFramesRef.current, measuredAudioMs); + visemeFramesRef.current = + source === 'procedural' + ? normalizeVisemeTimeline(measuredFrames, measuredAudioMs) + : measuredFrames; + visemeCursorRef.current = 0; + mascotLog('tts metadata duration ready — refined lipsync to %dms', measuredAudioMs); + } + } try { await handle.ended; } catch (err) { @@ -543,6 +632,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas } finally { if (isStillCurrent()) { playbackRef.current = null; + playbackStartedAtRef.current = 0; visemeFramesRef.current = []; visemeCodeRef.current = 'sil'; if (degraded) { @@ -569,7 +659,12 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas let visemeCode = 'sil'; const playback = playbackRef.current; if (playback) { - const ms = playback.currentMs(); + const audioMs = playback.currentMs(); + const wallClockMs = + playbackStartedAtRef.current > 0 + ? window.performance.now() - playbackStartedAtRef.current + : audioMs; + const ms = audioMs < 0 ? -1 : Math.max(audioMs, wallClockMs); if (ms >= 0) { const { frame, cursor } = findActiveFrame( visemeFramesRef.current, @@ -579,6 +674,16 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas visemeCursorRef.current = cursor; viseme = frame ? oculusVisemeToShape(frame.viseme) : VISEMES.REST; visemeCode = frame ? frame.viseme : 'sil'; + if (ms - lastLipsyncLogRef.current >= 500) { + lastLipsyncLogRef.current = ms; + mascotLog( + 'lipsync ms=%d cursor=%d/%d code=%s', + Math.round(ms), + cursor, + visemeFramesRef.current.length, + visemeCode + ); + } } } else if (face === 'speaking') { const since = window.performance.now() - lastDeltaAtRef.current; diff --git a/app/src/features/human/voice/ttsClient.test.ts b/app/src/features/human/voice/ttsClient.test.ts index b47547f44..e154df786 100644 --- a/app/src/features/human/voice/ttsClient.test.ts +++ b/app/src/features/human/voice/ttsClient.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it, vi } from 'vitest'; import { callCoreRpc } from '../../../services/coreRpcClient'; import { + hasUsableStarts, + normalizeVisemeTimeline, prepareForSpeech, proceduralVisemes, synthesizeSpeech, @@ -178,3 +180,71 @@ describe('proceduralVisemes', () => { expect(short[0].end_ms - short[0].start_ms).toBeGreaterThanOrEqual(60); }); }); + +describe('hasUsableStarts', () => { + it('is true for a spread-out, mostly-distinct timeline', () => { + expect( + hasUsableStarts([ + { viseme: 'aa', start_ms: 0, end_ms: 100 }, + { viseme: 'PP', start_ms: 100, end_ms: 200 }, + { viseme: 'E', start_ms: 200, end_ms: 300 }, + ]) + ).toBe(true); + }); + + it('is false when every start collapses to zero (degenerate backend timing)', () => { + expect( + hasUsableStarts([ + { viseme: 'aa', start_ms: 0, end_ms: 80 }, + { viseme: 'PP', start_ms: 0, end_ms: 80 }, + { viseme: 'E', start_ms: 0, end_ms: 80 }, + ]) + ).toBe(false); + }); + + it('is false for fewer than two frames', () => { + expect(hasUsableStarts([{ viseme: 'aa', start_ms: 10, end_ms: 90 }])).toBe(false); + expect(hasUsableStarts([])).toBe(false); + }); +}); + +describe('normalizeVisemeTimeline', () => { + it('preserves a real timeline and keeps gaps as pauses', () => { + // Real per-frame timing spanning the clip, with a gap 450→900 (a pause). + const track = [ + { viseme: 'aa', start_ms: 0, end_ms: 450 }, + { viseme: 'PP', start_ms: 900, end_ms: 1000 }, + ]; + const out = normalizeVisemeTimeline(track, 1000); + // First frame keeps its real end (450) — the gap to 900 stays a pause. + expect(out[0]).toEqual({ viseme: 'aa', start_ms: 0, end_ms: 450 }); + expect(out[1].start_ms).toBe(900); + }); + + it('clamps an overrunning end to the next cue start', () => { + const track = [ + { viseme: 'aa', start_ms: 0, end_ms: 9999 }, // overruns next + { viseme: 'PP', start_ms: 600, end_ms: 1000 }, + ]; + const out = normalizeVisemeTimeline(track, 1000); + expect(out[0].end_ms).toBe(600); + }); + + it('evenly distributes the sequence when timestamps are degenerate', () => { + // All-zero starts → unusable → spread evenly across the audio duration. + const track = [ + { viseme: 'aa', start_ms: 0, end_ms: 80 }, + { viseme: 'PP', start_ms: 0, end_ms: 80 }, + { viseme: 'E', start_ms: 0, end_ms: 80 }, + { viseme: 'oh', start_ms: 0, end_ms: 80 }, + ]; + const out = normalizeVisemeTimeline(track, 1000); + expect(out.map(f => f.viseme)).toEqual(['aa', 'PP', 'E', 'oh']); + expect(out[0]).toEqual({ viseme: 'aa', start_ms: 0, end_ms: 250 }); + expect(out[3]).toEqual({ viseme: 'oh', start_ms: 750, end_ms: 1000 }); + }); + + it('returns empty frames untouched', () => { + expect(normalizeVisemeTimeline([], 1000)).toEqual([]); + }); +}); diff --git a/app/src/features/human/voice/ttsClient.ts b/app/src/features/human/voice/ttsClient.ts index fd9819db5..4427ab26a 100644 --- a/app/src/features/human/voice/ttsClient.ts +++ b/app/src/features/human/voice/ttsClient.ts @@ -115,6 +115,69 @@ export function visemesFromAlignment(alignment: AlignmentFrame[]): VisemeFrame[] return out; } +/** + * Does this viseme track carry a genuinely usable per-frame start timeline? + * + * The cloud TTS path sometimes ships visemes with all-zero (or otherwise + * collapsed) `start_ms`, which can't represent the gaps between words. We treat + * the timing as usable only when there are at least two frames, the furthest + * start is non-zero, and most starts are distinct (a real, spread-out timeline). + */ +export function hasUsableStarts(frames: VisemeFrame[]): boolean { + if (frames.length < 2) return false; + const maxStart = Math.max(...frames.map(f => f.start_ms)); + if (!(maxStart > 0)) return false; + const distinct = new Set(frames.map(f => f.start_ms)).size; + return distinct > frames.length * 0.5; +} + +/** + * Turn a backend viseme list into a walkable timeline matched to the audio. + * + * The cloud TTS path ships viseme *codes* in order but with unreliable + * timestamps — observed shapes include a constant `end_ms` (= whole-utterance + * length) on every frame, and all-zero `start_ms`. Either way the naive + * `findActiveFrame` freezes the mouth (frame 0 "covers" everything, or every + * frame is zero-length so it snaps to the last `sil`). + * + * Strategy: + * - If the frames carry a genuinely usable timeline — strictly-ish increasing + * starts that span most of the audio — keep it, just rebuilding each end from + * the next start (visemes are cue points) and stretching the last to the end. + * - Otherwise distribute the viseme *sequence* evenly across the measured audio + * duration. We lose the natural per-phoneme rhythm but keep the real phonetic + * order, so the mouth animates start-to-finish in lockstep with the voice. + */ +export function normalizeVisemeTimeline(frames: VisemeFrame[], totalMs: number): VisemeFrame[] { + if (frames.length === 0) return frames; + const sorted = [...frames].sort((a, b) => a.start_ms - b.start_ms); + const maxStart = sorted[sorted.length - 1].start_ms; + const total = totalMs > 0 ? totalMs : Math.max(maxStart, frames.length * 80); + + // A real timeline's last cue starts deep into the clip. If the furthest start + // is in the front portion (e.g. every start is 0), the timestamps are junk — + // fall back to even distribution. + const hasUsableTimeline = maxStart > total * 0.5; + if (hasUsableTimeline) { + // Preserve the real per-frame end. A gap between one frame's end and the + // next frame's start is a genuine pause — findActiveFrame returns no frame + // there, so the mouth rests (sil) instead of holding a shape across the + // silence. We only clamp ends so a frame never overruns the next cue. + return sorted.map((f, i) => { + const next = sorted[i + 1]; + const end = next ? Math.min(f.end_ms, next.start_ms) : Math.max(f.end_ms, total); + return { viseme: f.viseme, start_ms: f.start_ms, end_ms: Math.max(f.start_ms, end) }; + }); + } + + const step = total / frames.length; + return sorted.map((f, i) => ({ + viseme: f.viseme, + start_ms: Math.round(i * step), + end_ms: Math.round((i + 1) * step), + })); +} + /** * Reshape an assistant message into something the TTS engine can read with * natural cadence. The agent's reply is markdown — raw `**bold**`, headings, diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 2640f0a3d..687eded19 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -4748,6 +4748,7 @@ const messages: TranslationMap = { 'اعرض ردود المساعد كنص بلا إطار مع إبقاء رسائلك داخل فقاعات.', 'settings.mascot.active': 'نشط', 'settings.mascot.characterDesc': 'وصف الشخصية', + 'settings.mascot.characterDraft': 'مسودة', 'settings.mascot.characterHeading': 'عنوان الشخصية', 'settings.mascot.customGifError': 'أدخل HTTPS .gif URL، أو الاسترجاع HTTP .gif URL، أو file:// .gif URL، أو مسار .gif المحلي.', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index 8308d21fc..e3de4cf01 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -4846,6 +4846,7 @@ const messages: TranslationMap = { 'আপনার বার্তাগুলি বাবলে রেখে অ্যাসিস্ট্যান্টের উত্তর ফ্রেমহীন টেক্সট হিসেবে দেখান।', 'settings.mascot.active': 'সক্রিয়', 'settings.mascot.characterDesc': 'চরিত্রের বিবরণ', + 'settings.mascot.characterDraft': 'খসড়া', 'settings.mascot.characterHeading': 'চরিত্রের শিরোনাম', 'settings.mascot.customGifError': 'একটি HTTPS .gif URL, লুপব্যাক HTTP .gif URL, file:// .gif URL, অথবা স্থানীয় .gif পাথ লিখুন।', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index 32f9944f5..da9d01143 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -4971,6 +4971,7 @@ const messages: TranslationMap = { 'Zeigt Assistentenantworten als ungerahmten Text an und lässt deine Nachrichten in Blasen.', 'settings.mascot.active': 'Aktiv', 'settings.mascot.characterDesc': 'Charakterbeschreibung', + 'settings.mascot.characterDraft': 'Entwurf', 'settings.mascot.characterHeading': 'Zeichenüberschrift', 'settings.mascot.customGifError': 'GIF konnte nicht geladen werden. Bitte überprüfe die URL und versuche es erneut.', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index a844317b8..29bb33d43 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -5483,6 +5483,7 @@ const en: TranslationMap = { 'Collapse the live step-by-step agent timeline in chat. A blinking “Processing” link still lets you open the full run.', 'settings.mascot.active': 'Active', 'settings.mascot.characterDesc': 'Choose your OpenHuman character.', + 'settings.mascot.characterDraft': 'Draft', 'settings.mascot.characterHeading': 'Character', 'settings.mascot.customGifError': 'Enter an HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL, or local .gif path.', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 8807dcbc0..a79172577 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -4938,6 +4938,7 @@ const messages: TranslationMap = { 'Muestra las respuestas del asistente como texto sin marco y mantiene tus mensajes en burbujas.', 'settings.mascot.active': 'Activo', 'settings.mascot.characterDesc': 'Descripción del personaje', + 'settings.mascot.characterDraft': 'Borrador', 'settings.mascot.characterHeading': 'Encabezado del personaje', 'settings.mascot.customGifError': 'Introduzca una ruta HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL o ruta .gif local.', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index eda813ef3..2668328d4 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -4957,6 +4957,7 @@ const messages: TranslationMap = { "Affiche les réponses de l'assistant en texte sans cadre tout en gardant vos messages en bulles.", 'settings.mascot.active': 'Actif', 'settings.mascot.characterDesc': 'Description du personnage', + 'settings.mascot.characterDraft': 'Brouillon', 'settings.mascot.characterHeading': 'Titre du personnage', 'settings.mascot.customGifError': 'Entrez un HTTPS .gif URL, un bouclage HTTP .gif URL, un fichier:// .gif URL ou un chemin local .gif.', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index 44c982213..cfcc0142f 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -4851,6 +4851,7 @@ const messages: TranslationMap = { 'असिस्टेंट के जवाबों को बिना फ्रेम वाले टेक्स्ट के रूप में दिखाएं और आपके संदेश बबल में रखें।', 'settings.mascot.active': 'एक्टिव', 'settings.mascot.characterDesc': 'कैरेक्टर विवरण', + 'settings.mascot.characterDraft': 'ड्राफ़्ट', 'settings.mascot.characterHeading': 'कैरेक्टर शीर्षक', 'settings.mascot.customGifError': 'एक HTTPS .gif URL, लूपबैक HTTP .gif URL, फ़ाइल:// .gif URL, या स्थानीय .gif पथ दर्ज करें।', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index a797f6186..752a68348 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -4863,6 +4863,7 @@ const messages: TranslationMap = { 'Tampilkan balasan asisten sebagai teks tanpa bingkai, sementara pesan Anda tetap dalam gelembung.', 'settings.mascot.active': 'Aktif', 'settings.mascot.characterDesc': 'Deskripsi karakter', + 'settings.mascot.characterDraft': 'Draf', 'settings.mascot.characterHeading': 'Judul karakter', 'settings.mascot.customGifError': 'Masukkan HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL, atau jalur .gif lokal.', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index 2a08afc32..f25805fd7 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -4928,6 +4928,7 @@ const messages: TranslationMap = { "Mostra le risposte dell'assistente come testo senza cornice mantenendo i tuoi messaggi nei fumetti.", 'settings.mascot.active': 'Attivo', 'settings.mascot.characterDesc': 'Descrizione personaggio', + 'settings.mascot.characterDraft': 'Bozza', 'settings.mascot.characterHeading': 'Intestazione personaggio', 'settings.mascot.customGifError': 'Immettere un HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL o un percorso .gif locale.', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index efcbfa98e..83c8da01c 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -4798,6 +4798,7 @@ const messages: TranslationMap = { '사용자 메시지는 말풍선으로 유지하고 어시스턴트 답변은 프레임 없는 텍스트로 표시합니다.', 'settings.mascot.active': '활성', 'settings.mascot.characterDesc': '캐릭터 설명', + 'settings.mascot.characterDraft': '초안', 'settings.mascot.characterHeading': '캐릭터 제목', 'settings.mascot.customGifError': 'HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL 또는 로컬 .gif 경로를 입력하세요.', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 07c2fb6c1..2771e409a 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -4917,6 +4917,7 @@ const messages: TranslationMap = { 'Wyświetla odpowiedzi asystenta jako tekst bez ramki, a Twoje wiadomości pozostawia w dymkach.', 'settings.mascot.active': 'Aktywny', 'settings.mascot.characterDesc': 'Wybierz charakter maskotki OpenHuman.', + 'settings.mascot.characterDraft': 'Wersja robocza', 'settings.mascot.characterHeading': 'Charakter', 'settings.mascot.customGifError': 'Wprowadź URL .gif HTTPS, URL .gif loopback HTTP, URL file:// .gif lub lokalną ścieżkę .gif.', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index 38816d45b..7dc5c8e92 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -4931,6 +4931,7 @@ const messages: TranslationMap = { 'Renderiza as respostas do assistente como texto sem moldura e mantém suas mensagens em balões.', 'settings.mascot.active': 'Ativo', 'settings.mascot.characterDesc': 'Descrição do personagem', + 'settings.mascot.characterDraft': 'Rascunho', 'settings.mascot.characterHeading': 'Título do personagem', 'settings.mascot.customGifError': 'Insira um caminho HTTPS .gif URL, loopback HTTP .gif URL, arquivo:// .gif URL ou .gif local.', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index 851d13ddf..f8beb5b8e 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -4891,6 +4891,7 @@ const messages: TranslationMap = { 'Показывает ответы ассистента как текст без рамки, оставляя ваши сообщения в пузырьках.', 'settings.mascot.active': 'Активно', 'settings.mascot.characterDesc': 'Описание персонажа', + 'settings.mascot.characterDraft': 'Черновик', 'settings.mascot.characterHeading': 'Персонаж', 'settings.mascot.customGifError': 'Введите HTTPS .gif URL, петлевой путь HTTP .gif URL, file:// .gif URL или локальный путь .gif.', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 543eb53c5..0b5f7ef1c 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -4608,6 +4608,7 @@ const messages: TranslationMap = { 'settings.appearance.assistantTextModeDesc': '将助手回复渲染为无边框文本,同时保留你的消息气泡。', 'settings.mascot.active': '活跃', 'settings.mascot.characterDesc': '选择你的 OpenHuman 角色', + 'settings.mascot.characterDraft': '草稿', 'settings.mascot.characterHeading': '角色', 'settings.mascot.customGifError': '输入 HTTPS .gif 链接、本地回环 HTTP .gif 链接、file:// .gif 链接或本地 .gif 路径。', diff --git a/app/src/store/index.ts b/app/src/store/index.ts index cda735b95..c6184febb 100644 --- a/app/src/store/index.ts +++ b/app/src/store/index.ts @@ -159,13 +159,14 @@ const persistedThreadReducer = persistReducer(threadPersistConfig, threadReducer const layoutPersistConfig = { key: 'layout', storage, whitelist: ['panels'] }; const persistedLayoutReducer = persistReducer(layoutPersistConfig, layoutReducer); -// Persist only previously persisted mascot appearance fields plus the custom -// GIF override added by this feature; leave existing non-persisted mascot -// fields as runtime state to avoid changing refresh behavior. +// Persist the mascot appearance fields, the custom GIF override, and the +// selected mascot id (so the chosen GitHub-manifest mascot survives a reload — +// the slice's REHYDRATE guard re-validates it). Other mascot fields stay as +// runtime state. const mascotPersistConfig = { key: 'mascot', storage, - whitelist: ['color', 'voiceId', 'customMascotGifUrl'], + whitelist: ['color', 'voiceId', 'customMascotGifUrl', 'selectedMascotId'], }; const persistedMascotReducer = persistReducer(mascotPersistConfig, mascotReducer); diff --git a/app/src/store/mascotSlice.ts b/app/src/store/mascotSlice.ts index 301cf8964..4aec33c6e 100644 --- a/app/src/store/mascotSlice.ts +++ b/app/src/store/mascotSlice.ts @@ -110,12 +110,12 @@ export interface MascotState { */ voiceUseLocaleDefault: boolean; /** - * Server-side mascot id selected from the backend mascot library - * (PR tinyhumansai/backend#770). `null` keeps the local YellowMascot - * renderer; any non-empty value tells `BackendMascot` (loaded via - * `mascotService`) to take over. The id is opaque server-side and - * length-capped at the same threshold as voiceId to keep the - * persisted blob bounded. + * Mascot id selected from the published GitHub manifest + * (`tinyhumansai/mascots`, resolved via `useMascotManifest`). `null` falls + * back to the manifest's default (first `ready`) mascot; any non-empty value + * pins that specific mascot. The id is the manifest entry id (e.g. + * `tiny-mascot`) and length-capped at the same threshold as voiceId to keep + * the persisted blob bounded. */ selectedMascotId: string | null; /** diff --git a/app/src/test/setup.ts b/app/src/test/setup.ts index a6bfe3cba..76009c30a 100644 --- a/app/src/test/setup.ts +++ b/app/src/test/setup.ts @@ -221,6 +221,8 @@ vi.mock('../utils/config', () => ({ DEV_JWT_TOKEN: undefined, MASCOT_VOICE_ID: 'JBFqnCBsd6RMkjVDRZzb', MASCOT_VOICE_MODEL_ID: 'eleven_multilingual_v2', + MASCOT_MANIFEST_URL: + 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/dist/mascots.json', })); vi.mock('../services/backendUrl', () => ({ diff --git a/app/src/utils/config.ts b/app/src/utils/config.ts index f1ea7e9b4..269e6e2b7 100644 --- a/app/src/utils/config.ts +++ b/app/src/utils/config.ts @@ -240,3 +240,16 @@ export const MASCOT_VOICE_ID = export const MASCOT_VOICE_MODEL_ID = (import.meta.env.VITE_MASCOT_VOICE_MODEL_ID as string | undefined)?.trim() || 'eleven_multilingual_v2'; + +/** + * URL of the published mascot manifest (`dist/mascots.json` from the + * `tinyhumansai/mascots` repo). This is the authoritative source for the + * in-app mascot library — each entry names a Rive `.riv` runtime file plus its + * `stateEngine` (poses, viseme codes, channels). Fetched directly over HTTPS + * (the `raw.githubusercontent.com` host is CORS-open and allowed by the + * webview CSP's `connect-src https:`). Override with `VITE_MASCOT_MANIFEST_URL` + * to point at a fork or a locally-served manifest during development. + */ +export const MASCOT_MANIFEST_URL = + (import.meta.env.VITE_MASCOT_MANIFEST_URL as string | undefined)?.trim() || + 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/dist/mascots.json'; diff --git a/app/test/playwright/specs/settings-feature-preferences.spec.ts b/app/test/playwright/specs/settings-feature-preferences.spec.ts index 8ecc3e76c..32cb4150f 100644 --- a/app/test/playwright/specs/settings-feature-preferences.spec.ts +++ b/app/test/playwright/specs/settings-feature-preferences.spec.ts @@ -42,7 +42,9 @@ async function getDefaultMessagingChannel(page: Page): Promise { async function getMascotVoiceId(page: Page): Promise { return page.evaluate(() => { const win = window as unknown as { - __OPENHUMAN_STORE__?: { getState?: () => { mascot: { voiceId?: string | null } } }; + __OPENHUMAN_STORE__?: { + getState?: () => { mascot: { selectedMascotId?: string | null; voiceId?: string | null } }; + }; }; const state = win.__OPENHUMAN_STORE__?.getState?.(); if (!state) { @@ -52,6 +54,19 @@ async function getMascotVoiceId(page: Page): Promise { }); } +async function getSelectedMascotId(page: Page): Promise { + return page.evaluate(() => { + const win = window as unknown as { + __OPENHUMAN_STORE__?: { getState?: () => { mascot: { selectedMascotId?: string | null } } }; + }; + const state = win.__OPENHUMAN_STORE__?.getState?.(); + if (!state) { + throw new Error('__OPENHUMAN_STORE__ is unavailable'); + } + return state.mascot.selectedMascotId ?? null; + }); +} + async function getPersistedMascotColor(page: Page): Promise { return page.evaluate(() => { const userId = localStorage.getItem('OPENHUMAN_ACTIVE_USER_ID'); @@ -71,11 +86,83 @@ async function getPersistedMascotColor(page: Page): Promise { }); } +async function getPersistedSelectedMascotId(page: Page): Promise { + return page.evaluate(() => { + const userId = localStorage.getItem('OPENHUMAN_ACTIVE_USER_ID'); + if (!userId) return null; + + const raw = localStorage.getItem(`${userId}:persist:mascot`); + if (!raw) return null; + + try { + const parsed = JSON.parse(raw) as { selectedMascotId?: unknown }; + if (typeof parsed.selectedMascotId !== 'string') return null; + const selectedMascotId = JSON.parse(parsed.selectedMascotId) as unknown; + return typeof selectedMascotId === 'string' ? selectedMascotId : null; + } catch { + return null; + } + }); +} + async function getAriaChecked(page: Page, label: string): Promise { const value = await page.getByRole('switch', { name: label }).getAttribute('aria-checked'); return value; } +async function installMascotManifestMock(page: Page): Promise { + const manifest = { + schemaVersion: 1, + generatedAt: '2026-06-29T00:00:00.000Z', + source: { repository: 'tinyhumansai/mascots', branch: 'main', commit: 'e2e' }, + mascots: [ + { + id: 'tiny-mascot', + name: 'Tiny Default', + status: 'ready', + files: [ + { + role: 'runtime', + path: 'dist/tiny.riv', + url: 'https://example.test/mascots/tiny.riv', + sha256: 'tiny-runtime-sha', + }, + ], + stateEngine: { + states: { idle: 'idle', thinking: 'thinking', speaking: 'speaking', writing: 'writing' }, + idlePoseCycle: ['idle'], + visemeCodes: ['sil', 'aa', 'oh'], + }, + }, + { + id: 'river-guide', + name: 'River Guide', + status: 'draft', + files: [ + { + role: 'runtime', + path: 'dist/river.riv', + url: 'https://example.test/mascots/river.riv', + sha256: 'river-runtime-sha', + }, + ], + stateEngine: { + states: { idle: 'idle', thinking: 'thinking', speaking: 'speaking', writing: 'writing' }, + idlePoseCycle: ['idle', 'wave'], + visemeCodes: ['sil', 'aa', 'oh'], + }, + }, + ], + }; + + await page.route('https://raw.githubusercontent.com/tinyhumansai/mascots/**', route => + route.fulfill({ contentType: 'application/json', body: JSON.stringify(manifest) }) + ); + await page.route('https://example.test/mascots/*.riv', route => + route.fulfill({ contentType: 'application/octet-stream', body: Buffer.from('not-a-rive-file') }) + ); +} + interface ToolsSnapshot { result?: { localState?: { onboardingTasks?: { enabledTools?: string[] | null } | null } | null }; localState?: { onboardingTasks?: { enabledTools?: string[] | null } | null } | null; @@ -211,6 +298,32 @@ test.describe('Settings - Feature Preferences', () => { await expect(page.getByTestId('mascot-color-burgundy')).toHaveAttribute('aria-checked', 'true'); }); + test('persists manifest mascot selection and uses it on the Human page', async ({ page }) => { + await installMascotManifestMock(page); + await openAuthenticatedRoute(page, 'pw-settings-manifest-mascot', '/settings/mascot'); + + await expect(page.getByRole('heading', { name: 'Character', exact: true })).toBeVisible(); + await expect(page.getByTestId('manifest-mascot-river-guide')).toContainText('River Guide'); + await page.getByTestId('manifest-mascot-river-guide').click(); + await expect(page.getByTestId('manifest-mascot-river-guide')).toHaveAttribute( + 'aria-pressed', + 'true' + ); + await expect.poll(() => getSelectedMascotId(page)).toBe('river-guide'); + await expect.poll(() => getPersistedSelectedMascotId(page)).toBe('river-guide'); + + await reloadAndWait(page); + await expect.poll(() => getSelectedMascotId(page)).toBe('river-guide'); + await expect(page.getByTestId('manifest-mascot-river-guide')).toHaveAttribute( + 'aria-pressed', + 'true' + ); + + await page.goto('/#/human'); + await waitForAppReady(page); + await expect.poll(() => getSelectedMascotId(page)).toBe('river-guide'); + }); + test('persists the custom mascot voice override on the voice panel', async ({ page }) => { await openAuthenticatedRoute(page, 'pw-settings-mascot-voice', '/settings/voice'); diff --git a/src/openhuman/about_app/catalog_data.rs b/src/openhuman/about_app/catalog_data.rs index 2a4b8ea27..cb8bbd787 100644 --- a/src/openhuman/about_app/catalog_data.rs +++ b/src/openhuman/about_app/catalog_data.rs @@ -76,6 +76,18 @@ const GITHUB_REPO_SOURCE: Option = Some(CapabilityPrivacy { destinations: &["GitHub API (api.github.com)"], }); +// Persona Pack fetches the published mascot manifest directly from GitHub raw +// content, then downloads the selected runtime asset from the manifest's +// declared file URL. The request is metadata-class (manifest and asset URLs), +// but it does leave the device and bypasses the managed backend. +const GITHUB_MASCOT_MANIFEST: Option = Some(CapabilityPrivacy { + leaves_device: true, + data_kind: PrivacyDataKind::Metadata, + destinations: &[ + "GitHub raw content (raw.githubusercontent.com) and manifest-declared mascot asset hosts", + ], +}); + const SEARXNG_RAW_TO_CONFIGURED_INSTANCE: Option = Some(CapabilityPrivacy { leaves_device: true, data_kind: PrivacyDataKind::Raw, @@ -1352,7 +1364,7 @@ pub(super) const CAPABILITIES: &[Capability] = &[ description: "Personalize the assistant as one identity: set a display name and description, edit or reset the SOUL.md personality prompt, and reach mascot avatar and voice settings — all from a single Persona surface.", how_to: "Settings > Persona", status: CapabilityStatus::Beta, - privacy: None, + privacy: GITHUB_MASCOT_MANIFEST, }, Capability { id: "settings.manage_privacy_analytics", diff --git a/src/openhuman/about_app/catalog_tests.rs b/src/openhuman/about_app/catalog_tests.rs index 1021ef57a..4c904e32d 100644 --- a/src/openhuman/about_app/catalog_tests.rs +++ b/src/openhuman/about_app/catalog_tests.rs @@ -102,6 +102,26 @@ fn local_only_capability_marks_no_destinations() { assert!(privacy.destinations.is_empty()); } +#[test] +fn persona_pack_reports_github_mascot_manifest_destination() { + let cap = lookup("settings.persona_pack").expect("persona pack capability exists"); + let privacy = cap.privacy.expect("persona pack is privacy-annotated"); + + assert!( + privacy.leaves_device, + "loading the mascot library fetches a GitHub-hosted manifest" + ); + assert_eq!(privacy.data_kind, PrivacyDataKind::Metadata); + let haystack = privacy.destinations.join(" | ").to_lowercase(); + assert!( + haystack.contains("github") + && haystack.contains("raw.githubusercontent.com") + && haystack.contains("asset"), + "destinations must disclose the GitHub raw manifest host and manifest asset hosts, got: {:?}", + privacy.destinations + ); +} + #[test] fn unannotated_capability_serializes_without_privacy_field() { let cap = lookup("conversation.create").expect("capability exists"); diff --git a/src/openhuman/voice/reply_speech.rs b/src/openhuman/voice/reply_speech.rs index f9980238a..9c16b7643 100644 --- a/src/openhuman/voice/reply_speech.rs +++ b/src/openhuman/voice/reply_speech.rs @@ -294,11 +294,28 @@ fn parse_cue(v: &Value) -> Option { if viseme.is_empty() { return None; } - let start = read_u64(v, &["start_ms", "time_ms", "t"]).unwrap_or(0); - let end = read_u64(v, &["end_ms"]) + // Accept millisecond keys, or seconds keys (`startSeconds`/`endSeconds`, the + // shape the cloud backend actually ships) converted to ms. Without the + // seconds keys every frame collapsed to start=0/end=80 — the mascot mouth + // then froze on the first viseme because the whole track had no real timing. + let start = read_ms( + v, + &["start_ms", "time_ms", "t"], + &["startSeconds", "start_seconds", "startSec"], + ) + .unwrap_or(0); + let end = read_ms(v, &["end_ms"], &["endSeconds", "end_seconds", "endSec"]) .or_else(|| { - let t = read_u64(v, &["time_ms", "t"])?; - let d = read_u64(v, &["duration_ms", "d"])?; + let t = read_ms( + v, + &["time_ms", "t"], + &["startSeconds", "start_seconds", "startSec"], + )?; + let d = read_ms( + v, + &["duration_ms", "d"], + &["durationSeconds", "duration_seconds", "durationSec"], + )?; Some(t + d) }) .unwrap_or(start + 80); @@ -314,8 +331,8 @@ fn parse_cue(v: &Value) -> Option { fn parse_alignment(v: &Value) -> Option { let ch = v.get("char").and_then(Value::as_str)?.to_string(); - let start = read_u64(v, &["start_ms"])?; - let end = read_u64(v, &["end_ms"])?; + let start = read_ms(v, &["start_ms"], &["startSeconds", "start_seconds"])?; + let end = read_ms(v, &["end_ms"], &["endSeconds", "end_seconds"])?; if end <= start { return None; } @@ -340,6 +357,27 @@ fn read_u64(v: &Value, keys: &[&str]) -> Option { None } +/// Read a time value in milliseconds. Tries `ms_keys` first (integer or float +/// ms), then `sec_keys` interpreted as seconds and converted to ms. This lets +/// the parser tolerate both the `*_ms` contract and the backend's +/// `*Seconds` shape without the caller caring which arrived. +fn read_ms(v: &Value, ms_keys: &[&str], sec_keys: &[&str]) -> Option { + if let Some(ms) = read_u64(v, ms_keys) { + return Some(ms); + } + for k in sec_keys { + if let Some(f) = v.get(*k).and_then(Value::as_f64) { + if f.is_finite() && f >= 0.0 { + return Some((f * 1000.0).round() as u64); + } + } + if let Some(n) = v.get(*k).and_then(Value::as_u64) { + return Some(n.saturating_mul(1000)); + } + } + None +} + #[cfg(test)] mod tests { use super::*; @@ -383,6 +421,75 @@ mod tests { ); } + #[test] + fn normalize_accepts_seconds_keys() { + // The cloud backend ships per-frame timing as seconds (`startSeconds`/ + // `endSeconds`); the parser must convert to ms rather than dropping it + // (which collapsed every frame to start=0/end=80 and froze the mouth). + let raw = json!({ + "audio_base64": "AAA=", + "visemes": [ + { "viseme": "sil", "startSeconds": 0.0, "endSeconds": 0.12 }, + { "viseme": "aa", "startSeconds": 0.12, "endSeconds": 0.45 }, + // A gap before the next cue (0.45 → 0.90) is a real pause: the + // mouth rests there. Preserved because we keep the true ends. + { "viseme": "PP", "startSeconds": 0.9, "endSeconds": 1.05 }, + ], + "alignment": [ + { "char": "h", "startSeconds": 0.0, "endSeconds": 0.05 }, + ], + }); + let r = normalize_response(&raw); + assert_eq!(r.visemes.len(), 3); + assert_eq!( + r.visemes[0], + VisemeFrame { + viseme: "sil".into(), + start_ms: 0, + end_ms: 120 + } + ); + assert_eq!( + r.visemes[1], + VisemeFrame { + viseme: "aa".into(), + start_ms: 120, + end_ms: 450 + } + ); + assert_eq!( + r.visemes[2], + VisemeFrame { + viseme: "PP".into(), + start_ms: 900, + end_ms: 1050 + } + ); + let alignment = r.alignment.expect("alignment present"); + assert_eq!(alignment.len(), 1); + assert_eq!(alignment[0].start_ms, 0); + assert_eq!(alignment[0].end_ms, 50); + } + + #[test] + fn normalize_accepts_short_seconds_duration_aliases() { + let raw = json!({ + "audio_base64": "AAA=", + "visemes": [ + { "viseme": "aa", "startSec": 1.2, "durationSec": 0.15 }, + ], + }); + let r = normalize_response(&raw); + assert_eq!( + r.visemes, + vec![VisemeFrame { + viseme: "aa".into(), + start_ms: 1200, + end_ms: 1350 + }] + ); + } + #[test] fn normalize_drops_malformed_cues() { let raw = json!({