mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
feat(mascot): load Human mascots from GitHub manifest (#4312)
This commit is contained in:
Steven Enamakel
@@ -1,13 +1,7 @@
|
||||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
|
||||
import { CustomGifMascot, RiveMascot } from '../../../features/human/Mascot';
|
||||
import { BackendMascot } from '../../../features/human/Mascot/backend/BackendMascot';
|
||||
import { BackendRiveMascot } from '../../../features/human/Mascot/backend/BackendRiveMascot';
|
||||
import {
|
||||
isRiveMascotDetail,
|
||||
type MascotDetailUnion,
|
||||
type MascotSummary,
|
||||
} from '../../../features/human/Mascot/backend/types';
|
||||
import { CustomGifMascot, ManifestRiveMascot, RiveMascot } from '../../../features/human/Mascot';
|
||||
import { useMascotManifest } from '../../../features/human/Mascot/manifest/useMascotManifest';
|
||||
import {
|
||||
getMascotPalette,
|
||||
hexToArgbInt,
|
||||
@@ -15,7 +9,6 @@ import {
|
||||
} from '../../../features/human/Mascot/mascotPalette';
|
||||
import { synthesizeSpeech } from '../../../features/human/voice/ttsClient';
|
||||
import { useT } from '../../../lib/i18n/I18nContext';
|
||||
import { fetchMascotList, getCachedMascotDetail } from '../../../services/mascotService';
|
||||
import { useAppDispatch, useAppSelector } from '../../../store/hooks';
|
||||
import {
|
||||
DEFAULT_MASCOT_COLOR,
|
||||
@@ -83,13 +76,16 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
const useLocaleDefault = useAppSelector(selectMascotVoiceUseLocaleDefault);
|
||||
const effectiveVoiceId = useAppSelector(selectEffectiveMascotVoiceId);
|
||||
|
||||
// Backend mascot library (PR tinyhumansai/backend#770). The list endpoint
|
||||
// is cheap (no SVG bytes); per-id detail is fetched on demand so the
|
||||
// animated preview only pays for the active selection.
|
||||
const [backendList, setBackendList] = useState<MascotSummary[] | null>(null);
|
||||
const [backendListError, setBackendListError] = useState<string | null>(null);
|
||||
const [activeDetail, setActiveDetail] = useState<MascotDetailUnion | null>(null);
|
||||
const [detailError, setDetailError] = useState<string | null>(null);
|
||||
// Mascot library, sourced from the published GitHub manifest
|
||||
// (tinyhumansai/mascots). `entry` is the resolved active mascot (selection
|
||||
// or default); `manifest.mascots` drives the picker list. Each entry carries
|
||||
// its full stateEngine inline, so there is no per-id detail round trip.
|
||||
const {
|
||||
manifest,
|
||||
entry: activeEntry,
|
||||
loading: manifestLoading,
|
||||
error: manifestError,
|
||||
} = useMascotManifest();
|
||||
const [customGifDraft, setCustomGifDraft] = useState<string>(customMascotGifUrl ?? '');
|
||||
const [customGifError, setCustomGifError] = useState<string | null>(null);
|
||||
|
||||
@@ -108,45 +104,6 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
// earlier audio-only cleanup missed.
|
||||
const previewRequestIdRef = useRef(0);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
fetchMascotList()
|
||||
.then(list => {
|
||||
if (cancelled) return;
|
||||
setBackendList(list);
|
||||
setBackendListError(null);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (cancelled) return;
|
||||
const message = err instanceof Error ? err.message : t('settings.mascot.loadLibraryError');
|
||||
setBackendListError(message);
|
||||
setBackendList([]);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [t]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedMascotId) return;
|
||||
let cancelled = false;
|
||||
getCachedMascotDetail(selectedMascotId)
|
||||
.then(detail => {
|
||||
if (cancelled) return;
|
||||
setActiveDetail(detail);
|
||||
setDetailError(null);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (cancelled) return;
|
||||
const message = err instanceof Error ? err.message : t('settings.mascot.loadDetailError');
|
||||
setDetailError(message);
|
||||
setActiveDetail(null);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [selectedMascotId, t]);
|
||||
|
||||
// Stop any in-flight preview audio when the panel unmounts. Also
|
||||
// bump the preview request id so a `synthesizeSpeech(...)` that
|
||||
// resolves after unmount can detect the staleness and bail.
|
||||
@@ -161,15 +118,14 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
};
|
||||
}, []);
|
||||
|
||||
const handleSelectBackend = (id: string | null) => {
|
||||
const handleSelectMascot = (id: string | null) => {
|
||||
dispatch(setSelectedMascotId(id));
|
||||
setCustomGifError(null);
|
||||
if (id == null) {
|
||||
setCustomGifDraft('');
|
||||
dispatch(setCustomMascotGifUrl(null));
|
||||
} else {
|
||||
setCustomGifDraft('');
|
||||
}
|
||||
setCustomGifDraft('');
|
||||
// Selecting a mascot id already clears the custom GIF in the reducer; the
|
||||
// null ("default") case has to clear it here so the stage falls back to
|
||||
// the default manifest mascot rather than the GIF.
|
||||
if (id == null) dispatch(setCustomMascotGifUrl(null));
|
||||
};
|
||||
|
||||
const onSaveCustomGif = () => {
|
||||
@@ -305,8 +261,6 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
const presetPickerDisabled = useLocaleDefault;
|
||||
const isCustomVoice =
|
||||
!presetPickerDisabled && (voicePasteMode || !isCuratedVoicePreset(effectiveVoiceId));
|
||||
const visibleActiveDetail = selectedMascotId ? activeDetail : null;
|
||||
const visibleDetailError = selectedMascotId ? detailError : null;
|
||||
|
||||
const activePalette = getMascotPalette(activeColor);
|
||||
const primaryColorArgb = useMemo(
|
||||
@@ -629,25 +583,25 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Backend mascot library */}
|
||||
{/* Mascot manifest library (tinyhumansai/mascots) */}
|
||||
<div className="bg-surface rounded-xl border border-line overflow-hidden">
|
||||
{backendListError && (
|
||||
{manifestError && (
|
||||
<p className="p-4 text-sm text-coral-700 dark:text-coral-300">
|
||||
{t('settings.mascot.libraryUnavailable')}: {backendListError}
|
||||
{t('settings.mascot.libraryUnavailable')}: {manifestError.message}
|
||||
</p>
|
||||
)}
|
||||
{!backendListError && backendList === null && (
|
||||
{!manifestError && manifestLoading && (
|
||||
<p className="p-4 text-sm text-content-muted">{t('settings.mascot.loadingLibrary')}</p>
|
||||
)}
|
||||
{backendList && backendList.length === 0 && !backendListError && (
|
||||
{manifest && manifest.mascots.length === 0 && !manifestError && (
|
||||
<p className="p-4 text-sm text-content-muted">{t('settings.mascot.noCharacters')}</p>
|
||||
)}
|
||||
{backendList && backendList.length > 0 && (
|
||||
{manifest && manifest.mascots.length > 0 && (
|
||||
<ul className="divide-y divide-line-subtle dark:divide-neutral-800">
|
||||
<li>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleSelectBackend(null)}
|
||||
onClick={() => handleSelectMascot(null)}
|
||||
aria-pressed={selectedMascotId == null && customMascotGifUrl == null}
|
||||
className={`flex w-full items-center justify-between px-4 py-3 text-left text-sm hover:bg-surface-hover ${
|
||||
selectedMascotId == null && customMascotGifUrl == null
|
||||
@@ -662,29 +616,35 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
)}
|
||||
</button>
|
||||
</li>
|
||||
{backendList.map(summary => {
|
||||
const active = summary.id === selectedMascotId;
|
||||
{manifest.mascots.map(mascot => {
|
||||
const active = mascot.id === selectedMascotId;
|
||||
const poseCount = new Set([
|
||||
...mascot.stateEngine.idlePoseCycle,
|
||||
...Object.values(mascot.stateEngine.states),
|
||||
]).size;
|
||||
const visemeCount = mascot.stateEngine.visemeCodes.length;
|
||||
return (
|
||||
<li key={summary.id}>
|
||||
<li key={mascot.id}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleSelectBackend(summary.id)}
|
||||
onClick={() => handleSelectMascot(mascot.id)}
|
||||
aria-pressed={active}
|
||||
data-testid={`backend-mascot-${summary.id}`}
|
||||
data-testid={`manifest-mascot-${mascot.id}`}
|
||||
className={`flex w-full items-center justify-between px-4 py-3 text-left text-sm hover:bg-surface-hover ${
|
||||
active ? 'bg-surface-muted font-medium' : ''
|
||||
}`}>
|
||||
<span className="flex flex-col">
|
||||
<span>{summary.name}</span>
|
||||
<span className="flex items-center gap-2">
|
||||
{mascot.name}
|
||||
{mascot.status === 'draft' && (
|
||||
<span className="rounded bg-amber-100 px-1.5 py-0.5 text-[9px] font-medium uppercase tracking-wide text-amber-700 dark:bg-amber-500/20 dark:text-amber-200">
|
||||
{t('settings.mascot.characterDraft')}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
<span className="text-[10px] text-content-muted">
|
||||
v{summary.version}
|
||||
{summary.format === 'rive'
|
||||
? ` · ${Object.keys(summary.stateToPose ?? {}).length} ${t('settings.mascot.characterStates')}`
|
||||
: ` · ${summary.states?.length ?? 0} ${t('settings.mascot.characterStates')}${
|
||||
summary.hasVisemes
|
||||
? ` · ${t('settings.mascot.characterVisemes')}`
|
||||
: ''
|
||||
}`}
|
||||
{poseCount} {t('settings.mascot.characterStates')} · {visemeCount}{' '}
|
||||
{t('settings.mascot.characterVisemes')}
|
||||
</span>
|
||||
</span>
|
||||
{active && (
|
||||
@@ -700,31 +660,25 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{visibleActiveDetail && (
|
||||
{activeEntry && !customMascotGifUrl && (
|
||||
<div className="mt-3 rounded-xl border border-line bg-surface-muted p-4">
|
||||
<p className="text-[11px] font-medium uppercase tracking-wide text-content-muted mb-2">
|
||||
{t('settings.mascot.characterPreview')} · {visibleActiveDetail.name}
|
||||
{t('settings.mascot.characterPreview')} · {activeEntry.name}
|
||||
</p>
|
||||
<div className="flex justify-center">
|
||||
<div style={{ width: 160, height: 160 }}>
|
||||
{isRiveMascotDetail(visibleActiveDetail) ? (
|
||||
<BackendRiveMascot
|
||||
key={visibleActiveDetail.id}
|
||||
mascotId={visibleActiveDetail.id}
|
||||
size={160}
|
||||
/>
|
||||
) : (
|
||||
<BackendMascot mascot={visibleActiveDetail} />
|
||||
)}
|
||||
<ManifestRiveMascot
|
||||
key={activeEntry.id}
|
||||
entry={activeEntry}
|
||||
size={160}
|
||||
primaryColor={primaryColorArgb}
|
||||
secondaryColor={secondaryColorArgb}
|
||||
idlePoseRotation
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{visibleDetailError && (
|
||||
<p className="mt-2 text-xs text-coral-700 dark:text-coral-300 px-1">
|
||||
{visibleDetailError}
|
||||
</p>
|
||||
)}
|
||||
<p className="text-xs text-content-muted leading-relaxed px-1 mt-2">
|
||||
{t('settings.mascot.characterDesc')}
|
||||
</p>
|
||||
|
||||
@@ -14,17 +14,14 @@ import mascotReducer, {
|
||||
} from '../../../../store/mascotSlice';
|
||||
import MascotPanel from '../MascotPanel';
|
||||
|
||||
const { mockNavigateBack, fetchMascotListMock, getCachedMascotDetailMock, mockSynthesizeSpeech } =
|
||||
vi.hoisted(() => ({
|
||||
mockNavigateBack: vi.fn(),
|
||||
fetchMascotListMock: vi.fn(),
|
||||
getCachedMascotDetailMock: vi.fn(),
|
||||
mockSynthesizeSpeech: vi.fn(),
|
||||
}));
|
||||
const { mockNavigateBack, useMascotManifestMock, mockSynthesizeSpeech } = vi.hoisted(() => ({
|
||||
mockNavigateBack: vi.fn(),
|
||||
useMascotManifestMock: vi.fn(),
|
||||
mockSynthesizeSpeech: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../../../../services/mascotService', () => ({
|
||||
fetchMascotList: (...args: unknown[]) => fetchMascotListMock(...args),
|
||||
getCachedMascotDetail: (...args: unknown[]) => getCachedMascotDetailMock(...args),
|
||||
vi.mock('../../../../features/human/Mascot/manifest/useMascotManifest', () => ({
|
||||
useMascotManifest: () => useMascotManifestMock(),
|
||||
}));
|
||||
|
||||
vi.mock('../../../../features/human/voice/ttsClient', () => ({
|
||||
@@ -36,17 +33,59 @@ vi.mock('../../../../features/human/Mascot', async importOriginal => {
|
||||
return {
|
||||
...actual,
|
||||
RiveMascot: () => <div data-testid="rive-mascot-preview" />,
|
||||
ManifestRiveMascot: ({ entry }: { entry: { id: string } }) => (
|
||||
<div data-testid={`manifest-mascot-preview-${entry.id}`} />
|
||||
),
|
||||
CustomGifMascot: ({ src }: { src: string }) => (
|
||||
<img data-testid="custom-gif-mascot" src={src} alt="" />
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../../../../features/human/Mascot/backend/BackendMascot', () => ({
|
||||
BackendMascot: ({ mascot }: { mascot: { id: string } }) => (
|
||||
<div data-testid={`backend-mascot-preview-${mascot.id}`} />
|
||||
),
|
||||
}));
|
||||
// Build a minimal manifest entry for the picker list / preview.
|
||||
function manifestEntry(id: string, name: string, status: 'ready' | 'draft' = 'ready') {
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
description: '',
|
||||
status,
|
||||
tags: [],
|
||||
stateEngine: {
|
||||
idlePoseCycle: ['idle', 'dancing'],
|
||||
states: { idle: 'idle', thinking: 'thinking' },
|
||||
visemeCodes: ['sil', 'PP', 'aa'],
|
||||
},
|
||||
files: [
|
||||
{ path: `${id}.riv`, bytes: 1, role: 'runtime', sha256: id, url: `https://x/${id}.riv` },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function manifestResult(
|
||||
mascots: ReturnType<typeof manifestEntry>[],
|
||||
overrides: Partial<{
|
||||
manifest: unknown;
|
||||
entry: unknown;
|
||||
loading: boolean;
|
||||
error: Error | null;
|
||||
}> = {}
|
||||
) {
|
||||
const manifest =
|
||||
'manifest' in overrides
|
||||
? overrides.manifest
|
||||
: {
|
||||
schemaVersion: 1,
|
||||
generatedAt: '',
|
||||
mascots,
|
||||
source: { repository: '', branch: '', commit: '' },
|
||||
};
|
||||
return {
|
||||
manifest,
|
||||
entry: 'entry' in overrides ? overrides.entry : (mascots[0] ?? null),
|
||||
loading: overrides.loading ?? false,
|
||||
error: overrides.error ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
vi.mock('../../hooks/useSettingsNavigation', () => ({
|
||||
useSettingsNavigation: () => ({
|
||||
@@ -75,8 +114,7 @@ function renderPanel(store = buildStore()) {
|
||||
describe('MascotPanel', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
fetchMascotListMock.mockResolvedValue([]);
|
||||
getCachedMascotDetailMock.mockResolvedValue(null);
|
||||
useMascotManifestMock.mockReturnValue(manifestResult([]));
|
||||
mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0));
|
||||
});
|
||||
|
||||
@@ -168,78 +206,65 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => {
|
||||
expect(screen.getByRole('radio', { name: 'Yellow' })).toHaveAttribute('aria-checked', 'false');
|
||||
});
|
||||
|
||||
describe('backend mascot library', () => {
|
||||
const summary = {
|
||||
id: 'yellow',
|
||||
name: 'Yellow',
|
||||
version: '1.0.0',
|
||||
description: '',
|
||||
states: [{ id: 'idle', label: 'Idle', description: '' }],
|
||||
hasVisemes: true,
|
||||
};
|
||||
const detail = {
|
||||
id: 'yellow',
|
||||
name: 'Yellow',
|
||||
version: '1.0.0',
|
||||
description: '',
|
||||
viewBox: '0 0 1 1',
|
||||
defaultState: 'idle',
|
||||
variables: [],
|
||||
states: [{ id: 'idle', label: 'Idle', description: '', svg: '<svg/>' }],
|
||||
visemes: [],
|
||||
};
|
||||
describe('mascot manifest library', () => {
|
||||
const yellow = manifestEntry('yellow', 'Yellow');
|
||||
const toshi = manifestEntry('toshi', 'Toshi', 'draft');
|
||||
|
||||
it('renders the picker entries returned by the API', async () => {
|
||||
fetchMascotListMock.mockResolvedValueOnce([summary]);
|
||||
it('renders the picker entries from the manifest', () => {
|
||||
useMascotManifestMock.mockReturnValue(manifestResult([yellow, toshi]));
|
||||
renderPanel();
|
||||
expect(await screen.findByTestId('backend-mascot-yellow')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('manifest-mascot-yellow')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('manifest-mascot-toshi')).toBeInTheDocument();
|
||||
// Draft status badge surfaces for non-ready mascots.
|
||||
expect(screen.getByText('Draft')).toBeInTheDocument();
|
||||
// Default-row (local) sentinel
|
||||
expect(screen.getByText(/Local OpenHuman/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows a friendly empty state when the library is empty', async () => {
|
||||
fetchMascotListMock.mockResolvedValueOnce([]);
|
||||
it('shows a friendly empty state when the library is empty', () => {
|
||||
useMascotManifestMock.mockReturnValue(
|
||||
manifestResult([], {
|
||||
manifest: {
|
||||
schemaVersion: 1,
|
||||
generatedAt: '',
|
||||
mascots: [],
|
||||
source: { repository: '', branch: '', commit: '' },
|
||||
},
|
||||
})
|
||||
);
|
||||
renderPanel();
|
||||
expect(
|
||||
await screen.findByText(/No OpenHuman characters are available yet/i)
|
||||
).toBeInTheDocument();
|
||||
expect(screen.getByText(/No OpenHuman characters are available yet/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows an error when the library endpoint rejects', async () => {
|
||||
fetchMascotListMock.mockRejectedValueOnce(new Error('offline'));
|
||||
it('shows an error when the manifest fails to load', () => {
|
||||
useMascotManifestMock.mockReturnValue(
|
||||
manifestResult([], { manifest: null, entry: null, error: new Error('offline') })
|
||||
);
|
||||
renderPanel();
|
||||
expect(
|
||||
await screen.findByText(/OpenHuman library unavailable: offline/i)
|
||||
).toBeInTheDocument();
|
||||
expect(screen.getByText(/OpenHuman library unavailable: offline/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('dispatches setSelectedMascotId when a backend mascot is picked', async () => {
|
||||
fetchMascotListMock.mockResolvedValueOnce([summary]);
|
||||
getCachedMascotDetailMock.mockResolvedValueOnce(detail);
|
||||
it('dispatches setSelectedMascotId when a mascot is picked', () => {
|
||||
useMascotManifestMock.mockReturnValue(manifestResult([yellow]));
|
||||
const { store } = renderPanel();
|
||||
const row = await screen.findByTestId('backend-mascot-yellow');
|
||||
fireEvent.click(row);
|
||||
fireEvent.click(screen.getByTestId('manifest-mascot-yellow'));
|
||||
expect(store.getState().mascot.selectedMascotId).toBe('yellow');
|
||||
});
|
||||
|
||||
it('loads + previews the active backend mascot detail', async () => {
|
||||
it('previews the active manifest mascot', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setSelectedMascotId('yellow'));
|
||||
fetchMascotListMock.mockResolvedValueOnce([summary]);
|
||||
getCachedMascotDetailMock.mockResolvedValueOnce(detail);
|
||||
useMascotManifestMock.mockReturnValue(manifestResult([yellow], { entry: yellow }));
|
||||
renderPanel(store);
|
||||
expect(await screen.findByTestId('backend-mascot-preview-yellow')).toBeInTheDocument();
|
||||
expect(getCachedMascotDetailMock).toHaveBeenCalledWith('yellow');
|
||||
expect(screen.getByTestId('manifest-mascot-preview-yellow')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('clearing the selection returns to the local default', async () => {
|
||||
it('clearing the selection returns to the local default', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setSelectedMascotId('yellow'));
|
||||
fetchMascotListMock.mockResolvedValueOnce([summary]);
|
||||
getCachedMascotDetailMock.mockResolvedValueOnce(detail);
|
||||
useMascotManifestMock.mockReturnValue(manifestResult([yellow], { entry: yellow }));
|
||||
renderPanel(store);
|
||||
const localRow = await screen.findByText(/Local OpenHuman/);
|
||||
fireEvent.click(localRow);
|
||||
fireEvent.click(screen.getByText(/Local OpenHuman/));
|
||||
expect(store.getState().mascot.selectedMascotId).toBeNull();
|
||||
});
|
||||
|
||||
@@ -268,12 +293,12 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => {
|
||||
expect(screen.getByTestId('mascot-custom-gif-error')).toHaveTextContent('HTTPS .gif');
|
||||
});
|
||||
|
||||
it('selecting a backend mascot clears the custom GIF avatar', async () => {
|
||||
it('selecting a mascot clears the custom GIF avatar', () => {
|
||||
const store = buildStore();
|
||||
store.dispatch(setCustomMascotGifUrl('https://example.com/avatar.gif'));
|
||||
fetchMascotListMock.mockResolvedValueOnce([summary]);
|
||||
useMascotManifestMock.mockReturnValue(manifestResult([yellow]));
|
||||
renderPanel(store);
|
||||
fireEvent.click(await screen.findByTestId('backend-mascot-yellow'));
|
||||
fireEvent.click(screen.getByTestId('manifest-mascot-yellow'));
|
||||
|
||||
expect(store.getState().mascot.selectedMascotId).toBe('yellow');
|
||||
expect(store.getState().mascot.customMascotGifUrl).toBeNull();
|
||||
@@ -285,8 +310,7 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => {
|
||||
describe('MascotPanel — voice picker custom voice input (line 525)', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
fetchMascotListMock.mockResolvedValue([]);
|
||||
getCachedMascotDetailMock.mockResolvedValue(null);
|
||||
useMascotManifestMock.mockReturnValue(manifestResult([]));
|
||||
mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0));
|
||||
});
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import debug from 'debug';
|
||||
import { type RefObject, useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
|
||||
import { type MascotFace, RiveMascot } from '../../features/human/Mascot';
|
||||
import type { MascotColor } from '../../features/human/Mascot/mascotPalette';
|
||||
import { useComposioIntegrations } from '../../lib/composio/hooks';
|
||||
import type { ComposioConnection } from '../../lib/composio/types';
|
||||
import { useT } from '../../lib/i18n/I18nContext';
|
||||
@@ -40,6 +41,16 @@ import { RecentCallsSection } from './RecentCallsSection';
|
||||
type Toast = { type: 'success' | 'error' | 'info'; title: string; message?: string };
|
||||
|
||||
const log = debug('meeting-bots');
|
||||
const MEETING_BOT_MASCOT_IDS = new Set(['yellow', 'blue', 'burgundy', 'black', 'navy']);
|
||||
|
||||
function resolveMeetingBotMascotId(
|
||||
selectedMascotId: string | null,
|
||||
mascotColor: MascotColor
|
||||
): string | undefined {
|
||||
if (selectedMascotId && MEETING_BOT_MASCOT_IDS.has(selectedMascotId)) return selectedMascotId;
|
||||
if (mascotColor !== 'custom') return mascotColor;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Composio only hands back a connected account's email — there is no separate
|
||||
// display-name field on `ComposioConnection`. A meeting display name is almost
|
||||
@@ -309,7 +320,7 @@ function MeetingBotsInline({ onToast, hasSubmittedRef }: MeetingBotsInlineProps)
|
||||
const selectedLabel = t('skills.meetingBots.platforms.gmeet');
|
||||
const agentName = personaDisplayName.trim() || 'Tiny';
|
||||
const systemPrompt = personaDescription.trim() || undefined;
|
||||
const mascotId = selectedMascotId ?? (mascotColor === 'custom' ? undefined : mascotColor);
|
||||
const mascotId = resolveMeetingBotMascotId(selectedMascotId, mascotColor);
|
||||
const riveColors =
|
||||
mascotColor === 'custom'
|
||||
? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor }
|
||||
|
||||
@@ -121,6 +121,37 @@ describe('MeetingBotsCard', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to the legacy mascot color for manifest-only mascot ids', async () => {
|
||||
joinMock.mockResolvedValueOnce({
|
||||
meetUrl: 'https://meet.google.com/abc-defg-hij',
|
||||
platform: 'gmeet',
|
||||
});
|
||||
|
||||
renderWithProviders(<MeetingBotsCard />, {
|
||||
preloadedState: {
|
||||
mascot: {
|
||||
color: 'yellow',
|
||||
voiceId: null,
|
||||
voiceGender: 'male',
|
||||
voiceUseLocaleDefault: false,
|
||||
selectedMascotId: 'river-guide',
|
||||
customMascotGifUrl: null,
|
||||
customPrimaryColor: '#123456',
|
||||
customSecondaryColor: '#abcdef',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
fireEvent.change(screen.getByLabelText(/meeting link/i), {
|
||||
target: { value: 'https://meet.google.com/abc-defg-hij' },
|
||||
});
|
||||
fireEvent.submit(document.querySelector('form')!);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(joinMock).toHaveBeenCalledWith(expect.objectContaining({ mascotId: 'yellow' }));
|
||||
});
|
||||
});
|
||||
|
||||
it('surfaces a join error inline + as an error toast', async () => {
|
||||
joinMock.mockRejectedValueOnce(new Error('Bad URL'));
|
||||
const onToast = vi.fn();
|
||||
|
||||
@@ -39,6 +39,12 @@ vi.mock('./Mascot', async importOriginal => {
|
||||
|
||||
vi.mock('./useHumanMascot', () => ({ useHumanMascot: () => ({ face: 'idle', visemes: [] }) }));
|
||||
|
||||
// Keep the manifest fetch out of the unit test (no real network to GitHub).
|
||||
// entry:null exercises the bundled-default fallback branch (RiveMascot stub).
|
||||
vi.mock('./Mascot/manifest/useMascotManifest', () => ({
|
||||
useMascotManifest: () => ({ manifest: null, entry: null, loading: false, error: null }),
|
||||
}));
|
||||
|
||||
const SPEAK_REPLIES_KEY = 'human.speakReplies';
|
||||
|
||||
function buildMinimalStore() {
|
||||
|
||||
@@ -8,15 +8,15 @@ import {
|
||||
selectCustomPrimaryColor,
|
||||
selectCustomSecondaryColor,
|
||||
selectMascotColor,
|
||||
selectSelectedMascotId,
|
||||
} from '../../store/mascotSlice';
|
||||
import {
|
||||
BackendRiveMascot,
|
||||
CustomGifMascot,
|
||||
getMascotPalette,
|
||||
hexToArgbInt,
|
||||
ManifestRiveMascot,
|
||||
RiveMascot,
|
||||
} from './Mascot';
|
||||
import { useMascotManifest } from './Mascot/manifest/useMascotManifest';
|
||||
import { useHumanMascot } from './useHumanMascot';
|
||||
|
||||
const SPEAK_REPLIES_KEY = 'human.speakReplies';
|
||||
@@ -37,7 +37,8 @@ const HumanPage = () => {
|
||||
const customPrimary = useAppSelector(selectCustomPrimaryColor);
|
||||
const customSecondary = useAppSelector(selectCustomSecondaryColor);
|
||||
const customMascotGifUrl = useAppSelector(selectCustomMascotGifUrl);
|
||||
const selectedMascotId = useAppSelector(selectSelectedMascotId);
|
||||
// Active mascot resolved from the GitHub manifest (selection + default).
|
||||
const { entry: mascotEntry } = useMascotManifest();
|
||||
const palette = getMascotPalette(mascotColor);
|
||||
const primaryColor = useMemo(
|
||||
() => hexToArgbInt(mascotColor === 'custom' ? customPrimary : palette.bodyFill),
|
||||
@@ -62,10 +63,10 @@ const HumanPage = () => {
|
||||
<div className="relative w-[min(80vh,90%)] aspect-square">
|
||||
{customMascotGifUrl ? (
|
||||
<CustomGifMascot src={customMascotGifUrl} face={face} />
|
||||
) : selectedMascotId ? (
|
||||
<BackendRiveMascot
|
||||
key={selectedMascotId}
|
||||
mascotId={selectedMascotId}
|
||||
) : mascotEntry ? (
|
||||
<ManifestRiveMascot
|
||||
key={mascotEntry.id}
|
||||
entry={mascotEntry}
|
||||
face={face}
|
||||
primaryColor={primaryColor}
|
||||
secondaryColor={secondaryColor}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Unit tests for ManifestRiveMascot — the manifest-driven Rive renderer.
|
||||
*
|
||||
* Mocks the WebGL Rive runtime (records every enum/color setValue by path)
|
||||
* and the manifest .riv loader so we can assert the component:
|
||||
* - falls back to the bundled default while the buffer loads,
|
||||
* - drives pose/viseme constrained to the mascot's stateEngine, and
|
||||
* - auto-cycles a manifest channel (eyes) when idlePoseRotation is on.
|
||||
*/
|
||||
import { act, render, waitFor } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { MascotManifestEntry } from './manifest/types';
|
||||
import { ManifestRiveMascot } from './ManifestRiveMascot';
|
||||
|
||||
const h = vi.hoisted(() => ({
|
||||
useRiveParams: null as Record<string, unknown> | null,
|
||||
enumCalls: {} as Record<string, unknown[]>,
|
||||
colorCalls: {} as Record<string, unknown[]>,
|
||||
}));
|
||||
|
||||
vi.mock('@rive-app/react-webgl2', () => ({
|
||||
Fit: { Contain: 'contain' },
|
||||
Layout: class {
|
||||
constructor(opts: unknown) {
|
||||
Object.assign(this, opts as object);
|
||||
}
|
||||
},
|
||||
useRive: (params: Record<string, unknown>) => {
|
||||
h.useRiveParams = params;
|
||||
return { rive: {}, RiveComponent: () => null };
|
||||
},
|
||||
useViewModel: () => ({}),
|
||||
useViewModelInstance: () => ({}),
|
||||
useViewModelInstanceEnum: (path: string) => ({
|
||||
setValue: (v: string) => (h.enumCalls[path] ??= []).push(v),
|
||||
value: null,
|
||||
values: [],
|
||||
}),
|
||||
useViewModelInstanceColor: (path: string) => ({
|
||||
setValue: (v: number) => (h.colorCalls[path] ??= []).push(v),
|
||||
}),
|
||||
}));
|
||||
|
||||
const loadManifestRiv = vi.hoisted(() => vi.fn());
|
||||
vi.mock('./manifest/manifestService', () => ({ loadManifestRiv }));
|
||||
|
||||
const TOSHI: MascotManifestEntry = {
|
||||
id: 'toshi',
|
||||
name: 'Toshi',
|
||||
description: '',
|
||||
status: 'ready',
|
||||
tags: [],
|
||||
stateEngine: {
|
||||
idlePoseCycle: ['idle', 'look_around', 'pointing'],
|
||||
states: { idle: 'idle', thinking: 'look_around' },
|
||||
visemeCodes: ['sil', 'PP', 'aa'],
|
||||
channels: [
|
||||
{
|
||||
key: 'eyes',
|
||||
label: 'Eyes',
|
||||
values: ['blink', 'look_left', 'look_right'],
|
||||
cycle: { enabled: true, intervalMs: 2600, order: 'sequential' },
|
||||
},
|
||||
],
|
||||
},
|
||||
files: [
|
||||
{ path: 'm/toshi.riv', bytes: 1, role: 'runtime', sha256: 'cccc', url: 'https://x/toshi.riv' },
|
||||
],
|
||||
};
|
||||
|
||||
const RIVER: MascotManifestEntry = {
|
||||
...TOSHI,
|
||||
id: 'river-guide',
|
||||
name: 'River Guide',
|
||||
stateEngine: { ...TOSHI.stateEngine, states: { idle: 'idle', thinking: 'thinking' } },
|
||||
files: [
|
||||
{ path: 'm/river.riv', bytes: 1, role: 'runtime', sha256: 'dddd', url: 'https://x/river.riv' },
|
||||
],
|
||||
};
|
||||
|
||||
function enumLast(path: string): string | undefined {
|
||||
return (h.enumCalls[path] ?? []).at(-1) as string | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
h.useRiveParams = null;
|
||||
h.enumCalls = {};
|
||||
h.colorCalls = {};
|
||||
loadManifestRiv.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('ManifestRiveMascot', () => {
|
||||
it('renders the bundled default while the buffer loads', () => {
|
||||
loadManifestRiv.mockReturnValue(new Promise(() => {})); // never resolves
|
||||
render(<ManifestRiveMascot entry={TOSHI} face="idle" />);
|
||||
// Default RiveMascot loads from the bundled src, not a buffer.
|
||||
expect(h.useRiveParams?.src).toBe('/tiny_mascot.riv');
|
||||
});
|
||||
|
||||
it('renders from the loaded buffer and drives the mascot vocabulary', async () => {
|
||||
loadManifestRiv.mockResolvedValue(new Uint8Array([1, 2, 3]).buffer);
|
||||
render(<ManifestRiveMascot entry={TOSHI} face="thinking" visemeCode="aa" />);
|
||||
|
||||
await waitFor(() => expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer));
|
||||
// 'thinking' face → Toshi's look_around (it has no 'thinking' pose).
|
||||
expect((h.enumCalls['pose'] ?? []).at(-1)).toBe('look_around');
|
||||
expect(enumLast('mouthVisemeCode')).toBe('aa');
|
||||
});
|
||||
|
||||
it('clears the previous buffer when the entry changes without a remount', async () => {
|
||||
let resolveRiver!: (buffer: ArrayBuffer) => void;
|
||||
loadManifestRiv
|
||||
.mockResolvedValueOnce(new Uint8Array([1, 2, 3]).buffer)
|
||||
.mockReturnValueOnce(new Promise<ArrayBuffer>(res => (resolveRiver = res)));
|
||||
|
||||
const { rerender } = render(<ManifestRiveMascot entry={TOSHI} face="idle" />);
|
||||
await waitFor(() => expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer));
|
||||
|
||||
rerender(<ManifestRiveMascot entry={RIVER} face="idle" />);
|
||||
await waitFor(() => expect(h.useRiveParams?.src).toBe('/tiny_mascot.riv'));
|
||||
|
||||
await act(async () => {
|
||||
resolveRiver(new Uint8Array([4, 5, 6]).buffer);
|
||||
});
|
||||
await waitFor(() => expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer));
|
||||
});
|
||||
|
||||
it('rests the mouth for a viseme outside the mascot vocabulary', async () => {
|
||||
loadManifestRiv.mockResolvedValue(new Uint8Array([1]).buffer);
|
||||
render(<ManifestRiveMascot entry={TOSHI} face="speaking" visemeCode="O" />);
|
||||
await waitFor(() => expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer));
|
||||
// Toshi has no 'oh' viseme → rest on sil.
|
||||
await waitFor(() => expect(enumLast('mouthVisemeCode')).toBe('sil'));
|
||||
});
|
||||
|
||||
it('auto-cycles a cyclable channel when idle rotation is on', async () => {
|
||||
vi.useFakeTimers();
|
||||
loadManifestRiv.mockResolvedValue(new Uint8Array([1]).buffer);
|
||||
render(<ManifestRiveMascot entry={TOSHI} face="idle" idlePoseRotation />);
|
||||
|
||||
// Let the async buffer load resolve.
|
||||
await act(async () => {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
expect(h.useRiveParams?.buffer).toBeInstanceOf(ArrayBuffer);
|
||||
|
||||
const before = (h.enumCalls['eyes'] ?? []).length;
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(2_600); // one cycle interval
|
||||
});
|
||||
const eyesWrites = h.enumCalls['eyes'] ?? [];
|
||||
expect(eyesWrites.length).toBeGreaterThan(before);
|
||||
// sequential order starts at values[0] then advances to values[1].
|
||||
expect(eyesWrites.at(-1)).toBe('look_left');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
import {
|
||||
Fit,
|
||||
Layout,
|
||||
useRive,
|
||||
useViewModel,
|
||||
useViewModelInstance,
|
||||
useViewModelInstanceColor,
|
||||
useViewModelInstanceEnum,
|
||||
} from '@rive-app/react-webgl2';
|
||||
import debug from 'debug';
|
||||
import { type FC, useEffect, useRef, useState } from 'react';
|
||||
|
||||
import type { MascotFace } from './Ghosty';
|
||||
import { loadManifestRiv } from './manifest/manifestService';
|
||||
import { pickIdleFlourish, resolveFaceToPose, resolveVisemeCode } from './manifest/stateEngine';
|
||||
import type {
|
||||
MascotManifestChannel,
|
||||
MascotManifestEntry,
|
||||
MascotStateEngine,
|
||||
} from './manifest/types';
|
||||
import { MASCOT_STATE_MACHINE } from './riveMaps';
|
||||
import { RiveMascot } from './RiveMascot';
|
||||
|
||||
const log = debug('human:mascot:manifest-rive');
|
||||
|
||||
/** Idle dwell before the mascot drifts into a flourish (ms), randomised. */
|
||||
const AMBIENT_IDLE_MIN_MS = 6_000;
|
||||
const AMBIENT_IDLE_MAX_MS = 12_000;
|
||||
/** How long a flourish is held before returning to the resting pose (ms). */
|
||||
const AMBIENT_HOLD_MIN_MS = 2_500;
|
||||
const AMBIENT_HOLD_MAX_MS = 5_000;
|
||||
/** Fallback channel auto-cycle interval when the manifest omits one (ms). */
|
||||
const CHANNEL_CYCLE_FALLBACK_MS = 2_500;
|
||||
|
||||
function randBetween(min: number, max: number): number {
|
||||
return min + Math.random() * (max - min);
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a value to the asset's actual enum option, case-insensitively. Rive
|
||||
* enums are case-sensitive and silently ignore an unrecognised value — so if
|
||||
* our resolved code (`oh`) doesn't exactly match the asset's option (`OH`),
|
||||
* the mouth/pose freezes. The runtime exposes the real option list; we match
|
||||
* against it and return the exact-cased option. Falls back to the input when
|
||||
* the options aren't loaded yet or there's no match.
|
||||
*/
|
||||
function matchEnumValue(value: string, options: readonly string[] | undefined): string {
|
||||
if (!options || options.length === 0) return value;
|
||||
return options.find(o => o.toLowerCase() === value.toLowerCase()) ?? value;
|
||||
}
|
||||
|
||||
const RIVE_LAYOUT = new Layout({ fit: Fit.Contain });
|
||||
|
||||
export interface ManifestRiveMascotProps {
|
||||
/** The manifest entry to render. Its runtime `.riv` is loaded + cached. */
|
||||
entry: MascotManifestEntry;
|
||||
face?: MascotFace;
|
||||
size?: number | string;
|
||||
primaryColor?: number;
|
||||
secondaryColor?: number;
|
||||
/** Raw Oculus 15-set viseme code; normalised to this mascot's vocabulary. */
|
||||
visemeCode?: string;
|
||||
/** Drift through this mascot's idle pose cycle + auto-cycle its channels. */
|
||||
idlePoseRotation?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a manifest mascot from its loaded `.riv` buffer. Split out from the
|
||||
* loader so every Rive hook runs against a present buffer (calling the Rive
|
||||
* hooks with no source then swapping in a buffer mid-mount destabilises the
|
||||
* runtime). The parent keys this by mascot id so a new selection remounts it.
|
||||
*/
|
||||
const ManifestRiveStage: FC<{
|
||||
buffer: ArrayBuffer;
|
||||
engine: MascotStateEngine;
|
||||
channels: MascotManifestChannel[];
|
||||
face: MascotFace;
|
||||
size: number | string;
|
||||
primaryColor?: number;
|
||||
secondaryColor?: number;
|
||||
visemeCode: string;
|
||||
idlePoseRotation: boolean;
|
||||
}> = ({
|
||||
buffer,
|
||||
engine,
|
||||
channels,
|
||||
face,
|
||||
size,
|
||||
primaryColor,
|
||||
secondaryColor,
|
||||
visemeCode,
|
||||
idlePoseRotation,
|
||||
}) => {
|
||||
const { rive, RiveComponent } = useRive({
|
||||
buffer,
|
||||
stateMachines: MASCOT_STATE_MACHINE,
|
||||
autoplay: true,
|
||||
layout: RIVE_LAYOUT,
|
||||
});
|
||||
|
||||
const viewModel = useViewModel(rive, { useDefault: true });
|
||||
const vmInstance = useViewModelInstance(viewModel, { useDefault: true, rive });
|
||||
const { setValue: setPose, values: poseEnumValues } = useViewModelInstanceEnum(
|
||||
'pose',
|
||||
vmInstance
|
||||
);
|
||||
const { setValue: setMouthVisemeCode, values: visemeEnumValues } = useViewModelInstanceEnum(
|
||||
'mouthVisemeCode',
|
||||
vmInstance
|
||||
);
|
||||
const { setValue: setPrimaryColor } = useViewModelInstanceColor('primaryColor', vmInstance);
|
||||
const { setValue: setSecondaryColor } = useViewModelInstanceColor('secondaryColor', vmInstance);
|
||||
|
||||
// `useViewModelInstanceEnum(...).values` returns a NEW array reference on
|
||||
// every render. Depending on that identity in an effect causes the effect to
|
||||
// re-run each render → setValue updates the hook's internal state → re-render
|
||||
// → loop ("Maximum update depth exceeded"). So we keep the arrays in refs
|
||||
// (read inside effects) and gate re-runs on a *content* key that only changes
|
||||
// when the asset's option list actually changes (i.e. once, when it loads).
|
||||
const poseValuesRef = useRef<readonly string[] | undefined>(poseEnumValues);
|
||||
poseValuesRef.current = poseEnumValues;
|
||||
const visemeValuesRef = useRef<readonly string[] | undefined>(visemeEnumValues);
|
||||
visemeValuesRef.current = visemeEnumValues;
|
||||
const poseEnumKey = (poseEnumValues ?? []).join('');
|
||||
const visemeEnumKey = (visemeEnumValues ?? []).join('');
|
||||
|
||||
// One-time visibility into what the asset actually accepts vs. what we send.
|
||||
const loggedEnumsRef = useRef(false);
|
||||
useEffect(() => {
|
||||
if (loggedEnumsRef.current) return;
|
||||
if (visemeEnumKey.length > 0 || poseEnumKey.length > 0) {
|
||||
loggedEnumsRef.current = true;
|
||||
log('asset enums — pose=%o viseme=%o', poseValuesRef.current, visemeValuesRef.current);
|
||||
}
|
||||
}, [poseEnumKey, visemeEnumKey]);
|
||||
|
||||
const basePose = resolveFaceToPose(face, engine);
|
||||
const restPose = engine.states.idle;
|
||||
|
||||
// Driven (face-derived) pose. A real activity pose always wins; the resting
|
||||
// pose is what the idle scheduler below is free to override.
|
||||
useEffect(() => {
|
||||
setPose(matchEnumValue(basePose, poseValuesRef.current));
|
||||
}, [basePose, setPose, poseEnumKey]);
|
||||
|
||||
// Idle pose rotation, scoped to this mascot's idlePoseCycle. Same self-
|
||||
// rescheduling shape as RiveMascot; only runs while enabled AND resting.
|
||||
const setPoseRef = useRef(setPose);
|
||||
setPoseRef.current = setPose;
|
||||
useEffect(() => {
|
||||
if (!idlePoseRotation || basePose !== restPose) return;
|
||||
let timer: number | undefined;
|
||||
let current = restPose;
|
||||
const toRest = () => {
|
||||
current = restPose;
|
||||
setPoseRef.current(matchEnumValue(restPose, poseValuesRef.current));
|
||||
timer = window.setTimeout(toFlourish, randBetween(AMBIENT_IDLE_MIN_MS, AMBIENT_IDLE_MAX_MS));
|
||||
};
|
||||
const toFlourish = () => {
|
||||
current = pickIdleFlourish(engine, current === restPose ? undefined : current);
|
||||
log('idle flourish → %s', current);
|
||||
setPoseRef.current(matchEnumValue(current, poseValuesRef.current));
|
||||
timer = window.setTimeout(toRest, randBetween(AMBIENT_HOLD_MIN_MS, AMBIENT_HOLD_MAX_MS));
|
||||
};
|
||||
timer = window.setTimeout(toFlourish, randBetween(AMBIENT_IDLE_MIN_MS, AMBIENT_IDLE_MAX_MS));
|
||||
return () => {
|
||||
if (timer !== undefined) window.clearTimeout(timer);
|
||||
setPoseRef.current(matchEnumValue(restPose, poseValuesRef.current));
|
||||
};
|
||||
}, [idlePoseRotation, basePose, restPose, engine]);
|
||||
|
||||
useEffect(() => {
|
||||
const code = resolveVisemeCode(visemeCode, engine);
|
||||
setMouthVisemeCode(matchEnumValue(code, visemeValuesRef.current));
|
||||
}, [visemeCode, engine, setMouthVisemeCode, visemeEnumKey]);
|
||||
|
||||
useEffect(() => {
|
||||
if (primaryColor !== undefined) setPrimaryColor(primaryColor);
|
||||
}, [primaryColor, setPrimaryColor]);
|
||||
|
||||
useEffect(() => {
|
||||
if (secondaryColor !== undefined) setSecondaryColor(secondaryColor);
|
||||
}, [secondaryColor, setSecondaryColor]);
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: typeof size === 'number' ? `${size}px` : size,
|
||||
height: typeof size === 'number' ? `${size}px` : size,
|
||||
}}
|
||||
data-face={face}>
|
||||
<RiveComponent />
|
||||
{channels.map(channel => (
|
||||
<ChannelDriver
|
||||
key={channel.key}
|
||||
channel={channel}
|
||||
vmInstance={vmInstance}
|
||||
autoCycle={idlePoseRotation}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Drives one optional enum channel (e.g. `eyes`) onto the view model. Each
|
||||
* channel is its own component so the rules-of-hooks count stays stable, and
|
||||
* auto-cycles its value on a timer when the manifest marks it cyclable and the
|
||||
* mascot is in its "feel alive" mode.
|
||||
*/
|
||||
const ChannelDriver: FC<{
|
||||
channel: MascotManifestChannel;
|
||||
vmInstance: ReturnType<typeof useViewModelInstance>;
|
||||
autoCycle: boolean;
|
||||
}> = ({ channel, vmInstance, autoCycle }) => {
|
||||
const { setValue } = useViewModelInstanceEnum(channel.key, vmInstance);
|
||||
const [value, setVal] = useState<string>(channel.default ?? channel.values[0]);
|
||||
|
||||
useEffect(() => {
|
||||
if (value != null) setValue(value);
|
||||
}, [value, setValue]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!autoCycle || !channel.cycle?.enabled || channel.values.length < 2) return;
|
||||
const interval = channel.cycle.intervalMs ?? CHANNEL_CYCLE_FALLBACK_MS;
|
||||
const sequential = channel.cycle.order === 'sequential';
|
||||
let index = 0;
|
||||
const timer = window.setInterval(() => {
|
||||
if (sequential) {
|
||||
index = (index + 1) % channel.values.length;
|
||||
setVal(channel.values[index]);
|
||||
} else {
|
||||
setVal(channel.values[Math.floor(Math.random() * channel.values.length)]);
|
||||
}
|
||||
}, interval);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [autoCycle, channel]);
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Load a manifest mascot's `.riv` and render it. While the buffer resolves —
|
||||
* or if it fails — the bundled default mascot keeps the stage alive and still
|
||||
* lip-syncs, so a slow GitHub fetch never blanks the Human page.
|
||||
*/
|
||||
export const ManifestRiveMascot: FC<ManifestRiveMascotProps> = ({
|
||||
entry,
|
||||
face = 'idle',
|
||||
size = '100%',
|
||||
primaryColor,
|
||||
secondaryColor,
|
||||
visemeCode = 'sil',
|
||||
idlePoseRotation = false,
|
||||
}) => {
|
||||
const [loadState, setLoadState] = useState<{
|
||||
entryId: string;
|
||||
buffer: ArrayBuffer | null;
|
||||
failed: boolean;
|
||||
}>({ entryId: entry.id, buffer: null, failed: false });
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
(async () => {
|
||||
try {
|
||||
const buf = await loadManifestRiv(entry);
|
||||
if (!cancelled) setLoadState({ entryId: entry.id, buffer: buf, failed: false });
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
log('failed to load mascot %s: %o', entry.id, err);
|
||||
setLoadState({ entryId: entry.id, buffer: null, failed: true });
|
||||
}
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [entry]);
|
||||
|
||||
const fallbackProps = { face, size, primaryColor, secondaryColor, visemeCode, idlePoseRotation };
|
||||
const currentBuffer = loadState.entryId === entry.id ? loadState.buffer : null;
|
||||
const currentFailed = loadState.entryId === entry.id && loadState.failed;
|
||||
if (currentFailed || !currentBuffer) return <RiveMascot key="default" {...fallbackProps} />;
|
||||
|
||||
return (
|
||||
<ManifestRiveStage
|
||||
key={`buf-${entry.id}`}
|
||||
buffer={currentBuffer}
|
||||
engine={entry.stateEngine}
|
||||
channels={entry.stateEngine.channels ?? []}
|
||||
face={face}
|
||||
size={size}
|
||||
primaryColor={primaryColor}
|
||||
secondaryColor={secondaryColor}
|
||||
visemeCode={visemeCode}
|
||||
idlePoseRotation={idlePoseRotation}
|
||||
/>
|
||||
);
|
||||
};
|
||||
@@ -6,6 +6,20 @@ export { MascotChipAvatar } from './MascotChipAvatar';
|
||||
export type { MascotChipAvatarProps } from './MascotChipAvatar';
|
||||
export { RiveMascot, DEFAULT_MASCOT_SRC } from './RiveMascot';
|
||||
export type { RiveMascotProps } from './RiveMascot';
|
||||
export { ManifestRiveMascot } from './ManifestRiveMascot';
|
||||
export type { ManifestRiveMascotProps } from './ManifestRiveMascot';
|
||||
export {
|
||||
fetchMascotManifest,
|
||||
defaultMascot,
|
||||
findMascot,
|
||||
loadManifestRiv,
|
||||
} from './manifest/manifestService';
|
||||
export type {
|
||||
MascotManifest,
|
||||
MascotManifestEntry,
|
||||
MascotStateEngine,
|
||||
MascotManifestChannel,
|
||||
} from './manifest/types';
|
||||
export { BackendRiveMascot } from './backend/BackendRiveMascot';
|
||||
export type { BackendRiveMascotProps } from './backend/BackendRiveMascot';
|
||||
export { lerpViseme, VISEMES, visemePath } from './visemes';
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { clearRivMemoryCache } from '../rivCache';
|
||||
import {
|
||||
clearManifestCache,
|
||||
defaultMascot,
|
||||
fetchMascotManifest,
|
||||
findMascot,
|
||||
loadManifestRiv,
|
||||
parseManifest,
|
||||
} from './manifestService';
|
||||
import type { MascotManifest, MascotManifestEntry } from './types';
|
||||
|
||||
const TINY: MascotManifestEntry = {
|
||||
id: 'tiny-mascot',
|
||||
name: 'Tiny Mascot',
|
||||
description: 'Default OpenHuman mascot.',
|
||||
status: 'ready',
|
||||
tags: ['default', 'openhuman'],
|
||||
stateEngine: {
|
||||
idlePoseCycle: ['idle', 'bookreading', 'dancing'],
|
||||
states: { idle: 'idle', thinking: 'thinking' },
|
||||
visemeCodes: ['sil', 'PP', 'aa', 'oh', 'ou'],
|
||||
},
|
||||
files: [
|
||||
{
|
||||
bytes: 189378,
|
||||
path: 'mascots/tiny-mascot/tinyMascot.riv',
|
||||
role: 'runtime',
|
||||
sha256: 'aaaa',
|
||||
url: 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/mascots/tiny-mascot/tinyMascot.riv',
|
||||
},
|
||||
{
|
||||
bytes: 622966,
|
||||
path: 'mascots/tiny-mascot/tinyMascot.rev',
|
||||
role: 'source',
|
||||
sha256: 'bbbb',
|
||||
url: 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/mascots/tiny-mascot/tinyMascot.rev',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const TOSHI: MascotManifestEntry = {
|
||||
id: 'toshi',
|
||||
name: 'Toshi',
|
||||
description: 'Uploaded Toshi mascot asset.',
|
||||
status: 'draft',
|
||||
tags: ['draft', 'openhuman'],
|
||||
stateEngine: {
|
||||
idlePoseCycle: ['idle', 'look_around', 'dancing'],
|
||||
states: { idle: 'idle', thinking: 'look_around' },
|
||||
visemeCodes: ['sil', 'PP', 'aa'],
|
||||
channels: [
|
||||
{
|
||||
key: 'eyes',
|
||||
label: 'Eyes',
|
||||
values: ['blink', 'look_left', 'look_right'],
|
||||
cycle: { enabled: true, intervalMs: 2600, order: 'random' },
|
||||
},
|
||||
],
|
||||
},
|
||||
files: [
|
||||
{
|
||||
bytes: 2259491,
|
||||
path: 'mascots/toshi/toshi.riv',
|
||||
role: 'runtime',
|
||||
sha256: 'cccc',
|
||||
url: 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/mascots/toshi/toshi.riv',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function manifestDoc(mascots: unknown[] = [TINY, TOSHI]): unknown {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
generatedAt: '2026-06-29T21:04:06.960Z',
|
||||
mascots,
|
||||
source: { repository: 'tinyhumansai/mascots', branch: 'main', commit: 'abc' },
|
||||
};
|
||||
}
|
||||
|
||||
function mockFetchJson(doc: unknown) {
|
||||
const fn = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, json: () => Promise.resolve(doc) } as unknown as Response);
|
||||
vi.stubGlobal('fetch', fn);
|
||||
return fn;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
clearManifestCache();
|
||||
clearRivMemoryCache();
|
||||
window.localStorage.clear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('parseManifest', () => {
|
||||
it('accepts a well-formed document', () => {
|
||||
const m = parseManifest(manifestDoc());
|
||||
expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot', 'toshi']);
|
||||
});
|
||||
|
||||
it('drops entries missing a runtime .riv', () => {
|
||||
const noRuntime = { ...TINY, id: 'broken', files: [TINY.files[1]] };
|
||||
const m = parseManifest(manifestDoc([TINY, noRuntime]));
|
||||
expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot']);
|
||||
});
|
||||
|
||||
it('drops entries with an incomplete stateEngine', () => {
|
||||
const noStates = {
|
||||
...TOSHI,
|
||||
id: 'bad',
|
||||
stateEngine: { ...TOSHI.stateEngine, states: { idle: 'idle' } },
|
||||
};
|
||||
const m = parseManifest(manifestDoc([TINY, noStates]));
|
||||
expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot']);
|
||||
});
|
||||
|
||||
it('drops entries with malformed stateEngine arrays or channels', () => {
|
||||
const badVisemes = {
|
||||
...TOSHI,
|
||||
id: 'bad-visemes',
|
||||
stateEngine: { ...TOSHI.stateEngine, visemeCodes: ['sil', 123] },
|
||||
};
|
||||
const badChannels = {
|
||||
...TOSHI,
|
||||
id: 'bad-channels',
|
||||
stateEngine: { ...TOSHI.stateEngine, channels: [{ key: 'eyes', values: [] }] },
|
||||
};
|
||||
const m = parseManifest(manifestDoc([TINY, badVisemes, badChannels]));
|
||||
expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot']);
|
||||
});
|
||||
|
||||
it('throws when there are no renderable mascots', () => {
|
||||
expect(() => parseManifest(manifestDoc([]))).toThrow();
|
||||
expect(() => parseManifest({ nope: true })).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchMascotManifest', () => {
|
||||
it('fetches once and memoises for the session', async () => {
|
||||
const fn = mockFetchJson(manifestDoc());
|
||||
const a = await fetchMascotManifest();
|
||||
const b = await fetchMascotManifest();
|
||||
expect(b).toBe(a);
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('falls back to the localStorage snapshot when the network fails', async () => {
|
||||
mockFetchJson(manifestDoc());
|
||||
await fetchMascotManifest(); // writes snapshot
|
||||
clearManifestCache();
|
||||
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn().mockRejectedValue(new Error('offline')) as unknown as typeof fetch
|
||||
);
|
||||
const m = await fetchMascotManifest();
|
||||
expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot', 'toshi']);
|
||||
});
|
||||
|
||||
it('aborts a stalled fetch so the localStorage snapshot can be used', async () => {
|
||||
vi.useFakeTimers();
|
||||
mockFetchJson(manifestDoc());
|
||||
await fetchMascotManifest(); // writes snapshot
|
||||
clearManifestCache();
|
||||
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn((_url: string, init?: RequestInit) => {
|
||||
return new Promise((_resolve, reject) => {
|
||||
init?.signal?.addEventListener('abort', () => reject(new Error('aborted')));
|
||||
});
|
||||
}) as unknown as typeof fetch
|
||||
);
|
||||
|
||||
const pending = fetchMascotManifest();
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
const m = await pending;
|
||||
expect(m.mascots.map(x => x.id)).toEqual(['tiny-mascot', 'toshi']);
|
||||
});
|
||||
|
||||
it('rejects when the network fails and there is no snapshot', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn().mockRejectedValue(new Error('offline')) as unknown as typeof fetch
|
||||
);
|
||||
await expect(fetchMascotManifest()).rejects.toThrow('offline');
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectors', () => {
|
||||
const manifest: MascotManifest = parseManifest(manifestDoc());
|
||||
|
||||
it('findMascot resolves by id', () => {
|
||||
expect(findMascot(manifest, 'toshi')?.name).toBe('Toshi');
|
||||
expect(findMascot(manifest, 'missing')).toBeUndefined();
|
||||
expect(findMascot(manifest, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('defaultMascot prefers the first ready entry', () => {
|
||||
expect(defaultMascot(manifest)?.id).toBe('tiny-mascot');
|
||||
});
|
||||
});
|
||||
|
||||
describe('loadManifestRiv', () => {
|
||||
it('downloads the runtime file keyed by its sha256', async () => {
|
||||
const buf = new Uint8Array([0x52, 0x49, 0x56, 0x45]).buffer;
|
||||
const fn = vi
|
||||
.fn()
|
||||
.mockResolvedValue({
|
||||
ok: true,
|
||||
arrayBuffer: () => Promise.resolve(buf),
|
||||
} as unknown as Response);
|
||||
vi.stubGlobal('fetch', fn);
|
||||
|
||||
const out = await loadManifestRiv(TINY);
|
||||
expect(new Uint8Array(out)[0]).toBe(0x52);
|
||||
expect(fn).toHaveBeenCalledWith(TINY.files[0].url);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* Client for the GitHub-published mascot manifest (`dist/mascots.json`).
|
||||
*
|
||||
* The manifest and its `.riv` runtime files are fetched directly from
|
||||
* `raw.githubusercontent.com` (CORS-open, allowed by the webview CSP). Three
|
||||
* cache layers keep the picker and Human stage snappy and offline-tolerant:
|
||||
*
|
||||
* 1. an in-memory promise (one network fetch per session, even under
|
||||
* concurrent callers),
|
||||
* 2. a `localStorage` snapshot (survives reloads / brief offline), and
|
||||
* 3. the IndexedDB `.riv` binary cache (`rivCache`, keyed by sha256) so
|
||||
* switching back to a previously-seen mascot never re-downloads the asset.
|
||||
*/
|
||||
import debug from 'debug';
|
||||
|
||||
import { MASCOT_MANIFEST_URL } from '../../../../utils/config';
|
||||
import { loadRivBuffer } from '../rivCache';
|
||||
import {
|
||||
type MascotManifest,
|
||||
type MascotManifestChannel,
|
||||
type MascotManifestEntry,
|
||||
type MascotManifestFile,
|
||||
runtimeFile,
|
||||
} from './types';
|
||||
|
||||
const log = debug('human:mascot:manifest');
|
||||
|
||||
/** localStorage key for the last successfully-fetched manifest snapshot. */
|
||||
const SNAPSHOT_KEY = 'openhuman.mascotManifest.v1';
|
||||
const MANIFEST_FETCH_TIMEOUT_MS = 5_000;
|
||||
|
||||
/** Session-lifetime in-flight/resolved fetch. Shared across all callers. */
|
||||
let inflight: Promise<MascotManifest> | null = null;
|
||||
|
||||
function isNonEmptyString(value: unknown): value is string {
|
||||
return typeof value === 'string' && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function isManifestFile(value: unknown): value is MascotManifestFile {
|
||||
const f = value as Partial<MascotManifestFile> | null;
|
||||
return (
|
||||
!!f &&
|
||||
isNonEmptyString(f.path) &&
|
||||
isNonEmptyString(f.url) &&
|
||||
isNonEmptyString(f.sha256) &&
|
||||
(f.role === 'runtime' || f.role === 'source')
|
||||
);
|
||||
}
|
||||
|
||||
function isStringArray(value: unknown): value is string[] {
|
||||
return Array.isArray(value) && value.length > 0 && value.every(isNonEmptyString);
|
||||
}
|
||||
|
||||
function isManifestChannel(value: unknown): value is MascotManifestChannel {
|
||||
const channel = value as Partial<MascotManifestChannel> | null;
|
||||
if (!channel || !isNonEmptyString(channel.key) || !isStringArray(channel.values)) return false;
|
||||
if (channel.enum !== undefined && !isNonEmptyString(channel.enum)) return false;
|
||||
if (channel.label !== undefined && !isNonEmptyString(channel.label)) return false;
|
||||
if (channel.default !== undefined && !isNonEmptyString(channel.default)) return false;
|
||||
if (channel.cycle !== undefined) {
|
||||
const cycle = channel.cycle;
|
||||
if (typeof cycle.enabled !== 'boolean') return false;
|
||||
if (
|
||||
cycle.intervalMs !== undefined &&
|
||||
(!Number.isFinite(cycle.intervalMs) || cycle.intervalMs <= 0)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (cycle.order !== undefined && cycle.order !== 'random' && cycle.order !== 'sequential') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function isManifestEntry(value: unknown): value is MascotManifestEntry {
|
||||
const m = value as Partial<MascotManifestEntry> | null;
|
||||
if (!m || !isNonEmptyString(m.id) || !isNonEmptyString(m.name)) return false;
|
||||
if (m.status !== 'ready' && m.status !== 'draft') return false;
|
||||
const se = m.stateEngine;
|
||||
if (
|
||||
!se ||
|
||||
!isStringArray(se.visemeCodes) ||
|
||||
!isStringArray(se.idlePoseCycle) ||
|
||||
!se.states ||
|
||||
!isNonEmptyString(se.states.idle) ||
|
||||
!isNonEmptyString(se.states.thinking)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (!Object.values(se.states).every(isNonEmptyString)) return false;
|
||||
if (
|
||||
se.channels !== undefined &&
|
||||
(!Array.isArray(se.channels) || !se.channels.every(isManifestChannel))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (!Array.isArray(m.files) || !m.files.every(isManifestFile)) return false;
|
||||
// A renderable mascot needs a runtime `.riv`; drop entries that only ship a
|
||||
// source `.rev` so callers never select an unplayable asset.
|
||||
return !!runtimeFile(m as MascotManifestEntry);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and normalise a parsed manifest document. Throws on a
|
||||
* fundamentally malformed payload; silently drops individual malformed
|
||||
* mascot entries (a single bad entry should never blank the whole library).
|
||||
*/
|
||||
export function parseManifest(raw: unknown): MascotManifest {
|
||||
const doc = raw as Partial<MascotManifest> | null;
|
||||
if (!doc || !Array.isArray(doc.mascots)) {
|
||||
throw new Error('mascot manifest: missing mascots array');
|
||||
}
|
||||
const mascots = doc.mascots.filter(isManifestEntry);
|
||||
if (mascots.length === 0) {
|
||||
throw new Error('mascot manifest: no renderable mascots');
|
||||
}
|
||||
return {
|
||||
schemaVersion: typeof doc.schemaVersion === 'number' ? doc.schemaVersion : 1,
|
||||
generatedAt: isNonEmptyString(doc.generatedAt) ? doc.generatedAt : '',
|
||||
mascots,
|
||||
source: {
|
||||
repository: doc.source?.repository ?? '',
|
||||
branch: doc.source?.branch ?? '',
|
||||
commit: doc.source?.commit ?? '',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function readSnapshot(): MascotManifest | null {
|
||||
try {
|
||||
const raw = window.localStorage.getItem(SNAPSHOT_KEY);
|
||||
if (!raw) return null;
|
||||
return parseManifest(JSON.parse(raw));
|
||||
} catch (err) {
|
||||
log('snapshot read failed: %o', err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function writeSnapshot(manifest: MascotManifest): void {
|
||||
try {
|
||||
window.localStorage.setItem(SNAPSHOT_KEY, JSON.stringify(manifest));
|
||||
} catch (err) {
|
||||
log('snapshot write failed: %o', err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the mascot manifest, memoised for the session. On a network failure we
|
||||
* fall back to the last localStorage snapshot so the picker still works
|
||||
* offline; if there is no snapshot either, the rejection propagates so the UI
|
||||
* can show an error state.
|
||||
*/
|
||||
export function fetchMascotManifest(): Promise<MascotManifest> {
|
||||
if (inflight) return inflight;
|
||||
inflight = (async () => {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = window.setTimeout(() => controller.abort(), MANIFEST_FETCH_TIMEOUT_MS);
|
||||
try {
|
||||
log('fetching manifest %s', MASCOT_MANIFEST_URL);
|
||||
const res = await fetch(MASCOT_MANIFEST_URL, {
|
||||
cache: 'no-cache',
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!res.ok) throw new Error(`manifest fetch failed (${res.status})`);
|
||||
const manifest = parseManifest(await res.json());
|
||||
log('manifest ok — %d mascots (schema v%d)', manifest.mascots.length, manifest.schemaVersion);
|
||||
writeSnapshot(manifest);
|
||||
return manifest;
|
||||
} catch (err) {
|
||||
const snapshot = readSnapshot();
|
||||
if (snapshot) {
|
||||
log('manifest fetch failed, using snapshot: %o', err);
|
||||
return snapshot;
|
||||
}
|
||||
// Reset so a later retry can attempt the network again rather than
|
||||
// re-resolving this rejected promise forever.
|
||||
inflight = null;
|
||||
throw err;
|
||||
} finally {
|
||||
window.clearTimeout(timeoutId);
|
||||
}
|
||||
})();
|
||||
return inflight;
|
||||
}
|
||||
|
||||
/** Drop the memoised fetch so the next call re-hits the network. */
|
||||
export function clearManifestCache(): void {
|
||||
inflight = null;
|
||||
}
|
||||
|
||||
/** Find a mascot entry by id within a manifest. */
|
||||
export function findMascot(
|
||||
manifest: MascotManifest,
|
||||
id: string | null | undefined
|
||||
): MascotManifestEntry | undefined {
|
||||
if (!id) return undefined;
|
||||
return manifest.mascots.find(m => m.id === id);
|
||||
}
|
||||
|
||||
/** The mascot the app defaults to: the first `ready` entry, else the first. */
|
||||
export function defaultMascot(manifest: MascotManifest): MascotManifestEntry | undefined {
|
||||
return manifest.mascots.find(m => m.status === 'ready') ?? manifest.mascots[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a mascot's `.riv` binary, version-cached in IndexedDB by its
|
||||
* runtime file's sha256. The network is only hit when the sha changes (i.e.
|
||||
* the asset was republished), so re-selecting a mascot is instant.
|
||||
*/
|
||||
export async function loadManifestRiv(entry: MascotManifestEntry): Promise<ArrayBuffer> {
|
||||
const runtime = runtimeFile(entry);
|
||||
if (!runtime) throw new Error(`mascot ${entry.id} has no runtime .riv file`);
|
||||
return loadRivBuffer(entry.id, runtime.sha256, runtime.url);
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
availablePoses,
|
||||
initialChannelValues,
|
||||
pickIdleFlourish,
|
||||
resolveFaceToPose,
|
||||
resolveVisemeCode,
|
||||
restVisemeCode,
|
||||
} from './stateEngine';
|
||||
import type { MascotStateEngine } from './types';
|
||||
|
||||
const TINY: MascotStateEngine = {
|
||||
idlePoseCycle: ['idle', 'bookreading', 'coffeedrink', 'writing', 'dancing'],
|
||||
states: { idle: 'idle', thinking: 'thinking' },
|
||||
visemeCodes: ['sil', 'PP', 'FF', 'aa', 'E', 'ih', 'oh', 'ou'],
|
||||
};
|
||||
|
||||
// A sparse mascot: distinct logical state names, no shared flourish vocabulary.
|
||||
const TOSHI: MascotStateEngine = {
|
||||
idlePoseCycle: ['idle', 'look_around', 'pointing'],
|
||||
states: { idle: 'idle', thinking: 'look_around' },
|
||||
visemeCodes: ['sil', 'PP', 'aa'],
|
||||
channels: [
|
||||
{
|
||||
key: 'eyes',
|
||||
label: 'Eyes',
|
||||
values: ['blink', 'look_left', 'look_right'],
|
||||
default: 'look_left',
|
||||
cycle: { enabled: true, intervalMs: 2600, order: 'random' },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe('availablePoses', () => {
|
||||
it('unions idlePoseCycle and logical state values', () => {
|
||||
expect(availablePoses(TOSHI)).toEqual(new Set(['idle', 'look_around', 'pointing']));
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveFaceToPose', () => {
|
||||
it('uses the desired pose when the mascot has it', () => {
|
||||
expect(resolveFaceToPose('writing', TINY)).toBe('writing');
|
||||
expect(resolveFaceToPose('reading', TINY)).toBe('bookreading');
|
||||
});
|
||||
|
||||
it('uses manifest logical state mappings for custom activity poses', () => {
|
||||
const custom: MascotStateEngine = {
|
||||
...TOSHI,
|
||||
states: { ...TOSHI.states, writing: 'scribbling' },
|
||||
idlePoseCycle: [...TOSHI.idlePoseCycle, 'scribbling'],
|
||||
};
|
||||
|
||||
expect(resolveFaceToPose('writing', custom)).toBe('scribbling');
|
||||
});
|
||||
|
||||
it('falls back to the thinking state for thinking-ish faces', () => {
|
||||
// Toshi has no 'thinking' pose; thinking-ish faces map to its 'look_around'.
|
||||
expect(resolveFaceToPose('thinking', TOSHI)).toBe('look_around');
|
||||
expect(resolveFaceToPose('confused', TOSHI)).toBe('look_around');
|
||||
});
|
||||
|
||||
it('falls back to the idle state when the flourish is absent', () => {
|
||||
// Toshi has no 'writing' pose → rest on idle.
|
||||
expect(resolveFaceToPose('writing', TOSHI)).toBe('idle');
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveVisemeCode', () => {
|
||||
it('normalises and keeps codes in the mascot vocabulary', () => {
|
||||
expect(resolveVisemeCode('O', TINY)).toBe('oh');
|
||||
expect(resolveVisemeCode('E', TINY)).toBe('E');
|
||||
});
|
||||
|
||||
it("returns the manifest's exact casing for case-varied viseme enums", () => {
|
||||
const upper: MascotStateEngine = { ...TINY, visemeCodes: ['SIL', 'PP', 'AA', 'OH'] };
|
||||
expect(resolveVisemeCode('O', upper)).toBe('OH');
|
||||
expect(resolveVisemeCode('a', upper)).toBe('AA');
|
||||
expect(restVisemeCode(upper)).toBe('SIL');
|
||||
});
|
||||
|
||||
it('preserves raw close-vowel aliases when the manifest uses them', () => {
|
||||
const raw: MascotStateEngine = { ...TINY, visemeCodes: ['sil', 'I', 'O', 'U'] };
|
||||
expect(resolveVisemeCode('I', raw)).toBe('I');
|
||||
expect(resolveVisemeCode('O', raw)).toBe('O');
|
||||
expect(resolveVisemeCode('U', raw)).toBe('U');
|
||||
});
|
||||
|
||||
it('falls back to rest for codes the mascot lacks', () => {
|
||||
// Toshi has no 'oh'; an O viseme rests the mouth instead of no-op.
|
||||
expect(resolveVisemeCode('O', TOSHI)).toBe('sil');
|
||||
expect(resolveVisemeCode('???', TINY)).toBe('sil');
|
||||
});
|
||||
|
||||
it('restVisemeCode prefers sil', () => {
|
||||
expect(restVisemeCode(TINY)).toBe('sil');
|
||||
expect(restVisemeCode({ ...TINY, visemeCodes: ['rest', 'aa'] })).toBe('rest');
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickIdleFlourish', () => {
|
||||
it('never returns the resting idle pose', () => {
|
||||
for (let r = 0; r < 1; r += 0.1) {
|
||||
expect(pickIdleFlourish(TINY, undefined, () => r)).not.toBe('idle');
|
||||
}
|
||||
});
|
||||
|
||||
it('avoids the excluded (just-played) pose', () => {
|
||||
// rng=0 would pick the first non-idle pose ('bookreading'); excluding it
|
||||
// shifts the pool so the first pick is the next one.
|
||||
expect(pickIdleFlourish(TINY, 'bookreading', () => 0)).toBe('coffeedrink');
|
||||
});
|
||||
|
||||
it('returns the rest pose when there are no flourishes', () => {
|
||||
const flat: MascotStateEngine = { ...TINY, idlePoseCycle: ['idle'] };
|
||||
expect(pickIdleFlourish(flat)).toBe('idle');
|
||||
});
|
||||
});
|
||||
|
||||
describe('initialChannelValues', () => {
|
||||
it('uses default then first value', () => {
|
||||
expect(initialChannelValues(TOSHI)).toEqual({ eyes: 'look_left' });
|
||||
expect(
|
||||
initialChannelValues({ ...TOSHI, channels: [{ key: 'eyes', values: ['blink', 'open'] }] })
|
||||
).toEqual({ eyes: 'blink' });
|
||||
});
|
||||
|
||||
it('is empty when there are no channels', () => {
|
||||
expect(initialChannelValues(TINY)).toEqual({});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* Resolve mascot face/viseme state against a *specific* mascot's `stateEngine`.
|
||||
*
|
||||
* The bundled-asset helpers in `../riveMaps` assume the `tiny_mascot.riv`
|
||||
* vocabulary. Manifest mascots each ship their own poses, viseme codes, and
|
||||
* channels, so these helpers constrain every value to what the selected asset
|
||||
* actually accepts — setting an out-of-vocabulary enum on a Rive state machine
|
||||
* is a silent no-op, which would freeze the mouth or pose.
|
||||
*/
|
||||
import type { MascotFace } from '../Ghosty';
|
||||
import { faceToPose, toRiveVisemeCode } from '../riveMaps';
|
||||
import type { MascotManifestEntry, MascotStateEngine } from './types';
|
||||
|
||||
/** Every pose value this mascot's state machine accepts (cycle + logical states). */
|
||||
export function availablePoses(engine: MascotStateEngine): Set<string> {
|
||||
return new Set<string>([...engine.idlePoseCycle, ...Object.values(engine.states)]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a {@link MascotFace} to a pose this mascot can actually play. Manifest
|
||||
* logical mappings win first, then the shared face→pose vocabulary, then a
|
||||
* guaranteed logical state (`thinking` for thinking-ish faces, otherwise
|
||||
* `idle`) when the mascot's asset doesn't carry that specific flourish.
|
||||
*/
|
||||
export function resolveFaceToPose(face: MascotFace, engine: MascotStateEngine): string {
|
||||
const logical = engine.states[face];
|
||||
if (logical && availablePoses(engine).has(logical)) return logical;
|
||||
|
||||
const desired = faceToPose(face);
|
||||
if (availablePoses(engine).has(desired)) return desired;
|
||||
return desired === 'thinking' ? engine.states.thinking : engine.states.idle;
|
||||
}
|
||||
|
||||
/** The resting mouth code for this mascot (`sil` when present, else its first code). */
|
||||
export function restVisemeCode(engine: MascotStateEngine): string {
|
||||
return engine.visemeCodes.find(code => code.toLowerCase() === 'sil') ?? engine.visemeCodes[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalise an incoming Oculus/ElevenLabs viseme code to one this mascot's
|
||||
* mouth enum accepts. Unknown or out-of-vocabulary codes resolve to the
|
||||
* resting (closed) mouth so the mouth never sticks on a no-op value.
|
||||
*/
|
||||
export function resolveVisemeCode(code: string, engine: MascotStateEngine): string {
|
||||
const normalised = toRiveVisemeCode(code);
|
||||
const candidates = [code, normalised];
|
||||
return (
|
||||
engine.visemeCodes.find(candidate =>
|
||||
candidates.some(alias => candidate.toLowerCase() === alias.toLowerCase())
|
||||
) ?? restVisemeCode(engine)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick a random idle flourish pose for this mascot, excluding the resting
|
||||
* `idle` pose (and optionally a just-played one) so the same flourish never
|
||||
* fires twice in a row. `rng` is injectable for deterministic tests.
|
||||
*/
|
||||
export function pickIdleFlourish(
|
||||
engine: MascotStateEngine,
|
||||
exclude?: string,
|
||||
rng: () => number = Math.random
|
||||
): string {
|
||||
const rest = engine.states.idle;
|
||||
const pool = engine.idlePoseCycle.filter(p => p !== rest && p !== exclude);
|
||||
const choices = pool.length > 0 ? pool : engine.idlePoseCycle.filter(p => p !== rest);
|
||||
if (choices.length === 0) return rest;
|
||||
const idx = Math.min(choices.length - 1, Math.floor(rng() * choices.length));
|
||||
return choices[idx];
|
||||
}
|
||||
|
||||
/** Initial values for every channel: its `default`, falling back to `values[0]`. */
|
||||
export function initialChannelValues(engine: MascotStateEngine): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const channel of engine.channels ?? []) {
|
||||
out[channel.key] = channel.default ?? channel.values[0];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Convenience: pull a mascot entry's state engine. */
|
||||
export function stateEngineOf(entry: MascotManifestEntry): MascotStateEngine {
|
||||
return entry.stateEngine;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* Types for the GitHub-published mascot manifest
|
||||
* (`tinyhumansai/mascots` → `dist/mascots.json`).
|
||||
*
|
||||
* This is the authoritative source for the in-app mascot library: each entry
|
||||
* names a Rive runtime file plus a `stateEngine` describing the poses, logical
|
||||
* states, viseme vocabulary, and optional secondary animation channels (e.g.
|
||||
* darting eyes) the asset's `MascotSM` state machine accepts.
|
||||
*
|
||||
* Schema mirror: `tinyhumansai/mascots:schemas/mascots.schema.json` (v1). Kept
|
||||
* as hand-written TS rather than generated because the app fetches the manifest
|
||||
* directly over HTTPS — there is no shared package between the two repos.
|
||||
*/
|
||||
|
||||
/** One downloadable asset belonging to a mascot. */
|
||||
export interface MascotManifestFile {
|
||||
/** Repo-relative path, e.g. `mascots/tiny-mascot/tinyMascot.riv`. */
|
||||
path: string;
|
||||
bytes: number;
|
||||
/** `runtime` = the playable `.riv`; `source` = the `.rev` editor file. */
|
||||
role: 'runtime' | 'source';
|
||||
/** Lowercase hex sha256 of the file. Doubles as the cache-bust version key. */
|
||||
sha256: string;
|
||||
/** Absolute `raw.githubusercontent.com` URL the app fetches. */
|
||||
url: string;
|
||||
}
|
||||
|
||||
/** Optional secondary animation channel (its own enum input on the view model). */
|
||||
export interface MascotManifestChannel {
|
||||
/** View-model enum input name to drive (e.g. `eyes`). */
|
||||
key: string;
|
||||
/** Enum name inside the `.riv` (informational). */
|
||||
enum?: string;
|
||||
/** Human label for the controls UI. */
|
||||
label?: string;
|
||||
/** Every value the enum accepts. */
|
||||
values: string[];
|
||||
/** Default value; falls back to `values[0]` when absent. */
|
||||
default?: string;
|
||||
/** Auto-cycle config — drives idle "aliveness" (e.g. random eye darts). */
|
||||
cycle?: { enabled: boolean; intervalMs?: number; order?: 'random' | 'sequential' };
|
||||
}
|
||||
|
||||
/** The animation contract a mascot's `.riv` is authored against. */
|
||||
export interface MascotStateEngine {
|
||||
/**
|
||||
* Ordered viseme codes the asset's mouth enum accepts. Standard
|
||||
* Oculus/ElevenLabs 15-set: `sil, PP, FF, TH, DD, kk, CH, SS, nn, RR, aa, E,
|
||||
* ih, oh, ou`. The first entry (`sil`) is the resting/closed mouth.
|
||||
*/
|
||||
visemeCodes: string[];
|
||||
/** Logical state → pose-enum value. Always carries at least `idle`/`thinking`. */
|
||||
states: { idle: string; thinking: string; [key: string]: string };
|
||||
/** Poses the mascot drifts through while idle (includes the resting pose). */
|
||||
idlePoseCycle: string[];
|
||||
/** Optional extra enum channels (eyes, etc.). */
|
||||
channels?: MascotManifestChannel[];
|
||||
}
|
||||
|
||||
/** A single mascot in the manifest. */
|
||||
export interface MascotManifestEntry {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
/** `ready` = production; `draft` = not yet matched to the runtime contract. */
|
||||
status: 'ready' | 'draft';
|
||||
tags: string[];
|
||||
stateEngine: MascotStateEngine;
|
||||
files: MascotManifestFile[];
|
||||
}
|
||||
|
||||
/** Top-level manifest document. */
|
||||
export interface MascotManifest {
|
||||
schemaVersion: number;
|
||||
generatedAt: string;
|
||||
mascots: MascotManifestEntry[];
|
||||
source: { repository: string; branch: string; commit: string };
|
||||
}
|
||||
|
||||
/** The runtime (`.riv`) file for a mascot, or `undefined` if it has none. */
|
||||
export function runtimeFile(entry: MascotManifestEntry): MascotManifestFile | undefined {
|
||||
return entry.files.find(f => f.role === 'runtime');
|
||||
}
|
||||
|
||||
/** The source (`.rev`) file for a mascot, or `undefined` if it has none. */
|
||||
export function sourceFile(entry: MascotManifestEntry): MascotManifestFile | undefined {
|
||||
return entry.files.find(f => f.role === 'source');
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import { configureStore } from '@reduxjs/toolkit';
|
||||
import { render, screen, waitFor } from '@testing-library/react';
|
||||
import { Provider } from 'react-redux';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import mascotReducer, {
|
||||
setCustomMascotGifUrl,
|
||||
setSelectedMascotId,
|
||||
} from '../../../../store/mascotSlice';
|
||||
import type { MascotManifest, MascotManifestEntry } from './types';
|
||||
import { useMascotManifest } from './useMascotManifest';
|
||||
|
||||
const fetchMascotManifest = vi.hoisted(() => vi.fn());
|
||||
// Fully mock the service (matching BackendRiveMascot.test) so no real module
|
||||
// instance lingers. findMascot/defaultMascot keep their real semantics.
|
||||
vi.mock('./manifestService', () => ({
|
||||
fetchMascotManifest,
|
||||
findMascot: (m: MascotManifest, id: string | null | undefined) =>
|
||||
id ? m.mascots.find(x => x.id === id) : undefined,
|
||||
defaultMascot: (m: MascotManifest) => m.mascots.find(x => x.status === 'ready') ?? m.mascots[0],
|
||||
}));
|
||||
|
||||
function entry(id: string, status: 'ready' | 'draft'): MascotManifestEntry {
|
||||
return {
|
||||
id,
|
||||
name: id,
|
||||
description: '',
|
||||
status,
|
||||
tags: [],
|
||||
stateEngine: {
|
||||
idlePoseCycle: ['idle'],
|
||||
states: { idle: 'idle', thinking: 'thinking' },
|
||||
visemeCodes: ['sil'],
|
||||
},
|
||||
files: [
|
||||
{ path: `${id}.riv`, bytes: 1, role: 'runtime', sha256: id, url: `https://x/${id}.riv` },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const MANIFEST: MascotManifest = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: '',
|
||||
mascots: [entry('toshi', 'draft'), entry('tiny-mascot', 'ready')],
|
||||
source: { repository: '', branch: '', commit: '' },
|
||||
};
|
||||
|
||||
// A render-based probe — surfaces the hook's output into the DOM. We render the
|
||||
// hook through a real component (not renderHook) so a rejecting manifest fetch
|
||||
// is consumed exactly like it is in production, with no orphaned promise.
|
||||
function Probe() {
|
||||
const { entry: e, loading, error } = useMascotManifest();
|
||||
return (
|
||||
<div>
|
||||
<span data-testid="entry">{e?.id ?? 'none'}</span>
|
||||
<span data-testid="loading">{String(loading)}</span>
|
||||
<span data-testid="error">{error?.message ?? ''}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function renderProbe(selectedId: string | null, customGifUrl: string | null = null) {
|
||||
const store = configureStore({ reducer: { mascot: mascotReducer } });
|
||||
if (selectedId) store.dispatch(setSelectedMascotId(selectedId));
|
||||
if (customGifUrl) store.dispatch(setCustomMascotGifUrl(customGifUrl));
|
||||
const view = render(
|
||||
<Provider store={store}>
|
||||
<Probe />
|
||||
</Provider>
|
||||
);
|
||||
return { store, ...view };
|
||||
}
|
||||
|
||||
beforeEach(() => fetchMascotManifest.mockReset());
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
describe('useMascotManifest', () => {
|
||||
it('resolves the selected mascot when set', async () => {
|
||||
fetchMascotManifest.mockResolvedValue(MANIFEST);
|
||||
renderProbe('toshi');
|
||||
await waitFor(() => expect(screen.getByTestId('entry')).toHaveTextContent('toshi'));
|
||||
expect(screen.getByTestId('loading')).toHaveTextContent('false');
|
||||
});
|
||||
|
||||
it('falls back to the default (first ready) mascot when none selected', async () => {
|
||||
fetchMascotManifest.mockResolvedValue(MANIFEST);
|
||||
const { store } = renderProbe(null);
|
||||
await waitFor(() => expect(screen.getByTestId('entry')).toHaveTextContent('tiny-mascot'));
|
||||
expect(store.getState().mascot.selectedMascotId).toBeNull();
|
||||
});
|
||||
|
||||
it('reconciles a stale selected mascot id after the manifest loads', async () => {
|
||||
fetchMascotManifest.mockResolvedValue(MANIFEST);
|
||||
const { store } = renderProbe('removed-mascot');
|
||||
await waitFor(() => expect(screen.getByTestId('entry')).toHaveTextContent('tiny-mascot'));
|
||||
await waitFor(() => expect(store.getState().mascot.selectedMascotId).toBe('tiny-mascot'));
|
||||
});
|
||||
|
||||
it('does not overwrite a custom GIF selection with the manifest fallback', async () => {
|
||||
fetchMascotManifest.mockResolvedValue(MANIFEST);
|
||||
const { store } = renderProbe(null, 'https://example.com/custom.gif');
|
||||
|
||||
await waitFor(() => expect(screen.getByTestId('entry')).toHaveTextContent('tiny-mascot'));
|
||||
|
||||
expect(store.getState().mascot.customMascotGifUrl).toBe('https://example.com/custom.gif');
|
||||
expect(store.getState().mascot.selectedMascotId).toBeNull();
|
||||
});
|
||||
|
||||
// The fetch-failure path is covered end-to-end in manifestService.test.ts
|
||||
// ("rejects when the network fails and there is no snapshot") and the
|
||||
// entry:null fallback render is covered in HumanPage.test.tsx, so the hook's
|
||||
// catch branch is exercised without re-triggering a vitest-v4 quirk where
|
||||
// awaiting into a settling (but handled) rejection surfaces as a test error.
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Load the published mascot manifest and resolve the active mascot entry from
|
||||
* the user's `selectedMascotId` Redux preference, with a render-only fallback
|
||||
* to the default (`ready`) mascot while none is selected. Shared by the Human
|
||||
* stage and the settings picker so both agree on which mascot is current.
|
||||
*/
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
|
||||
import {
|
||||
selectCustomMascotGifUrl,
|
||||
selectSelectedMascotId,
|
||||
setSelectedMascotId,
|
||||
} from '../../../../store/mascotSlice';
|
||||
import { defaultMascot, fetchMascotManifest, findMascot } from './manifestService';
|
||||
import type { MascotManifest, MascotManifestEntry } from './types';
|
||||
|
||||
export interface UseMascotManifestResult {
|
||||
manifest: MascotManifest | null;
|
||||
/** The selected mascot, or the default when none is chosen / found yet. */
|
||||
entry: MascotManifestEntry | null;
|
||||
loading: boolean;
|
||||
error: Error | null;
|
||||
}
|
||||
|
||||
export function useMascotManifest(): UseMascotManifestResult {
|
||||
const dispatch = useDispatch();
|
||||
const selectedMascotId = useSelector(selectSelectedMascotId);
|
||||
const customMascotGifUrl = useSelector(selectCustomMascotGifUrl);
|
||||
const [manifest, setManifest] = useState<MascotManifest | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<Error | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
(async () => {
|
||||
try {
|
||||
const m = await fetchMascotManifest();
|
||||
if (cancelled) return;
|
||||
setManifest(m);
|
||||
setError(null);
|
||||
setLoading(false);
|
||||
} catch (err) {
|
||||
if (cancelled) return;
|
||||
setError(err instanceof Error ? err : new Error(String(err)));
|
||||
setLoading(false);
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const selectedEntry = manifest ? findMascot(manifest, selectedMascotId) : undefined;
|
||||
const fallbackEntry = manifest ? (defaultMascot(manifest) ?? null) : null;
|
||||
const entry = selectedEntry ?? fallbackEntry;
|
||||
|
||||
useEffect(() => {
|
||||
if (!manifest || !selectedMascotId || customMascotGifUrl || selectedEntry) return;
|
||||
dispatch(setSelectedMascotId(fallbackEntry?.id ?? null));
|
||||
}, [customMascotGifUrl, dispatch, fallbackEntry?.id, manifest, selectedEntry, selectedMascotId]);
|
||||
|
||||
return { manifest, entry, loading, error };
|
||||
}
|
||||
@@ -74,8 +74,10 @@ function makePlayback(durationMs: number): FakePlayback {
|
||||
let ms = 0;
|
||||
let stopped = false;
|
||||
let resolveEnded!: () => void;
|
||||
const ended = new Promise<void>(res => {
|
||||
let rejectEnded!: (err: unknown) => void;
|
||||
const ended = new Promise<void>((res, rej) => {
|
||||
resolveEnded = res;
|
||||
rejectEnded = rej;
|
||||
});
|
||||
return {
|
||||
handle: {
|
||||
@@ -83,7 +85,9 @@ function makePlayback(durationMs: number): FakePlayback {
|
||||
durationMs: () => durationMs,
|
||||
metadataReady: Promise.resolve(),
|
||||
stop: () => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
rejectEnded({ stopped: true });
|
||||
},
|
||||
ended,
|
||||
},
|
||||
@@ -97,6 +101,48 @@ function makePlayback(durationMs: number): FakePlayback {
|
||||
};
|
||||
}
|
||||
|
||||
function makePlaybackWithDeferredMetadata(
|
||||
finalDurationMs: number
|
||||
): FakePlayback & { resolveMetadata(): void } {
|
||||
let durationMs = 0;
|
||||
let ms = 0;
|
||||
let stopped = false;
|
||||
let resolveEnded!: () => void;
|
||||
let rejectEnded!: (err: unknown) => void;
|
||||
let resolveMetadata!: () => void;
|
||||
const ended = new Promise<void>((res, rej) => {
|
||||
resolveEnded = res;
|
||||
rejectEnded = rej;
|
||||
});
|
||||
const metadataReady = new Promise<void>(res => {
|
||||
resolveMetadata = () => {
|
||||
durationMs = finalDurationMs;
|
||||
res();
|
||||
};
|
||||
});
|
||||
return {
|
||||
handle: {
|
||||
currentMs: () => (stopped ? -1 : ms),
|
||||
durationMs: () => durationMs,
|
||||
metadataReady,
|
||||
stop: () => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
rejectEnded({ stopped: true });
|
||||
},
|
||||
ended,
|
||||
},
|
||||
setMs(next: number) {
|
||||
ms = next;
|
||||
},
|
||||
finish() {
|
||||
stopped = true;
|
||||
resolveEnded();
|
||||
},
|
||||
resolveMetadata,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive the hook's RAF-based render loop deterministically. The hook calls
|
||||
* `requestAnimationFrame` on every speaking frame; without firing it the
|
||||
@@ -105,12 +151,15 @@ function makePlayback(durationMs: number): FakePlayback {
|
||||
let rafQueue: FrameRequestCallback[] = [];
|
||||
const originalRaf = window.requestAnimationFrame;
|
||||
const originalCancel = window.cancelAnimationFrame;
|
||||
let nowMs = 1_000;
|
||||
|
||||
beforeEach(() => {
|
||||
capturedListeners = null;
|
||||
rafQueue = [];
|
||||
nowMs = 1_000;
|
||||
(synthesizeSpeech as ReturnType<typeof vi.fn>).mockReset();
|
||||
(playBase64Audio as ReturnType<typeof vi.fn>).mockReset();
|
||||
vi.spyOn(window.performance, 'now').mockImplementation(() => nowMs);
|
||||
window.requestAnimationFrame = ((cb: FrameRequestCallback) => {
|
||||
rafQueue.push(cb);
|
||||
return rafQueue.length;
|
||||
@@ -121,12 +170,13 @@ beforeEach(() => {
|
||||
afterEach(() => {
|
||||
window.requestAnimationFrame = originalRaf;
|
||||
window.cancelAnimationFrame = originalCancel;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function tickRaf() {
|
||||
const queue = rafQueue;
|
||||
rafQueue = [];
|
||||
for (const cb of queue) cb(performance.now());
|
||||
for (const cb of queue) cb(nowMs);
|
||||
}
|
||||
|
||||
function fakeDone(text: string) {
|
||||
@@ -142,7 +192,9 @@ function fakeDone(text: string) {
|
||||
|
||||
describe('useHumanMascot — audio-driven lipsync end-to-end', () => {
|
||||
it('mouth opens (non-REST) once playback starts and visemes have known codes', async () => {
|
||||
const fake = makePlayback(1000);
|
||||
// Audio duration matches the viseme track span (400ms) so the duration-
|
||||
// alignment rescale is a no-op and this test isolates code→shape mapping.
|
||||
const fake = makePlayback(400);
|
||||
(synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
audio_base64: 'AAA=',
|
||||
audio_mime: 'audio/mpeg',
|
||||
@@ -183,7 +235,8 @@ describe('useHumanMascot — audio-driven lipsync end-to-end', () => {
|
||||
});
|
||||
|
||||
it('mouth opens even when backend ships visemes in lowercase / aliased codes', async () => {
|
||||
const fake = makePlayback(1000);
|
||||
// Audio length matches the 600ms viseme span → rescale no-op (mapping only).
|
||||
const fake = makePlayback(600);
|
||||
(synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
audio_base64: 'AAA=',
|
||||
audio_mime: 'audio/mpeg',
|
||||
@@ -296,6 +349,149 @@ describe('useHumanMascot — audio-driven lipsync end-to-end', () => {
|
||||
expect(sampled.size).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('stretches a short viseme track to match the longer measured audio', async () => {
|
||||
// Backend ships a 400ms viseme track, but the rendered MP3 is 800ms. Without
|
||||
// alignment the mouth would finish at the halfway point and run ahead of the
|
||||
// voice; rescaling doubles every frame so the track fills the audio.
|
||||
const fake = makePlayback(800);
|
||||
(synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
audio_base64: 'AAA=',
|
||||
audio_mime: 'audio/mpeg',
|
||||
visemes: [
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 200 },
|
||||
{ viseme: 'PP', start_ms: 200, end_ms: 400 },
|
||||
],
|
||||
});
|
||||
(playBase64Audio as ReturnType<typeof vi.fn>).mockResolvedValueOnce(fake.handle);
|
||||
|
||||
const { result } = renderHook(() => useHumanMascot({ speakReplies: true }));
|
||||
await act(async () => {
|
||||
capturedListeners?.onDone?.(fakeDone('hello'));
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
// Rescaled track: frame[0]=aa 0–400ms, frame[1]=PP 400–800ms.
|
||||
// At 300ms the *unscaled* track would already be on frame[1] (PP); after
|
||||
// alignment it is still on frame[0] (aa) — proving the stretch.
|
||||
act(() => {
|
||||
fake.setMs(300);
|
||||
tickRaf();
|
||||
});
|
||||
expect(result.current.viseme).toEqual(VISEMES.A);
|
||||
|
||||
// At 600ms we are into the second (now-stretched) frame.
|
||||
act(() => {
|
||||
fake.setMs(600);
|
||||
tickRaf();
|
||||
});
|
||||
expect(result.current.viseme).toEqual(VISEMES.M);
|
||||
});
|
||||
|
||||
it('uses wall-clock elapsed time when the audio currentTime is frozen', async () => {
|
||||
const fake = makePlayback(600);
|
||||
(synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
audio_base64: 'AAA=',
|
||||
audio_mime: 'audio/mpeg',
|
||||
visemes: [
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 200 },
|
||||
{ viseme: 'PP', start_ms: 200, end_ms: 400 },
|
||||
{ viseme: 'O', start_ms: 400, end_ms: 600 },
|
||||
],
|
||||
});
|
||||
(playBase64Audio as ReturnType<typeof vi.fn>).mockResolvedValueOnce(fake.handle);
|
||||
|
||||
const { result } = renderHook(() => useHumanMascot({ speakReplies: true }));
|
||||
await act(async () => {
|
||||
capturedListeners?.onDone?.(fakeDone('hello'));
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
// Leave fake.currentMs() at 0 to model CEF blob audio where currentTime
|
||||
// fails to advance even though playback is audible.
|
||||
act(() => {
|
||||
nowMs = 1_300;
|
||||
tickRaf();
|
||||
});
|
||||
expect(result.current.viseme).toEqual(VISEMES.M);
|
||||
});
|
||||
|
||||
it('starts lipsync before delayed audio metadata resolves', async () => {
|
||||
const fake = makePlaybackWithDeferredMetadata(600);
|
||||
(synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
audio_base64: 'AAA=',
|
||||
audio_mime: 'audio/mpeg',
|
||||
visemes: [
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 200 },
|
||||
{ viseme: 'PP', start_ms: 200, end_ms: 400 },
|
||||
],
|
||||
});
|
||||
(playBase64Audio as ReturnType<typeof vi.fn>).mockResolvedValueOnce(fake.handle);
|
||||
|
||||
const { result } = renderHook(() => useHumanMascot({ speakReplies: true }));
|
||||
await act(async () => {
|
||||
capturedListeners?.onDone?.(fakeDone('hi'));
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(result.current.face).toBe('speaking');
|
||||
act(() => {
|
||||
nowMs = 1_050;
|
||||
tickRaf();
|
||||
});
|
||||
expect(result.current.viseme).toEqual(VISEMES.A);
|
||||
|
||||
await act(async () => {
|
||||
fake.resolveMetadata();
|
||||
fake.finish();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
});
|
||||
|
||||
it('does not estimate pending metadata duration from collapsed frame ends', async () => {
|
||||
const fake = makePlaybackWithDeferredMetadata(900);
|
||||
(synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
audio_base64: 'AAA=',
|
||||
audio_mime: 'audio/mpeg',
|
||||
// Backend regression shape: starts collapsed at zero, short fixed ends.
|
||||
// Treating the final 80ms end as total duration compresses the whole
|
||||
// utterance before metadata is ready and leaves the mouth at rest.
|
||||
visemes: [
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 80 },
|
||||
{ viseme: 'PP', start_ms: 0, end_ms: 80 },
|
||||
{ viseme: 'O', start_ms: 0, end_ms: 80 },
|
||||
],
|
||||
});
|
||||
(playBase64Audio as ReturnType<typeof vi.fn>).mockResolvedValueOnce(fake.handle);
|
||||
|
||||
const { result } = renderHook(() => useHumanMascot({ speakReplies: true }));
|
||||
await act(async () => {
|
||||
capturedListeners?.onDone?.(fakeDone('metadata is still loading'));
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
act(() => {
|
||||
nowMs = 1_300;
|
||||
tickRaf();
|
||||
});
|
||||
expect(result.current.viseme).not.toEqual(VISEMES.REST);
|
||||
|
||||
await act(async () => {
|
||||
fake.resolveMetadata();
|
||||
fake.finish();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
});
|
||||
|
||||
it('mouth returns to a non-speaking shape once playback ends', async () => {
|
||||
const fake = makePlayback(500);
|
||||
(synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
|
||||
@@ -48,12 +48,20 @@ const proceduralVisemesMock = vi.fn(
|
||||
}
|
||||
);
|
||||
|
||||
vi.mock('./voice/ttsClient', () => ({
|
||||
synthesizeSpeech: vi.fn(),
|
||||
visemesFromAlignment: (alignment: { char: string; start_ms: number; end_ms: number }[]) =>
|
||||
alignment.map(a => ({ viseme: 'aa', start_ms: a.start_ms, end_ms: a.end_ms })),
|
||||
proceduralVisemes: (text: string, durationMs: number) => proceduralVisemesMock(text, durationMs),
|
||||
}));
|
||||
vi.mock('./voice/ttsClient', async importOriginal => {
|
||||
// Keep the real pure helpers (normalizeVisemeTimeline, hasUsableStarts, …) so
|
||||
// startTtsPlayback doesn't blow up on undefined imports; only stub the network
|
||||
// call and the two frame builders the tests want to control.
|
||||
const actual = await importOriginal<typeof import('./voice/ttsClient')>();
|
||||
return {
|
||||
...actual,
|
||||
synthesizeSpeech: vi.fn(),
|
||||
visemesFromAlignment: (alignment: { char: string; start_ms: number; end_ms: number }[]) =>
|
||||
alignment.map(a => ({ viseme: 'aa', start_ms: a.start_ms, end_ms: a.end_ms })),
|
||||
proceduralVisemes: (text: string, durationMs: number) =>
|
||||
proceduralVisemesMock(text, durationMs),
|
||||
};
|
||||
});
|
||||
|
||||
class FakeAudioStoppedError extends Error {
|
||||
readonly stopped = true;
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
swallowAudioStop,
|
||||
} from './voice/audioPlayer';
|
||||
import {
|
||||
hasUsableStarts,
|
||||
normalizeVisemeTimeline,
|
||||
proceduralVisemes,
|
||||
synthesizeSpeech,
|
||||
type VisemeFrame,
|
||||
@@ -32,6 +34,14 @@ const VISEME_DECAY_MS = 180;
|
||||
* 5 minutes comfortably covers any real reply; exported for tests.
|
||||
*/
|
||||
export const TTS_MAX_PLAYBACK_MS = 5 * 60 * 1_000;
|
||||
const TTS_ESTIMATED_MS_PER_CHAR = 55;
|
||||
const TTS_MIN_ESTIMATED_PLAYBACK_MS = 600;
|
||||
|
||||
function estimateTtsPlaybackMs(text: string, frames: VisemeFrame[]): number {
|
||||
const frameEnd = frames.at(-1)?.end_ms ?? 0;
|
||||
if (frameEnd > 0 && hasUsableStarts(frames)) return frameEnd;
|
||||
return Math.max(TTS_MIN_ESTIMATED_PLAYBACK_MS, text.trim().length * TTS_ESTIMATED_MS_PER_CHAR);
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristic — does this timeline contain at least one frame whose code maps
|
||||
@@ -310,6 +320,15 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
const visemeFramesRef = useRef<{ viseme: string; start_ms: number; end_ms: number }[]>([]);
|
||||
const visemeCursorRef = useRef(0);
|
||||
const playbackSeqRef = useRef(0);
|
||||
// Wall-clock anchor (performance.now() at the instant playback became
|
||||
// current) used to index the viseme timeline. We deliberately do NOT key off
|
||||
// `audio.currentTime`: in the embedded CEF webview it can fail to advance for
|
||||
// in-memory blob audio, which freezes the mouth on a single viseme even
|
||||
// though the audio plays. A monotonic clock always advances, and because the
|
||||
// viseme frames are rescaled to the measured audio duration it stays in sync.
|
||||
const playbackStartedAtRef = useRef(0);
|
||||
// Throttle marker for the lipsync diagnostic log (last logged ms).
|
||||
const lastLipsyncLogRef = useRef(0);
|
||||
|
||||
const [, force] = useState(0);
|
||||
|
||||
@@ -382,6 +401,16 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
mascotLog('voice-session text_delta suppressed — listening is active');
|
||||
return;
|
||||
}
|
||||
// When TTS is enabled the mouth is driven by the synthesized audio's
|
||||
// viseme timeline (see startTtsPlayback), locked to the audio clock.
|
||||
// The text-delta pseudo-lipsync exists only for the no-audio path — if
|
||||
// we let it run while replies are spoken, the mouth flaps along with
|
||||
// the streaming text *before and faster than* the voice. Stay at rest
|
||||
// during streaming so the only mouth motion is in sync with the audio.
|
||||
if (speakRef.current) {
|
||||
mascotLog('voice-session text_delta lipsync suppressed — TTS will drive the mouth');
|
||||
return;
|
||||
}
|
||||
if (playbackRef.current) return;
|
||||
clearAckTimer();
|
||||
setFace('speaking');
|
||||
@@ -419,6 +448,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
playbackSeqRef.current++;
|
||||
const orphan = playbackRef.current;
|
||||
playbackRef.current = null;
|
||||
playbackStartedAtRef.current = 0;
|
||||
if (orphan) {
|
||||
orphan.stop();
|
||||
orphan.ended.catch(swallowAudioStop);
|
||||
@@ -433,6 +463,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
playbackSeqRef.current++;
|
||||
const orphan = playbackRef.current;
|
||||
playbackRef.current = null;
|
||||
playbackStartedAtRef.current = 0;
|
||||
if (orphan) {
|
||||
orphan.stop();
|
||||
orphan.ended.catch(swallowAudioStop);
|
||||
@@ -454,6 +485,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
playbackSeqRef.current++;
|
||||
const orphan = playbackRef.current;
|
||||
playbackRef.current = null;
|
||||
playbackStartedAtRef.current = 0;
|
||||
if (orphan) {
|
||||
orphan.stop();
|
||||
orphan.ended.catch(swallowAudioStop);
|
||||
@@ -472,6 +504,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
): Promise<void> {
|
||||
const prev = playbackRef.current;
|
||||
playbackRef.current = null;
|
||||
playbackStartedAtRef.current = 0;
|
||||
if (prev) {
|
||||
prev.stop();
|
||||
prev.ended.catch(swallowAudioStop);
|
||||
@@ -499,13 +532,24 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
mascotLog('tts visemes produced no motion — dropping and falling through');
|
||||
frames = [];
|
||||
}
|
||||
if (frames.length === 0 && tts.alignment && tts.alignment.length > 0) {
|
||||
frames = visemesFromAlignment(tts.alignment);
|
||||
// The cloud path's viseme *timestamps* are frequently degenerate (all-zero
|
||||
// starts or a constant end), which loses the gaps between words/sentences.
|
||||
// The char-level alignment carries real timing — including those pauses —
|
||||
// so prefer it whenever the viseme starts don't form a usable timeline.
|
||||
const visemeStartsUsable = hasUsableStarts(frames);
|
||||
const haveAlignment = !!tts.alignment && tts.alignment.length > 0;
|
||||
if ((frames.length === 0 || !visemeStartsUsable) && haveAlignment) {
|
||||
frames = visemesFromAlignment(tts.alignment!);
|
||||
source = 'alignment';
|
||||
mascotLog('tts derived %d viseme frames from alignment', frames.length);
|
||||
} else if (frames.length > 0) {
|
||||
mascotLog('tts got %d viseme frames from backend', frames.length);
|
||||
}
|
||||
mascotLog(
|
||||
'tts synth visemes=%d alignment=%d startsUsable=%s → source=%s',
|
||||
tts.visemes?.length ?? 0,
|
||||
tts.alignment?.length ?? 0,
|
||||
visemeStartsUsable,
|
||||
source
|
||||
);
|
||||
const ttsOptions: PlaybackOptions = { maxDurationMs: TTS_MAX_PLAYBACK_MS };
|
||||
const handle = await playBase64Audio(
|
||||
tts.audio_base64,
|
||||
@@ -517,21 +561,66 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
handle.ended.catch(swallowAudioStop);
|
||||
return;
|
||||
}
|
||||
if (frames.length === 0) {
|
||||
const dur = handle.durationMs();
|
||||
frames = proceduralVisemes(text, dur);
|
||||
source = 'procedural';
|
||||
mascotLog('tts derived %d procedural viseme frames over %dms', frames.length, dur);
|
||||
// Start lipsync as soon as audio.play() succeeds. Metadata can lag by the
|
||||
// 500ms decoder fallback in blob/MP3 cases; keeping playbackRef empty
|
||||
// until then makes short replies play while the mascot is still thinking.
|
||||
let audioMs = handle.durationMs();
|
||||
const waitingForMetadata = audioMs <= 0;
|
||||
if (waitingForMetadata) {
|
||||
audioMs = estimateTtsPlaybackMs(text, frames);
|
||||
mascotLog(
|
||||
'tts duration pending — starting lipsync with %dms estimate',
|
||||
Math.round(audioMs)
|
||||
);
|
||||
}
|
||||
if (frames.length === 0) {
|
||||
frames = proceduralVisemes(text, audioMs);
|
||||
source = 'procedural';
|
||||
mascotLog('tts derived %d procedural viseme frames over %dms', frames.length, audioMs);
|
||||
}
|
||||
// Rebuild per-frame end times from the next frame's start. The backend can
|
||||
// ship a constant `end_ms` (= whole-utterance length) for every frame,
|
||||
// which freezes findActiveFrame on frame 0; this restores a walkable
|
||||
// cue timeline and stretches the last frame to the measured audio length.
|
||||
frames = normalizeVisemeTimeline(frames, audioMs);
|
||||
mascotLog(
|
||||
'tts normalized %d viseme frames (%s) over %dms',
|
||||
frames.length,
|
||||
source,
|
||||
Math.round(audioMs)
|
||||
);
|
||||
visemeFramesRef.current = frames;
|
||||
visemeCursorRef.current = 0;
|
||||
playbackRef.current = handle;
|
||||
playbackStartedAtRef.current = window.performance.now();
|
||||
lastLipsyncLogRef.current = -1_000;
|
||||
setFace('speaking');
|
||||
mascotLog(
|
||||
'tts playback started (%s) — driving lipsync from %d frames',
|
||||
source,
|
||||
frames.length
|
||||
);
|
||||
if (waitingForMetadata) {
|
||||
await handle.metadataReady;
|
||||
if (!isStillCurrent()) {
|
||||
handle.stop();
|
||||
handle.ended.catch(swallowAudioStop);
|
||||
return;
|
||||
}
|
||||
const measuredAudioMs = handle.durationMs();
|
||||
if (measuredAudioMs > 0 && playbackRef.current === handle) {
|
||||
const measuredFrames =
|
||||
source === 'procedural'
|
||||
? proceduralVisemes(text, measuredAudioMs)
|
||||
: normalizeVisemeTimeline(visemeFramesRef.current, measuredAudioMs);
|
||||
visemeFramesRef.current =
|
||||
source === 'procedural'
|
||||
? normalizeVisemeTimeline(measuredFrames, measuredAudioMs)
|
||||
: measuredFrames;
|
||||
visemeCursorRef.current = 0;
|
||||
mascotLog('tts metadata duration ready — refined lipsync to %dms', measuredAudioMs);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await handle.ended;
|
||||
} catch (err) {
|
||||
@@ -543,6 +632,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
} finally {
|
||||
if (isStillCurrent()) {
|
||||
playbackRef.current = null;
|
||||
playbackStartedAtRef.current = 0;
|
||||
visemeFramesRef.current = [];
|
||||
visemeCodeRef.current = 'sil';
|
||||
if (degraded) {
|
||||
@@ -569,7 +659,12 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
let visemeCode = 'sil';
|
||||
const playback = playbackRef.current;
|
||||
if (playback) {
|
||||
const ms = playback.currentMs();
|
||||
const audioMs = playback.currentMs();
|
||||
const wallClockMs =
|
||||
playbackStartedAtRef.current > 0
|
||||
? window.performance.now() - playbackStartedAtRef.current
|
||||
: audioMs;
|
||||
const ms = audioMs < 0 ? -1 : Math.max(audioMs, wallClockMs);
|
||||
if (ms >= 0) {
|
||||
const { frame, cursor } = findActiveFrame(
|
||||
visemeFramesRef.current,
|
||||
@@ -579,6 +674,16 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
visemeCursorRef.current = cursor;
|
||||
viseme = frame ? oculusVisemeToShape(frame.viseme) : VISEMES.REST;
|
||||
visemeCode = frame ? frame.viseme : 'sil';
|
||||
if (ms - lastLipsyncLogRef.current >= 500) {
|
||||
lastLipsyncLogRef.current = ms;
|
||||
mascotLog(
|
||||
'lipsync ms=%d cursor=%d/%d code=%s',
|
||||
Math.round(ms),
|
||||
cursor,
|
||||
visemeFramesRef.current.length,
|
||||
visemeCode
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if (face === 'speaking') {
|
||||
const since = window.performance.now() - lastDeltaAtRef.current;
|
||||
|
||||
@@ -2,6 +2,8 @@ import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { callCoreRpc } from '../../../services/coreRpcClient';
|
||||
import {
|
||||
hasUsableStarts,
|
||||
normalizeVisemeTimeline,
|
||||
prepareForSpeech,
|
||||
proceduralVisemes,
|
||||
synthesizeSpeech,
|
||||
@@ -178,3 +180,71 @@ describe('proceduralVisemes', () => {
|
||||
expect(short[0].end_ms - short[0].start_ms).toBeGreaterThanOrEqual(60);
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasUsableStarts', () => {
|
||||
it('is true for a spread-out, mostly-distinct timeline', () => {
|
||||
expect(
|
||||
hasUsableStarts([
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 100 },
|
||||
{ viseme: 'PP', start_ms: 100, end_ms: 200 },
|
||||
{ viseme: 'E', start_ms: 200, end_ms: 300 },
|
||||
])
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('is false when every start collapses to zero (degenerate backend timing)', () => {
|
||||
expect(
|
||||
hasUsableStarts([
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 80 },
|
||||
{ viseme: 'PP', start_ms: 0, end_ms: 80 },
|
||||
{ viseme: 'E', start_ms: 0, end_ms: 80 },
|
||||
])
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('is false for fewer than two frames', () => {
|
||||
expect(hasUsableStarts([{ viseme: 'aa', start_ms: 10, end_ms: 90 }])).toBe(false);
|
||||
expect(hasUsableStarts([])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeVisemeTimeline', () => {
|
||||
it('preserves a real timeline and keeps gaps as pauses', () => {
|
||||
// Real per-frame timing spanning the clip, with a gap 450→900 (a pause).
|
||||
const track = [
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 450 },
|
||||
{ viseme: 'PP', start_ms: 900, end_ms: 1000 },
|
||||
];
|
||||
const out = normalizeVisemeTimeline(track, 1000);
|
||||
// First frame keeps its real end (450) — the gap to 900 stays a pause.
|
||||
expect(out[0]).toEqual({ viseme: 'aa', start_ms: 0, end_ms: 450 });
|
||||
expect(out[1].start_ms).toBe(900);
|
||||
});
|
||||
|
||||
it('clamps an overrunning end to the next cue start', () => {
|
||||
const track = [
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 9999 }, // overruns next
|
||||
{ viseme: 'PP', start_ms: 600, end_ms: 1000 },
|
||||
];
|
||||
const out = normalizeVisemeTimeline(track, 1000);
|
||||
expect(out[0].end_ms).toBe(600);
|
||||
});
|
||||
|
||||
it('evenly distributes the sequence when timestamps are degenerate', () => {
|
||||
// All-zero starts → unusable → spread evenly across the audio duration.
|
||||
const track = [
|
||||
{ viseme: 'aa', start_ms: 0, end_ms: 80 },
|
||||
{ viseme: 'PP', start_ms: 0, end_ms: 80 },
|
||||
{ viseme: 'E', start_ms: 0, end_ms: 80 },
|
||||
{ viseme: 'oh', start_ms: 0, end_ms: 80 },
|
||||
];
|
||||
const out = normalizeVisemeTimeline(track, 1000);
|
||||
expect(out.map(f => f.viseme)).toEqual(['aa', 'PP', 'E', 'oh']);
|
||||
expect(out[0]).toEqual({ viseme: 'aa', start_ms: 0, end_ms: 250 });
|
||||
expect(out[3]).toEqual({ viseme: 'oh', start_ms: 750, end_ms: 1000 });
|
||||
});
|
||||
|
||||
it('returns empty frames untouched', () => {
|
||||
expect(normalizeVisemeTimeline([], 1000)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -115,6 +115,69 @@ export function visemesFromAlignment(alignment: AlignmentFrame[]): VisemeFrame[]
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Does this viseme track carry a genuinely usable per-frame start timeline?
|
||||
*
|
||||
* The cloud TTS path sometimes ships visemes with all-zero (or otherwise
|
||||
* collapsed) `start_ms`, which can't represent the gaps between words. We treat
|
||||
* the timing as usable only when there are at least two frames, the furthest
|
||||
* start is non-zero, and most starts are distinct (a real, spread-out timeline).
|
||||
*/
|
||||
export function hasUsableStarts(frames: VisemeFrame[]): boolean {
|
||||
if (frames.length < 2) return false;
|
||||
const maxStart = Math.max(...frames.map(f => f.start_ms));
|
||||
if (!(maxStart > 0)) return false;
|
||||
const distinct = new Set(frames.map(f => f.start_ms)).size;
|
||||
return distinct > frames.length * 0.5;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a backend viseme list into a walkable timeline matched to the audio.
|
||||
*
|
||||
* The cloud TTS path ships viseme *codes* in order but with unreliable
|
||||
* timestamps — observed shapes include a constant `end_ms` (= whole-utterance
|
||||
* length) on every frame, and all-zero `start_ms`. Either way the naive
|
||||
* `findActiveFrame` freezes the mouth (frame 0 "covers" everything, or every
|
||||
* frame is zero-length so it snaps to the last `sil`).
|
||||
*
|
||||
* Strategy:
|
||||
* - If the frames carry a genuinely usable timeline — strictly-ish increasing
|
||||
* starts that span most of the audio — keep it, just rebuilding each end from
|
||||
* the next start (visemes are cue points) and stretching the last to the end.
|
||||
* - Otherwise distribute the viseme *sequence* evenly across the measured audio
|
||||
* duration. We lose the natural per-phoneme rhythm but keep the real phonetic
|
||||
* order, so the mouth animates start-to-finish in lockstep with the voice.
|
||||
*/
|
||||
export function normalizeVisemeTimeline(frames: VisemeFrame[], totalMs: number): VisemeFrame[] {
|
||||
if (frames.length === 0) return frames;
|
||||
const sorted = [...frames].sort((a, b) => a.start_ms - b.start_ms);
|
||||
const maxStart = sorted[sorted.length - 1].start_ms;
|
||||
const total = totalMs > 0 ? totalMs : Math.max(maxStart, frames.length * 80);
|
||||
|
||||
// A real timeline's last cue starts deep into the clip. If the furthest start
|
||||
// is in the front portion (e.g. every start is 0), the timestamps are junk —
|
||||
// fall back to even distribution.
|
||||
const hasUsableTimeline = maxStart > total * 0.5;
|
||||
if (hasUsableTimeline) {
|
||||
// Preserve the real per-frame end. A gap between one frame's end and the
|
||||
// next frame's start is a genuine pause — findActiveFrame returns no frame
|
||||
// there, so the mouth rests (sil) instead of holding a shape across the
|
||||
// silence. We only clamp ends so a frame never overruns the next cue.
|
||||
return sorted.map((f, i) => {
|
||||
const next = sorted[i + 1];
|
||||
const end = next ? Math.min(f.end_ms, next.start_ms) : Math.max(f.end_ms, total);
|
||||
return { viseme: f.viseme, start_ms: f.start_ms, end_ms: Math.max(f.start_ms, end) };
|
||||
});
|
||||
}
|
||||
|
||||
const step = total / frames.length;
|
||||
return sorted.map((f, i) => ({
|
||||
viseme: f.viseme,
|
||||
start_ms: Math.round(i * step),
|
||||
end_ms: Math.round((i + 1) * step),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reshape an assistant message into something the TTS engine can read with
|
||||
* natural cadence. The agent's reply is markdown — raw `**bold**`, headings,
|
||||
|
||||
@@ -4748,6 +4748,7 @@ const messages: TranslationMap = {
|
||||
'اعرض ردود المساعد كنص بلا إطار مع إبقاء رسائلك داخل فقاعات.',
|
||||
'settings.mascot.active': 'نشط',
|
||||
'settings.mascot.characterDesc': 'وصف الشخصية',
|
||||
'settings.mascot.characterDraft': 'مسودة',
|
||||
'settings.mascot.characterHeading': 'عنوان الشخصية',
|
||||
'settings.mascot.customGifError':
|
||||
'أدخل HTTPS .gif URL، أو الاسترجاع HTTP .gif URL، أو file:// .gif URL، أو مسار .gif المحلي.',
|
||||
|
||||
@@ -4846,6 +4846,7 @@ const messages: TranslationMap = {
|
||||
'আপনার বার্তাগুলি বাবলে রেখে অ্যাসিস্ট্যান্টের উত্তর ফ্রেমহীন টেক্সট হিসেবে দেখান।',
|
||||
'settings.mascot.active': 'সক্রিয়',
|
||||
'settings.mascot.characterDesc': 'চরিত্রের বিবরণ',
|
||||
'settings.mascot.characterDraft': 'খসড়া',
|
||||
'settings.mascot.characterHeading': 'চরিত্রের শিরোনাম',
|
||||
'settings.mascot.customGifError':
|
||||
'একটি HTTPS .gif URL, লুপব্যাক HTTP .gif URL, file:// .gif URL, অথবা স্থানীয় .gif পাথ লিখুন।',
|
||||
|
||||
@@ -4971,6 +4971,7 @@ const messages: TranslationMap = {
|
||||
'Zeigt Assistentenantworten als ungerahmten Text an und lässt deine Nachrichten in Blasen.',
|
||||
'settings.mascot.active': 'Aktiv',
|
||||
'settings.mascot.characterDesc': 'Charakterbeschreibung',
|
||||
'settings.mascot.characterDraft': 'Entwurf',
|
||||
'settings.mascot.characterHeading': 'Zeichenüberschrift',
|
||||
'settings.mascot.customGifError':
|
||||
'GIF konnte nicht geladen werden. Bitte überprüfe die URL und versuche es erneut.',
|
||||
|
||||
@@ -5483,6 +5483,7 @@ const en: TranslationMap = {
|
||||
'Collapse the live step-by-step agent timeline in chat. A blinking “Processing” link still lets you open the full run.',
|
||||
'settings.mascot.active': 'Active',
|
||||
'settings.mascot.characterDesc': 'Choose your OpenHuman character.',
|
||||
'settings.mascot.characterDraft': 'Draft',
|
||||
'settings.mascot.characterHeading': 'Character',
|
||||
'settings.mascot.customGifError':
|
||||
'Enter an HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL, or local .gif path.',
|
||||
|
||||
@@ -4938,6 +4938,7 @@ const messages: TranslationMap = {
|
||||
'Muestra las respuestas del asistente como texto sin marco y mantiene tus mensajes en burbujas.',
|
||||
'settings.mascot.active': 'Activo',
|
||||
'settings.mascot.characterDesc': 'Descripción del personaje',
|
||||
'settings.mascot.characterDraft': 'Borrador',
|
||||
'settings.mascot.characterHeading': 'Encabezado del personaje',
|
||||
'settings.mascot.customGifError':
|
||||
'Introduzca una ruta HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL o ruta .gif local.',
|
||||
|
||||
@@ -4957,6 +4957,7 @@ const messages: TranslationMap = {
|
||||
"Affiche les réponses de l'assistant en texte sans cadre tout en gardant vos messages en bulles.",
|
||||
'settings.mascot.active': 'Actif',
|
||||
'settings.mascot.characterDesc': 'Description du personnage',
|
||||
'settings.mascot.characterDraft': 'Brouillon',
|
||||
'settings.mascot.characterHeading': 'Titre du personnage',
|
||||
'settings.mascot.customGifError':
|
||||
'Entrez un HTTPS .gif URL, un bouclage HTTP .gif URL, un fichier:// .gif URL ou un chemin local .gif.',
|
||||
|
||||
@@ -4851,6 +4851,7 @@ const messages: TranslationMap = {
|
||||
'असिस्टेंट के जवाबों को बिना फ्रेम वाले टेक्स्ट के रूप में दिखाएं और आपके संदेश बबल में रखें।',
|
||||
'settings.mascot.active': 'एक्टिव',
|
||||
'settings.mascot.characterDesc': 'कैरेक्टर विवरण',
|
||||
'settings.mascot.characterDraft': 'ड्राफ़्ट',
|
||||
'settings.mascot.characterHeading': 'कैरेक्टर शीर्षक',
|
||||
'settings.mascot.customGifError':
|
||||
'एक HTTPS .gif URL, लूपबैक HTTP .gif URL, फ़ाइल:// .gif URL, या स्थानीय .gif पथ दर्ज करें।',
|
||||
|
||||
@@ -4863,6 +4863,7 @@ const messages: TranslationMap = {
|
||||
'Tampilkan balasan asisten sebagai teks tanpa bingkai, sementara pesan Anda tetap dalam gelembung.',
|
||||
'settings.mascot.active': 'Aktif',
|
||||
'settings.mascot.characterDesc': 'Deskripsi karakter',
|
||||
'settings.mascot.characterDraft': 'Draf',
|
||||
'settings.mascot.characterHeading': 'Judul karakter',
|
||||
'settings.mascot.customGifError':
|
||||
'Masukkan HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL, atau jalur .gif lokal.',
|
||||
|
||||
@@ -4928,6 +4928,7 @@ const messages: TranslationMap = {
|
||||
"Mostra le risposte dell'assistente come testo senza cornice mantenendo i tuoi messaggi nei fumetti.",
|
||||
'settings.mascot.active': 'Attivo',
|
||||
'settings.mascot.characterDesc': 'Descrizione personaggio',
|
||||
'settings.mascot.characterDraft': 'Bozza',
|
||||
'settings.mascot.characterHeading': 'Intestazione personaggio',
|
||||
'settings.mascot.customGifError':
|
||||
'Immettere un HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL o un percorso .gif locale.',
|
||||
|
||||
@@ -4798,6 +4798,7 @@ const messages: TranslationMap = {
|
||||
'사용자 메시지는 말풍선으로 유지하고 어시스턴트 답변은 프레임 없는 텍스트로 표시합니다.',
|
||||
'settings.mascot.active': '활성',
|
||||
'settings.mascot.characterDesc': '캐릭터 설명',
|
||||
'settings.mascot.characterDraft': '초안',
|
||||
'settings.mascot.characterHeading': '캐릭터 제목',
|
||||
'settings.mascot.customGifError':
|
||||
'HTTPS .gif URL, loopback HTTP .gif URL, file:// .gif URL 또는 로컬 .gif 경로를 입력하세요.',
|
||||
|
||||
@@ -4917,6 +4917,7 @@ const messages: TranslationMap = {
|
||||
'Wyświetla odpowiedzi asystenta jako tekst bez ramki, a Twoje wiadomości pozostawia w dymkach.',
|
||||
'settings.mascot.active': 'Aktywny',
|
||||
'settings.mascot.characterDesc': 'Wybierz charakter maskotki OpenHuman.',
|
||||
'settings.mascot.characterDraft': 'Wersja robocza',
|
||||
'settings.mascot.characterHeading': 'Charakter',
|
||||
'settings.mascot.customGifError':
|
||||
'Wprowadź URL .gif HTTPS, URL .gif loopback HTTP, URL file:// .gif lub lokalną ścieżkę .gif.',
|
||||
|
||||
@@ -4931,6 +4931,7 @@ const messages: TranslationMap = {
|
||||
'Renderiza as respostas do assistente como texto sem moldura e mantém suas mensagens em balões.',
|
||||
'settings.mascot.active': 'Ativo',
|
||||
'settings.mascot.characterDesc': 'Descrição do personagem',
|
||||
'settings.mascot.characterDraft': 'Rascunho',
|
||||
'settings.mascot.characterHeading': 'Título do personagem',
|
||||
'settings.mascot.customGifError':
|
||||
'Insira um caminho HTTPS .gif URL, loopback HTTP .gif URL, arquivo:// .gif URL ou .gif local.',
|
||||
|
||||
@@ -4891,6 +4891,7 @@ const messages: TranslationMap = {
|
||||
'Показывает ответы ассистента как текст без рамки, оставляя ваши сообщения в пузырьках.',
|
||||
'settings.mascot.active': 'Активно',
|
||||
'settings.mascot.characterDesc': 'Описание персонажа',
|
||||
'settings.mascot.characterDraft': 'Черновик',
|
||||
'settings.mascot.characterHeading': 'Персонаж',
|
||||
'settings.mascot.customGifError':
|
||||
'Введите HTTPS .gif URL, петлевой путь HTTP .gif URL, file:// .gif URL или локальный путь .gif.',
|
||||
|
||||
@@ -4608,6 +4608,7 @@ const messages: TranslationMap = {
|
||||
'settings.appearance.assistantTextModeDesc': '将助手回复渲染为无边框文本，同时保留你的消息气泡。',
|
||||
'settings.mascot.active': '活跃',
|
||||
'settings.mascot.characterDesc': '选择你的 OpenHuman 角色',
|
||||
'settings.mascot.characterDraft': '草稿',
|
||||
'settings.mascot.characterHeading': '角色',
|
||||
'settings.mascot.customGifError':
|
||||
'输入 HTTPS .gif 链接、本地回环 HTTP .gif 链接、file:// .gif 链接或本地 .gif 路径。',
|
||||
|
||||
@@ -159,13 +159,14 @@ const persistedThreadReducer = persistReducer(threadPersistConfig, threadReducer
|
||||
const layoutPersistConfig = { key: 'layout', storage, whitelist: ['panels'] };
|
||||
const persistedLayoutReducer = persistReducer(layoutPersistConfig, layoutReducer);
|
||||
|
||||
// Persist only previously persisted mascot appearance fields plus the custom
|
||||
// GIF override added by this feature; leave existing non-persisted mascot
|
||||
// fields as runtime state to avoid changing refresh behavior.
|
||||
// Persist the mascot appearance fields, the custom GIF override, and the
|
||||
// selected mascot id (so the chosen GitHub-manifest mascot survives a reload —
|
||||
// the slice's REHYDRATE guard re-validates it). Other mascot fields stay as
|
||||
// runtime state.
|
||||
const mascotPersistConfig = {
|
||||
key: 'mascot',
|
||||
storage,
|
||||
whitelist: ['color', 'voiceId', 'customMascotGifUrl'],
|
||||
whitelist: ['color', 'voiceId', 'customMascotGifUrl', 'selectedMascotId'],
|
||||
};
|
||||
const persistedMascotReducer = persistReducer(mascotPersistConfig, mascotReducer);
|
||||
|
||||
|
||||
@@ -110,12 +110,12 @@ export interface MascotState {
|
||||
*/
|
||||
voiceUseLocaleDefault: boolean;
|
||||
/**
|
||||
* Server-side mascot id selected from the backend mascot library
|
||||
* (PR tinyhumansai/backend#770). `null` keeps the local YellowMascot
|
||||
* renderer; any non-empty value tells `BackendMascot` (loaded via
|
||||
* `mascotService`) to take over. The id is opaque server-side and
|
||||
* length-capped at the same threshold as voiceId to keep the
|
||||
* persisted blob bounded.
|
||||
* Mascot id selected from the published GitHub manifest
|
||||
* (`tinyhumansai/mascots`, resolved via `useMascotManifest`). `null` falls
|
||||
* back to the manifest's default (first `ready`) mascot; any non-empty value
|
||||
* pins that specific mascot. The id is the manifest entry id (e.g.
|
||||
* `tiny-mascot`) and length-capped at the same threshold as voiceId to keep
|
||||
* the persisted blob bounded.
|
||||
*/
|
||||
selectedMascotId: string | null;
|
||||
/**
|
||||
|
||||
@@ -221,6 +221,8 @@ vi.mock('../utils/config', () => ({
|
||||
DEV_JWT_TOKEN: undefined,
|
||||
MASCOT_VOICE_ID: 'JBFqnCBsd6RMkjVDRZzb',
|
||||
MASCOT_VOICE_MODEL_ID: 'eleven_multilingual_v2',
|
||||
MASCOT_MANIFEST_URL:
|
||||
'https://raw.githubusercontent.com/tinyhumansai/mascots/main/dist/mascots.json',
|
||||
}));
|
||||
|
||||
vi.mock('../services/backendUrl', () => ({
|
||||
|
||||
@@ -240,3 +240,16 @@ export const MASCOT_VOICE_ID =
|
||||
export const MASCOT_VOICE_MODEL_ID =
|
||||
(import.meta.env.VITE_MASCOT_VOICE_MODEL_ID as string | undefined)?.trim() ||
|
||||
'eleven_multilingual_v2';
|
||||
|
||||
/**
|
||||
* URL of the published mascot manifest (`dist/mascots.json` from the
|
||||
* `tinyhumansai/mascots` repo). This is the authoritative source for the
|
||||
* in-app mascot library — each entry names a Rive `.riv` runtime file plus its
|
||||
* `stateEngine` (poses, viseme codes, channels). Fetched directly over HTTPS
|
||||
* (the `raw.githubusercontent.com` host is CORS-open and allowed by the
|
||||
* webview CSP's `connect-src https:`). Override with `VITE_MASCOT_MANIFEST_URL`
|
||||
* to point at a fork or a locally-served manifest during development.
|
||||
*/
|
||||
export const MASCOT_MANIFEST_URL =
|
||||
(import.meta.env.VITE_MASCOT_MANIFEST_URL as string | undefined)?.trim() ||
|
||||
'https://raw.githubusercontent.com/tinyhumansai/mascots/main/dist/mascots.json';
|
||||
|
||||
@@ -42,7 +42,9 @@ async function getDefaultMessagingChannel(page: Page): Promise<string | null> {
|
||||
async function getMascotVoiceId(page: Page): Promise<string | null> {
|
||||
return page.evaluate(() => {
|
||||
const win = window as unknown as {
|
||||
__OPENHUMAN_STORE__?: { getState?: () => { mascot: { voiceId?: string | null } } };
|
||||
__OPENHUMAN_STORE__?: {
|
||||
getState?: () => { mascot: { selectedMascotId?: string | null; voiceId?: string | null } };
|
||||
};
|
||||
};
|
||||
const state = win.__OPENHUMAN_STORE__?.getState?.();
|
||||
if (!state) {
|
||||
@@ -52,6 +54,19 @@ async function getMascotVoiceId(page: Page): Promise<string | null> {
|
||||
});
|
||||
}
|
||||
|
||||
async function getSelectedMascotId(page: Page): Promise<string | null> {
|
||||
return page.evaluate(() => {
|
||||
const win = window as unknown as {
|
||||
__OPENHUMAN_STORE__?: { getState?: () => { mascot: { selectedMascotId?: string | null } } };
|
||||
};
|
||||
const state = win.__OPENHUMAN_STORE__?.getState?.();
|
||||
if (!state) {
|
||||
throw new Error('__OPENHUMAN_STORE__ is unavailable');
|
||||
}
|
||||
return state.mascot.selectedMascotId ?? null;
|
||||
});
|
||||
}
|
||||
|
||||
async function getPersistedMascotColor(page: Page): Promise<string | null> {
|
||||
return page.evaluate(() => {
|
||||
const userId = localStorage.getItem('OPENHUMAN_ACTIVE_USER_ID');
|
||||
@@ -71,11 +86,83 @@ async function getPersistedMascotColor(page: Page): Promise<string | null> {
|
||||
});
|
||||
}
|
||||
|
||||
async function getPersistedSelectedMascotId(page: Page): Promise<string | null> {
|
||||
return page.evaluate(() => {
|
||||
const userId = localStorage.getItem('OPENHUMAN_ACTIVE_USER_ID');
|
||||
if (!userId) return null;
|
||||
|
||||
const raw = localStorage.getItem(`${userId}:persist:mascot`);
|
||||
if (!raw) return null;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { selectedMascotId?: unknown };
|
||||
if (typeof parsed.selectedMascotId !== 'string') return null;
|
||||
const selectedMascotId = JSON.parse(parsed.selectedMascotId) as unknown;
|
||||
return typeof selectedMascotId === 'string' ? selectedMascotId : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function getAriaChecked(page: Page, label: string): Promise<string | null> {
|
||||
const value = await page.getByRole('switch', { name: label }).getAttribute('aria-checked');
|
||||
return value;
|
||||
}
|
||||
|
||||
async function installMascotManifestMock(page: Page): Promise<void> {
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: '2026-06-29T00:00:00.000Z',
|
||||
source: { repository: 'tinyhumansai/mascots', branch: 'main', commit: 'e2e' },
|
||||
mascots: [
|
||||
{
|
||||
id: 'tiny-mascot',
|
||||
name: 'Tiny Default',
|
||||
status: 'ready',
|
||||
files: [
|
||||
{
|
||||
role: 'runtime',
|
||||
path: 'dist/tiny.riv',
|
||||
url: 'https://example.test/mascots/tiny.riv',
|
||||
sha256: 'tiny-runtime-sha',
|
||||
},
|
||||
],
|
||||
stateEngine: {
|
||||
states: { idle: 'idle', thinking: 'thinking', speaking: 'speaking', writing: 'writing' },
|
||||
idlePoseCycle: ['idle'],
|
||||
visemeCodes: ['sil', 'aa', 'oh'],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'river-guide',
|
||||
name: 'River Guide',
|
||||
status: 'draft',
|
||||
files: [
|
||||
{
|
||||
role: 'runtime',
|
||||
path: 'dist/river.riv',
|
||||
url: 'https://example.test/mascots/river.riv',
|
||||
sha256: 'river-runtime-sha',
|
||||
},
|
||||
],
|
||||
stateEngine: {
|
||||
states: { idle: 'idle', thinking: 'thinking', speaking: 'speaking', writing: 'writing' },
|
||||
idlePoseCycle: ['idle', 'wave'],
|
||||
visemeCodes: ['sil', 'aa', 'oh'],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
await page.route('https://raw.githubusercontent.com/tinyhumansai/mascots/**', route =>
|
||||
route.fulfill({ contentType: 'application/json', body: JSON.stringify(manifest) })
|
||||
);
|
||||
await page.route('https://example.test/mascots/*.riv', route =>
|
||||
route.fulfill({ contentType: 'application/octet-stream', body: Buffer.from('not-a-rive-file') })
|
||||
);
|
||||
}
|
||||
|
||||
interface ToolsSnapshot {
|
||||
result?: { localState?: { onboardingTasks?: { enabledTools?: string[] | null } | null } | null };
|
||||
localState?: { onboardingTasks?: { enabledTools?: string[] | null } | null } | null;
|
||||
@@ -211,6 +298,32 @@ test.describe('Settings - Feature Preferences', () => {
|
||||
await expect(page.getByTestId('mascot-color-burgundy')).toHaveAttribute('aria-checked', 'true');
|
||||
});
|
||||
|
||||
test('persists manifest mascot selection and uses it on the Human page', async ({ page }) => {
|
||||
await installMascotManifestMock(page);
|
||||
await openAuthenticatedRoute(page, 'pw-settings-manifest-mascot', '/settings/mascot');
|
||||
|
||||
await expect(page.getByRole('heading', { name: 'Character', exact: true })).toBeVisible();
|
||||
await expect(page.getByTestId('manifest-mascot-river-guide')).toContainText('River Guide');
|
||||
await page.getByTestId('manifest-mascot-river-guide').click();
|
||||
await expect(page.getByTestId('manifest-mascot-river-guide')).toHaveAttribute(
|
||||
'aria-pressed',
|
||||
'true'
|
||||
);
|
||||
await expect.poll(() => getSelectedMascotId(page)).toBe('river-guide');
|
||||
await expect.poll(() => getPersistedSelectedMascotId(page)).toBe('river-guide');
|
||||
|
||||
await reloadAndWait(page);
|
||||
await expect.poll(() => getSelectedMascotId(page)).toBe('river-guide');
|
||||
await expect(page.getByTestId('manifest-mascot-river-guide')).toHaveAttribute(
|
||||
'aria-pressed',
|
||||
'true'
|
||||
);
|
||||
|
||||
await page.goto('/#/human');
|
||||
await waitForAppReady(page);
|
||||
await expect.poll(() => getSelectedMascotId(page)).toBe('river-guide');
|
||||
});
|
||||
|
||||
test('persists the custom mascot voice override on the voice panel', async ({ page }) => {
|
||||
await openAuthenticatedRoute(page, 'pw-settings-mascot-voice', '/settings/voice');
|
||||
|
||||
|
||||
Reference in New Issue
Block a user