feat(mascot): load Human mascots from GitHub manifest (#4312)

This commit is contained in:
Steven Enamakel
2026-06-29 20:46:20 -07:00
committed by GitHub
parent 89be906d20
commit d746fec3ef
43 changed files with 2367 additions and 217 deletions
@@ -1,13 +1,7 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { CustomGifMascot, RiveMascot } from '../../../features/human/Mascot';
import { BackendMascot } from '../../../features/human/Mascot/backend/BackendMascot';
import { BackendRiveMascot } from '../../../features/human/Mascot/backend/BackendRiveMascot';
import {
isRiveMascotDetail,
type MascotDetailUnion,
type MascotSummary,
} from '../../../features/human/Mascot/backend/types';
import { CustomGifMascot, ManifestRiveMascot, RiveMascot } from '../../../features/human/Mascot';
import { useMascotManifest } from '../../../features/human/Mascot/manifest/useMascotManifest';
import {
getMascotPalette,
hexToArgbInt,
@@ -15,7 +9,6 @@ import {
} from '../../../features/human/Mascot/mascotPalette';
import { synthesizeSpeech } from '../../../features/human/voice/ttsClient';
import { useT } from '../../../lib/i18n/I18nContext';
import { fetchMascotList, getCachedMascotDetail } from '../../../services/mascotService';
import { useAppDispatch, useAppSelector } from '../../../store/hooks';
import {
DEFAULT_MASCOT_COLOR,
@@ -83,13 +76,16 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
const useLocaleDefault = useAppSelector(selectMascotVoiceUseLocaleDefault);
const effectiveVoiceId = useAppSelector(selectEffectiveMascotVoiceId);
// Backend mascot library (PR tinyhumansai/backend#770). The list endpoint
// is cheap (no SVG bytes); per-id detail is fetched on demand so the
// animated preview only pays for the active selection.
const [backendList, setBackendList] = useState<MascotSummary[] | null>(null);
const [backendListError, setBackendListError] = useState<string | null>(null);
const [activeDetail, setActiveDetail] = useState<MascotDetailUnion | null>(null);
const [detailError, setDetailError] = useState<string | null>(null);
// Mascot library, sourced from the published GitHub manifest
// (tinyhumansai/mascots). `entry` is the resolved active mascot (selection
// or default); `manifest.mascots` drives the picker list. Each entry carries
// its full stateEngine inline, so there is no per-id detail round trip.
const {
manifest,
entry: activeEntry,
loading: manifestLoading,
error: manifestError,
} = useMascotManifest();
const [customGifDraft, setCustomGifDraft] = useState<string>(customMascotGifUrl ?? '');
const [customGifError, setCustomGifError] = useState<string | null>(null);
@@ -108,45 +104,6 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
// earlier audio-only cleanup missed.
const previewRequestIdRef = useRef(0);
useEffect(() => {
let cancelled = false;
fetchMascotList()
.then(list => {
if (cancelled) return;
setBackendList(list);
setBackendListError(null);
})
.catch((err: unknown) => {
if (cancelled) return;
const message = err instanceof Error ? err.message : t('settings.mascot.loadLibraryError');
setBackendListError(message);
setBackendList([]);
});
return () => {
cancelled = true;
};
}, [t]);
useEffect(() => {
if (!selectedMascotId) return;
let cancelled = false;
getCachedMascotDetail(selectedMascotId)
.then(detail => {
if (cancelled) return;
setActiveDetail(detail);
setDetailError(null);
})
.catch((err: unknown) => {
if (cancelled) return;
const message = err instanceof Error ? err.message : t('settings.mascot.loadDetailError');
setDetailError(message);
setActiveDetail(null);
});
return () => {
cancelled = true;
};
}, [selectedMascotId, t]);
// Stop any in-flight preview audio when the panel unmounts. Also
// bump the preview request id so a `synthesizeSpeech(...)` that
// resolves after unmount can detect the staleness and bail.
@@ -161,15 +118,14 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
};
}, []);
const handleSelectBackend = (id: string | null) => {
const handleSelectMascot = (id: string | null) => {
dispatch(setSelectedMascotId(id));
setCustomGifError(null);
if (id == null) {
setCustomGifDraft('');
dispatch(setCustomMascotGifUrl(null));
} else {
setCustomGifDraft('');
}
setCustomGifDraft('');
// Selecting a mascot id already clears the custom GIF in the reducer; the
// null ("default") case has to clear it here so the stage falls back to
// the default manifest mascot rather than the GIF.
if (id == null) dispatch(setCustomMascotGifUrl(null));
};
const onSaveCustomGif = () => {
@@ -305,8 +261,6 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
const presetPickerDisabled = useLocaleDefault;
const isCustomVoice =
!presetPickerDisabled && (voicePasteMode || !isCuratedVoicePreset(effectiveVoiceId));
const visibleActiveDetail = selectedMascotId ? activeDetail : null;
const visibleDetailError = selectedMascotId ? detailError : null;
const activePalette = getMascotPalette(activeColor);
const primaryColorArgb = useMemo(
@@ -629,25 +583,25 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
)}
</div>
{/* Backend mascot library */}
{/* Mascot manifest library (tinyhumansai/mascots) */}
<div className="bg-surface rounded-xl border border-line overflow-hidden">
{backendListError && (
{manifestError && (
<p className="p-4 text-sm text-coral-700 dark:text-coral-300">
{t('settings.mascot.libraryUnavailable')}: {backendListError}
{t('settings.mascot.libraryUnavailable')}: {manifestError.message}
</p>
)}
{!backendListError && backendList === null && (
{!manifestError && manifestLoading && (
<p className="p-4 text-sm text-content-muted">{t('settings.mascot.loadingLibrary')}</p>
)}
{backendList && backendList.length === 0 && !backendListError && (
{manifest && manifest.mascots.length === 0 && !manifestError && (
<p className="p-4 text-sm text-content-muted">{t('settings.mascot.noCharacters')}</p>
)}
{backendList && backendList.length > 0 && (
{manifest && manifest.mascots.length > 0 && (
<ul className="divide-y divide-line-subtle dark:divide-neutral-800">
<li>
<button
type="button"
onClick={() => handleSelectBackend(null)}
onClick={() => handleSelectMascot(null)}
aria-pressed={selectedMascotId == null && customMascotGifUrl == null}
className={`flex w-full items-center justify-between px-4 py-3 text-left text-sm hover:bg-surface-hover ${
selectedMascotId == null && customMascotGifUrl == null
@@ -662,29 +616,35 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
)}
</button>
</li>
{backendList.map(summary => {
const active = summary.id === selectedMascotId;
{manifest.mascots.map(mascot => {
const active = mascot.id === selectedMascotId;
const poseCount = new Set([
...mascot.stateEngine.idlePoseCycle,
...Object.values(mascot.stateEngine.states),
]).size;
const visemeCount = mascot.stateEngine.visemeCodes.length;
return (
<li key={summary.id}>
<li key={mascot.id}>
<button
type="button"
onClick={() => handleSelectBackend(summary.id)}
onClick={() => handleSelectMascot(mascot.id)}
aria-pressed={active}
data-testid={`backend-mascot-${summary.id}`}
data-testid={`manifest-mascot-${mascot.id}`}
className={`flex w-full items-center justify-between px-4 py-3 text-left text-sm hover:bg-surface-hover ${
active ? 'bg-surface-muted font-medium' : ''
}`}>
<span className="flex flex-col">
<span>{summary.name}</span>
<span className="flex items-center gap-2">
{mascot.name}
{mascot.status === 'draft' && (
<span className="rounded bg-amber-100 px-1.5 py-0.5 text-[9px] font-medium uppercase tracking-wide text-amber-700 dark:bg-amber-500/20 dark:text-amber-200">
{t('settings.mascot.characterDraft')}
</span>
)}
</span>
<span className="text-[10px] text-content-muted">
v{summary.version}
{summary.format === 'rive'
? ` · ${Object.keys(summary.stateToPose ?? {}).length} ${t('settings.mascot.characterStates')}`
: ` · ${summary.states?.length ?? 0} ${t('settings.mascot.characterStates')}${
summary.hasVisemes
? ` · ${t('settings.mascot.characterVisemes')}`
: ''
}`}
{poseCount} {t('settings.mascot.characterStates')} · {visemeCount}{' '}
{t('settings.mascot.characterVisemes')}
</span>
</span>
{active && (
@@ -700,31 +660,25 @@ const MascotPanel = ({ embedded = false }: MascotPanelProps) => {
)}
</div>
{visibleActiveDetail && (
{activeEntry && !customMascotGifUrl && (
<div className="mt-3 rounded-xl border border-line bg-surface-muted p-4">
<p className="text-[11px] font-medium uppercase tracking-wide text-content-muted mb-2">
{t('settings.mascot.characterPreview')} · {visibleActiveDetail.name}
{t('settings.mascot.characterPreview')} · {activeEntry.name}
</p>
<div className="flex justify-center">
<div style={{ width: 160, height: 160 }}>
{isRiveMascotDetail(visibleActiveDetail) ? (
<BackendRiveMascot
key={visibleActiveDetail.id}
mascotId={visibleActiveDetail.id}
size={160}
/>
) : (
<BackendMascot mascot={visibleActiveDetail} />
)}
<ManifestRiveMascot
key={activeEntry.id}
entry={activeEntry}
size={160}
primaryColor={primaryColorArgb}
secondaryColor={secondaryColorArgb}
idlePoseRotation
/>
</div>
</div>
</div>
)}
{visibleDetailError && (
<p className="mt-2 text-xs text-coral-700 dark:text-coral-300 px-1">
{visibleDetailError}
</p>
)}
<p className="text-xs text-content-muted leading-relaxed px-1 mt-2">
{t('settings.mascot.characterDesc')}
</p>
@@ -14,17 +14,14 @@ import mascotReducer, {
} from '../../../../store/mascotSlice';
import MascotPanel from '../MascotPanel';
const { mockNavigateBack, fetchMascotListMock, getCachedMascotDetailMock, mockSynthesizeSpeech } =
vi.hoisted(() => ({
mockNavigateBack: vi.fn(),
fetchMascotListMock: vi.fn(),
getCachedMascotDetailMock: vi.fn(),
mockSynthesizeSpeech: vi.fn(),
}));
const { mockNavigateBack, useMascotManifestMock, mockSynthesizeSpeech } = vi.hoisted(() => ({
mockNavigateBack: vi.fn(),
useMascotManifestMock: vi.fn(),
mockSynthesizeSpeech: vi.fn(),
}));
vi.mock('../../../../services/mascotService', () => ({
fetchMascotList: (...args: unknown[]) => fetchMascotListMock(...args),
getCachedMascotDetail: (...args: unknown[]) => getCachedMascotDetailMock(...args),
vi.mock('../../../../features/human/Mascot/manifest/useMascotManifest', () => ({
useMascotManifest: () => useMascotManifestMock(),
}));
vi.mock('../../../../features/human/voice/ttsClient', () => ({
@@ -36,17 +33,59 @@ vi.mock('../../../../features/human/Mascot', async importOriginal => {
return {
...actual,
RiveMascot: () => <div data-testid="rive-mascot-preview" />,
ManifestRiveMascot: ({ entry }: { entry: { id: string } }) => (
<div data-testid={`manifest-mascot-preview-${entry.id}`} />
),
CustomGifMascot: ({ src }: { src: string }) => (
<img data-testid="custom-gif-mascot" src={src} alt="" />
),
};
});
vi.mock('../../../../features/human/Mascot/backend/BackendMascot', () => ({
BackendMascot: ({ mascot }: { mascot: { id: string } }) => (
<div data-testid={`backend-mascot-preview-${mascot.id}`} />
),
}));
// Build a minimal manifest entry for the picker list / preview.
function manifestEntry(id: string, name: string, status: 'ready' | 'draft' = 'ready') {
return {
id,
name,
description: '',
status,
tags: [],
stateEngine: {
idlePoseCycle: ['idle', 'dancing'],
states: { idle: 'idle', thinking: 'thinking' },
visemeCodes: ['sil', 'PP', 'aa'],
},
files: [
{ path: `${id}.riv`, bytes: 1, role: 'runtime', sha256: id, url: `https://x/${id}.riv` },
],
};
}
function manifestResult(
mascots: ReturnType<typeof manifestEntry>[],
overrides: Partial<{
manifest: unknown;
entry: unknown;
loading: boolean;
error: Error | null;
}> = {}
) {
const manifest =
'manifest' in overrides
? overrides.manifest
: {
schemaVersion: 1,
generatedAt: '',
mascots,
source: { repository: '', branch: '', commit: '' },
};
return {
manifest,
entry: 'entry' in overrides ? overrides.entry : (mascots[0] ?? null),
loading: overrides.loading ?? false,
error: overrides.error ?? null,
};
}
vi.mock('../../hooks/useSettingsNavigation', () => ({
useSettingsNavigation: () => ({
@@ -75,8 +114,7 @@ function renderPanel(store = buildStore()) {
describe('MascotPanel', () => {
beforeEach(() => {
vi.clearAllMocks();
fetchMascotListMock.mockResolvedValue([]);
getCachedMascotDetailMock.mockResolvedValue(null);
useMascotManifestMock.mockReturnValue(manifestResult([]));
mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0));
});
@@ -168,78 +206,65 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => {
expect(screen.getByRole('radio', { name: 'Yellow' })).toHaveAttribute('aria-checked', 'false');
});
describe('backend mascot library', () => {
const summary = {
id: 'yellow',
name: 'Yellow',
version: '1.0.0',
description: '',
states: [{ id: 'idle', label: 'Idle', description: '' }],
hasVisemes: true,
};
const detail = {
id: 'yellow',
name: 'Yellow',
version: '1.0.0',
description: '',
viewBox: '0 0 1 1',
defaultState: 'idle',
variables: [],
states: [{ id: 'idle', label: 'Idle', description: '', svg: '<svg/>' }],
visemes: [],
};
describe('mascot manifest library', () => {
const yellow = manifestEntry('yellow', 'Yellow');
const toshi = manifestEntry('toshi', 'Toshi', 'draft');
it('renders the picker entries returned by the API', async () => {
fetchMascotListMock.mockResolvedValueOnce([summary]);
it('renders the picker entries from the manifest', () => {
useMascotManifestMock.mockReturnValue(manifestResult([yellow, toshi]));
renderPanel();
expect(await screen.findByTestId('backend-mascot-yellow')).toBeInTheDocument();
expect(screen.getByTestId('manifest-mascot-yellow')).toBeInTheDocument();
expect(screen.getByTestId('manifest-mascot-toshi')).toBeInTheDocument();
// Draft status badge surfaces for non-ready mascots.
expect(screen.getByText('Draft')).toBeInTheDocument();
// Default-row (local) sentinel
expect(screen.getByText(/Local OpenHuman/)).toBeInTheDocument();
});
it('shows a friendly empty state when the library is empty', async () => {
fetchMascotListMock.mockResolvedValueOnce([]);
it('shows a friendly empty state when the library is empty', () => {
useMascotManifestMock.mockReturnValue(
manifestResult([], {
manifest: {
schemaVersion: 1,
generatedAt: '',
mascots: [],
source: { repository: '', branch: '', commit: '' },
},
})
);
renderPanel();
expect(
await screen.findByText(/No OpenHuman characters are available yet/i)
).toBeInTheDocument();
expect(screen.getByText(/No OpenHuman characters are available yet/i)).toBeInTheDocument();
});
it('shows an error when the library endpoint rejects', async () => {
fetchMascotListMock.mockRejectedValueOnce(new Error('offline'));
it('shows an error when the manifest fails to load', () => {
useMascotManifestMock.mockReturnValue(
manifestResult([], { manifest: null, entry: null, error: new Error('offline') })
);
renderPanel();
expect(
await screen.findByText(/OpenHuman library unavailable: offline/i)
).toBeInTheDocument();
expect(screen.getByText(/OpenHuman library unavailable: offline/i)).toBeInTheDocument();
});
it('dispatches setSelectedMascotId when a backend mascot is picked', async () => {
fetchMascotListMock.mockResolvedValueOnce([summary]);
getCachedMascotDetailMock.mockResolvedValueOnce(detail);
it('dispatches setSelectedMascotId when a mascot is picked', () => {
useMascotManifestMock.mockReturnValue(manifestResult([yellow]));
const { store } = renderPanel();
const row = await screen.findByTestId('backend-mascot-yellow');
fireEvent.click(row);
fireEvent.click(screen.getByTestId('manifest-mascot-yellow'));
expect(store.getState().mascot.selectedMascotId).toBe('yellow');
});
it('loads + previews the active backend mascot detail', async () => {
it('previews the active manifest mascot', () => {
const store = buildStore();
store.dispatch(setSelectedMascotId('yellow'));
fetchMascotListMock.mockResolvedValueOnce([summary]);
getCachedMascotDetailMock.mockResolvedValueOnce(detail);
useMascotManifestMock.mockReturnValue(manifestResult([yellow], { entry: yellow }));
renderPanel(store);
expect(await screen.findByTestId('backend-mascot-preview-yellow')).toBeInTheDocument();
expect(getCachedMascotDetailMock).toHaveBeenCalledWith('yellow');
expect(screen.getByTestId('manifest-mascot-preview-yellow')).toBeInTheDocument();
});
it('clearing the selection returns to the local default', async () => {
it('clearing the selection returns to the local default', () => {
const store = buildStore();
store.dispatch(setSelectedMascotId('yellow'));
fetchMascotListMock.mockResolvedValueOnce([summary]);
getCachedMascotDetailMock.mockResolvedValueOnce(detail);
useMascotManifestMock.mockReturnValue(manifestResult([yellow], { entry: yellow }));
renderPanel(store);
const localRow = await screen.findByText(/Local OpenHuman/);
fireEvent.click(localRow);
fireEvent.click(screen.getByText(/Local OpenHuman/));
expect(store.getState().mascot.selectedMascotId).toBeNull();
});
@@ -268,12 +293,12 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => {
expect(screen.getByTestId('mascot-custom-gif-error')).toHaveTextContent('HTTPS .gif');
});
it('selecting a backend mascot clears the custom GIF avatar', async () => {
it('selecting a mascot clears the custom GIF avatar', () => {
const store = buildStore();
store.dispatch(setCustomMascotGifUrl('https://example.com/avatar.gif'));
fetchMascotListMock.mockResolvedValueOnce([summary]);
useMascotManifestMock.mockReturnValue(manifestResult([yellow]));
renderPanel(store);
fireEvent.click(await screen.findByTestId('backend-mascot-yellow'));
fireEvent.click(screen.getByTestId('manifest-mascot-yellow'));
expect(store.getState().mascot.selectedMascotId).toBe('yellow');
expect(store.getState().mascot.customMascotGifUrl).toBeNull();
@@ -285,8 +310,7 @@ describe('MascotPanel — mascotSlice rehydrate guard', () => {
describe('MascotPanel — voice picker custom voice input (line 525)', () => {
beforeEach(() => {
vi.clearAllMocks();
fetchMascotListMock.mockResolvedValue([]);
getCachedMascotDetailMock.mockResolvedValue(null);
useMascotManifestMock.mockReturnValue(manifestResult([]));
mockSynthesizeSpeech.mockResolvedValue(new Uint8Array(0));
});
+12 -1
View File
@@ -2,6 +2,7 @@ import debug from 'debug';
import { type RefObject, useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { type MascotFace, RiveMascot } from '../../features/human/Mascot';
import type { MascotColor } from '../../features/human/Mascot/mascotPalette';
import { useComposioIntegrations } from '../../lib/composio/hooks';
import type { ComposioConnection } from '../../lib/composio/types';
import { useT } from '../../lib/i18n/I18nContext';
@@ -40,6 +41,16 @@ import { RecentCallsSection } from './RecentCallsSection';
type Toast = { type: 'success' | 'error' | 'info'; title: string; message?: string };
const log = debug('meeting-bots');
const MEETING_BOT_MASCOT_IDS = new Set(['yellow', 'blue', 'burgundy', 'black', 'navy']);
function resolveMeetingBotMascotId(
selectedMascotId: string | null,
mascotColor: MascotColor
): string | undefined {
if (selectedMascotId && MEETING_BOT_MASCOT_IDS.has(selectedMascotId)) return selectedMascotId;
if (mascotColor !== 'custom') return mascotColor;
return undefined;
}
// Composio only hands back a connected account's email — there is no separate
// display-name field on `ComposioConnection`. A meeting display name is almost
@@ -309,7 +320,7 @@ function MeetingBotsInline({ onToast, hasSubmittedRef }: MeetingBotsInlineProps)
const selectedLabel = t('skills.meetingBots.platforms.gmeet');
const agentName = personaDisplayName.trim() || 'Tiny';
const systemPrompt = personaDescription.trim() || undefined;
const mascotId = selectedMascotId ?? (mascotColor === 'custom' ? undefined : mascotColor);
const mascotId = resolveMeetingBotMascotId(selectedMascotId, mascotColor);
const riveColors =
mascotColor === 'custom'
? { primaryColor: customPrimaryColor, secondaryColor: customSecondaryColor }
@@ -121,6 +121,37 @@ describe('MeetingBotsCard', () => {
});
});
it('falls back to the legacy mascot color for manifest-only mascot ids', async () => {
joinMock.mockResolvedValueOnce({
meetUrl: 'https://meet.google.com/abc-defg-hij',
platform: 'gmeet',
});
renderWithProviders(<MeetingBotsCard />, {
preloadedState: {
mascot: {
color: 'yellow',
voiceId: null,
voiceGender: 'male',
voiceUseLocaleDefault: false,
selectedMascotId: 'river-guide',
customMascotGifUrl: null,
customPrimaryColor: '#123456',
customSecondaryColor: '#abcdef',
},
},
});
fireEvent.change(screen.getByLabelText(/meeting link/i), {
target: { value: 'https://meet.google.com/abc-defg-hij' },
});
fireEvent.submit(document.querySelector('form')!);
await vi.waitFor(() => {
expect(joinMock).toHaveBeenCalledWith(expect.objectContaining({ mascotId: 'yellow' }));
});
});
it('surfaces a join error inline + as an error toast', async () => {
joinMock.mockRejectedValueOnce(new Error('Bad URL'));
const onToast = vi.fn();