diff --git a/app/src/features/human/HumanPage.tsx b/app/src/features/human/HumanPage.tsx index 53955bfc8..490576f98 100644 --- a/app/src/features/human/HumanPage.tsx +++ b/app/src/features/human/HumanPage.tsx @@ -28,7 +28,7 @@ const HumanPage = () => { window.localStorage.setItem(SPEAK_REPLIES_KEY, speakReplies ? '1' : '0'); }, [speakReplies]); - const { face } = useHumanMascot({ speakReplies }); + const { face, visemeCode } = useHumanMascot({ speakReplies }); const mascotColor = useAppSelector(selectMascotColor); const customPrimary = useAppSelector(selectCustomPrimaryColor); const customSecondary = useAppSelector(selectCustomSecondaryColor); @@ -58,7 +58,12 @@ const HumanPage = () => { {customMascotGifUrl ? ( ) : ( - + )} diff --git a/app/src/features/human/Mascot/Ghosty.tsx b/app/src/features/human/Mascot/Ghosty.tsx index bf02a1784..b92296534 100644 --- a/app/src/features/human/Mascot/Ghosty.tsx +++ b/app/src/features/human/Mascot/Ghosty.tsx @@ -22,6 +22,16 @@ import { visemePath, VISEMES, type VisemeShape } from './visemes'; * - `proud` — task fully completed after meaningful tool/subagent work. * - `cautious` — gentle warning; less severe than `concerned`. * + * Activity poses — driven by what the agent is actively doing: + * - `celebrating` — success animation after meaningful work. + * - `writing` — agent is editing/creating files. + * - `reading` — agent is browsing or reading content. + * - `recording` — agent is capturing screen or recording. + * - `waving` — greeting or hello gesture. + * - `dancing` — celebratory/playful animation. + * - `drinking_coffee` — agent is processing / long-running task. + * - `drinking_boba` — relaxed variant of processing. + * * `normal` is the legacy alias for `idle` and stays accepted for backwards * compatibility with older callers. */ @@ -37,6 +47,14 @@ export type MascotFace = | 'curious' | 'proud' | 'cautious' + | 'celebrating' + | 'writing' + | 'reading' + | 'recording' + | 'waving' + | 'dancing' + | 'drinking_coffee' + | 'drinking_boba' | 'normal'; export interface GhostyProps { @@ -162,6 +180,70 @@ const FACE_PRESETS: Record, FacePreset> = { showBrows: true, blushOpacity: 0.65, }, + celebrating: { + eyeScaleY: 0.4, + eyeScaleX: 1.15, + browTilt: -8, + browDy: -8, + showBrows: false, + blushOpacity: 1, + }, + writing: { + eyeScaleY: 0.75, + eyeScaleX: 1, + browTilt: -2, + browDy: -1, + showBrows: false, + blushOpacity: 0.7, + }, + reading: { + eyeScaleY: 0.85, + eyeScaleX: 1.05, + browTilt: -6, + browDy: -4, + showBrows: true, + blushOpacity: 0.75, + }, + recording: { + eyeScaleY: 1, + eyeScaleX: 1, + browTilt: 0, + browDy: 0, + showBrows: false, + blushOpacity: 0.9, + }, + waving: { + eyeScaleY: 0.5, + eyeScaleX: 1.1, + browTilt: -6, + browDy: -6, + showBrows: false, + blushOpacity: 1, + }, + dancing: { + eyeScaleY: 0.4, + eyeScaleX: 1.15, + browTilt: -8, + browDy: -8, + showBrows: false, + blushOpacity: 1, + }, + drinking_coffee: { + eyeScaleY: 0.6, + eyeScaleX: 1, + browTilt: 0, + browDy: 0, + showBrows: false, + blushOpacity: 0.8, + }, + drinking_boba: { + eyeScaleY: 0.55, + eyeScaleX: 1.05, + browTilt: 0, + browDy: 0, + showBrows: false, + blushOpacity: 0.85, + }, }; function presetFor(face: MascotFace): FacePreset { @@ -375,6 +457,19 @@ function restMouthPath(face: MascotFace): string { case 'cautious': // Slight downward turn — concern lite. return 'M482,600 Q520,568 558,600 Q520,585 482,600 Z'; + case 'celebrating': + case 'dancing': + return 'M460,565 Q520,635 580,565 Q520,605 460,565 Z'; + case 'waving': + return 'M460,565 Q520,635 580,565 Q520,605 460,565 Z'; + case 'writing': + case 'reading': + return 'M488,585 Q520,595 552,585 Q520,592 488,585 Z'; + case 'recording': + return 'M495,580 Q520,600 545,580 Q520,615 495,580 Z'; + case 'drinking_coffee': + case 'drinking_boba': + return 'M500,578 Q520,598 540,578 Q520,610 500,578 Z'; default: return visemePath(VISEMES.REST); } diff --git a/app/src/features/human/Mascot/RiveMascot.tsx b/app/src/features/human/Mascot/RiveMascot.tsx index fb264cbb5..c16895385 100644 --- a/app/src/features/human/Mascot/RiveMascot.tsx +++ b/app/src/features/human/Mascot/RiveMascot.tsx @@ -4,42 +4,72 @@ import { useRive, useViewModel, useViewModelInstance, - useViewModelInstanceBoolean, useViewModelInstanceColor, useViewModelInstanceString, } from '@rive-app/react-webgl2'; import { type FC, useEffect } from 'react'; import type { MascotFace } from './Ghosty'; -import type { VisemeId } from './visemes'; export interface RiveMascotProps { face?: MascotFace; size?: number | string; primaryColor?: number; secondaryColor?: number; - viseme?: VisemeId; + /** Raw Oculus 15-set viseme code (e.g. 'sil', 'PP', 'aa') sent directly to + * the Rive state machine's `mouthVisemeCode` input. When omitted, defaults + * to 'sil' (mouth closed). */ + visemeCode?: string; } -const SPEAKING_FACES: ReadonlySet = new Set(['speaking', 'happy']); - +/** + * Maps every MascotFace to the closest Rive pose animation. The Rive asset + * supports: idle, thinking, celebration, bookreading, coffeedrink, writing, + * bobbateadrink, recording, hand_wave, dancing. + */ const FACE_TO_POSE: Record = { idle: 'idle', normal: 'idle', - sleep: 'sleeping', + sleep: 'idle', listening: 'idle', thinking: 'thinking', confused: 'thinking', speaking: 'idle', happy: 'idle', - concerned: 'idle', - // New emotional states — map to the nearest available Rive pose until - // dedicated animations are added to the .riv asset. - curious: 'thinking', - proud: 'idle', - cautious: 'idle', + concerned: 'thinking', + curious: 'bookreading', + proud: 'celebration', + cautious: 'thinking', + celebrating: 'celebration', + writing: 'writing', + reading: 'bookreading', + recording: 'recording', + waving: 'hand_wave', + dancing: 'dancing', + drinking_coffee: 'coffeedrink', + drinking_boba: 'bobbateadrink', }; +/** + * ElevenLabs / Oculus 15-set → Rive asset's `visme_codes` vocabulary. + * The Rive file uses `ih`/`oh`/`ou` for vowels instead of `E`/`O`/`U`. + * Codes already in the Rive vocabulary pass through unchanged. + */ +const OCULUS_TO_RIVE_VISEME: Record = { + E: 'ih', + I: 'ih', + O: 'oh', + U: 'ou', + e: 'ih', + i: 'ih', + o: 'oh', + u: 'ou', +}; + +function toRiveVisemeCode(oculusCode: string): string { + return OCULUS_TO_RIVE_VISEME[oculusCode] ?? oculusCode; +} + const RIVE_LAYOUT = new Layout({ fit: Fit.Contain }); export const RiveMascot: FC = ({ @@ -47,7 +77,7 @@ export const RiveMascot: FC = ({ size = '100%', primaryColor, secondaryColor, - viseme = 'REST', + visemeCode = 'sil', }) => { const { rive, RiveComponent } = useRive({ src: '/tiny_mascot.riv', @@ -58,20 +88,21 @@ export const RiveMascot: FC = ({ const viewModel = useViewModel(rive, { useDefault: true }); const vmInstance = useViewModelInstance(viewModel, { useDefault: true, rive }); - const { setValue: setMouthOpen } = useViewModelInstanceBoolean('mouthOpen', vmInstance); const { setValue: setPose } = useViewModelInstanceString('pose', vmInstance); - const { setValue: setViseme } = useViewModelInstanceString('viseme', vmInstance); + const { setValue: setMouthVisemeCode } = useViewModelInstanceString( + 'mouthVisemeCode', + vmInstance + ); const { setValue: setPrimaryColor } = useViewModelInstanceColor('primaryColor', vmInstance); const { setValue: setSecondaryColor } = useViewModelInstanceColor('secondaryColor', vmInstance); useEffect(() => { - setMouthOpen(SPEAKING_FACES.has(face!)); - setPose(FACE_TO_POSE[face!] ?? 'idle'); - }, [face, setMouthOpen, setPose]); + setPose(FACE_TO_POSE[face] ?? 'idle'); + }, [face, setPose]); useEffect(() => { - setViseme(viseme); - }, [viseme, setViseme]); + setMouthVisemeCode(toRiveVisemeCode(visemeCode)); + }, [visemeCode, setMouthVisemeCode]); useEffect(() => { if (primaryColor !== undefined) setPrimaryColor(primaryColor); diff --git a/app/src/features/human/useHumanMascot.test.ts b/app/src/features/human/useHumanMascot.test.ts index c07270e8d..e577dc981 100644 --- a/app/src/features/human/useHumanMascot.test.ts +++ b/app/src/features/human/useHumanMascot.test.ts @@ -7,6 +7,7 @@ import { ACK_FACE_HOLD_MS, pickConversationAckFace, pickViseme, + pickVisemeCode, TTS_MAX_PLAYBACK_MS, useHumanMascot, } from './useHumanMascot'; @@ -139,6 +140,54 @@ describe('pickViseme', () => { }); }); +describe('pickVisemeCode', () => { + it('maps vowels to Oculus 15-set codes', () => { + expect(pickVisemeCode('a')).toBe('aa'); + expect(pickVisemeCode('e')).toBe('E'); + expect(pickVisemeCode('i')).toBe('I'); + expect(pickVisemeCode('o')).toBe('O'); + expect(pickVisemeCode('u')).toBe('U'); + }); + + it('maps labials to PP', () => { + expect(pickVisemeCode('m')).toBe('PP'); + expect(pickVisemeCode('b')).toBe('PP'); + expect(pickVisemeCode('p')).toBe('PP'); + }); + + it('maps fricatives to FF', () => { + expect(pickVisemeCode('f')).toBe('FF'); + expect(pickVisemeCode('v')).toBe('FF'); + }); + + it('maps sibilants to SS', () => { + expect(pickVisemeCode('s')).toBe('SS'); + expect(pickVisemeCode('z')).toBe('SS'); + }); + + it('maps other consonants to their Oculus codes', () => { + expect(pickVisemeCode('n')).toBe('nn'); + expect(pickVisemeCode('t')).toBe('DD'); + expect(pickVisemeCode('k')).toBe('kk'); + expect(pickVisemeCode('r')).toBe('RR'); + }); + + it('uses the trailing letter of multi-char deltas', () => { + expect(pickVisemeCode('hello')).toBe('O'); + expect(pickVisemeCode('world')).toBe('DD'); + }); + + it('ignores punctuation when picking the trailing letter', () => { + expect(pickVisemeCode('Hi!')).toBe('I'); + }); + + it('falls back to E for unmapped consonants and empty input', () => { + expect(pickVisemeCode('x')).toBe('E'); + expect(pickVisemeCode('')).toBe('E'); + expect(pickVisemeCode('...')).toBe('E'); + }); +}); + describe('pickConversationAckFace', () => { it('prefers explicit reaction emoji from chat_done', () => { expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '✅' })).toBe('happy'); @@ -257,24 +306,46 @@ describe('useHumanMascot state machine', () => { expect(result.current.face).toBe('thinking'); }); - it('moves to confused on tool_call', () => { + it('maps tool_call to activity face when tool has a visual association', () => { const { result } = renderHook(() => useHumanMascot()); act(() => { capturedListeners?.onInferenceStart?.(fakeEvent({})); capturedListeners?.onToolCall?.( - fakeEvent({ tool_name: 'search', skill_id: 's', args: {}, round: 1 }) + fakeEvent({ tool_name: 'file_write', skill_id: 's', args: {}, round: 1 }) ); }); - expect(result.current.face).toBe('confused'); + expect(result.current.face).toBe('writing'); }); - it('moves to confused on iteration_start beyond round 1', () => { + it('falls back to thinking on tool_call for unmapped tools', () => { + const { result } = renderHook(() => useHumanMascot()); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onToolCall?.( + fakeEvent({ tool_name: 'custom_tool', skill_id: 's', args: {}, round: 1 }) + ); + }); + expect(result.current.face).toBe('thinking'); + }); + + it('maps reading tools to reading face', () => { + const { result } = renderHook(() => useHumanMascot()); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onToolCall?.( + fakeEvent({ tool_name: 'web_search', skill_id: 's', args: {}, round: 1 }) + ); + }); + expect(result.current.face).toBe('reading'); + }); + + it('moves to drinking_coffee on iteration_start beyond round 1', () => { const { result } = renderHook(() => useHumanMascot()); act(() => { capturedListeners?.onInferenceStart?.(fakeEvent({})); capturedListeners?.onIterationStart?.(fakeEvent({ round: 2, message: '' })); }); - expect(result.current.face).toBe('confused'); + expect(result.current.face).toBe('drinking_coffee'); }); it('does not flip to confused on iteration_start round 1', () => { @@ -309,7 +380,7 @@ describe('useHumanMascot state machine', () => { act(() => { capturedListeners?.onDone?.( fakeEvent({ - full_response: 'hello', + full_response: 'sure thing', rounds_used: 1, total_input_tokens: 1, total_output_tokens: 1, @@ -418,7 +489,7 @@ describe('useHumanMascot state machine', () => { expect(result.current.face).toBe('thinking'); }); - it('promotes to proud on chat_done when a tool succeeded in the same turn', () => { + it('promotes to celebrating on chat_done when a tool succeeded in the same turn', () => { const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); act(() => { capturedListeners?.onInferenceStart?.(fakeEvent({})); @@ -435,7 +506,7 @@ describe('useHumanMascot state machine', () => { }) ); }); - expect(result.current.face).toBe('proud'); + expect(result.current.face).toBe('celebrating'); act(() => { vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1); }); @@ -459,7 +530,7 @@ describe('useHumanMascot state machine', () => { expect(result.current.face).toBe('happy'); }); - it('promotes to proud on chat_done when a subagent succeeded in the same turn', () => { + it('promotes to celebrating on chat_done when a subagent succeeded in the same turn', () => { const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); act(() => { capturedListeners?.onInferenceStart?.(fakeEvent({})); @@ -482,7 +553,7 @@ describe('useHumanMascot state machine', () => { }) ); }); - expect(result.current.face).toBe('proud'); + expect(result.current.face).toBe('celebrating'); }); it('shows concerned when a subagent fails', () => { @@ -503,7 +574,7 @@ describe('useHumanMascot state machine', () => { it('resets work tracking on each new turn', () => { const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); - // Turn 1: tool succeeded → proud + // Turn 1: tool succeeded → celebrating act(() => { capturedListeners?.onInferenceStart?.(fakeEvent({})); capturedListeners?.onToolResult?.( @@ -519,7 +590,7 @@ describe('useHumanMascot state machine', () => { }) ); }); - expect(result.current.face).toBe('proud'); + expect(result.current.face).toBe('celebrating'); act(() => { vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1); }); @@ -621,7 +692,7 @@ describe('useHumanMascot TTS playback', () => { const { result } = renderHook(() => useHumanMascot({ speakReplies: true })); await act(async () => { - capturedListeners?.onDone?.(fakeDone('hello')); + capturedListeners?.onDone?.(fakeDone('sure thing')); // Let synthesizeSpeech and playBase64Audio resolve. await Promise.resolve(); await Promise.resolve(); diff --git a/app/src/features/human/useHumanMascot.ts b/app/src/features/human/useHumanMascot.ts index 851469ea6..693b3a2d6 100644 --- a/app/src/features/human/useHumanMascot.ts +++ b/app/src/features/human/useHumanMascot.ts @@ -89,9 +89,67 @@ export function pickViseme(delta: string): VisemeShape { } } +/** + * Pick a raw Oculus viseme code from a text delta character. Used to drive + * `mouthVisemeCode` on the Rive state machine during pseudo-lipsync (no TTS). + * Returns Oculus 15-set codes; `RiveMascot` translates vowels (`E`→`ih`, + * `O`→`oh`, `U`→`ou`) to the Rive asset's vocabulary at render time. + */ +export function pickVisemeCode(delta: string): string { + const ch = delta + .replace(/[^a-zA-Z]/g, '') + .slice(-1) + .toLowerCase(); + switch (ch) { + case 'a': + return 'aa'; + case 'e': + return 'E'; + case 'i': + case 'y': + return 'I'; + case 'o': + return 'O'; + case 'u': + case 'w': + return 'U'; + case 'm': + case 'b': + case 'p': + return 'PP'; + case 'f': + case 'v': + return 'FF'; + case 's': + case 'z': + return 'SS'; + case 'n': + case 'l': + return 'nn'; + case 't': + case 'd': + return 'DD'; + case 'k': + case 'g': + return 'kk'; + case 'r': + return 'RR'; + default: + return 'E'; + } +} + type ConversationAckFace = Extract< MascotFace, - 'happy' | 'confused' | 'concerned' | 'curious' | 'proud' | 'cautious' + | 'happy' + | 'confused' + | 'concerned' + | 'curious' + | 'proud' + | 'cautious' + | 'celebrating' + | 'dancing' + | 'waving' >; type ConversationAckEvent = { full_response?: string | null; reaction_emoji?: string | null }; @@ -99,9 +157,11 @@ const HAPPY_REACTION_EMOJIS = new Set(['✅', '🎉', '🙌', '😊', '😄', ' const PROUD_REACTION_EMOJIS = new Set(['⭐', '🌟', '🏆', '🎯', '💯', '🚀', '✨', '🥇']); const CURIOUS_REACTION_EMOJIS = new Set(['🔍', '💭', '🧐', '🤓', '👀']); const CONFUSED_REACTION_EMOJIS = new Set(['🤔', '❓', '❔']); -// ⚠️ is cautious (heads-up), ❌/🚨 are concerned (failure). const CAUTIOUS_REACTION_EMOJIS = new Set(['⚠️', '⚠', '💡', '⚡']); const CONCERNED_REACTION_EMOJIS = new Set(['🚨', '❌', '😕', '😟']); +const CELEBRATING_REACTION_EMOJIS = new Set(['🥳', '🍾', '🎊', '🎈', '🪅']); +const DANCING_REACTION_EMOJIS = new Set(['💃', '🕺', '🎵', '🎶', '🎸']); +const WAVING_REACTION_EMOJIS = new Set(['👋', '🤝', '🫡']); const CONCERNED_TEXT_RE = /\b(sorry|apolog(?:y|ize|ise)|failed|failure|error|cannot|can't|unable|blocked|problem)\b/i; @@ -114,6 +174,10 @@ const CURIOUS_TEXT_RE = /\b(interesting|fascinating|curious(ly)?|let me (check|look|investigate)|i('ll)? (look|check) into|actually|turns? out)\b/i; const CAUTIOUS_TEXT_RE = /\b(be careful|warning|caution|heads? up|please note|make sure|important(ly)?|note that|worth (noting|mentioning))\b/i; +const CELEBRATING_TEXT_RE = + /\b(congrat(ulations|s)?|well done|bravo|hooray|woohoo|amazing|fantastic|incredible|awesome work)\b/i; +const GREETING_TEXT_RE = + /^(hello|hey|hi there|good (morning|afternoon|evening)|welcome back|greetings|howdy)[!.,]?(?:\s|$)/i; /** * Map conversation-level meaning into the short acknowledgement face that @@ -123,6 +187,9 @@ const CAUTIOUS_TEXT_RE = export function pickConversationAckFace(event: ConversationAckEvent): ConversationAckFace | null { const reaction = event.reaction_emoji?.trim(); if (reaction) { + if (CELEBRATING_REACTION_EMOJIS.has(reaction)) return 'celebrating'; + if (DANCING_REACTION_EMOJIS.has(reaction)) return 'dancing'; + if (WAVING_REACTION_EMOJIS.has(reaction)) return 'waving'; if (PROUD_REACTION_EMOJIS.has(reaction)) return 'proud'; if (HAPPY_REACTION_EMOJIS.has(reaction)) return 'happy'; if (CURIOUS_REACTION_EMOJIS.has(reaction)) return 'curious'; @@ -133,18 +200,53 @@ export function pickConversationAckFace(event: ConversationAckEvent): Conversati const text = event.full_response?.trim() ?? ''; if (!text) return null; - // Priority: concerned > cautious > proud > confused > curious > happy. - // Concerned and cautious share some vocabulary; check concerned first so - // outright failures don't get softened to a heads-up. if (CONCERNED_TEXT_RE.test(text)) return 'concerned'; if (CAUTIOUS_TEXT_RE.test(text)) return 'cautious'; + if (CELEBRATING_TEXT_RE.test(text)) return 'celebrating'; if (PROUD_TEXT_RE.test(text)) return 'proud'; if (CONFUSED_TEXT_RE.test(text)) return 'confused'; if (CURIOUS_TEXT_RE.test(text)) return 'curious'; + if (GREETING_TEXT_RE.test(text)) return 'waving'; if (HAPPY_TEXT_RE.test(text)) return 'happy'; return null; } +/** + * Map a tool name to an activity pose. Returns null when the tool doesn't + * have a strong visual association — the caller falls back to a generic face. + */ +function toolToActivityFace(toolName: string): MascotFace | null { + const name = toolName.toLowerCase(); + + if ( + name.includes('file_write') || + name.includes('edit_file') || + name.includes('apply_patch') || + name.includes('create_file') || + name.includes('write') + ) { + return 'writing'; + } + + if ( + name.includes('browser') || + name.includes('web_search') || + name.includes('web_fetch') || + name.includes('read_file') || + name.includes('search') || + name.includes('grep') || + name.includes('find') + ) { + return 'reading'; + } + + if (name.includes('screen') || name.includes('screenshot') || name.includes('capture')) { + return 'recording'; + } + + return null; +} + export interface UseHumanMascotOptions { /** When true, post-stream replies are sent to ElevenLabs and the mouth * follows the returned viseme timeline while the audio plays. */ @@ -157,6 +259,8 @@ export interface UseHumanMascotOptions { export interface UseHumanMascotResult { face: MascotFace; viseme: VisemeShape; + /** Raw Oculus 15-set viseme code for Rive's `mouthVisemeCode` input. */ + visemeCode: string; } /** @@ -165,7 +269,8 @@ export interface UseHumanMascotResult { * Mapping (kept in one place so the visual model stays coherent): * * - `inference_start` → `thinking` - * - `iteration_start` round > 1 or `tool_call` → `confused` (heavy reasoning) + * - `iteration_start` round > 1 or `tool_call` → activity pose based on tool + * name (writing/reading/recording) or `confused` as fallback * - `tool_result success=false` → `concerned` (held briefly) * - `text_delta` → `speaking`, pseudo-lipsync from the trailing letter * - `chat_done` (no TTS) → message-aware ack face (held briefly), then `idle` @@ -184,31 +289,22 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas const listeningRef = useRef(listening); listeningRef.current = listening; - // Effective mascot voice id: resolves the manual override, the - // locale-default toggle, and the build-time fallback into a single - // string (see `selectEffectiveMascotVoiceId`). Mirrored into a ref so - // the inner `startTtsPlayback` closure always reads the latest value - // without having to re-create the callback on every re-render. const effectiveMascotVoiceId = useSelector(selectEffectiveMascotVoiceId); const mascotVoiceIdRef = useRef(effectiveMascotVoiceId); mascotVoiceIdRef.current = effectiveMascotVoiceId; const [face, setFace] = useState('idle'); const targetRef = useRef(VISEMES.REST); + const visemeCodeRef = useRef('sil'); const lastDeltaAtRef = useRef(0); const ackTimerRef = useRef(null); - // Track meaningful work performed in the current turn so onDone can - // distinguish a proud completion from a routine happy acknowledgement. const toolSucceededRef = useRef(false); const subagentSucceededRef = useRef(false); - // TTS playback state — non-null while audio is mid-flight. const playbackRef = useRef(null); const visemeFramesRef = useRef<{ viseme: string; start_ms: number; end_ms: number }[]>([]); const visemeCursorRef = useRef(0); - // Monotonic counter — only the latest startTtsPlayback's callbacks may - // mutate idle state; older invocations bail out. const playbackSeqRef = useRef(0); const [, force] = useState(0); @@ -239,22 +335,26 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas setFace('thinking'); }, onIterationStart: e => { - // Subsequent iterations mean the agent is grinding through tool rounds. if (e.round > 1) { clearAckTimer(); - mascotLog('voice-session transition → confused (iteration round=%d)', e.round); - setFace('confused'); + mascotLog('voice-session transition → drinking_coffee (iteration round=%d)', e.round); + setFace('drinking_coffee'); } }, - onToolCall: () => { + onToolCall: e => { clearAckTimer(); - mascotLog('voice-session transition → confused (tool_call)'); - setFace('confused'); + const activityFace = toolToActivityFace(e.tool_name); + if (activityFace) { + mascotLog('voice-session transition → %s (tool_call %s)', activityFace, e.tool_name); + setFace(activityFace); + } else { + mascotLog('voice-session transition → thinking (tool_call %s)', e.tool_name); + setFace('thinking'); + } }, onToolResult: e => { if (!e.success) { mascotLog('voice-session transition → concerned (tool_result failed)'); - // Don't fully derail — let the next inference step take over. setFace('concerned'); } else { toolSucceededRef.current = true; @@ -274,7 +374,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas } }, onTextDelta: e => { - // Pseudo-lipsync only kicks in if no real audio is playing. if (listeningRef.current) { mascotLog('voice-session text_delta suppressed — listening is active'); return; @@ -283,6 +382,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas clearAckTimer(); setFace('speaking'); targetRef.current = pickViseme(e.delta); + visemeCodeRef.current = pickVisemeCode(e.delta); lastDeltaAtRef.current = window.performance.now(); }, onDone: e => { @@ -290,14 +390,11 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas mascotLog('voice-session onDone suppressed — listening is active'); return; } - // Upgrade to `proud` when the turn involved real tool/subagent work. - // A happy/null text cue paired with actual execution is a completion — - // that reads as proud, not a routine acknowledgement. const didMeaningfulWork = toolSucceededRef.current || subagentSucceededRef.current; const explicitAck = pickConversationAckFace(e); const ackFace: ConversationAckFace = (explicitAck === 'happy' || explicitAck === null) && didMeaningfulWork - ? 'proud' + ? 'celebrating' : (explicitAck ?? 'happy'); toolSucceededRef.current = false; subagentSucceededRef.current = false; @@ -308,24 +405,18 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas didMeaningfulWork ); if (!speakRef.current || !e.full_response?.trim()) { - // Soft acknowledgement beat instead of snapping back to idle. holdThenIdle(ackFace); return; } - // Fire-and-forget — startTtsPlayback owns its cleanup via finally. void startTtsPlayback(e.full_response, ackFace).catch(() => {}); }, onError: () => { mascotLog('voice-session transition → concerned (chat_error), cancelling in-flight TTS'); - // Bump seq to invalidate any in-flight startTtsPlayback awaiters. playbackSeqRef.current++; const orphan = playbackRef.current; playbackRef.current = null; if (orphan) { orphan.stop(); - // We're early-returning instead of awaiting `orphan.ended`, so the - // stop()-sentinel rejection has no handler — attach one explicitly - // or it surfaces as an unhandledrejection in Sentry (#1472). orphan.ended.catch(swallowAudioStop); } visemeFramesRef.current = []; @@ -335,7 +426,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas return () => { unsub(); clearAckTimer(); - // Same — invalidate in-flight callbacks before tearing down. playbackSeqRef.current++; const orphan = playbackRef.current; playbackRef.current = null; @@ -357,8 +447,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas ? 'user started recording while TTS was playing (interrupted)' : 'mic activated, no TTS to cancel' ); - // Treat mic-hot as an explicit interruption: stale synthesis/playback - // callbacks must not switch the mascot back to speaking after we listen. playbackSeqRef.current++; const orphan = playbackRef.current; playbackRef.current = null; @@ -369,6 +457,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas visemeFramesRef.current = []; visemeCursorRef.current = 0; targetRef.current = VISEMES.REST; + visemeCodeRef.current = 'sil'; lastDeltaAtRef.current = 0; setFace('idle'); }, [listening]); @@ -377,8 +466,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas text: string, ackFace: ConversationAckFace = 'happy' ): Promise { - // Cancel any in-flight playback so its handle.ended callback can't reset - // state belonging to the new run. const prev = playbackRef.current; playbackRef.current = null; if (prev) { @@ -396,13 +483,8 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas setFace('thinking'); let tts; try { - // Always pass the effective voice id — the selector already - // resolves manual override / locale default / build-time - // fallback to a single string, so `synthesizeSpeech` doesn't - // need its own fallback branch here. tts = await synthesizeSpeech(text, { voiceId: mascotVoiceIdRef.current }); } catch (err) { - // Voice path unavailable — degrade cleanly to text-only behavior. if (isStillCurrent()) degraded = true; throw err; } @@ -410,26 +492,16 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas let frames: VisemeFrame[] = tts.visemes ?? []; let source: 'visemes' | 'alignment' | 'procedural' = 'visemes'; if (frames.length > 0 && !framesProduceMotion(frames)) { - // Backend shipped frames but every code maps to REST — usually means - // the codes are in a vocabulary `oculusVisemeToShape` doesn't know. - // Drop them and let the alignment / procedural path take over so the - // mouth doesn't sit on the rest-smile path for the whole clip. mascotLog('tts visemes produced no motion — dropping and falling through'); frames = []; } if (frames.length === 0 && tts.alignment && tts.alignment.length > 0) { - // Backend didn't ship viseme cues — derive a coarse track from char timings - // so the mouth still animates in sync with the audio. frames = visemesFromAlignment(tts.alignment); source = 'alignment'; mascotLog('tts derived %d viseme frames from alignment', frames.length); } else if (frames.length > 0) { mascotLog('tts got %d viseme frames from backend', frames.length); } - // Start audio first — `playBase64Audio` calls `audio.play()` directly so - // the user-gesture chain that authorized speech stays intact. If we - // awaited anything else between the user click and play(), CEF would - // reject playback under its autoplay policy. const ttsOptions: PlaybackOptions = { maxDurationMs: TTS_MAX_PLAYBACK_MS }; const handle = await playBase64Audio( tts.audio_base64, @@ -442,12 +514,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas return; } if (frames.length === 0) { - // Last-resort fallback: backend shipped neither viseme cues nor - // alignment (e.g. the new public `tts-v1` model on the hosted - // backend). Use whatever duration the decoder has reported so far — - // `proceduralVisemes` falls back to a text-length estimate when the - // metadata hasn't loaded yet, so we don't await it on the critical - // path (waiting opens a window where audio plays under a static face). const dur = handle.durationMs(); frames = proceduralVisemes(text, dur); source = 'procedural'; @@ -465,8 +531,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas try { await handle.ended; } catch (err) { - // Stop sentinel is expected when a newer turn cancels playback — - // rethrow anything else so real decoder errors aren't masked. swallowAudioStop(err); } } catch (err) { @@ -476,6 +540,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas if (isStillCurrent()) { playbackRef.current = null; visemeFramesRef.current = []; + visemeCodeRef.current = 'sil'; if (degraded) { holdThenIdle('concerned'); } else { @@ -485,9 +550,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas } } - // RAF loop while we're speaking. TTS playback always sets face to - // 'speaking' before awaiting the audio, so this also covers the audio-driven - // viseme path. useEffect(() => { if (face !== 'speaking') return; let raf = 0; @@ -500,6 +562,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas }, [face]); let viseme: VisemeShape = VISEMES.REST; + let visemeCode = 'sil'; const playback = playbackRef.current; if (playback) { const ms = playback.currentMs(); @@ -511,17 +574,18 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas ); visemeCursorRef.current = cursor; viseme = frame ? oculusVisemeToShape(frame.viseme) : VISEMES.REST; + visemeCode = frame ? frame.viseme : 'sil'; } } else if (face === 'speaking') { const since = window.performance.now() - lastDeltaAtRef.current; const decay = Math.max(0, Math.min(1, since / VISEME_DECAY_MS)); viseme = lerpViseme(targetRef.current, VISEMES.REST, decay); + visemeCode = decay > 0.5 ? 'sil' : visemeCodeRef.current; } - // `listening` is an external override so callers wiring dictation state - // can reflect mic-on without racing the chat event subscription. const effectiveFace: MascotFace = listening ? 'listening' : face; const effectiveViseme: VisemeShape = listening ? VISEMES.REST : viseme; + const effectiveVisemeCode: string = listening ? 'sil' : visemeCode; - return { face: effectiveFace, viseme: effectiveViseme }; + return { face: effectiveFace, viseme: effectiveViseme, visemeCode: effectiveVisemeCode }; } diff --git a/app/src/pages/ios/MascotScreen.tsx b/app/src/pages/ios/MascotScreen.tsx index 627d493a1..987643ef7 100644 --- a/app/src/pages/ios/MascotScreen.tsx +++ b/app/src/pages/ios/MascotScreen.tsx @@ -205,7 +205,7 @@ export const MascotScreen: FC = () => { // without a stale closure over the pttActive state variable. const pttActiveRef = useRef(false); - const { face } = useHumanMascot({ listening: pttActive }); + const { face, visemeCode } = useHumanMascot({ listening: pttActive }); // Derive label from stored profile. const pairedLabel = (() => { @@ -431,7 +431,7 @@ export const MascotScreen: FC = () => { {/* Mascot canvas */}
- +
diff --git a/tiny_mascot.riv b/tiny_mascot.riv index 5e259e66f..8818a1fab 100644 Binary files a/tiny_mascot.riv and b/tiny_mascot.riv differ