feat(mascot): sync Rive mascot with new poses, viseme codes & mood detection (#3173)

app/src/features/human/HumanPage.tsx

@@ -28,7 +28,7 @@ const HumanPage = () => {
     window.localStorage.setItem(SPEAK_REPLIES_KEY, speakReplies ? '1' : '0');
   }, [speakReplies]);
 
-  const { face } = useHumanMascot({ speakReplies });
+  const { face, visemeCode } = useHumanMascot({ speakReplies });
   const mascotColor = useAppSelector(selectMascotColor);
   const customPrimary = useAppSelector(selectCustomPrimaryColor);
   const customSecondary = useAppSelector(selectCustomSecondaryColor);
@@ -58,7 +58,12 @@ const HumanPage = () => {
           {customMascotGifUrl ? (
             <CustomGifMascot src={customMascotGifUrl} face={face} />
           ) : (
-            <RiveMascot face={face} primaryColor={primaryColor} secondaryColor={secondaryColor} />
+            <RiveMascot
+              face={face}
+              primaryColor={primaryColor}
+              secondaryColor={secondaryColor}
+              visemeCode={visemeCode}
+            />
           )}
         </div>
       </div>

app/src/features/human/Mascot/Ghosty.tsx

@@ -22,6 +22,16 @@ import { visemePath, VISEMES, type VisemeShape } from './visemes';
  * - `proud` — task fully completed after meaningful tool/subagent work.
  * - `cautious` — gentle warning; less severe than `concerned`.
  *
+ * Activity poses — driven by what the agent is actively doing:
+ * - `celebrating` — success animation after meaningful work.
+ * - `writing` — agent is editing/creating files.
+ * - `reading` — agent is browsing or reading content.
+ * - `recording` — agent is capturing screen or recording.
+ * - `waving` — greeting or hello gesture.
+ * - `dancing` — celebratory/playful animation.
+ * - `drinking_coffee` — agent is processing / long-running task.
+ * - `drinking_boba` — relaxed variant of processing.
+ *
  * `normal` is the legacy alias for `idle` and stays accepted for backwards
  * compatibility with older callers.
  */
@@ -37,6 +47,14 @@ export type MascotFace =
   | 'curious'
   | 'proud'
   | 'cautious'
+  | 'celebrating'
+  | 'writing'
+  | 'reading'
+  | 'recording'
+  | 'waving'
+  | 'dancing'
+  | 'drinking_coffee'
+  | 'drinking_boba'
   | 'normal';
 
 export interface GhostyProps {
@@ -162,6 +180,70 @@ const FACE_PRESETS: Record<Exclude<MascotFace, 'normal'>, FacePreset> = {
     showBrows: true,
     blushOpacity: 0.65,
   },
+  celebrating: {
+    eyeScaleY: 0.4,
+    eyeScaleX: 1.15,
+    browTilt: -8,
+    browDy: -8,
+    showBrows: false,
+    blushOpacity: 1,
+  },
+  writing: {
+    eyeScaleY: 0.75,
+    eyeScaleX: 1,
+    browTilt: -2,
+    browDy: -1,
+    showBrows: false,
+    blushOpacity: 0.7,
+  },
+  reading: {
+    eyeScaleY: 0.85,
+    eyeScaleX: 1.05,
+    browTilt: -6,
+    browDy: -4,
+    showBrows: true,
+    blushOpacity: 0.75,
+  },
+  recording: {
+    eyeScaleY: 1,
+    eyeScaleX: 1,
+    browTilt: 0,
+    browDy: 0,
+    showBrows: false,
+    blushOpacity: 0.9,
+  },
+  waving: {
+    eyeScaleY: 0.5,
+    eyeScaleX: 1.1,
+    browTilt: -6,
+    browDy: -6,
+    showBrows: false,
+    blushOpacity: 1,
+  },
+  dancing: {
+    eyeScaleY: 0.4,
+    eyeScaleX: 1.15,
+    browTilt: -8,
+    browDy: -8,
+    showBrows: false,
+    blushOpacity: 1,
+  },
+  drinking_coffee: {
+    eyeScaleY: 0.6,
+    eyeScaleX: 1,
+    browTilt: 0,
+    browDy: 0,
+    showBrows: false,
+    blushOpacity: 0.8,
+  },
+  drinking_boba: {
+    eyeScaleY: 0.55,
+    eyeScaleX: 1.05,
+    browTilt: 0,
+    browDy: 0,
+    showBrows: false,
+    blushOpacity: 0.85,
+  },
 };
 
 function presetFor(face: MascotFace): FacePreset {
@@ -375,6 +457,19 @@ function restMouthPath(face: MascotFace): string {
     case 'cautious':
       // Slight downward turn — concern lite.
       return 'M482,600 Q520,568 558,600 Q520,585 482,600 Z';
+    case 'celebrating':
+    case 'dancing':
+      return 'M460,565 Q520,635 580,565 Q520,605 460,565 Z';
+    case 'waving':
+      return 'M460,565 Q520,635 580,565 Q520,605 460,565 Z';
+    case 'writing':
+    case 'reading':
+      return 'M488,585 Q520,595 552,585 Q520,592 488,585 Z';
+    case 'recording':
+      return 'M495,580 Q520,600 545,580 Q520,615 495,580 Z';
+    case 'drinking_coffee':
+    case 'drinking_boba':
+      return 'M500,578 Q520,598 540,578 Q520,610 500,578 Z';
     default:
       return visemePath(VISEMES.REST);
   }

app/src/features/human/Mascot/RiveMascot.tsx

@@ -4,42 +4,72 @@ import {
   useRive,
   useViewModel,
   useViewModelInstance,
-  useViewModelInstanceBoolean,
   useViewModelInstanceColor,
   useViewModelInstanceString,
 } from '@rive-app/react-webgl2';
 import { type FC, useEffect } from 'react';
 
 import type { MascotFace } from './Ghosty';
-import type { VisemeId } from './visemes';
 
 export interface RiveMascotProps {
   face?: MascotFace;
   size?: number | string;
   primaryColor?: number;
   secondaryColor?: number;
-  viseme?: VisemeId;
+  /** Raw Oculus 15-set viseme code (e.g. 'sil', 'PP', 'aa') sent directly to
+   *  the Rive state machine's `mouthVisemeCode` input. When omitted, defaults
+   *  to 'sil' (mouth closed). */
+  visemeCode?: string;
 }
 
-const SPEAKING_FACES: ReadonlySet<MascotFace> = new Set(['speaking', 'happy']);
-
+/**
+ * Maps every MascotFace to the closest Rive pose animation. The Rive asset
+ * supports: idle, thinking, celebration, bookreading, coffeedrink, writing,
+ * bobbateadrink, recording, hand_wave, dancing.
+ */
 const FACE_TO_POSE: Record<MascotFace, string> = {
   idle: 'idle',
   normal: 'idle',
-  sleep: 'sleeping',
+  sleep: 'idle',
   listening: 'idle',
   thinking: 'thinking',
   confused: 'thinking',
   speaking: 'idle',
   happy: 'idle',
-  concerned: 'idle',
-  // New emotional states — map to the nearest available Rive pose until
-  // dedicated animations are added to the .riv asset.
-  curious: 'thinking',
-  proud: 'idle',
-  cautious: 'idle',
+  concerned: 'thinking',
+  curious: 'bookreading',
+  proud: 'celebration',
+  cautious: 'thinking',
+  celebrating: 'celebration',
+  writing: 'writing',
+  reading: 'bookreading',
+  recording: 'recording',
+  waving: 'hand_wave',
+  dancing: 'dancing',
+  drinking_coffee: 'coffeedrink',
+  drinking_boba: 'bobbateadrink',
 };
 
+/**
+ * ElevenLabs / Oculus 15-set → Rive asset's `visme_codes` vocabulary.
+ * The Rive file uses `ih`/`oh`/`ou` for vowels instead of `E`/`O`/`U`.
+ * Codes already in the Rive vocabulary pass through unchanged.
+ */
+const OCULUS_TO_RIVE_VISEME: Record<string, string> = {
+  E: 'ih',
+  I: 'ih',
+  O: 'oh',
+  U: 'ou',
+  e: 'ih',
+  i: 'ih',
+  o: 'oh',
+  u: 'ou',
+};
+
+function toRiveVisemeCode(oculusCode: string): string {
+  return OCULUS_TO_RIVE_VISEME[oculusCode] ?? oculusCode;
+}
+
 const RIVE_LAYOUT = new Layout({ fit: Fit.Contain });
 
 export const RiveMascot: FC<RiveMascotProps> = ({
@@ -47,7 +77,7 @@ export const RiveMascot: FC<RiveMascotProps> = ({
   size = '100%',
   primaryColor,
   secondaryColor,
-  viseme = 'REST',
+  visemeCode = 'sil',
 }) => {
   const { rive, RiveComponent } = useRive({
     src: '/tiny_mascot.riv',
@@ -58,20 +88,21 @@ export const RiveMascot: FC<RiveMascotProps> = ({
 
   const viewModel = useViewModel(rive, { useDefault: true });
   const vmInstance = useViewModelInstance(viewModel, { useDefault: true, rive });
-  const { setValue: setMouthOpen } = useViewModelInstanceBoolean('mouthOpen', vmInstance);
   const { setValue: setPose } = useViewModelInstanceString('pose', vmInstance);
-  const { setValue: setViseme } = useViewModelInstanceString('viseme', vmInstance);
+  const { setValue: setMouthVisemeCode } = useViewModelInstanceString(
+    'mouthVisemeCode',
+    vmInstance
+  );
   const { setValue: setPrimaryColor } = useViewModelInstanceColor('primaryColor', vmInstance);
   const { setValue: setSecondaryColor } = useViewModelInstanceColor('secondaryColor', vmInstance);
 
   useEffect(() => {
-    setMouthOpen(SPEAKING_FACES.has(face!));
-    setPose(FACE_TO_POSE[face!] ?? 'idle');
-  }, [face, setMouthOpen, setPose]);
+    setPose(FACE_TO_POSE[face] ?? 'idle');
+  }, [face, setPose]);
 
   useEffect(() => {
-    setViseme(viseme);
-  }, [viseme, setViseme]);
+    setMouthVisemeCode(toRiveVisemeCode(visemeCode));
+  }, [visemeCode, setMouthVisemeCode]);
 
   useEffect(() => {
     if (primaryColor !== undefined) setPrimaryColor(primaryColor);

app/src/features/human/useHumanMascot.test.ts

@@ -7,6 +7,7 @@ import {
   ACK_FACE_HOLD_MS,
   pickConversationAckFace,
   pickViseme,
+  pickVisemeCode,
   TTS_MAX_PLAYBACK_MS,
   useHumanMascot,
 } from './useHumanMascot';
@@ -139,6 +140,54 @@ describe('pickViseme', () => {
   });
 });
 
+describe('pickVisemeCode', () => {
+  it('maps vowels to Oculus 15-set codes', () => {
+    expect(pickVisemeCode('a')).toBe('aa');
+    expect(pickVisemeCode('e')).toBe('E');
+    expect(pickVisemeCode('i')).toBe('I');
+    expect(pickVisemeCode('o')).toBe('O');
+    expect(pickVisemeCode('u')).toBe('U');
+  });
+
+  it('maps labials to PP', () => {
+    expect(pickVisemeCode('m')).toBe('PP');
+    expect(pickVisemeCode('b')).toBe('PP');
+    expect(pickVisemeCode('p')).toBe('PP');
+  });
+
+  it('maps fricatives to FF', () => {
+    expect(pickVisemeCode('f')).toBe('FF');
+    expect(pickVisemeCode('v')).toBe('FF');
+  });
+
+  it('maps sibilants to SS', () => {
+    expect(pickVisemeCode('s')).toBe('SS');
+    expect(pickVisemeCode('z')).toBe('SS');
+  });
+
+  it('maps other consonants to their Oculus codes', () => {
+    expect(pickVisemeCode('n')).toBe('nn');
+    expect(pickVisemeCode('t')).toBe('DD');
+    expect(pickVisemeCode('k')).toBe('kk');
+    expect(pickVisemeCode('r')).toBe('RR');
+  });
+
+  it('uses the trailing letter of multi-char deltas', () => {
+    expect(pickVisemeCode('hello')).toBe('O');
+    expect(pickVisemeCode('world')).toBe('DD');
+  });
+
+  it('ignores punctuation when picking the trailing letter', () => {
+    expect(pickVisemeCode('Hi!')).toBe('I');
+  });
+
+  it('falls back to E for unmapped consonants and empty input', () => {
+    expect(pickVisemeCode('x')).toBe('E');
+    expect(pickVisemeCode('')).toBe('E');
+    expect(pickVisemeCode('...')).toBe('E');
+  });
+});
+
 describe('pickConversationAckFace', () => {
   it('prefers explicit reaction emoji from chat_done', () => {
     expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '✅' })).toBe('happy');
@@ -257,24 +306,46 @@ describe('useHumanMascot state machine', () => {
     expect(result.current.face).toBe('thinking');
   });
 
-  it('moves to confused on tool_call', () => {
+  it('maps tool_call to activity face when tool has a visual association', () => {
     const { result } = renderHook(() => useHumanMascot());
     act(() => {
       capturedListeners?.onInferenceStart?.(fakeEvent({}));
       capturedListeners?.onToolCall?.(
-        fakeEvent({ tool_name: 'search', skill_id: 's', args: {}, round: 1 })
+        fakeEvent({ tool_name: 'file_write', skill_id: 's', args: {}, round: 1 })
       );
     });
-    expect(result.current.face).toBe('confused');
+    expect(result.current.face).toBe('writing');
   });
 
-  it('moves to confused on iteration_start beyond round 1', () => {
+  it('falls back to thinking on tool_call for unmapped tools', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onToolCall?.(
+        fakeEvent({ tool_name: 'custom_tool', skill_id: 's', args: {}, round: 1 })
+      );
+    });
+    expect(result.current.face).toBe('thinking');
+  });
+
+  it('maps reading tools to reading face', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onToolCall?.(
+        fakeEvent({ tool_name: 'web_search', skill_id: 's', args: {}, round: 1 })
+      );
+    });
+    expect(result.current.face).toBe('reading');
+  });
+
+  it('moves to drinking_coffee on iteration_start beyond round 1', () => {
     const { result } = renderHook(() => useHumanMascot());
     act(() => {
       capturedListeners?.onInferenceStart?.(fakeEvent({}));
       capturedListeners?.onIterationStart?.(fakeEvent({ round: 2, message: '' }));
     });
-    expect(result.current.face).toBe('confused');
+    expect(result.current.face).toBe('drinking_coffee');
   });
 
   it('does not flip to confused on iteration_start round 1', () => {
@@ -309,7 +380,7 @@ describe('useHumanMascot state machine', () => {
     act(() => {
       capturedListeners?.onDone?.(
         fakeEvent({
-          full_response: 'hello',
+          full_response: 'sure thing',
           rounds_used: 1,
           total_input_tokens: 1,
           total_output_tokens: 1,
@@ -418,7 +489,7 @@ describe('useHumanMascot state machine', () => {
     expect(result.current.face).toBe('thinking');
   });
 
-  it('promotes to proud on chat_done when a tool succeeded in the same turn', () => {
+  it('promotes to celebrating on chat_done when a tool succeeded in the same turn', () => {
     const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
     act(() => {
       capturedListeners?.onInferenceStart?.(fakeEvent({}));
@@ -435,7 +506,7 @@ describe('useHumanMascot state machine', () => {
         })
       );
     });
-    expect(result.current.face).toBe('proud');
+    expect(result.current.face).toBe('celebrating');
     act(() => {
       vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
     });
@@ -459,7 +530,7 @@ describe('useHumanMascot state machine', () => {
     expect(result.current.face).toBe('happy');
   });
 
-  it('promotes to proud on chat_done when a subagent succeeded in the same turn', () => {
+  it('promotes to celebrating on chat_done when a subagent succeeded in the same turn', () => {
     const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
     act(() => {
       capturedListeners?.onInferenceStart?.(fakeEvent({}));
@@ -482,7 +553,7 @@ describe('useHumanMascot state machine', () => {
         })
       );
     });
-    expect(result.current.face).toBe('proud');
+    expect(result.current.face).toBe('celebrating');
   });
 
   it('shows concerned when a subagent fails', () => {
@@ -503,7 +574,7 @@ describe('useHumanMascot state machine', () => {
 
   it('resets work tracking on each new turn', () => {
     const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
-    // Turn 1: tool succeeded → proud
+    // Turn 1: tool succeeded → celebrating
     act(() => {
       capturedListeners?.onInferenceStart?.(fakeEvent({}));
       capturedListeners?.onToolResult?.(
@@ -519,7 +590,7 @@ describe('useHumanMascot state machine', () => {
         })
       );
     });
-    expect(result.current.face).toBe('proud');
+    expect(result.current.face).toBe('celebrating');
     act(() => {
       vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
     });
@@ -621,7 +692,7 @@ describe('useHumanMascot TTS playback', () => {
 
     const { result } = renderHook(() => useHumanMascot({ speakReplies: true }));
     await act(async () => {
-      capturedListeners?.onDone?.(fakeDone('hello'));
+      capturedListeners?.onDone?.(fakeDone('sure thing'));
       // Let synthesizeSpeech and playBase64Audio resolve.
       await Promise.resolve();
       await Promise.resolve();

app/src/features/human/useHumanMascot.ts

@@ -89,9 +89,67 @@ export function pickViseme(delta: string): VisemeShape {
   }
 }
 
+/**
+ * Pick a raw Oculus viseme code from a text delta character. Used to drive
+ * `mouthVisemeCode` on the Rive state machine during pseudo-lipsync (no TTS).
+ * Returns Oculus 15-set codes; `RiveMascot` translates vowels (`E`→`ih`,
+ * `O`→`oh`, `U`→`ou`) to the Rive asset's vocabulary at render time.
+ */
+export function pickVisemeCode(delta: string): string {
+  const ch = delta
+    .replace(/[^a-zA-Z]/g, '')
+    .slice(-1)
+    .toLowerCase();
+  switch (ch) {
+    case 'a':
+      return 'aa';
+    case 'e':
+      return 'E';
+    case 'i':
+    case 'y':
+      return 'I';
+    case 'o':
+      return 'O';
+    case 'u':
+    case 'w':
+      return 'U';
+    case 'm':
+    case 'b':
+    case 'p':
+      return 'PP';
+    case 'f':
+    case 'v':
+      return 'FF';
+    case 's':
+    case 'z':
+      return 'SS';
+    case 'n':
+    case 'l':
+      return 'nn';
+    case 't':
+    case 'd':
+      return 'DD';
+    case 'k':
+    case 'g':
+      return 'kk';
+    case 'r':
+      return 'RR';
+    default:
+      return 'E';
+  }
+}
+
 type ConversationAckFace = Extract<
   MascotFace,
-  'happy' | 'confused' | 'concerned' | 'curious' | 'proud' | 'cautious'
+  | 'happy'
+  | 'confused'
+  | 'concerned'
+  | 'curious'
+  | 'proud'
+  | 'cautious'
+  | 'celebrating'
+  | 'dancing'
+  | 'waving'
 >;
 type ConversationAckEvent = { full_response?: string | null; reaction_emoji?: string | null };
 
@@ -99,9 +157,11 @@ const HAPPY_REACTION_EMOJIS = new Set(['✅', '🎉', '🙌', '😊', '😄', '
 const PROUD_REACTION_EMOJIS = new Set(['⭐', '🌟', '🏆', '🎯', '💯', '🚀', '✨', '🥇']);
 const CURIOUS_REACTION_EMOJIS = new Set(['🔍', '💭', '🧐', '🤓', '👀']);
 const CONFUSED_REACTION_EMOJIS = new Set(['🤔', '❓', '❔']);
-// ⚠️ is cautious (heads-up), ❌/🚨 are concerned (failure).
 const CAUTIOUS_REACTION_EMOJIS = new Set(['⚠️', '⚠', '💡', '⚡']);
 const CONCERNED_REACTION_EMOJIS = new Set(['🚨', '❌', '😕', '😟']);
+const CELEBRATING_REACTION_EMOJIS = new Set(['🥳', '🍾', '🎊', '🎈', '🪅']);
+const DANCING_REACTION_EMOJIS = new Set(['💃', '🕺', '🎵', '🎶', '🎸']);
+const WAVING_REACTION_EMOJIS = new Set(['👋', '🤝', '🫡']);
 
 const CONCERNED_TEXT_RE =
   /\b(sorry|apolog(?:y|ize|ise)|failed|failure|error|cannot|can't|unable|blocked|problem)\b/i;
@@ -114,6 +174,10 @@ const CURIOUS_TEXT_RE =
   /\b(interesting|fascinating|curious(ly)?|let me (check|look|investigate)|i('ll)? (look|check) into|actually|turns? out)\b/i;
 const CAUTIOUS_TEXT_RE =
   /\b(be careful|warning|caution|heads? up|please note|make sure|important(ly)?|note that|worth (noting|mentioning))\b/i;
+const CELEBRATING_TEXT_RE =
+  /\b(congrat(ulations|s)?|well done|bravo|hooray|woohoo|amazing|fantastic|incredible|awesome work)\b/i;
+const GREETING_TEXT_RE =
+  /^(hello|hey|hi there|good (morning|afternoon|evening)|welcome back|greetings|howdy)[!.,]?(?:\s|$)/i;
 
 /**
  * Map conversation-level meaning into the short acknowledgement face that
@@ -123,6 +187,9 @@ const CAUTIOUS_TEXT_RE =
 export function pickConversationAckFace(event: ConversationAckEvent): ConversationAckFace | null {
   const reaction = event.reaction_emoji?.trim();
   if (reaction) {
+    if (CELEBRATING_REACTION_EMOJIS.has(reaction)) return 'celebrating';
+    if (DANCING_REACTION_EMOJIS.has(reaction)) return 'dancing';
+    if (WAVING_REACTION_EMOJIS.has(reaction)) return 'waving';
     if (PROUD_REACTION_EMOJIS.has(reaction)) return 'proud';
     if (HAPPY_REACTION_EMOJIS.has(reaction)) return 'happy';
     if (CURIOUS_REACTION_EMOJIS.has(reaction)) return 'curious';
@@ -133,18 +200,53 @@ export function pickConversationAckFace(event: ConversationAckEvent): Conversati
 
   const text = event.full_response?.trim() ?? '';
   if (!text) return null;
-  // Priority: concerned > cautious > proud > confused > curious > happy.
-  // Concerned and cautious share some vocabulary; check concerned first so
-  // outright failures don't get softened to a heads-up.
   if (CONCERNED_TEXT_RE.test(text)) return 'concerned';
   if (CAUTIOUS_TEXT_RE.test(text)) return 'cautious';
+  if (CELEBRATING_TEXT_RE.test(text)) return 'celebrating';
   if (PROUD_TEXT_RE.test(text)) return 'proud';
   if (CONFUSED_TEXT_RE.test(text)) return 'confused';
   if (CURIOUS_TEXT_RE.test(text)) return 'curious';
+  if (GREETING_TEXT_RE.test(text)) return 'waving';
   if (HAPPY_TEXT_RE.test(text)) return 'happy';
   return null;
 }
 
+/**
+ * Map a tool name to an activity pose. Returns null when the tool doesn't
+ * have a strong visual association — the caller falls back to a generic face.
+ */
+function toolToActivityFace(toolName: string): MascotFace | null {
+  const name = toolName.toLowerCase();
+
+  if (
+    name.includes('file_write') ||
+    name.includes('edit_file') ||
+    name.includes('apply_patch') ||
+    name.includes('create_file') ||
+    name.includes('write')
+  ) {
+    return 'writing';
+  }
+
+  if (
+    name.includes('browser') ||
+    name.includes('web_search') ||
+    name.includes('web_fetch') ||
+    name.includes('read_file') ||
+    name.includes('search') ||
+    name.includes('grep') ||
+    name.includes('find')
+  ) {
+    return 'reading';
+  }
+
+  if (name.includes('screen') || name.includes('screenshot') || name.includes('capture')) {
+    return 'recording';
+  }
+
+  return null;
+}
+
 export interface UseHumanMascotOptions {
   /** When true, post-stream replies are sent to ElevenLabs and the mouth
    *  follows the returned viseme timeline while the audio plays. */
@@ -157,6 +259,8 @@ export interface UseHumanMascotOptions {
 export interface UseHumanMascotResult {
   face: MascotFace;
   viseme: VisemeShape;
+  /** Raw Oculus 15-set viseme code for Rive's `mouthVisemeCode` input. */
+  visemeCode: string;
 }
 
 /**
@@ -165,7 +269,8 @@ export interface UseHumanMascotResult {
  * Mapping (kept in one place so the visual model stays coherent):
  *
  * - `inference_start` → `thinking`
- * - `iteration_start` round > 1 or `tool_call` → `confused` (heavy reasoning)
+ * - `iteration_start` round > 1 or `tool_call` → activity pose based on tool
+ *   name (writing/reading/recording) or `confused` as fallback
  * - `tool_result success=false` → `concerned` (held briefly)
  * - `text_delta` → `speaking`, pseudo-lipsync from the trailing letter
  * - `chat_done` (no TTS) → message-aware ack face (held briefly), then `idle`
@@ -184,31 +289,22 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
   const listeningRef = useRef(listening);
   listeningRef.current = listening;
 
-  // Effective mascot voice id: resolves the manual override, the
-  // locale-default toggle, and the build-time fallback into a single
-  // string (see `selectEffectiveMascotVoiceId`). Mirrored into a ref so
-  // the inner `startTtsPlayback` closure always reads the latest value
-  // without having to re-create the callback on every re-render.
   const effectiveMascotVoiceId = useSelector(selectEffectiveMascotVoiceId);
   const mascotVoiceIdRef = useRef<string>(effectiveMascotVoiceId);
   mascotVoiceIdRef.current = effectiveMascotVoiceId;
 
   const [face, setFace] = useState<MascotFace>('idle');
   const targetRef = useRef<VisemeShape>(VISEMES.REST);
+  const visemeCodeRef = useRef<string>('sil');
   const lastDeltaAtRef = useRef(0);
   const ackTimerRef = useRef<number | null>(null);
 
-  // Track meaningful work performed in the current turn so onDone can
-  // distinguish a proud completion from a routine happy acknowledgement.
   const toolSucceededRef = useRef(false);
   const subagentSucceededRef = useRef(false);
 
-  // TTS playback state — non-null while audio is mid-flight.
   const playbackRef = useRef<PlaybackHandle | null>(null);
   const visemeFramesRef = useRef<{ viseme: string; start_ms: number; end_ms: number }[]>([]);
   const visemeCursorRef = useRef(0);
-  // Monotonic counter — only the latest startTtsPlayback's callbacks may
-  // mutate idle state; older invocations bail out.
   const playbackSeqRef = useRef(0);
 
   const [, force] = useState(0);
@@ -239,22 +335,26 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
         setFace('thinking');
       },
       onIterationStart: e => {
-        // Subsequent iterations mean the agent is grinding through tool rounds.
         if (e.round > 1) {
           clearAckTimer();
-          mascotLog('voice-session transition → confused (iteration round=%d)', e.round);
-          setFace('confused');
+          mascotLog('voice-session transition → drinking_coffee (iteration round=%d)', e.round);
+          setFace('drinking_coffee');
         }
       },
-      onToolCall: () => {
+      onToolCall: e => {
         clearAckTimer();
-        mascotLog('voice-session transition → confused (tool_call)');
-        setFace('confused');
+        const activityFace = toolToActivityFace(e.tool_name);
+        if (activityFace) {
+          mascotLog('voice-session transition → %s (tool_call %s)', activityFace, e.tool_name);
+          setFace(activityFace);
+        } else {
+          mascotLog('voice-session transition → thinking (tool_call %s)', e.tool_name);
+          setFace('thinking');
+        }
       },
       onToolResult: e => {
         if (!e.success) {
           mascotLog('voice-session transition → concerned (tool_result failed)');
-          // Don't fully derail — let the next inference step take over.
           setFace('concerned');
         } else {
           toolSucceededRef.current = true;
@@ -274,7 +374,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
         }
       },
       onTextDelta: e => {
-        // Pseudo-lipsync only kicks in if no real audio is playing.
         if (listeningRef.current) {
           mascotLog('voice-session text_delta suppressed — listening is active');
           return;
@@ -283,6 +382,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
         clearAckTimer();
         setFace('speaking');
         targetRef.current = pickViseme(e.delta);
+        visemeCodeRef.current = pickVisemeCode(e.delta);
         lastDeltaAtRef.current = window.performance.now();
       },
       onDone: e => {
@@ -290,14 +390,11 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
           mascotLog('voice-session onDone suppressed — listening is active');
           return;
         }
-        // Upgrade to `proud` when the turn involved real tool/subagent work.
-        // A happy/null text cue paired with actual execution is a completion —
-        // that reads as proud, not a routine acknowledgement.
         const didMeaningfulWork = toolSucceededRef.current || subagentSucceededRef.current;
         const explicitAck = pickConversationAckFace(e);
         const ackFace: ConversationAckFace =
           (explicitAck === 'happy' || explicitAck === null) && didMeaningfulWork
-            ? 'proud'
+            ? 'celebrating'
             : (explicitAck ?? 'happy');
         toolSucceededRef.current = false;
         subagentSucceededRef.current = false;
@@ -308,24 +405,18 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
           didMeaningfulWork
         );
         if (!speakRef.current || !e.full_response?.trim()) {
-          // Soft acknowledgement beat instead of snapping back to idle.
           holdThenIdle(ackFace);
           return;
         }
-        // Fire-and-forget — startTtsPlayback owns its cleanup via finally.
         void startTtsPlayback(e.full_response, ackFace).catch(() => {});
       },
       onError: () => {
         mascotLog('voice-session transition → concerned (chat_error), cancelling in-flight TTS');
-        // Bump seq to invalidate any in-flight startTtsPlayback awaiters.
         playbackSeqRef.current++;
         const orphan = playbackRef.current;
         playbackRef.current = null;
         if (orphan) {
           orphan.stop();
-          // We're early-returning instead of awaiting `orphan.ended`, so the
-          // stop()-sentinel rejection has no handler — attach one explicitly
-          // or it surfaces as an unhandledrejection in Sentry (#1472).
           orphan.ended.catch(swallowAudioStop);
         }
         visemeFramesRef.current = [];
@@ -335,7 +426,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
     return () => {
       unsub();
       clearAckTimer();
-      // Same — invalidate in-flight callbacks before tearing down.
       playbackSeqRef.current++;
       const orphan = playbackRef.current;
       playbackRef.current = null;
@@ -357,8 +447,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
         ? 'user started recording while TTS was playing (interrupted)'
         : 'mic activated, no TTS to cancel'
     );
-    // Treat mic-hot as an explicit interruption: stale synthesis/playback
-    // callbacks must not switch the mascot back to speaking after we listen.
     playbackSeqRef.current++;
     const orphan = playbackRef.current;
     playbackRef.current = null;
@@ -369,6 +457,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
     visemeFramesRef.current = [];
     visemeCursorRef.current = 0;
     targetRef.current = VISEMES.REST;
+    visemeCodeRef.current = 'sil';
     lastDeltaAtRef.current = 0;
     setFace('idle');
   }, [listening]);
@@ -377,8 +466,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
     text: string,
     ackFace: ConversationAckFace = 'happy'
   ): Promise<void> {
-    // Cancel any in-flight playback so its handle.ended callback can't reset
-    // state belonging to the new run.
     const prev = playbackRef.current;
     playbackRef.current = null;
     if (prev) {
@@ -396,13 +483,8 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
       setFace('thinking');
       let tts;
       try {
-        // Always pass the effective voice id — the selector already
-        // resolves manual override / locale default / build-time
-        // fallback to a single string, so `synthesizeSpeech` doesn't
-        // need its own fallback branch here.
         tts = await synthesizeSpeech(text, { voiceId: mascotVoiceIdRef.current });
       } catch (err) {
-        // Voice path unavailable — degrade cleanly to text-only behavior.
         if (isStillCurrent()) degraded = true;
         throw err;
       }
@@ -410,26 +492,16 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
       let frames: VisemeFrame[] = tts.visemes ?? [];
       let source: 'visemes' | 'alignment' | 'procedural' = 'visemes';
       if (frames.length > 0 && !framesProduceMotion(frames)) {
-        // Backend shipped frames but every code maps to REST — usually means
-        // the codes are in a vocabulary `oculusVisemeToShape` doesn't know.
-        // Drop them and let the alignment / procedural path take over so the
-        // mouth doesn't sit on the rest-smile path for the whole clip.
         mascotLog('tts visemes produced no motion — dropping and falling through');
         frames = [];
       }
       if (frames.length === 0 && tts.alignment && tts.alignment.length > 0) {
-        // Backend didn't ship viseme cues — derive a coarse track from char timings
-        // so the mouth still animates in sync with the audio.
         frames = visemesFromAlignment(tts.alignment);
         source = 'alignment';
         mascotLog('tts derived %d viseme frames from alignment', frames.length);
       } else if (frames.length > 0) {
         mascotLog('tts got %d viseme frames from backend', frames.length);
       }
-      // Start audio first — `playBase64Audio` calls `audio.play()` directly so
-      // the user-gesture chain that authorized speech stays intact. If we
-      // awaited anything else between the user click and play(), CEF would
-      // reject playback under its autoplay policy.
       const ttsOptions: PlaybackOptions = { maxDurationMs: TTS_MAX_PLAYBACK_MS };
       const handle = await playBase64Audio(
         tts.audio_base64,
@@ -442,12 +514,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
         return;
       }
       if (frames.length === 0) {
-        // Last-resort fallback: backend shipped neither viseme cues nor
-        // alignment (e.g. the new public `tts-v1` model on the hosted
-        // backend). Use whatever duration the decoder has reported so far —
-        // `proceduralVisemes` falls back to a text-length estimate when the
-        // metadata hasn't loaded yet, so we don't await it on the critical
-        // path (waiting opens a window where audio plays under a static face).
         const dur = handle.durationMs();
         frames = proceduralVisemes(text, dur);
         source = 'procedural';
@@ -465,8 +531,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
       try {
         await handle.ended;
       } catch (err) {
-        // Stop sentinel is expected when a newer turn cancels playback —
-        // rethrow anything else so real decoder errors aren't masked.
         swallowAudioStop(err);
       }
     } catch (err) {
@@ -476,6 +540,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
       if (isStillCurrent()) {
         playbackRef.current = null;
         visemeFramesRef.current = [];
+        visemeCodeRef.current = 'sil';
         if (degraded) {
           holdThenIdle('concerned');
         } else {
@@ -485,9 +550,6 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
     }
   }
 
-  // RAF loop while we're speaking. TTS playback always sets face to
-  // 'speaking' before awaiting the audio, so this also covers the audio-driven
-  // viseme path.
   useEffect(() => {
     if (face !== 'speaking') return;
     let raf = 0;
@@ -500,6 +562,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
   }, [face]);
 
   let viseme: VisemeShape = VISEMES.REST;
+  let visemeCode = 'sil';
   const playback = playbackRef.current;
   if (playback) {
     const ms = playback.currentMs();
@@ -511,17 +574,18 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
       );
       visemeCursorRef.current = cursor;
       viseme = frame ? oculusVisemeToShape(frame.viseme) : VISEMES.REST;
+      visemeCode = frame ? frame.viseme : 'sil';
     }
   } else if (face === 'speaking') {
     const since = window.performance.now() - lastDeltaAtRef.current;
     const decay = Math.max(0, Math.min(1, since / VISEME_DECAY_MS));
     viseme = lerpViseme(targetRef.current, VISEMES.REST, decay);
+    visemeCode = decay > 0.5 ? 'sil' : visemeCodeRef.current;
   }
 
-  // `listening` is an external override so callers wiring dictation state
-  // can reflect mic-on without racing the chat event subscription.
   const effectiveFace: MascotFace = listening ? 'listening' : face;
   const effectiveViseme: VisemeShape = listening ? VISEMES.REST : viseme;
+  const effectiveVisemeCode: string = listening ? 'sil' : visemeCode;
 
-  return { face: effectiveFace, viseme: effectiveViseme };
+  return { face: effectiveFace, viseme: effectiveViseme, visemeCode: effectiveVisemeCode };
 }

app/src/pages/ios/MascotScreen.tsx

@@ -205,7 +205,7 @@ export const MascotScreen: FC = () => {
   // without a stale closure over the pttActive state variable.
   const pttActiveRef = useRef(false);
 
-  const { face } = useHumanMascot({ listening: pttActive });
+  const { face, visemeCode } = useHumanMascot({ listening: pttActive });
 
   // Derive label from stored profile.
   const pairedLabel = (() => {
@@ -431,7 +431,7 @@ export const MascotScreen: FC = () => {
       {/* Mascot canvas */}
       <div className="flex-1 flex items-center justify-center overflow-hidden min-h-0 py-4">
         <div className="w-full max-w-xs aspect-square">
-          <RiveMascot face={face} />
+          <RiveMascot face={face} visemeCode={visemeCode} />
         </div>
       </div>