feat(mascot): add emotional reaction layer for conversation state (#2900)

Co-authored-by: Taimoor <astikkosapparel009@gmail.com>

app/src/features/human/Mascot/Ghosty.tsx

@@ -17,6 +17,10 @@ import { visemePath, VISEMES, type VisemeShape } from './visemes';
  *   mouth from `viseme` rather than from `face`.
  * - `happy` — short post-turn acknowledgement before falling back to `idle`.
  * - `concerned` — error / failed tool / unavailable voice path.
+ * - `curious` — attentive/interested; user asked something engaging or agent
+ *   is exploring an interesting problem.
+ * - `proud` — task fully completed after meaningful tool/subagent work.
+ * - `cautious` — gentle warning; less severe than `concerned`.
  *
  * `normal` is the legacy alias for `idle` and stays accepted for backwards
  * compatibility with older callers.
@@ -30,6 +34,9 @@ export type MascotFace =
   | 'speaking'
   | 'happy'
   | 'concerned'
+  | 'curious'
+  | 'proud'
+  | 'cautious'
   | 'normal';
 
 export interface GhostyProps {
@@ -124,6 +131,37 @@ const FACE_PRESETS: Record<Exclude<MascotFace, 'normal'>, FacePreset> = {
     showBrows: true,
     blushOpacity: 0.5,
   },
+  // Wide, attentive eyes with raised brows — the mascot is engaged and
+  // interested in what is happening.
+  curious: {
+    eyeScaleY: 1.1,
+    eyeScaleX: 1.05,
+    browTilt: -10,
+    browDy: -8,
+    showBrows: true,
+    blushOpacity: 0.8,
+  },
+  // Squinted-happy with full blush — task completed after real work.
+  // Visually distinct from `happy` (less squint, brows soft) so it reads
+  // as satisfaction rather than a quick acknowledgement.
+  proud: {
+    eyeScaleY: 0.55,
+    eyeScaleX: 1.15,
+    browTilt: -4,
+    browDy: -4,
+    showBrows: false,
+    blushOpacity: 1,
+  },
+  // Gentle worry — a heads-up rather than a failure. Softer than `concerned`
+  // (less brow tilt, lighter blush reduction).
+  cautious: {
+    eyeScaleY: 0.9,
+    eyeScaleX: 0.95,
+    browTilt: 10,
+    browDy: -3,
+    showBrows: true,
+    blushOpacity: 0.65,
+  },
 };
 
 function presetFor(face: MascotFace): FacePreset {
@@ -328,6 +366,15 @@ function restMouthPath(face: MascotFace): string {
     case 'listening':
       // Open soft "o".
       return 'M495,580 Q520,600 545,580 Q520,615 495,580 Z';
+    case 'curious':
+      // Small open oval — slight "oh?" shape.
+      return 'M500,578 Q520,598 540,578 Q520,610 500,578 Z';
+    case 'proud':
+      // Relaxed upward curve, wider than happy but not a full grin.
+      return 'M468,570 Q520,625 572,570 Q520,600 468,570 Z';
+    case 'cautious':
+      // Slight downward turn — concern lite.
+      return 'M482,600 Q520,568 558,600 Q520,585 482,600 Z';
     default:
       return visemePath(VISEMES.REST);
   }

app/src/features/human/Mascot/RiveMascot.tsx

@@ -33,6 +33,11 @@ const FACE_TO_POSE: Record<MascotFace, string> = {
   speaking: 'idle',
   happy: 'idle',
   concerned: 'idle',
+  // New emotional states — map to the nearest available Rive pose until
+  // dedicated animations are added to the .riv asset.
+  curious: 'thinking',
+  proud: 'idle',
+  cautious: 'idle',
 };
 
 const RIVE_LAYOUT = new Layout({ fit: Fit.Contain });

app/src/features/human/useHumanMascot.test.ts

@@ -144,11 +144,26 @@ describe('pickConversationAckFace', () => {
     expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '🤔' })).toBe(
       'confused'
     );
+    // ⚠️ is now cautious (heads-up), not concerned.
     expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '⚠️' })).toBe(
+      'cautious'
+    );
+    expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '❌' })).toBe(
       'concerned'
     );
   });
 
+  it('maps proud and curious reaction emojis', () => {
+    expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '🏆' })).toBe('proud');
+    expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '⭐' })).toBe('proud');
+    expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '🔍' })).toBe(
+      'curious'
+    );
+    expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '🧐' })).toBe(
+      'curious'
+    );
+  });
+
   it('falls back to deterministic response text cues', () => {
     expect(
       pickConversationAckFace({ full_response: 'All set, this is fixed.', reaction_emoji: null })
@@ -167,6 +182,42 @@ describe('pickConversationAckFace', () => {
     ).toBe('concerned');
   });
 
+  it('maps proud text cues', () => {
+    expect(
+      pickConversationAckFace({
+        full_response: 'Successfully completed all tasks done!',
+        reaction_emoji: null,
+      })
+    ).toBe('proud');
+  });
+
+  it('maps cautious text cues', () => {
+    expect(
+      pickConversationAckFace({
+        full_response: 'Heads up, this might cause unexpected side effects.',
+        reaction_emoji: null,
+      })
+    ).toBe('cautious');
+  });
+
+  it('maps curious text cues', () => {
+    expect(
+      pickConversationAckFace({
+        full_response: 'Interesting — let me check what is happening here.',
+        reaction_emoji: null,
+      })
+    ).toBe('curious');
+  });
+
+  it('concerned takes priority over cautious when both patterns match', () => {
+    expect(
+      pickConversationAckFace({
+        full_response: 'Sorry, this failed. Make sure you try again.',
+        reaction_emoji: null,
+      })
+    ).toBe('concerned');
+  });
+
   it('returns null when there is no strong cue', () => {
     expect(
       pickConversationAckFace({ full_response: 'Here is the summary.', reaction_emoji: null })
@@ -366,6 +417,127 @@ describe('useHumanMascot state machine', () => {
     expect(result.current.face).toBe('thinking');
   });
 
+  it('promotes to proud on chat_done when a tool succeeded in the same turn', () => {
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onToolResult?.(
+        fakeEvent({ tool_name: 'run', skill_id: 's', output: 'ok', success: true, round: 1 })
+      );
+      capturedListeners?.onDone?.(
+        fakeEvent({
+          full_response: 'Here is the result.',
+          reaction_emoji: null,
+          rounds_used: 2,
+          total_input_tokens: 1,
+          total_output_tokens: 1,
+        })
+      );
+    });
+    expect(result.current.face).toBe('proud');
+    act(() => {
+      vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
+    });
+    expect(result.current.face).toBe('idle');
+  });
+
+  it('uses happy (not proud) when no tool work was done', () => {
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onDone?.(
+        fakeEvent({
+          full_response: 'Here is the result.',
+          reaction_emoji: null,
+          rounds_used: 1,
+          total_input_tokens: 1,
+          total_output_tokens: 1,
+        })
+      );
+    });
+    expect(result.current.face).toBe('happy');
+  });
+
+  it('promotes to proud on chat_done when a subagent succeeded in the same turn', () => {
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onSubagentDone?.(
+        fakeEvent({
+          tool_name: 'researcher',
+          skill_id: 'sa1',
+          message: 'done',
+          success: true,
+          round: 1,
+        })
+      );
+      capturedListeners?.onDone?.(
+        fakeEvent({
+          full_response: 'Research complete.',
+          reaction_emoji: null,
+          rounds_used: 1,
+          total_input_tokens: 1,
+          total_output_tokens: 1,
+        })
+      );
+    });
+    expect(result.current.face).toBe('proud');
+  });
+
+  it('shows concerned when a subagent fails', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onSubagentDone?.(
+        fakeEvent({
+          tool_name: 'researcher',
+          skill_id: 'sa1',
+          message: 'failed',
+          success: false,
+          round: 1,
+        })
+      );
+    });
+    expect(result.current.face).toBe('concerned');
+  });
+
+  it('resets work tracking on each new turn', () => {
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
+    // Turn 1: tool succeeded → proud
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onToolResult?.(
+        fakeEvent({ tool_name: 'run', skill_id: 's', output: 'ok', success: true, round: 1 })
+      );
+      capturedListeners?.onDone?.(
+        fakeEvent({
+          full_response: 'Done.',
+          reaction_emoji: null,
+          rounds_used: 2,
+          total_input_tokens: 1,
+          total_output_tokens: 1,
+        })
+      );
+    });
+    expect(result.current.face).toBe('proud');
+    act(() => {
+      vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
+    });
+    // Turn 2: no tool work → happy
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onDone?.(
+        fakeEvent({
+          full_response: 'Here you go.',
+          reaction_emoji: null,
+          rounds_used: 1,
+          total_input_tokens: 1,
+          total_output_tokens: 1,
+        })
+      );
+    });
+    expect(result.current.face).toBe('happy');
+  });
+
   it('listening overrides streaming speech deltas', () => {
     const { result, rerender } = renderHook(
       ({ listening }: { listening: boolean }) => useHumanMascot({ listening }),

app/src/features/human/useHumanMascot.ts

@@ -2,7 +2,7 @@ import debug from 'debug';
 import { useEffect, useRef, useState } from 'react';
 import { useSelector } from 'react-redux';
 
-import { subscribeChatEvents } from '../../services/chatService';
+import { type ChatSubagentDoneEvent, subscribeChatEvents } from '../../services/chatService';
 import { selectEffectiveMascotVoiceId } from '../../store/mascotSlice';
 import type { MascotFace } from './Mascot';
 import { lerpViseme, VISEMES, type VisemeShape } from './Mascot/visemes';
@@ -76,18 +76,31 @@ export function pickViseme(delta: string): VisemeShape {
   }
 }
 
-type ConversationAckFace = Extract<MascotFace, 'happy' | 'confused' | 'concerned'>;
+type ConversationAckFace = Extract<
+  MascotFace,
+  'happy' | 'confused' | 'concerned' | 'curious' | 'proud' | 'cautious'
+>;
 type ConversationAckEvent = { full_response?: string | null; reaction_emoji?: string | null };
 
 const HAPPY_REACTION_EMOJIS = new Set(['✅', '🎉', '🙌', '😊', '😄', '👍', '💪']);
+const PROUD_REACTION_EMOJIS = new Set(['⭐', '🌟', '🏆', '🎯', '💯', '🚀', '✨', '🥇']);
+const CURIOUS_REACTION_EMOJIS = new Set(['🔍', '💭', '🧐', '🤓', '👀']);
 const CONFUSED_REACTION_EMOJIS = new Set(['🤔', '❓', '❔']);
-const CONCERNED_REACTION_EMOJIS = new Set(['⚠️', '⚠', '🚨', '❌', '😕', '😟']);
+// ⚠️ is cautious (heads-up), ❌/🚨 are concerned (failure).
+const CAUTIOUS_REACTION_EMOJIS = new Set(['⚠️', '⚠', '💡', '⚡']);
+const CONCERNED_REACTION_EMOJIS = new Set(['🚨', '❌', '😕', '😟']);
 
 const CONCERNED_TEXT_RE =
   /\b(sorry|apolog(?:y|ize|ise)|failed|failure|error|cannot|can't|unable|blocked|problem)\b/i;
 const CONFUSED_TEXT_RE =
   /\b(not sure|unclear|ambiguous|clarify|which one|need more|can you confirm|maybe)\b/i;
 const HAPPY_TEXT_RE = /\b(done|completed|fixed|success|successful|ready|all set|great|nice)\b/i;
+const PROUD_TEXT_RE =
+  /\b(successfully completed|all tasks? (done|finished)|mission accomplished|everything (works?|is working)|all (checks?|tests?) pass(ed)?)\b/i;
+const CURIOUS_TEXT_RE =
+  /\b(interesting|fascinating|curious(ly)?|let me (check|look|investigate)|i('ll)? (look|check) into|actually|turns? out)\b/i;
+const CAUTIOUS_TEXT_RE =
+  /\b(be careful|warning|caution|heads? up|please note|make sure|important(ly)?|note that|worth (noting|mentioning))\b/i;
 
 /**
  * Map conversation-level meaning into the short acknowledgement face that
@@ -97,15 +110,24 @@ const HAPPY_TEXT_RE = /\b(done|completed|fixed|success|successful|ready|all set|
 export function pickConversationAckFace(event: ConversationAckEvent): ConversationAckFace | null {
   const reaction = event.reaction_emoji?.trim();
   if (reaction) {
+    if (PROUD_REACTION_EMOJIS.has(reaction)) return 'proud';
     if (HAPPY_REACTION_EMOJIS.has(reaction)) return 'happy';
+    if (CURIOUS_REACTION_EMOJIS.has(reaction)) return 'curious';
     if (CONFUSED_REACTION_EMOJIS.has(reaction)) return 'confused';
+    if (CAUTIOUS_REACTION_EMOJIS.has(reaction)) return 'cautious';
     if (CONCERNED_REACTION_EMOJIS.has(reaction)) return 'concerned';
   }
 
   const text = event.full_response?.trim() ?? '';
   if (!text) return null;
+  // Priority: concerned > cautious > proud > confused > curious > happy.
+  // Concerned and cautious share some vocabulary; check concerned first so
+  // outright failures don't get softened to a heads-up.
   if (CONCERNED_TEXT_RE.test(text)) return 'concerned';
+  if (CAUTIOUS_TEXT_RE.test(text)) return 'cautious';
+  if (PROUD_TEXT_RE.test(text)) return 'proud';
   if (CONFUSED_TEXT_RE.test(text)) return 'confused';
+  if (CURIOUS_TEXT_RE.test(text)) return 'curious';
   if (HAPPY_TEXT_RE.test(text)) return 'happy';
   return null;
 }
@@ -163,6 +185,11 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
   const lastDeltaAtRef = useRef(0);
   const ackTimerRef = useRef<number | null>(null);
 
+  // Track meaningful work performed in the current turn so onDone can
+  // distinguish a proud completion from a routine happy acknowledgement.
+  const toolSucceededRef = useRef(false);
+  const subagentSucceededRef = useRef(false);
+
   // TTS playback state — non-null while audio is mid-flight.
   const playbackRef = useRef<PlaybackHandle | null>(null);
   const visemeFramesRef = useRef<{ viseme: string; start_ms: number; end_ms: number }[]>([]);
@@ -193,6 +220,8 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
     const unsub = subscribeChatEvents({
       onInferenceStart: () => {
         clearAckTimer();
+        toolSucceededRef.current = false;
+        subagentSucceededRef.current = false;
         mascotLog('voice-session transition → thinking (inference_start)');
         setFace('thinking');
       },
@@ -215,9 +244,22 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
           // Don't fully derail — let the next inference step take over.
           setFace('concerned');
         } else {
+          toolSucceededRef.current = true;
           setFace('thinking');
         }
       },
+      onSubagentDone: (e: ChatSubagentDoneEvent) => {
+        if (e.success) {
+          mascotLog('voice-session subagent_done success tool=%s', e.tool_name);
+          subagentSucceededRef.current = true;
+        } else {
+          mascotLog(
+            'voice-session transition → concerned (subagent_done failed tool=%s)',
+            e.tool_name
+          );
+          setFace('concerned');
+        }
+      },
       onTextDelta: e => {
         // Pseudo-lipsync only kicks in if no real audio is playing.
         if (listeningRef.current) {
@@ -235,7 +277,23 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
           mascotLog('voice-session onDone suppressed — listening is active');
           return;
         }
-        const ackFace = pickConversationAckFace(e) ?? 'happy';
+        // Upgrade to `proud` when the turn involved real tool/subagent work.
+        // A happy/null text cue paired with actual execution is a completion —
+        // that reads as proud, not a routine acknowledgement.
+        const didMeaningfulWork = toolSucceededRef.current || subagentSucceededRef.current;
+        const explicitAck = pickConversationAckFace(e);
+        const ackFace: ConversationAckFace =
+          (explicitAck === 'happy' || explicitAck === null) && didMeaningfulWork
+            ? 'proud'
+            : (explicitAck ?? 'happy');
+        toolSucceededRef.current = false;
+        subagentSucceededRef.current = false;
+        mascotLog(
+          'voice-session onDone ackFace=%s (explicit=%s didWork=%s)',
+          ackFace,
+          explicitAck ?? 'none',
+          didMeaningfulWork
+        );
         if (!speakRef.current || !e.full_response?.trim()) {
           // Soft acknowledgement beat instead of snapping back to idle.
           holdThenIdle(ackFace);