diff --git a/app/src/features/human/Mascot/Ghosty.tsx b/app/src/features/human/Mascot/Ghosty.tsx index ba982f9f0..bf02a1784 100644 --- a/app/src/features/human/Mascot/Ghosty.tsx +++ b/app/src/features/human/Mascot/Ghosty.tsx @@ -17,6 +17,10 @@ import { visemePath, VISEMES, type VisemeShape } from './visemes'; * mouth from `viseme` rather than from `face`. * - `happy` — short post-turn acknowledgement before falling back to `idle`. * - `concerned` — error / failed tool / unavailable voice path. + * - `curious` — attentive/interested; user asked something engaging or agent + * is exploring an interesting problem. + * - `proud` — task fully completed after meaningful tool/subagent work. + * - `cautious` — gentle warning; less severe than `concerned`. * * `normal` is the legacy alias for `idle` and stays accepted for backwards * compatibility with older callers. @@ -30,6 +34,9 @@ export type MascotFace = | 'speaking' | 'happy' | 'concerned' + | 'curious' + | 'proud' + | 'cautious' | 'normal'; export interface GhostyProps { @@ -124,6 +131,37 @@ const FACE_PRESETS: Record, FacePreset> = { showBrows: true, blushOpacity: 0.5, }, + // Wide, attentive eyes with raised brows — the mascot is engaged and + // interested in what is happening. + curious: { + eyeScaleY: 1.1, + eyeScaleX: 1.05, + browTilt: -10, + browDy: -8, + showBrows: true, + blushOpacity: 0.8, + }, + // Squinted-happy with full blush — task completed after real work. + // Visually distinct from `happy` (less squint, brows soft) so it reads + // as satisfaction rather than a quick acknowledgement. + proud: { + eyeScaleY: 0.55, + eyeScaleX: 1.15, + browTilt: -4, + browDy: -4, + showBrows: false, + blushOpacity: 1, + }, + // Gentle worry — a heads-up rather than a failure. Softer than `concerned` + // (less brow tilt, lighter blush reduction). + cautious: { + eyeScaleY: 0.9, + eyeScaleX: 0.95, + browTilt: 10, + browDy: -3, + showBrows: true, + blushOpacity: 0.65, + }, }; function presetFor(face: MascotFace): FacePreset { @@ -328,6 +366,15 @@ function restMouthPath(face: MascotFace): string { case 'listening': // Open soft "o". return 'M495,580 Q520,600 545,580 Q520,615 495,580 Z'; + case 'curious': + // Small open oval — slight "oh?" shape. + return 'M500,578 Q520,598 540,578 Q520,610 500,578 Z'; + case 'proud': + // Relaxed upward curve, wider than happy but not a full grin. + return 'M468,570 Q520,625 572,570 Q520,600 468,570 Z'; + case 'cautious': + // Slight downward turn — concern lite. + return 'M482,600 Q520,568 558,600 Q520,585 482,600 Z'; default: return visemePath(VISEMES.REST); } diff --git a/app/src/features/human/Mascot/RiveMascot.tsx b/app/src/features/human/Mascot/RiveMascot.tsx index a2349f070..fb264cbb5 100644 --- a/app/src/features/human/Mascot/RiveMascot.tsx +++ b/app/src/features/human/Mascot/RiveMascot.tsx @@ -33,6 +33,11 @@ const FACE_TO_POSE: Record = { speaking: 'idle', happy: 'idle', concerned: 'idle', + // New emotional states — map to the nearest available Rive pose until + // dedicated animations are added to the .riv asset. + curious: 'thinking', + proud: 'idle', + cautious: 'idle', }; const RIVE_LAYOUT = new Layout({ fit: Fit.Contain }); diff --git a/app/src/features/human/useHumanMascot.test.ts b/app/src/features/human/useHumanMascot.test.ts index 2d1b07037..7528de15a 100644 --- a/app/src/features/human/useHumanMascot.test.ts +++ b/app/src/features/human/useHumanMascot.test.ts @@ -144,11 +144,26 @@ describe('pickConversationAckFace', () => { expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '🤔' })).toBe( 'confused' ); + // ⚠️ is now cautious (heads-up), not concerned. expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '⚠️' })).toBe( + 'cautious' + ); + expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '❌' })).toBe( 'concerned' ); }); + it('maps proud and curious reaction emojis', () => { + expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '🏆' })).toBe('proud'); + expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '⭐' })).toBe('proud'); + expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '🔍' })).toBe( + 'curious' + ); + expect(pickConversationAckFace({ full_response: 'Done', reaction_emoji: '🧐' })).toBe( + 'curious' + ); + }); + it('falls back to deterministic response text cues', () => { expect( pickConversationAckFace({ full_response: 'All set, this is fixed.', reaction_emoji: null }) @@ -167,6 +182,42 @@ describe('pickConversationAckFace', () => { ).toBe('concerned'); }); + it('maps proud text cues', () => { + expect( + pickConversationAckFace({ + full_response: 'Successfully completed all tasks done!', + reaction_emoji: null, + }) + ).toBe('proud'); + }); + + it('maps cautious text cues', () => { + expect( + pickConversationAckFace({ + full_response: 'Heads up, this might cause unexpected side effects.', + reaction_emoji: null, + }) + ).toBe('cautious'); + }); + + it('maps curious text cues', () => { + expect( + pickConversationAckFace({ + full_response: 'Interesting — let me check what is happening here.', + reaction_emoji: null, + }) + ).toBe('curious'); + }); + + it('concerned takes priority over cautious when both patterns match', () => { + expect( + pickConversationAckFace({ + full_response: 'Sorry, this failed. Make sure you try again.', + reaction_emoji: null, + }) + ).toBe('concerned'); + }); + it('returns null when there is no strong cue', () => { expect( pickConversationAckFace({ full_response: 'Here is the summary.', reaction_emoji: null }) @@ -366,6 +417,127 @@ describe('useHumanMascot state machine', () => { expect(result.current.face).toBe('thinking'); }); + it('promotes to proud on chat_done when a tool succeeded in the same turn', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onToolResult?.( + fakeEvent({ tool_name: 'run', skill_id: 's', output: 'ok', success: true, round: 1 }) + ); + capturedListeners?.onDone?.( + fakeEvent({ + full_response: 'Here is the result.', + reaction_emoji: null, + rounds_used: 2, + total_input_tokens: 1, + total_output_tokens: 1, + }) + ); + }); + expect(result.current.face).toBe('proud'); + act(() => { + vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1); + }); + expect(result.current.face).toBe('idle'); + }); + + it('uses happy (not proud) when no tool work was done', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onDone?.( + fakeEvent({ + full_response: 'Here is the result.', + reaction_emoji: null, + rounds_used: 1, + total_input_tokens: 1, + total_output_tokens: 1, + }) + ); + }); + expect(result.current.face).toBe('happy'); + }); + + it('promotes to proud on chat_done when a subagent succeeded in the same turn', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onSubagentDone?.( + fakeEvent({ + tool_name: 'researcher', + skill_id: 'sa1', + message: 'done', + success: true, + round: 1, + }) + ); + capturedListeners?.onDone?.( + fakeEvent({ + full_response: 'Research complete.', + reaction_emoji: null, + rounds_used: 1, + total_input_tokens: 1, + total_output_tokens: 1, + }) + ); + }); + expect(result.current.face).toBe('proud'); + }); + + it('shows concerned when a subagent fails', () => { + const { result } = renderHook(() => useHumanMascot()); + act(() => { + capturedListeners?.onSubagentDone?.( + fakeEvent({ + tool_name: 'researcher', + skill_id: 'sa1', + message: 'failed', + success: false, + round: 1, + }) + ); + }); + expect(result.current.face).toBe('concerned'); + }); + + it('resets work tracking on each new turn', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + // Turn 1: tool succeeded → proud + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onToolResult?.( + fakeEvent({ tool_name: 'run', skill_id: 's', output: 'ok', success: true, round: 1 }) + ); + capturedListeners?.onDone?.( + fakeEvent({ + full_response: 'Done.', + reaction_emoji: null, + rounds_used: 2, + total_input_tokens: 1, + total_output_tokens: 1, + }) + ); + }); + expect(result.current.face).toBe('proud'); + act(() => { + vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1); + }); + // Turn 2: no tool work → happy + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onDone?.( + fakeEvent({ + full_response: 'Here you go.', + reaction_emoji: null, + rounds_used: 1, + total_input_tokens: 1, + total_output_tokens: 1, + }) + ); + }); + expect(result.current.face).toBe('happy'); + }); + it('listening overrides streaming speech deltas', () => { const { result, rerender } = renderHook( ({ listening }: { listening: boolean }) => useHumanMascot({ listening }), diff --git a/app/src/features/human/useHumanMascot.ts b/app/src/features/human/useHumanMascot.ts index ecaab022f..4bf6a4226 100644 --- a/app/src/features/human/useHumanMascot.ts +++ b/app/src/features/human/useHumanMascot.ts @@ -2,7 +2,7 @@ import debug from 'debug'; import { useEffect, useRef, useState } from 'react'; import { useSelector } from 'react-redux'; -import { subscribeChatEvents } from '../../services/chatService'; +import { type ChatSubagentDoneEvent, subscribeChatEvents } from '../../services/chatService'; import { selectEffectiveMascotVoiceId } from '../../store/mascotSlice'; import type { MascotFace } from './Mascot'; import { lerpViseme, VISEMES, type VisemeShape } from './Mascot/visemes'; @@ -76,18 +76,31 @@ export function pickViseme(delta: string): VisemeShape { } } -type ConversationAckFace = Extract; +type ConversationAckFace = Extract< + MascotFace, + 'happy' | 'confused' | 'concerned' | 'curious' | 'proud' | 'cautious' +>; type ConversationAckEvent = { full_response?: string | null; reaction_emoji?: string | null }; const HAPPY_REACTION_EMOJIS = new Set(['✅', '🎉', '🙌', '😊', '😄', '👍', '💪']); +const PROUD_REACTION_EMOJIS = new Set(['⭐', '🌟', '🏆', '🎯', '💯', '🚀', '✨', '🥇']); +const CURIOUS_REACTION_EMOJIS = new Set(['🔍', '💭', '🧐', '🤓', '👀']); const CONFUSED_REACTION_EMOJIS = new Set(['🤔', '❓', '❔']); -const CONCERNED_REACTION_EMOJIS = new Set(['⚠️', '⚠', '🚨', '❌', '😕', '😟']); +// ⚠️ is cautious (heads-up), ❌/🚨 are concerned (failure). +const CAUTIOUS_REACTION_EMOJIS = new Set(['⚠️', '⚠', '💡', '⚡']); +const CONCERNED_REACTION_EMOJIS = new Set(['🚨', '❌', '😕', '😟']); const CONCERNED_TEXT_RE = /\b(sorry|apolog(?:y|ize|ise)|failed|failure|error|cannot|can't|unable|blocked|problem)\b/i; const CONFUSED_TEXT_RE = /\b(not sure|unclear|ambiguous|clarify|which one|need more|can you confirm|maybe)\b/i; const HAPPY_TEXT_RE = /\b(done|completed|fixed|success|successful|ready|all set|great|nice)\b/i; +const PROUD_TEXT_RE = + /\b(successfully completed|all tasks? (done|finished)|mission accomplished|everything (works?|is working)|all (checks?|tests?) pass(ed)?)\b/i; +const CURIOUS_TEXT_RE = + /\b(interesting|fascinating|curious(ly)?|let me (check|look|investigate)|i('ll)? (look|check) into|actually|turns? out)\b/i; +const CAUTIOUS_TEXT_RE = + /\b(be careful|warning|caution|heads? up|please note|make sure|important(ly)?|note that|worth (noting|mentioning))\b/i; /** * Map conversation-level meaning into the short acknowledgement face that @@ -97,15 +110,24 @@ const HAPPY_TEXT_RE = /\b(done|completed|fixed|success|successful|ready|all set| export function pickConversationAckFace(event: ConversationAckEvent): ConversationAckFace | null { const reaction = event.reaction_emoji?.trim(); if (reaction) { + if (PROUD_REACTION_EMOJIS.has(reaction)) return 'proud'; if (HAPPY_REACTION_EMOJIS.has(reaction)) return 'happy'; + if (CURIOUS_REACTION_EMOJIS.has(reaction)) return 'curious'; if (CONFUSED_REACTION_EMOJIS.has(reaction)) return 'confused'; + if (CAUTIOUS_REACTION_EMOJIS.has(reaction)) return 'cautious'; if (CONCERNED_REACTION_EMOJIS.has(reaction)) return 'concerned'; } const text = event.full_response?.trim() ?? ''; if (!text) return null; + // Priority: concerned > cautious > proud > confused > curious > happy. + // Concerned and cautious share some vocabulary; check concerned first so + // outright failures don't get softened to a heads-up. if (CONCERNED_TEXT_RE.test(text)) return 'concerned'; + if (CAUTIOUS_TEXT_RE.test(text)) return 'cautious'; + if (PROUD_TEXT_RE.test(text)) return 'proud'; if (CONFUSED_TEXT_RE.test(text)) return 'confused'; + if (CURIOUS_TEXT_RE.test(text)) return 'curious'; if (HAPPY_TEXT_RE.test(text)) return 'happy'; return null; } @@ -163,6 +185,11 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas const lastDeltaAtRef = useRef(0); const ackTimerRef = useRef(null); + // Track meaningful work performed in the current turn so onDone can + // distinguish a proud completion from a routine happy acknowledgement. + const toolSucceededRef = useRef(false); + const subagentSucceededRef = useRef(false); + // TTS playback state — non-null while audio is mid-flight. const playbackRef = useRef(null); const visemeFramesRef = useRef<{ viseme: string; start_ms: number; end_ms: number }[]>([]); @@ -193,6 +220,8 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas const unsub = subscribeChatEvents({ onInferenceStart: () => { clearAckTimer(); + toolSucceededRef.current = false; + subagentSucceededRef.current = false; mascotLog('voice-session transition → thinking (inference_start)'); setFace('thinking'); }, @@ -215,9 +244,22 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas // Don't fully derail — let the next inference step take over. setFace('concerned'); } else { + toolSucceededRef.current = true; setFace('thinking'); } }, + onSubagentDone: (e: ChatSubagentDoneEvent) => { + if (e.success) { + mascotLog('voice-session subagent_done success tool=%s', e.tool_name); + subagentSucceededRef.current = true; + } else { + mascotLog( + 'voice-session transition → concerned (subagent_done failed tool=%s)', + e.tool_name + ); + setFace('concerned'); + } + }, onTextDelta: e => { // Pseudo-lipsync only kicks in if no real audio is playing. if (listeningRef.current) { @@ -235,7 +277,23 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas mascotLog('voice-session onDone suppressed — listening is active'); return; } - const ackFace = pickConversationAckFace(e) ?? 'happy'; + // Upgrade to `proud` when the turn involved real tool/subagent work. + // A happy/null text cue paired with actual execution is a completion — + // that reads as proud, not a routine acknowledgement. + const didMeaningfulWork = toolSucceededRef.current || subagentSucceededRef.current; + const explicitAck = pickConversationAckFace(e); + const ackFace: ConversationAckFace = + (explicitAck === 'happy' || explicitAck === null) && didMeaningfulWork + ? 'proud' + : (explicitAck ?? 'happy'); + toolSucceededRef.current = false; + subagentSucceededRef.current = false; + mascotLog( + 'voice-session onDone ackFace=%s (explicit=%s didWork=%s)', + ackFace, + explicitAck ?? 'none', + didMeaningfulWork + ); if (!speakRef.current || !e.full_response?.trim()) { // Soft acknowledgement beat instead of snapping back to idle. holdThenIdle(ackFace);