feat(human): expand mascot expressions and tighten reply-speech state machine (#1147)

app/src/features/human/HumanPage.tsx

@@ -18,18 +18,20 @@ const HumanPage = () => {
 
   const { face, viseme } = useHumanMascot({ speakReplies });
 
+  // Sidebar reserves ~436px (420px panel + 16px gutter) on the right; the
+  // mascot stage takes the remaining width so the two never overlap.
   return (
     <div className="absolute inset-0 bg-stone-100 overflow-hidden">
-      {/* Mascot stage — full bleed under the floating sidebar. */}
-      <div className="absolute inset-0 flex items-center justify-center">
-        <div
-          className="pointer-events-none absolute inset-0"
-          style={{
-            background:
-              'radial-gradient(ellipse at 50% 40%, rgba(74,131,221,0.10), transparent 60%)',
-          }}
-        />
-        <div className="relative w-[min(80vh,80vw)] aspect-square">
+      <div
+        className="pointer-events-none absolute inset-0"
+        style={{
+          background: 'radial-gradient(ellipse at 35% 40%, rgba(74,131,221,0.10), transparent 60%)',
+        }}
+      />
+
+      {/* Mascot stage — fills the area to the left of the reserved sidebar column. */}
+      <div className="absolute inset-y-0 left-0 right-[436px] flex items-center justify-center">
+        <div className="relative w-[min(80vh,90%)] aspect-square">
           <Ghosty face={face} viseme={viseme} />
         </div>
       </div>
@@ -44,9 +46,9 @@ const HumanPage = () => {
         Speak replies
       </label>
 
-      {/* Floating chat sidebar — vertically centered above the BottomTabBar (~80px). */}
-      <div className="pointer-events-none absolute right-4 top-0 bottom-20 z-10 flex items-center">
-        <aside className="pointer-events-auto w-[420px] h-[min(720px,calc(100vh-160px))] rounded-2xl border border-stone-300 bg-white shadow-soft flex flex-col overflow-hidden">
+      {/* Chat sidebar — vertically centered above the BottomTabBar (~80px). */}
+      <div className="absolute right-4 top-0 bottom-20 z-10 flex items-center">
+        <aside className="w-[420px] h-[min(720px,calc(100vh-160px))] rounded-2xl border border-stone-300 bg-white shadow-soft flex flex-col overflow-hidden">
           <Conversations variant="sidebar" />
         </aside>
       </div>

app/src/features/human/Mascot/Ghosty.test.tsx

@@ -0,0 +1,58 @@
+import { render } from '@testing-library/react';
+import { describe, expect, it } from 'vitest';
+
+import { Ghosty, type MascotFace } from './Ghosty';
+import { VISEMES } from './visemes';
+
+const FACES: MascotFace[] = [
+  'idle',
+  'normal',
+  'listening',
+  'thinking',
+  'confused',
+  'speaking',
+  'happy',
+  'concerned',
+];
+
+describe('Ghosty', () => {
+  it.each(FACES)('renders the %s face preset without crashing', face => {
+    const { container } = render(<Ghosty face={face} />);
+    const svg = container.querySelector('svg[data-face]');
+    expect(svg).not.toBeNull();
+    expect(svg!.getAttribute('data-face')).toBe(face);
+  });
+
+  it('renders eyebrows for states that signal worry / focus', () => {
+    for (const face of ['listening', 'thinking', 'confused', 'concerned'] as MascotFace[]) {
+      const { container } = render(<Ghosty face={face} />);
+      expect(container.querySelector(`g[data-face-brows="${face}"]`)).not.toBeNull();
+    }
+  });
+
+  it('omits eyebrows for neutral / acknowledgement states', () => {
+    for (const face of ['idle', 'normal', 'speaking', 'happy'] as MascotFace[]) {
+      const { container } = render(<Ghosty face={face} />);
+      expect(container.querySelector('g[data-face-brows]')).toBeNull();
+    }
+  });
+
+  it('renders a viseme-driven mouth when speaking, distinct from the rest mouth', () => {
+    const { container: speaking } = render(
+      <Ghosty face="speaking" viseme={VISEMES.A} idPrefix="m1" />
+    );
+    const { container: idle } = render(<Ghosty face="idle" idPrefix="m2" />);
+    const speakingMouth = speaking.querySelector('path[data-face="speaking"]')?.getAttribute('d');
+    const idleMouth = idle.querySelector('path[data-face="idle"]')?.getAttribute('d');
+    expect(speakingMouth).toBeTruthy();
+    expect(idleMouth).toBeTruthy();
+    expect(speakingMouth).not.toBe(idleMouth);
+  });
+
+  it('respects the size override', () => {
+    const { container } = render(<Ghosty face="idle" size={42} />);
+    const svg = container.querySelector('svg');
+    expect(svg?.getAttribute('width')).toBe('42');
+    expect(svg?.getAttribute('height')).toBe('42');
+  });
+});

app/src/features/human/Mascot/Ghosty.tsx

@@ -5,7 +5,31 @@ import { ARM_PATH, BODY_PATH, LEFT_LEG_PATH, RIGHT_LEG_PATH, VIEWBOX } from './p
 import { useMascotClock } from './useMascotClock';
 import { visemePath, VISEMES, type VisemeShape } from './visemes';
 
-export type MascotFace = 'normal' | 'listening' | 'thinking' | 'speaking';
+/**
+ * Discrete face presets the mascot can wear. The state vocabulary mirrors the
+ * agent + voice lifecycle so the renderer stays presentation-only:
+ *
+ * - `idle` — at rest, no active turn.
+ * - `listening` — user is dictating / mic is hot.
+ * - `thinking` — first inference call in flight.
+ * - `confused` — agent is iterating, calling tools, or otherwise burning rounds.
+ * - `speaking` — text or audio is streaming back; the renderer drives the
+ *   mouth from `viseme` rather than from `face`.
+ * - `happy` — short post-turn acknowledgement before falling back to `idle`.
+ * - `concerned` — error / failed tool / unavailable voice path.
+ *
+ * `normal` is the legacy alias for `idle` and stays accepted for backwards
+ * compatibility with older callers.
+ */
+export type MascotFace =
+  | 'idle'
+  | 'listening'
+  | 'thinking'
+  | 'confused'
+  | 'speaking'
+  | 'happy'
+  | 'concerned'
+  | 'normal';
 
 export interface GhostyProps {
   bodyColor?: string;
@@ -19,16 +43,95 @@ export interface GhostyProps {
   idPrefix?: string;
 }
 
+interface FacePreset {
+  /** Vertical squash of the eyes (1 = round, < 1 = squinted). */
+  eyeScaleY: number;
+  /** Horizontal scale of the eyes. */
+  eyeScaleX: number;
+  /** Eyebrow tilt in degrees — positive points the inner brow up (worried). */
+  browTilt: number;
+  /** Vertical brow offset — negative is higher (raised). */
+  browDy: number;
+  /** Whether to render eyebrows at all. */
+  showBrows: boolean;
+  /** Blush intensity multiplier. */
+  blushOpacity: number;
+}
+
+const FACE_PRESETS: Record<Exclude<MascotFace, 'normal'>, FacePreset> = {
+  idle: {
+    eyeScaleY: 1,
+    eyeScaleX: 1,
+    browTilt: 0,
+    browDy: 0,
+    showBrows: false,
+    blushOpacity: 0.85,
+  },
+  listening: {
+    eyeScaleY: 1.05,
+    eyeScaleX: 1.05,
+    browTilt: -8,
+    browDy: -10,
+    showBrows: true,
+    blushOpacity: 0.9,
+  },
+  thinking: {
+    eyeScaleY: 0.7,
+    eyeScaleX: 1,
+    browTilt: -4,
+    browDy: -2,
+    showBrows: true,
+    blushOpacity: 0.6,
+  },
+  confused: {
+    eyeScaleY: 0.85,
+    eyeScaleX: 0.95,
+    browTilt: 14,
+    browDy: -4,
+    showBrows: true,
+    blushOpacity: 0.55,
+  },
+  speaking: {
+    eyeScaleY: 1,
+    eyeScaleX: 1,
+    browTilt: 0,
+    browDy: 0,
+    showBrows: false,
+    blushOpacity: 0.95,
+  },
+  happy: {
+    eyeScaleY: 0.45,
+    eyeScaleX: 1.1,
+    browTilt: -6,
+    browDy: -6,
+    showBrows: false,
+    blushOpacity: 1,
+  },
+  concerned: {
+    eyeScaleY: 0.95,
+    eyeScaleX: 0.95,
+    browTilt: 22,
+    browDy: -2,
+    showBrows: true,
+    blushOpacity: 0.5,
+  },
+};
+
+function presetFor(face: MascotFace): FacePreset {
+  return FACE_PRESETS[face === 'normal' ? 'idle' : face];
+}
+
 export const Ghosty: React.FC<GhostyProps> = ({
   bodyColor = '#2a3a55',
   blushColor = '#f4a3a3',
   arm = 'none',
-  face = 'normal',
+  face = 'idle',
   viseme,
   size = '100%',
   idPrefix = 'mascot',
 }) => {
   const t = useMascotClock();
+  const preset = presetFor(face);
 
   // Gentle bob for the whole character.
   const bob = Math.sin(t * Math.PI * 1.2) * 14;
@@ -43,8 +146,9 @@ export const Ghosty: React.FC<GhostyProps> = ({
 
   const wave = arm === 'wave' ? Math.sin(t * Math.PI * 2.4) * 12 : 0;
 
-  // Blink ~0.2s every 2.6s, offset so frame 0 is eyes open.
-  const blinkMs = 2600;
+  // Blink ~0.2s every 2.6s, offset so frame 0 is eyes open. While `thinking`
+  // we slow the blink down a touch so the squint reads as a sustained pose.
+  const blinkMs = face === 'thinking' ? 4200 : 2600;
   const blinkOffset = blinkMs / 2;
   const tMs = t * 1000;
   const inBlink = (tMs + blinkOffset) % blinkMs < 200;
@@ -54,12 +158,16 @@ export const Ghosty: React.FC<GhostyProps> = ({
   const bodyFill = `url(#${id('body')})`;
   const dotFill = `url(#${id('dot')})`;
 
+  // Restful mouth path varies by face so a non-speaking expression still reads.
+  const restMouth = restMouthPath(face);
+
   return (
     <svg
       width={size}
       height={size}
       viewBox={`0 0 ${VIEWBOX} ${VIEWBOX}`}
-      style={{ overflow: 'visible', display: 'block' }}>
+      style={{ overflow: 'visible', display: 'block' }}
+      data-face={face}>
       <GhostyDefs idPrefix={idPrefix} bodyColor={bodyColor} />
 
       <g
@@ -99,12 +207,59 @@ export const Ghosty: React.FC<GhostyProps> = ({
           <rect x={0} y={0} width={1000} height={1000} filter={`url(#${id('grain')})`} />
         </g>
 
-        <ellipse cx={360} cy={545} rx={48} ry={22} fill={blushColor} opacity={0.85} />
-        <ellipse cx={680} cy={545} rx={48} ry={22} fill={blushColor} opacity={0.85} />
+        <ellipse
+          cx={360}
+          cy={545}
+          rx={48}
+          ry={22}
+          fill={blushColor}
+          opacity={0.85 * preset.blushOpacity}
+        />
+        <ellipse
+          cx={680}
+          cy={545}
+          rx={48}
+          ry={22}
+          fill={blushColor}
+          opacity={0.85 * preset.blushOpacity}
+        />
+
+        {preset.showBrows && (
+          <g fill="#0a0a0a" data-face-brows={face}>
+            <rect
+              x={385}
+              y={455 + preset.browDy}
+              width={60}
+              height={9}
+              rx={4}
+              transform={`rotate(${-preset.browTilt} 415 ${460 + preset.browDy})`}
+            />
+            <rect
+              x={595}
+              y={455 + preset.browDy}
+              width={60}
+              height={9}
+              rx={4}
+              transform={`rotate(${preset.browTilt} 625 ${460 + preset.browDy})`}
+            />
+          </g>
+        )}
 
         <g>
-          <ellipse cx={415} cy={515} rx={30} ry={40 * blinkScale} fill="#0a0a0a" />
-          <ellipse cx={625} cy={515} rx={30} ry={40 * blinkScale} fill="#0a0a0a" />
+          <ellipse
+            cx={415}
+            cy={515}
+            rx={30 * preset.eyeScaleX}
+            ry={40 * preset.eyeScaleY * blinkScale}
+            fill="#0a0a0a"
+          />
+          <ellipse
+            cx={625}
+            cy={515}
+            rx={30 * preset.eyeScaleX}
+            ry={40 * preset.eyeScaleY * blinkScale}
+            fill="#0a0a0a"
+          />
           {!inBlink && (
             <>
               <circle cx={425} cy={501} r={7} fill="#ffffff" />
@@ -113,11 +268,55 @@ export const Ghosty: React.FC<GhostyProps> = ({
           )}
         </g>
 
-        <path d={visemePath(viseme ?? VISEMES.REST)} fill="#0a0a0a" data-face={face} />
+        {face === 'speaking' ? (
+          <path d={visemePath(viseme ?? VISEMES.REST)} fill="#0a0a0a" data-face={face} />
+        ) : (
+          <path d={restMouth} fill="#0a0a0a" data-face={face} />
+        )}
 
-        <ellipse cx={360} cy={545} rx={56} ry={26} fill={blushColor} opacity={0.18} />
-        <ellipse cx={680} cy={545} rx={56} ry={26} fill={blushColor} opacity={0.18} />
+        <ellipse
+          cx={360}
+          cy={545}
+          rx={56}
+          ry={26}
+          fill={blushColor}
+          opacity={0.18 * preset.blushOpacity}
+        />
+        <ellipse
+          cx={680}
+          cy={545}
+          rx={56}
+          ry={26}
+          fill={blushColor}
+          opacity={0.18 * preset.blushOpacity}
+        />
       </g>
     </svg>
   );
 };
+
+/**
+ * Closed-mouth shape for non-speaking states. Speaking is handled separately
+ * via `visemePath` so the mouth tracks the audio.
+ */
+function restMouthPath(face: MascotFace): string {
+  switch (face) {
+    case 'happy':
+      // Wider grin.
+      return 'M460,565 Q520,635 580,565 Q520,605 460,565 Z';
+    case 'concerned':
+      // Inverted curve — frown.
+      return 'M478,605 Q520,560 562,605 Q520,590 478,605 Z';
+    case 'confused':
+      // Slight side-tilt.
+      return 'M478,580 Q520,610 562,575 Q520,597 478,580 Z';
+    case 'thinking':
+      // Small straight pursed line.
+      return 'M488,585 Q520,595 552,585 Q520,592 488,585 Z';
+    case 'listening':
+      // Open soft "o".
+      return 'M495,580 Q520,600 545,580 Q520,615 495,580 Z';
+    default:
+      return visemePath(VISEMES.REST);
+  }
+}

app/src/features/human/useHumanMascot.test.ts

@@ -1,7 +1,55 @@
-import { describe, expect, it } from 'vitest';
+import { act, renderHook } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
+import type { ChatEventListeners } from '../../services/chatService';
 import { VISEMES } from './Mascot/visemes';
-import { pickViseme } from './useHumanMascot';
+import { ACK_FACE_HOLD_MS, pickViseme, useHumanMascot } from './useHumanMascot';
+import { playBase64Audio } from './voice/audioPlayer';
+import { synthesizeSpeech } from './voice/ttsClient';
+
+vi.mock('../../services/chatService', () => ({
+  subscribeChatEvents: (listeners: ChatEventListeners) => {
+    capturedListeners = listeners;
+    return () => {
+      capturedListeners = null;
+    };
+  },
+}));
+
+vi.mock('./voice/ttsClient', () => ({
+  synthesizeSpeech: vi.fn(),
+  visemesFromAlignment: (alignment: { char: string; start_ms: number; end_ms: number }[]) =>
+    alignment.map(a => ({ viseme: 'aa', start_ms: a.start_ms, end_ms: a.end_ms })),
+}));
+
+vi.mock('./voice/audioPlayer', () => ({ playBase64Audio: vi.fn() }));
+
+function makeFakePlayback(durationMs = 100) {
+  let stopped = false;
+  let resolveEnded!: () => void;
+  let rejectEnded!: (e: Error) => void;
+  const ended = new Promise<void>((res, rej) => {
+    resolveEnded = res;
+    rejectEnded = rej;
+  });
+  return {
+    handle: {
+      currentMs: () => (stopped ? -1 : 0),
+      stop: () => {
+        stopped = true;
+        rejectEnded(new Error('stopped'));
+      },
+      ended,
+    },
+    finishNaturally: () => {
+      stopped = true;
+      resolveEnded();
+    },
+    durationMs,
+  };
+}
+
+let capturedListeners: ChatEventListeners | null = null;
 
 describe('pickViseme', () => {
   it('maps vowels to their viseme', () => {
@@ -38,3 +86,266 @@ describe('pickViseme', () => {
     expect(pickViseme('')).toBe(VISEMES.E);
   });
 });
+
+describe('useHumanMascot state machine', () => {
+  beforeEach(() => {
+    capturedListeners = null;
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  function fakeEvent<T>(extra: T): T & { thread_id: string; request_id: string } {
+    return { thread_id: 't', request_id: 'r', ...extra };
+  }
+
+  it('starts idle', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    expect(result.current.face).toBe('idle');
+  });
+
+  it('moves to thinking on inference_start', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+    });
+    expect(result.current.face).toBe('thinking');
+  });
+
+  it('moves to confused on tool_call', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onToolCall?.(
+        fakeEvent({ tool_name: 'search', skill_id: 's', args: {}, round: 1 })
+      );
+    });
+    expect(result.current.face).toBe('confused');
+  });
+
+  it('moves to confused on iteration_start beyond round 1', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onIterationStart?.(fakeEvent({ round: 2, message: '' }));
+    });
+    expect(result.current.face).toBe('confused');
+  });
+
+  it('does not flip to confused on iteration_start round 1', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+      capturedListeners?.onIterationStart?.(fakeEvent({ round: 1, message: '' }));
+    });
+    expect(result.current.face).toBe('thinking');
+  });
+
+  it('moves to concerned on failed tool result', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onToolResult?.(
+        fakeEvent({ tool_name: 'search', skill_id: 's', output: 'oops', success: false, round: 1 })
+      );
+    });
+    expect(result.current.face).toBe('concerned');
+  });
+
+  it('moves to speaking on text_delta', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onTextDelta?.(fakeEvent({ round: 1, delta: 'hello' }));
+    });
+    expect(result.current.face).toBe('speaking');
+  });
+
+  it('holds happy briefly on chat_done without speakReplies, then idles', () => {
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
+    act(() => {
+      capturedListeners?.onDone?.(
+        fakeEvent({
+          full_response: 'hello',
+          rounds_used: 1,
+          total_input_tokens: 1,
+          total_output_tokens: 1,
+        })
+      );
+    });
+    expect(result.current.face).toBe('happy');
+    act(() => {
+      vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
+    });
+    expect(result.current.face).toBe('idle');
+  });
+
+  it('holds concerned briefly on chat_error, then idles', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onError?.(
+        fakeEvent({ message: 'boom', error_type: 'inference', round: 1 })
+      );
+    });
+    expect(result.current.face).toBe('concerned');
+    act(() => {
+      vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
+    });
+    expect(result.current.face).toBe('idle');
+  });
+
+  it('listening option overrides non-speaking faces', () => {
+    const { result, rerender } = renderHook(
+      ({ listening }: { listening: boolean }) => useHumanMascot({ listening }),
+      { initialProps: { listening: false } }
+    );
+    expect(result.current.face).toBe('idle');
+    rerender({ listening: true });
+    expect(result.current.face).toBe('listening');
+  });
+
+  it('clears the ack timer when a new turn starts before the hold finishes', () => {
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: false }));
+    act(() => {
+      capturedListeners?.onDone?.(
+        fakeEvent({
+          full_response: 'hi',
+          rounds_used: 1,
+          total_input_tokens: 1,
+          total_output_tokens: 1,
+        })
+      );
+    });
+    expect(result.current.face).toBe('happy');
+    act(() => {
+      capturedListeners?.onInferenceStart?.(fakeEvent({}));
+    });
+    expect(result.current.face).toBe('thinking');
+    // Advancing past the original hold must NOT flip back to idle since the
+    // timer was cleared by the new turn.
+    act(() => {
+      vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
+    });
+    expect(result.current.face).toBe('thinking');
+  });
+
+  it('successful tool result returns the face to thinking', () => {
+    const { result } = renderHook(() => useHumanMascot());
+    act(() => {
+      capturedListeners?.onToolResult?.(
+        fakeEvent({ tool_name: 'search', skill_id: 's', output: 'ok', success: true, round: 1 })
+      );
+    });
+    expect(result.current.face).toBe('thinking');
+  });
+
+  it('listening does not override speaking', () => {
+    const { result, rerender } = renderHook(
+      ({ listening }: { listening: boolean }) => useHumanMascot({ listening }),
+      { initialProps: { listening: false } }
+    );
+    act(() => {
+      capturedListeners?.onTextDelta?.(fakeEvent({ round: 1, delta: 'hi' }));
+    });
+    rerender({ listening: true });
+    expect(result.current.face).toBe('speaking');
+  });
+});
+
+describe('useHumanMascot TTS playback', () => {
+  beforeEach(() => {
+    capturedListeners = null;
+    vi.useFakeTimers();
+    (synthesizeSpeech as ReturnType<typeof vi.fn>).mockReset();
+    (playBase64Audio as ReturnType<typeof vi.fn>).mockReset();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  function fakeDone(text: string) {
+    return {
+      thread_id: 't',
+      request_id: 'r',
+      full_response: text,
+      rounds_used: 1,
+      total_input_tokens: 1,
+      total_output_tokens: 1,
+    };
+  }
+
+  it('runs a full TTS playback flow: thinking → speaking → happy → idle', async () => {
+    const fake = makeFakePlayback();
+    (synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
+      audio_base64: 'AAA=',
+      audio_mime: 'audio/mpeg',
+      visemes: [{ viseme: 'aa', start_ms: 0, end_ms: 100 }],
+    });
+    (playBase64Audio as ReturnType<typeof vi.fn>).mockResolvedValueOnce(fake.handle);
+
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: true }));
+    await act(async () => {
+      capturedListeners?.onDone?.(fakeDone('hello'));
+      // Let synthesizeSpeech and playBase64Audio resolve.
+      await Promise.resolve();
+      await Promise.resolve();
+      await Promise.resolve();
+    });
+    expect(result.current.face).toBe('speaking');
+
+    await act(async () => {
+      fake.finishNaturally();
+      await Promise.resolve();
+      await Promise.resolve();
+    });
+    expect(result.current.face).toBe('happy');
+
+    act(() => {
+      vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
+    });
+    expect(result.current.face).toBe('idle');
+  });
+
+  it('falls back to alignment-derived visemes when backend ships no cues', async () => {
+    const fake = makeFakePlayback();
+    (synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
+      audio_base64: 'AAA=',
+      audio_mime: 'audio/mpeg',
+      visemes: [],
+      alignment: [{ char: 'h', start_ms: 0, end_ms: 50 }],
+    });
+    (playBase64Audio as ReturnType<typeof vi.fn>).mockResolvedValueOnce(fake.handle);
+
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: true }));
+    await act(async () => {
+      capturedListeners?.onDone?.(fakeDone('hi'));
+      await Promise.resolve();
+      await Promise.resolve();
+      await Promise.resolve();
+    });
+    expect(result.current.face).toBe('speaking');
+    await act(async () => {
+      fake.finishNaturally();
+      await Promise.resolve();
+      await Promise.resolve();
+    });
+  });
+
+  it('shows concerned (not happy) when synthesizeSpeech rejects', async () => {
+    (synthesizeSpeech as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error('voice down'));
+
+    const { result } = renderHook(() => useHumanMascot({ speakReplies: true }));
+    await act(async () => {
+      capturedListeners?.onDone?.(fakeDone('hello'));
+      await Promise.resolve();
+      await Promise.resolve();
+      await Promise.resolve();
+    });
+    expect(result.current.face).toBe('concerned');
+    act(() => {
+      vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
+    });
+    expect(result.current.face).toBe('idle');
+  });
+});

app/src/features/human/useHumanMascot.ts

@@ -1,15 +1,25 @@
+import debug from 'debug';
 import { useEffect, useRef, useState } from 'react';
 
 import { subscribeChatEvents } from '../../services/chatService';
 import type { MascotFace } from './Mascot';
 import { lerpViseme, VISEMES, type VisemeShape } from './Mascot/visemes';
 import { type PlaybackHandle, playBase64Audio } from './voice/audioPlayer';
-import { synthesizeSpeech } from './voice/ttsClient';
+import { synthesizeSpeech, visemesFromAlignment } from './voice/ttsClient';
 import { findActiveFrame, oculusVisemeToShape } from './voice/visemeMap';
 
+const mascotLog = debug('human:mascot');
+
 /** ms the mouth holds the target viseme before decaying back to rest. */
 const VISEME_DECAY_MS = 180;
 
+/**
+ * How long to hold a transient acknowledgement face (`happy`, `concerned`)
+ * before decaying back to `idle`. Tuned to feel like a soft beat rather than
+ * a snap. Exported for tests.
+ */
+export const ACK_FACE_HOLD_MS = 700;
+
 /**
  * Pick a viseme from the trailing letter of a text delta. Heuristic — we
  * have no phoneme data — but it gives the mouth varied motion that tracks
@@ -49,26 +59,43 @@ export interface UseHumanMascotOptions {
   /** When true, post-stream replies are sent to ElevenLabs and the mouth
    *  follows the returned viseme timeline while the audio plays. */
   speakReplies?: boolean;
+  /** When true, force the mascot into a `listening` pose. Caller is responsible
+   *  for setting this while the mic is hot (e.g. from voice dictation state). */
+  listening?: boolean;
+}
+
+export interface UseHumanMascotResult {
+  face: MascotFace;
+  viseme: VisemeShape;
 }
 
 /**
- * Drives the mascot's face/mouth from chat events, with three phases:
- * - inference_start → thinking
- * - text_delta → speaking, pseudo-lipsync from the trailing letter of each delta
- * - chat_done (with `speakReplies`) → speaking, real visemes from TTS audio
- *   for the full response; falls back to neutral when audio ends or fails
+ * Drives the mascot's face/mouth from agent + voice lifecycle events.
+ *
+ * Mapping (kept in one place so the visual model stays coherent):
+ *
+ * - `inference_start` → `thinking`
+ * - `iteration_start` round > 1 or `tool_call` → `confused` (heavy reasoning)
+ * - `tool_result success=false` → `concerned` (held briefly)
+ * - `text_delta` → `speaking`, pseudo-lipsync from the trailing letter
+ * - `chat_done` (no TTS) → `happy` (held briefly), then `idle`
+ * - `chat_done` (TTS enabled) → `thinking` while synthesizing → `speaking`
+ *   with real visemes → `idle` when the audio ends
+ * - `chat_error`, TTS failure → `concerned` (held briefly), then `idle`
+ * - `listening` option override → `listening` (highest priority)
+ *
+ * Errors and unavailable voice degrade cleanly: speech failures fall through
+ * to text-only behavior and surface as a brief `concerned` beat.
  */
-export function useHumanMascot(options: UseHumanMascotOptions = {}): {
-  face: MascotFace;
-  viseme: VisemeShape;
-} {
-  const { speakReplies = false } = options;
+export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMascotResult {
+  const { speakReplies = false, listening = false } = options;
   const speakRef = useRef(speakReplies);
   speakRef.current = speakReplies;
 
-  const [face, setFace] = useState<MascotFace>('normal');
+  const [face, setFace] = useState<MascotFace>('idle');
   const targetRef = useRef<VisemeShape>(VISEMES.REST);
   const lastDeltaAtRef = useRef(0);
+  const ackTimerRef = useRef<number | null>(null);
 
   // TTS playback state — non-null while audio is mid-flight.
   const playbackRef = useRef<PlaybackHandle | null>(null);
@@ -80,19 +107,59 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): {
 
   const [, force] = useState(0);
 
+  function clearAckTimer() {
+    if (ackTimerRef.current != null) {
+      window.clearTimeout(ackTimerRef.current);
+      ackTimerRef.current = null;
+    }
+  }
+
+  function holdThenIdle(ackFace: MascotFace, ms = ACK_FACE_HOLD_MS) {
+    clearAckTimer();
+    setFace(ackFace);
+    ackTimerRef.current = window.setTimeout(() => {
+      ackTimerRef.current = null;
+      setFace('idle');
+    }, ms);
+  }
+
   useEffect(() => {
     const unsub = subscribeChatEvents({
-      onInferenceStart: () => setFace('thinking'),
+      onInferenceStart: () => {
+        clearAckTimer();
+        setFace('thinking');
+      },
+      onIterationStart: e => {
+        // Subsequent iterations mean the agent is grinding through tool rounds.
+        if (e.round > 1) {
+          clearAckTimer();
+          setFace('confused');
+        }
+      },
+      onToolCall: () => {
+        clearAckTimer();
+        setFace('confused');
+      },
+      onToolResult: e => {
+        if (!e.success) {
+          // Don't fully derail — let the next inference step take over.
+          setFace('concerned');
+        } else {
+          setFace('thinking');
+        }
+      },
       onTextDelta: e => {
         // Pseudo-lipsync only kicks in if no real audio is playing.
         if (playbackRef.current) return;
+        clearAckTimer();
         setFace('speaking');
         targetRef.current = pickViseme(e.delta);
         lastDeltaAtRef.current = window.performance.now();
       },
       onDone: e => {
         if (!speakRef.current || !e.full_response?.trim()) {
-          setFace('normal');
+          // Soft acknowledgement beat instead of snapping back to idle.
+          holdThenIdle('happy');
           return;
         }
         // Fire-and-forget — startTtsPlayback owns its cleanup via finally.
@@ -104,11 +171,12 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): {
         playbackRef.current?.stop();
         playbackRef.current = null;
         visemeFramesRef.current = [];
-        setFace('normal');
+        holdThenIdle('concerned');
       },
     });
     return () => {
       unsub();
+      clearAckTimer();
       // Same — invalidate in-flight callbacks before tearing down.
       playbackSeqRef.current++;
       playbackRef.current?.stop();
@@ -123,22 +191,43 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): {
     playbackRef.current = null;
     visemeFramesRef.current = [];
     visemeCursorRef.current = 0;
+    clearAckTimer();
     const seq = ++playbackSeqRef.current;
     const isStillCurrent = () => playbackSeqRef.current === seq;
+    let degraded = false;
 
     try {
       setFace('thinking');
-      const tts = await synthesizeSpeech(text);
+      let tts;
+      try {
+        tts = await synthesizeSpeech(text);
+      } catch (err) {
+        // Voice path unavailable — degrade cleanly to text-only behavior.
+        if (isStillCurrent()) degraded = true;
+        throw err;
+      }
       if (!isStillCurrent()) return;
-      visemeFramesRef.current = tts.visemes ?? [];
+      let frames = tts.visemes ?? [];
+      if (frames.length === 0 && tts.alignment && tts.alignment.length > 0) {
+        // Backend didn't ship viseme cues — derive a coarse track from char timings
+        // so the mouth still animates in sync with the audio.
+        frames = visemesFromAlignment(tts.alignment);
+        mascotLog('tts derived %d viseme frames from alignment', frames.length);
+      } else {
+        mascotLog('tts got %d viseme frames from backend', frames.length);
+      }
+      visemeFramesRef.current = frames;
       visemeCursorRef.current = 0;
+      // Flip face → speaking before starting playback so the RAF render loop
+      // is already running by the time the first viseme frame is due.
+      setFace('speaking');
       const handle = await playBase64Audio(tts.audio_base64, tts.audio_mime ?? 'audio/mpeg');
       if (!isStillCurrent()) {
         handle.stop();
         return;
       }
       playbackRef.current = handle;
-      setFace('speaking');
+      mascotLog('tts playback started — driving lipsync from %d frames', frames.length);
       try {
         await handle.ended;
       } catch {
@@ -148,12 +237,18 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): {
       if (isStillCurrent()) {
         playbackRef.current = null;
         visemeFramesRef.current = [];
-        setFace('normal');
+        if (degraded) {
+          holdThenIdle('concerned');
+        } else {
+          holdThenIdle('happy');
+        }
       }
     }
   }
 
-  // RAF loop while we're speaking (either pseudo-lipsync decay or audio-driven).
+  // RAF loop while we're speaking. TTS playback always sets face to
+  // 'speaking' before awaiting the audio, so this also covers the audio-driven
+  // viseme path.
   useEffect(() => {
     if (face !== 'speaking') return;
     let raf = 0;
@@ -184,5 +279,9 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): {
     viseme = lerpViseme(targetRef.current, VISEMES.REST, decay);
   }
 
-  return { face, viseme };
+  // `listening` is an external override so callers wiring dictation state
+  // can reflect mic-on without racing the chat event subscription.
+  const effectiveFace: MascotFace = listening && face !== 'speaking' ? 'listening' : face;
+
+  return { face: effectiveFace, viseme };
 }

app/src/features/human/voice/ttsClient.test.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { callCoreRpc } from '../../../services/coreRpcClient';
+import { synthesizeSpeech, visemesFromAlignment } from './ttsClient';
+
+vi.mock('../../../services/coreRpcClient', () => ({ callCoreRpc: vi.fn() }));
+
+describe('synthesizeSpeech (core RPC)', () => {
+  it('routes through openhuman.voice_reply_synthesize and forwards options', async () => {
+    const mock = callCoreRpc as ReturnType<typeof vi.fn>;
+    mock.mockResolvedValueOnce({
+      audio_base64: 'AAA=',
+      audio_mime: 'audio/mpeg',
+      visemes: [{ viseme: 'aa', start_ms: 0, end_ms: 100 }],
+    });
+    const r = await synthesizeSpeech('hello', { voiceId: 'v1', modelId: 'm1' });
+    expect(mock).toHaveBeenCalledWith({
+      method: 'openhuman.voice_reply_synthesize',
+      params: { text: 'hello', voice_id: 'v1', model_id: 'm1' },
+    });
+    expect(r.audio_base64).toBe('AAA=');
+    expect(r.visemes).toHaveLength(1);
+  });
+
+  it('omits options that were not provided', async () => {
+    const mock = callCoreRpc as ReturnType<typeof vi.fn>;
+    mock.mockResolvedValueOnce({ audio_base64: 'BBB=', audio_mime: 'audio/mpeg', visemes: [] });
+    await synthesizeSpeech('hi');
+    expect(mock).toHaveBeenCalledWith({
+      method: 'openhuman.voice_reply_synthesize',
+      params: { text: 'hi' },
+    });
+  });
+
+  it('propagates RPC errors so the caller can degrade cleanly', async () => {
+    const mock = callCoreRpc as ReturnType<typeof vi.fn>;
+    mock.mockRejectedValueOnce(new Error('voice unavailable'));
+    await expect(synthesizeSpeech('hi')).rejects.toThrow('voice unavailable');
+  });
+});
+
+describe('visemesFromAlignment', () => {
+  it('returns empty for empty input', () => {
+    expect(visemesFromAlignment([])).toEqual([]);
+  });
+
+  it('buckets alignment chars into ~80ms windows', () => {
+    const alignment = [
+      { char: 'h', start_ms: 0, end_ms: 30 },
+      { char: 'e', start_ms: 30, end_ms: 60 },
+      { char: 'l', start_ms: 90, end_ms: 120 },
+      { char: 'o', start_ms: 200, end_ms: 240 },
+    ];
+    const frames = visemesFromAlignment(alignment);
+    expect(frames.length).toBeGreaterThan(0);
+    const last = frames[frames.length - 1];
+    expect(last.viseme).toBe('O');
+  });
+
+  it.each([
+    ['a', 'aa'],
+    ['e', 'E'],
+    ['i', 'I'],
+    ['y', 'I'],
+    ['o', 'O'],
+    ['u', 'U'],
+    ['w', 'U'],
+    ['m', 'PP'],
+    ['b', 'PP'],
+    ['p', 'PP'],
+    ['f', 'FF'],
+    ['v', 'FF'],
+    ['s', 'SS'],
+    ['z', 'SS'],
+    ['r', 'RR'],
+    ['n', 'nn'],
+    ['l', 'DD'],
+    ['d', 'DD'],
+    ['t', 'DD'],
+    ['k', 'kk'],
+    ['g', 'kk'],
+    ['h', 'CH'],
+    ['c', 'CH'],
+    ['j', 'CH'],
+    ['x', 'sil'],
+  ])('maps trailing letter %s in a window to %s', (ch, code) => {
+    // Each char goes into its own 80ms+ window so the bucket flushes per char.
+    const alignment = [
+      { char: 'a', start_ms: 0, end_ms: 40 },
+      { char: ch, start_ms: 100, end_ms: 140 },
+    ];
+    const frames = visemesFromAlignment(alignment);
+    expect(frames[frames.length - 1].viseme).toBe(code);
+  });
+});

app/src/features/human/voice/ttsClient.ts

@@ -1,8 +1,12 @@
-import { apiClient } from '../../../services/apiClient';
+import debug from 'debug';
+
+import { callCoreRpc } from '../../../services/coreRpcClient';
+
+const ttsLog = debug('human:tts');
 
 /**
- * One frame on the ElevenLabs viseme timeline. Backend emits the Oculus /
- * Microsoft 15-set: `sil, PP, FF, TH, DD, kk, CH, SS, nn, RR, aa, E, I, O, U`.
+ * One frame on the viseme timeline. Backend emits the Oculus / Microsoft
+ * 15-set: `sil, PP, FF, TH, DD, kk, CH, SS, nn, RR, aa, E, I, O, U`.
  */
 export interface VisemeFrame {
   viseme: string;
@@ -16,10 +20,14 @@ export interface AlignmentFrame {
   end_ms: number;
 }
 
+/**
+ * Normalized response from the core RPC `openhuman.voice_reply_synthesize`.
+ * The core does the messy "tolerate multiple backend response shapes" work
+ * (see `src/openhuman/voice/reply_speech.rs`) so the UI can stay strict.
+ */
 export interface TtsResponse {
   audio_base64: string;
-  /** mime, e.g. "audio/mpeg". */
-  audio_mime?: string;
+  audio_mime: string;
   visemes: VisemeFrame[];
   alignment?: AlignmentFrame[];
 }
@@ -30,10 +38,115 @@ export interface TtsOptions {
   outputFormat?: string;
 }
 
+/**
+ * Synthesize agent reply speech via the Rust core. The core proxies the
+ * hosted backend's `/openai/v1/audio/speech` endpoint so the WebView never
+ * touches it directly, which sidesteps a class of "Load failed" CORS/TLS
+ * issues and keeps auth in one place.
+ */
 export async function synthesizeSpeech(text: string, opts: TtsOptions = {}): Promise<TtsResponse> {
-  const body: Record<string, unknown> = { text, with_visemes: true };
-  if (opts.voiceId) body.voice_id = opts.voiceId;
-  if (opts.modelId) body.model_id = opts.modelId;
-  if (opts.outputFormat) body.output_format = opts.outputFormat;
-  return apiClient.post<TtsResponse>('/openai/v1/audio/speech', body);
+  const params: Record<string, unknown> = { text };
+  if (opts.voiceId) params.voice_id = opts.voiceId;
+  if (opts.modelId) params.model_id = opts.modelId;
+  if (opts.outputFormat) params.output_format = opts.outputFormat;
+  ttsLog('synthesize chars=%d voice=%s', text.length, opts.voiceId ?? 'default');
+
+  const result = await callCoreRpc<TtsResponse>({
+    method: 'openhuman.voice_reply_synthesize',
+    params,
+  });
+
+  ttsLog(
+    'synthesize done audio_bytes=%d visemes=%d alignment=%d',
+    result.audio_base64.length,
+    result.visemes.length,
+    result.alignment?.length ?? 0
+  );
+  return result;
+}
+
+/**
+ * Fall back to deriving rough visemes from char-level alignment if the backend
+ * didn't return them. Uses the same heuristic as text-stream pseudo-lipsync —
+ * picks a mouth shape from the last letter in each ~80ms window. Kept on the
+ * client so it can run after the audio arrives without an extra round trip.
+ */
+export function visemesFromAlignment(alignment: AlignmentFrame[]): VisemeFrame[] {
+  if (alignment.length === 0) return [];
+  const WINDOW_MS = 80;
+  const out: VisemeFrame[] = [];
+  let bucketStart = alignment[0].start_ms;
+  let bucketEnd = bucketStart + WINDOW_MS;
+  let bucketChars = '';
+  for (const a of alignment) {
+    if (a.start_ms >= bucketEnd) {
+      if (bucketChars.length > 0) {
+        out.push({
+          viseme: alignmentLetterToCode(bucketChars),
+          start_ms: bucketStart,
+          end_ms: bucketEnd,
+        });
+      }
+      bucketStart = a.start_ms;
+      bucketEnd = bucketStart + WINDOW_MS;
+      bucketChars = '';
+    }
+    bucketChars += a.char;
+  }
+  if (bucketChars.length > 0) {
+    out.push({
+      viseme: alignmentLetterToCode(bucketChars),
+      start_ms: bucketStart,
+      end_ms: bucketEnd,
+    });
+  }
+  return out;
+}
+
+function alignmentLetterToCode(chunk: string): string {
+  const ch = chunk
+    .replace(/[^a-zA-Z]/g, '')
+    .slice(-1)
+    .toLowerCase();
+  switch (ch) {
+    case 'a':
+      return 'aa';
+    case 'e':
+      return 'E';
+    case 'i':
+    case 'y':
+      return 'I';
+    case 'o':
+      return 'O';
+    case 'u':
+    case 'w':
+      return 'U';
+    case 'm':
+    case 'b':
+    case 'p':
+      return 'PP';
+    case 'f':
+    case 'v':
+      return 'FF';
+    case 's':
+    case 'z':
+      return 'SS';
+    case 'r':
+      return 'RR';
+    case 'n':
+      return 'nn';
+    case 'l':
+    case 'd':
+    case 't':
+      return 'DD';
+    case 'k':
+    case 'g':
+      return 'kk';
+    case 'h':
+    case 'c':
+    case 'j':
+      return 'CH';
+    default:
+      return 'sil';
+  }
 }

src/openhuman/voice/mod.rs

@@ -11,6 +11,7 @@ pub mod hallucination;
 pub mod hotkey;
 mod ops;
 mod postprocess;
+pub mod reply_speech;
 mod schemas;
 pub mod server;
 pub mod streaming;

src/openhuman/voice/reply_speech.rs

@@ -0,0 +1,314 @@
+//! Reply-speech synthesis — proxies the hosted backend's
+//! `/openai/v1/audio/speech` endpoint (ElevenLabs under the hood) so the
+//! desktop UI does not have to talk to it directly. Returns base64-encoded
+//! audio + an Oculus-15 viseme alignment timeline the mascot uses for
+//! lip-sync.
+//!
+//! Lives in the voice domain because the response is consumed by the
+//! mascot's lipsync pipeline (`useHumanMascot` → `findActiveFrame` →
+//! `oculusVisemeToShape`).
+
+use log::debug;
+use reqwest::Method;
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+
+use crate::api::config::effective_api_url;
+use crate::api::jwt::get_session_token;
+use crate::api::BackendOAuthClient;
+use crate::openhuman::config::Config;
+use crate::rpc::RpcOutcome;
+
+const LOG_PREFIX: &str = "[voice_reply]";
+
+/// One frame on the viseme timeline. `viseme` is an Oculus / Microsoft
+/// 15-set code (`sil, PP, FF, TH, DD, kk, CH, SS, nn, RR, aa, E, I, O, U`).
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct VisemeFrame {
+    pub viseme: String,
+    pub start_ms: u64,
+    pub end_ms: u64,
+}
+
+/// Char-level timing returned by some backends (e.g. ElevenLabs alignment).
+/// Not directly rendered, but kept so the UI can derive a fallback timeline
+/// when the backend does not ship visemes.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct AlignmentFrame {
+    pub char: String,
+    pub start_ms: u64,
+    pub end_ms: u64,
+}
+
+/// Normalized response handed to the UI — matches the existing TS shape so
+/// the frontend swap is a one-line change.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ReplySpeechResult {
+    pub audio_base64: String,
+    pub audio_mime: String,
+    pub visemes: Vec<VisemeFrame>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub alignment: Option<Vec<AlignmentFrame>>,
+}
+
+/// Caller-tunable knobs.
+#[derive(Debug, Default, Clone)]
+pub struct ReplySpeechOptions {
+    pub voice_id: Option<String>,
+    pub model_id: Option<String>,
+    pub output_format: Option<String>,
+}
+
+/// Synthesize the agent's reply through the hosted backend.
+///
+/// Uses [`BackendOAuthClient`] for the same reason `referral` does: the
+/// desktop WebView's `fetch` to the backend can fail with an opaque
+/// "Load failed" (CORS/TLS quirks), and routing through the core gives us
+/// a consistent auth + retry surface.
+pub async fn synthesize_reply(
+    config: &Config,
+    text: &str,
+    opts: &ReplySpeechOptions,
+) -> Result<RpcOutcome<ReplySpeechResult>, String> {
+    let trimmed = text.trim();
+    if trimmed.is_empty() {
+        return Err("text is required".to_string());
+    }
+
+    let token = get_session_token(config)
+        .map_err(|e| e.to_string())?
+        .and_then(|t| {
+            let s = t.trim().to_string();
+            if s.is_empty() {
+                None
+            } else {
+                Some(s)
+            }
+        })
+        .ok_or_else(|| "no backend session token; sign in first".to_string())?;
+
+    let api_url = effective_api_url(&config.api_url);
+    let client = BackendOAuthClient::new(&api_url).map_err(|e| e.to_string())?;
+
+    let mut body = serde_json::Map::new();
+    body.insert("text".to_string(), json!(trimmed));
+    body.insert("with_visemes".to_string(), json!(true));
+    if let Some(v) = opts
+        .voice_id
+        .as_deref()
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+    {
+        body.insert("voice_id".to_string(), json!(v));
+    }
+    if let Some(v) = opts
+        .model_id
+        .as_deref()
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+    {
+        body.insert("model_id".to_string(), json!(v));
+    }
+    if let Some(v) = opts
+        .output_format
+        .as_deref()
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+    {
+        body.insert("output_format".to_string(), json!(v));
+    }
+
+    debug!(
+        "{LOG_PREFIX} synthesize chars={} voice={}",
+        trimmed.len(),
+        opts.voice_id.as_deref().unwrap_or("default")
+    );
+
+    let raw = client
+        .authed_json(
+            &token,
+            Method::POST,
+            "/openai/v1/audio/speech",
+            Some(Value::Object(body)),
+        )
+        .await
+        .map_err(|e| e.to_string())?;
+
+    let result = normalize_response(&raw);
+    debug!(
+        "{LOG_PREFIX} synthesized audio_bytes={} visemes={} alignment={}",
+        result.audio_base64.len(),
+        result.visemes.len(),
+        result.alignment.as_ref().map_or(0, Vec::len)
+    );
+
+    Ok(RpcOutcome::single_log(
+        result,
+        "voice reply synthesized via POST /openai/v1/audio/speech",
+    ))
+}
+
+/// Translate the backend's tolerant response shape into the UI contract.
+/// Accepts `visemes` / `cues` / `viseme_cues`, and per-frame
+/// `start_ms`+`end_ms` or `time_ms`+`duration_ms`.
+fn normalize_response(raw: &Value) -> ReplySpeechResult {
+    let audio_base64 = raw
+        .get("audio_base64")
+        .or_else(|| raw.get("audio"))
+        .and_then(Value::as_str)
+        .unwrap_or("")
+        .to_string();
+    let audio_mime = raw
+        .get("audio_mime")
+        .or_else(|| raw.get("mime"))
+        .and_then(Value::as_str)
+        .unwrap_or("audio/mpeg")
+        .to_string();
+
+    let cues = raw
+        .get("visemes")
+        .or_else(|| raw.get("cues"))
+        .or_else(|| raw.get("viseme_cues"));
+    let visemes = cues
+        .and_then(Value::as_array)
+        .map(|arr| arr.iter().filter_map(parse_cue).collect::<Vec<_>>())
+        .unwrap_or_default();
+
+    let alignment = raw
+        .get("alignment")
+        .or_else(|| raw.get("characters"))
+        .and_then(Value::as_array)
+        .map(|arr| arr.iter().filter_map(parse_alignment).collect::<Vec<_>>());
+
+    ReplySpeechResult {
+        audio_base64,
+        audio_mime,
+        visemes,
+        alignment,
+    }
+}
+
+fn parse_cue(v: &Value) -> Option<VisemeFrame> {
+    let viseme = v
+        .get("viseme")
+        .or_else(|| v.get("v"))
+        .or_else(|| v.get("code"))
+        .and_then(Value::as_str)?
+        .to_string();
+    if viseme.is_empty() {
+        return None;
+    }
+    let start = read_u64(v, &["start_ms", "time_ms", "t"]).unwrap_or(0);
+    let end = read_u64(v, &["end_ms"])
+        .or_else(|| {
+            let t = read_u64(v, &["time_ms", "t"])?;
+            let d = read_u64(v, &["duration_ms", "d"])?;
+            Some(t + d)
+        })
+        .unwrap_or(start + 80);
+    if end <= start {
+        return None;
+    }
+    Some(VisemeFrame {
+        viseme,
+        start_ms: start,
+        end_ms: end,
+    })
+}
+
+fn parse_alignment(v: &Value) -> Option<AlignmentFrame> {
+    let ch = v.get("char").and_then(Value::as_str)?.to_string();
+    let start = read_u64(v, &["start_ms"])?;
+    let end = read_u64(v, &["end_ms"])?;
+    if end <= start {
+        return None;
+    }
+    Some(AlignmentFrame {
+        char: ch,
+        start_ms: start,
+        end_ms: end,
+    })
+}
+
+fn read_u64(v: &Value, keys: &[&str]) -> Option<u64> {
+    for k in keys {
+        if let Some(n) = v.get(*k).and_then(Value::as_u64) {
+            return Some(n);
+        }
+        if let Some(f) = v.get(*k).and_then(Value::as_f64) {
+            if f.is_finite() && f >= 0.0 {
+                return Some(f as u64);
+            }
+        }
+    }
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn normalize_canonical_shape() {
+        let raw = json!({
+            "audio_base64": "AAA=",
+            "audio_mime": "audio/mpeg",
+            "visemes": [
+                { "viseme": "sil", "start_ms": 0, "end_ms": 100 },
+                { "viseme": "aa", "start_ms": 100, "end_ms": 250 },
+            ],
+        });
+        let r = normalize_response(&raw);
+        assert_eq!(r.audio_base64, "AAA=");
+        assert_eq!(r.audio_mime, "audio/mpeg");
+        assert_eq!(r.visemes.len(), 2);
+        assert_eq!(r.visemes[1].viseme, "aa");
+        assert_eq!(r.visemes[1].end_ms, 250);
+    }
+
+    #[test]
+    fn normalize_accepts_cues_and_short_keys() {
+        let raw = json!({
+            "audio": "BBB=",
+            "mime": "audio/wav",
+            "cues": [{ "v": "PP", "t": 0, "d": 80 }],
+        });
+        let r = normalize_response(&raw);
+        assert_eq!(r.audio_base64, "BBB=");
+        assert_eq!(r.audio_mime, "audio/wav");
+        assert_eq!(
+            r.visemes,
+            vec![VisemeFrame {
+                viseme: "PP".into(),
+                start_ms: 0,
+                end_ms: 80
+            }]
+        );
+    }
+
+    #[test]
+    fn normalize_drops_malformed_cues() {
+        let raw = json!({
+            "audio_base64": "CCC=",
+            "visemes": [
+                { "viseme": "aa", "start_ms": 0, "end_ms": 100 },
+                { "viseme": "",   "start_ms": 100, "end_ms": 200 },
+                { "viseme": "PP", "start_ms": 200, "end_ms": 200 },
+            ],
+        });
+        let r = normalize_response(&raw);
+        assert_eq!(r.visemes.len(), 1);
+        assert_eq!(r.visemes[0].viseme, "aa");
+    }
+
+    #[test]
+    fn normalize_passes_through_alignment() {
+        let raw = json!({
+            "audio_base64": "DDD=",
+            "alignment": [{ "char": "h", "start_ms": 0, "end_ms": 50 }],
+        });
+        let r = normalize_response(&raw);
+        assert_eq!(r.alignment.as_deref().unwrap()[0].char, "h");
+    }
+}

src/openhuman/voice/schemas.rs

@@ -44,6 +44,17 @@ struct TtsParams {
     output_path: Option<String>,
 }
 
+#[derive(Debug, Deserialize)]
+struct ReplySynthesizeParams {
+    text: String,
+    #[serde(default)]
+    voice_id: Option<String>,
+    #[serde(default)]
+    model_id: Option<String>,
+    #[serde(default)]
+    output_format: Option<String>,
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "snake_case")]
 enum OverlaySttState {
@@ -72,6 +83,7 @@ pub fn all_voice_controller_schemas() -> Vec<ControllerSchema> {
         voice_schemas("voice_transcribe"),
         voice_schemas("voice_transcribe_bytes"),
         voice_schemas("voice_tts"),
+        voice_schemas("voice_reply_synthesize"),
         voice_schemas("voice_server_start"),
         voice_schemas("voice_server_stop"),
         voice_schemas("voice_server_status"),
@@ -97,6 +109,10 @@ pub fn all_voice_registered_controllers() -> Vec<RegisteredController> {
             schema: voice_schemas("voice_tts"),
             handler: handle_voice_tts,
         },
+        RegisteredController {
+            schema: voice_schemas("voice_reply_synthesize"),
+            handler: handle_voice_reply_synthesize,
+        },
         RegisteredController {
             schema: voice_schemas("voice_server_start"),
             handler: handle_voice_server_start,
@@ -177,6 +193,26 @@ pub fn voice_schemas(function: &str) -> ControllerSchema {
             ],
             outputs: vec![json_output("tts", "TTS result with output path.")],
         },
+        "voice_reply_synthesize" => ControllerSchema {
+            namespace: "voice",
+            function: "reply_synthesize",
+            description:
+                "Synthesize an agent reply via the hosted backend (ElevenLabs) and return \
+                 base64 audio plus an Oculus-15 viseme alignment for mascot lip-sync.",
+            inputs: vec![
+                required_string("text", "Text to synthesize."),
+                optional_string(
+                    "voice_id",
+                    "Override voice id (defaults to backend selection).",
+                ),
+                optional_string("model_id", "Override model id."),
+                optional_string("output_format", "Override audio format (e.g. mp3_44100)."),
+            ],
+            outputs: vec![json_output(
+                "reply",
+                "ReplySpeechResult: { audio_base64, audio_mime, visemes, alignment? }.",
+            )],
+        },
         "voice_server_start" => ControllerSchema {
             namespace: "voice",
             function: "server_start",
@@ -292,6 +328,22 @@ fn handle_voice_tts(params: Map<String, Value>) -> ControllerFuture {
     })
 }
 
+fn handle_voice_reply_synthesize(params: Map<String, Value>) -> ControllerFuture {
+    Box::pin(async move {
+        let config = config_rpc::load_config_with_timeout().await?;
+        let p = deserialize_params::<ReplySynthesizeParams>(params)?;
+        let opts = crate::openhuman::voice::reply_speech::ReplySpeechOptions {
+            voice_id: p.voice_id,
+            model_id: p.model_id,
+            output_format: p.output_format,
+        };
+        to_json(
+            crate::openhuman::voice::reply_speech::synthesize_reply(&config, &p.text, &opts)
+                .await?,
+        )
+    })
+}
+
 fn handle_voice_server_start(params: Map<String, Value>) -> ControllerFuture {
     Box::pin(async move {
         use crate::openhuman::voice::hotkey::ActivationMode;