fix(voice): interrupt mascot speech when listening starts (#2678)

Signed-off-by: sunilkumarvalmiki <g.sunilkumarvalmiki@gmail.com> Co-authored-by: shanu <shanu@tinyhumans.ai> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: sanil-23 <sanil@vezures.xyz>
2026-07-27 21:08:00 +00:00 · 2026-05-28 20:41:37 +05:30
parent d7e8daa156
commit e7bc3c9fd9
31 changed files with 570 additions and 22 deletions
@@ -1,7 +1,7 @@
 import { fireEvent, render, screen, waitFor } from '@testing-library/react';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';

-import { MicComposer } from './MicComposer';
+import { isLowConfidenceTranscript, MAX_RECORDING_MS, MicComposer } from './MicComposer';

 // transcribeWithFactory + encodeBlobToWav are the network/heavy boundaries —
 // mock them here so we can drive the state machine without touching real APIs.
@@ -84,6 +84,7 @@ describe('MicComposer', () => {
  });

  afterEach(() => {
+    vi.useRealTimers();
    if (originalMediaDevicesDescriptor) {
      Object.defineProperty(globalThis.navigator, 'mediaDevices', originalMediaDevicesDescriptor);
    } else {
@@ -187,7 +188,11 @@ describe('MicComposer', () => {
  });

  it('falls back to wav re-encode when the native attempt fails', async () => {
+    vi.useFakeTimers({ shouldAdvanceTime: true });
+    // Native path: all 3 attempts (initial + 2 retries) fail, then WAV succeeds.
    transcribeWithFactoryMock
+      .mockRejectedValueOnce(new Error('codec not accepted'))
+      .mockRejectedValueOnce(new Error('codec not accepted'))
      .mockRejectedValueOnce(new Error('codec not accepted'))
      .mockResolvedValueOnce('after fallback');
    encodeBlobToWavMock.mockResolvedValueOnce(
@@ -200,9 +205,14 @@ describe('MicComposer', () => {
      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
    );
    fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
+    await waitFor(() => expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(1));
+    await vi.advanceTimersByTimeAsync(500);
+    await waitFor(() => expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(2));
+    await vi.advanceTimersByTimeAsync(1000);
    await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('after fallback'));
    expect(encodeBlobToWavMock).toHaveBeenCalledTimes(1);
-    expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(2);
+    // 3 native attempts + 1 WAV attempt
+    expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(4);
  });

  it('reports an error when transcription returns empty text', async () => {
@@ -485,6 +495,47 @@ describe('MicComposer', () => {
    expect(screen.getByRole('menuitemradio', { name: /USB Headset/i })).toBeInTheDocument();
  });

+  // ── Recording timeout (#1206) ──────────────────────────────────────────────
+
+  it('auto-stops recording after MAX_RECORDING_MS', async () => {
+    vi.useFakeTimers({ shouldAdvanceTime: true });
+    transcribeWithFactoryMock.mockResolvedValueOnce('timeout transcript');
+    const onSubmit = vi.fn();
+    render(<MicComposer disabled={false} onSubmit={onSubmit} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+
+    // Advance past MAX_RECORDING_MS — the timeout should auto-stop.
+    vi.advanceTimersByTime(MAX_RECORDING_MS + 100);
+    await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('timeout transcript'));
+  });
+
+  it('shows countdown label when remaining time <= 10s', async () => {
+    vi.useFakeTimers({ shouldAdvanceTime: true });
+    transcribeWithFactoryMock.mockResolvedValueOnce('ok');
+    render(<MicComposer disabled={false} onSubmit={vi.fn()} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+
+    // Initially shows "Tap to send" (not the countdown — remainingSecs=60 > 10).
+    expect(screen.getByText('Tap to send')).toBeInTheDocument();
+
+    // Advance 51 interval ticks (51s). The decrementing counter goes
+    // from 60 → 9, which is <= 10 so the countdown label appears.
+    vi.advanceTimersByTime(51_000);
+
+    // The label pattern: "Tap to send (9s)" — match the parenthesized digit+s.
+    await waitFor(() => expect(screen.getByText(/\d+s\)/)).toBeInTheDocument());
+  });
+
  it('handles enumerateDevices throwing gracefully (no crash, selector hidden)', async () => {
    const enumerateDevicesMock = vi.fn().mockRejectedValue(new Error('NotAllowed'));
    Object.defineProperty(globalThis.navigator, 'mediaDevices', {
@@ -501,4 +552,203 @@ describe('MicComposer', () => {
    // Composer still functional
    expect(screen.getByText('Tap and speak')).toBeInTheDocument();
  });
+
+  // ── STT retry (#1206) ──────────────────────────────────────────────────────
+
+  it('retries transient STT failures before falling back to WAV', async () => {
+    vi.useFakeTimers({ shouldAdvanceTime: true });
+    // Native path: fail twice (transient), then succeed
+    transcribeWithFactoryMock
+      .mockRejectedValueOnce(new Error('network timeout'))
+      .mockRejectedValueOnce(new Error('network timeout'))
+      .mockResolvedValueOnce('retry success');
+    const onSubmit = vi.fn();
+    render(<MicComposer disabled={false} onSubmit={onSubmit} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+    fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
+    await waitFor(() => expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(1));
+    await vi.advanceTimersByTimeAsync(500);
+    await waitFor(() => expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(2));
+    await vi.advanceTimersByTimeAsync(1000);
+    await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('retry success'));
+    // Should have called transcribe 3 times (initial + 2 retries), no WAV fallback
+    expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(3);
+    expect(encodeBlobToWavMock).not.toHaveBeenCalled();
+  });
+
+  it('falls back to WAV after exhausting native retries', async () => {
+    vi.useFakeTimers({ shouldAdvanceTime: true });
+    // Native path: all 3 attempts fail, then WAV path succeeds
+    transcribeWithFactoryMock
+      .mockRejectedValueOnce(new Error('server error'))
+      .mockRejectedValueOnce(new Error('server error'))
+      .mockRejectedValueOnce(new Error('server error'))
+      .mockResolvedValueOnce('wav retry ok');
+    encodeBlobToWavMock.mockResolvedValueOnce(
+      new Blob([new Uint8Array([0])], { type: 'audio/wav' })
+    );
+    const onSubmit = vi.fn();
+    render(<MicComposer disabled={false} onSubmit={onSubmit} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+    fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
+    await waitFor(() => expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(1));
+    await vi.advanceTimersByTimeAsync(500);
+    await waitFor(() => expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(2));
+    await vi.advanceTimersByTimeAsync(1000);
+    await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('wav retry ok'));
+    // 3 native attempts + 1 WAV attempt
+    expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(4);
+    expect(encodeBlobToWavMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not continue retrying after unmount during STT backoff', async () => {
+    vi.useFakeTimers({ shouldAdvanceTime: true });
+    transcribeWithFactoryMock
+      .mockRejectedValueOnce(new Error('network timeout'))
+      .mockResolvedValueOnce('late transcript');
+    const onSubmit = vi.fn();
+    const onError = vi.fn();
+    const view = render(<MicComposer disabled={false} onSubmit={onSubmit} onError={onError} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+    fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
+    await waitFor(() => expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(1));
+
+    view.unmount();
+    await vi.advanceTimersByTimeAsync(500);
+
+    expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(1);
+    expect(onSubmit).not.toHaveBeenCalled();
+    expect(onError).not.toHaveBeenCalled();
+  });
+
+  it('does not retry permanent errors (stale sidecar)', async () => {
+    transcribeWithFactoryMock.mockRejectedValueOnce(
+      new Error(
+        'Voice transcription is unavailable in this build. Restart the OpenHuman desktop app to pick up the latest core sidecar.'
+      )
+    );
+    const onError = vi.fn();
+    const onSubmit = vi.fn();
+    render(<MicComposer disabled={false} onSubmit={onSubmit} onError={onError} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+    fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
+    await waitFor(() => expect(onError).toHaveBeenCalledWith(expect.stringMatching(/failed/i)));
+    // Only 1 attempt — no retries for permanent errors
+    expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(1);
+    expect(onSubmit).not.toHaveBeenCalled();
+  });
+
+  // ── Low-confidence transcript detection (#1206) ────────────────────────────
+
+  it('rejects a single-character transcript as low confidence', async () => {
+    transcribeWithFactoryMock.mockResolvedValueOnce('I');
+    const onError = vi.fn();
+    const onSubmit = vi.fn();
+    render(<MicComposer disabled={false} onSubmit={onSubmit} onError={onError} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+    fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
+    await waitFor(() =>
+      expect(onError).toHaveBeenCalledWith(expect.stringMatching(/could not understand/i))
+    );
+    expect(onSubmit).not.toHaveBeenCalled();
+  });
+
+  it('rejects a repeated-character transcript as low confidence', async () => {
+    transcribeWithFactoryMock.mockResolvedValueOnce('aaa');
+    const onError = vi.fn();
+    const onSubmit = vi.fn();
+    render(<MicComposer disabled={false} onSubmit={onSubmit} onError={onError} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+    fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
+    await waitFor(() =>
+      expect(onError).toHaveBeenCalledWith(expect.stringMatching(/could not understand/i))
+    );
+    expect(onSubmit).not.toHaveBeenCalled();
+  });
+
+  it('accepts a normal transcript and submits it', async () => {
+    transcribeWithFactoryMock.mockResolvedValueOnce('Hello, how are you?');
+    const onSubmit = vi.fn();
+    render(<MicComposer disabled={false} onSubmit={onSubmit} />);
+
+    fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
+    await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
+    await waitFor(() =>
+      expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
+    );
+    fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
+    await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('Hello, how are you?'));
+  });
+});
+
+// ── isLowConfidenceTranscript unit tests ──────────────────────────────────
+
+describe('isLowConfidenceTranscript', () => {
+  it('flags empty strings', () => {
+    expect(isLowConfidenceTranscript('')).toBe(true);
+    expect(isLowConfidenceTranscript('  ')).toBe(true);
+  });
+
+  it('flags single characters', () => {
+    expect(isLowConfidenceTranscript('I')).toBe(true);
+    expect(isLowConfidenceTranscript('A')).toBe(true);
+  });
+
+  it('flags repeated single characters', () => {
+    expect(isLowConfidenceTranscript('aaa')).toBe(true);
+    expect(isLowConfidenceTranscript('mmm')).toBe(true);
+    expect(isLowConfidenceTranscript('...')).toBe(true);
+  });
+
+  it('flags punctuation-only strings', () => {
+    expect(isLowConfidenceTranscript('...')).toBe(true);
+    expect(isLowConfidenceTranscript('!?')).toBe(true);
+    expect(isLowConfidenceTranscript('---')).toBe(true);
+  });
+
+  it('accepts normal text', () => {
+    expect(isLowConfidenceTranscript('Hello')).toBe(false);
+    expect(isLowConfidenceTranscript('Hi there')).toBe(false);
+    expect(isLowConfidenceTranscript('OK')).toBe(false);
+  });
+
+  it('accepts non-Latin scripts', () => {
+    expect(isLowConfidenceTranscript('Hola')).toBe(false);
+    expect(isLowConfidenceTranscript('Привет')).toBe(false);
+    expect(isLowConfidenceTranscript('こんにちは')).toBe(false);
+    expect(isLowConfidenceTranscript('안녕하세요')).toBe(false);
+    expect(isLowConfidenceTranscript('مرحبا')).toBe(false);
+    expect(isLowConfidenceTranscript('नमस्ते')).toBe(false);
+    expect(isLowConfidenceTranscript('বাংলা')).toBe(false);
+  });
 });
@@ -14,6 +14,65 @@ interface AudioInputDevice {

 const composerLog = debug('human:mic-composer');

+/**
+ * Maximum recording duration in milliseconds. Auto-stops the recorder when
+ * reached so accidental or forgotten recordings don't run indefinitely.
+ * 60 seconds matches the practical ceiling for single-utterance STT accuracy.
+ * Exported for test assertions.
+ */
+export const MAX_RECORDING_MS = 60_000;
+
+/** Maximum number of retries for transient STT failures per encoding path. */
+const STT_MAX_RETRIES = 2;
+
+/** Base delay for exponential backoff (ms). Delay = base * 2^attempt. */
+const STT_RETRY_BASE_MS = 500;
+
+/**
+ * Errors that indicate a permanent failure — retrying won't help.
+ * Matched case-insensitively against the error message.
+ */
+const PERMANENT_ERROR_PATTERNS = [
+  'unknown method', // stale sidecar
+  'audio blob is empty',
+  'unavailable in this build',
+];
+
+class TranscriptionCancelledError extends Error {
+  constructor() {
+    super('transcription cancelled');
+    this.name = 'TranscriptionCancelledError';
+  }
+}
+
+function isTranscriptionCancelledError(err: unknown): err is TranscriptionCancelledError {
+  return err instanceof TranscriptionCancelledError;
+}
+
+function isTransientError(err: unknown): boolean {
+  const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+  return !PERMANENT_ERROR_PATTERNS.some(p => msg.includes(p));
+}
+
+/**
+ * Heuristic check for low-confidence transcripts. The backend doesn't expose
+ * a confidence score, so we detect likely bad results from text patterns:
+ * - Single character (often a stray noise → "I" or "A")
+ * - Repeated single character ("aaa", "mmm")
+ * - Only punctuation / whitespace
+ * Exported for testing.
+ */
+export function isLowConfidenceTranscript(text: string): boolean {
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return true;
+  if (trimmed.length === 1) return true;
+  // All same character repeated: "aaa", "mmm", "..."
+  if (/^(.)\1+$/i.test(trimmed)) return true;
+  // Only punctuation, whitespace, or symbols — no actual words
+  if (!/\p{L}/u.test(trimmed)) return true;
+  return false;
+}
+
 /** MIME types MediaRecorder will be asked to use, in priority order.
 *
 *  AAC-in-MP4 is preferred because the hosted STT upstream (GMI Whisper)
@@ -90,6 +149,10 @@ export function MicComposer({
  // Without this, two awaited `getUserMedia` calls can resolve back-to-back
  // and leave one of the granted streams orphaned (mic indicator stuck on).
  const startInFlightRef = useRef(false);
+  // Recording timeout — auto-stops after MAX_RECORDING_MS.
+  const recordingTimerRef = useRef<number | null>(null);
+  const countdownRef = useRef<number | null>(null);
+  const [remainingSecs, setRemainingSecs] = useState<number | null>(null);

  // If the component unmounts mid-record, release the mic so the OS indicator
  // doesn't get stuck on.
@@ -97,6 +160,7 @@ export function MicComposer({
    disposedRef.current = false;
    return () => {
      disposedRef.current = true;
+      clearRecordingTimer();
      // Detach onstop first — `recorder.stop()` below is what would fire it,
      // and we don't want finalizeRecording running post-unmount.
      if (recorderRef.current) recorderRef.current.onstop = null;
@@ -217,6 +281,33 @@ export function MicComposer({
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [state, disabled]);

+  function clearRecordingTimer() {
+    if (recordingTimerRef.current != null) {
+      window.clearTimeout(recordingTimerRef.current);
+      recordingTimerRef.current = null;
+    }
+    if (countdownRef.current != null) {
+      window.clearInterval(countdownRef.current);
+      countdownRef.current = null;
+    }
+    setRemainingSecs(null);
+  }
+
+  function startRecordingTimer() {
+    clearRecordingTimer();
+    const totalSecs = Math.ceil(MAX_RECORDING_MS / 1000);
+    let tick = totalSecs;
+    setRemainingSecs(tick);
+    countdownRef.current = window.setInterval(() => {
+      tick = Math.max(0, tick - 1);
+      setRemainingSecs(tick);
+    }, 1000);
+    recordingTimerRef.current = window.setTimeout(() => {
+      composerLog('recording auto-stopped after %dms', MAX_RECORDING_MS);
+      stopRecording();
+    }, MAX_RECORDING_MS);
+  }
+
  function stopStream() {
    if (streamRef.current) {
      for (const track of streamRef.current.getTracks()) {
@@ -323,12 +414,14 @@ export function MicComposer({
    recorder.start();
    setState('recording');
    startInFlightRef.current = false;
+    startRecordingTimer();
    composerLog('recording started mime=%s', recorder.mimeType || '(default)');
  }

  function stopRecording() {
    const recorder = recorderRef.current;
    if (!recorder || recorder.state === 'inactive') return;
+    clearRecordingTimer();
    setState('transcribing');
    try {
      recorder.stop();
@@ -372,44 +465,104 @@ export function MicComposer({

    try {
      const transcript = await transcribeWithFallback(blob);
+      if (disposedRef.current) return;
      if (!transcript) {
        onError?.(t('mic.noSpeechDetected'));
        setState('idle');
        return;
      }
+      if (isLowConfidenceTranscript(transcript)) {
+        composerLog('low-confidence transcript detected length=%d', transcript.length);
+        onError?.(t('mic.lowConfidenceResult'));
+        setState('idle');
+        return;
+      }
      await onSubmit(transcript);
    } catch (err) {
+      if (disposedRef.current || isTranscriptionCancelledError(err)) {
+        composerLog('transcribe cancelled after composer disposal');
+        return;
+      }
      const msg = err instanceof Error ? err.message : String(err);
      composerLog('transcribe failed: %s', msg);
      onError?.(t('mic.transcriptionFailed').replace('{message}', msg));
    } finally {
-      setState('idle');
+      if (!disposedRef.current) setState('idle');
    }
  }

+  /**
+   * Retry a transcription call up to `STT_MAX_RETRIES` times with
+   * exponential backoff for transient errors. Permanent errors (stale
+   * sidecar, empty blob) are rethrown immediately.
+   */
+  async function transcribeWithRetry(blob: Blob, label: string): Promise<string> {
+    const opts = language ? { language } : undefined;
+    let lastErr: unknown;
+    const throwIfDisposed = () => {
+      if (disposedRef.current) throw new TranscriptionCancelledError();
+    };
+    for (let attempt = 0; attempt <= STT_MAX_RETRIES; attempt++) {
+      throwIfDisposed();
+      try {
+        composerLog(
+          'transcribe attempt=%s/%d bytes=%d mime=%s lang=%s',
+          label,
+          attempt,
+          blob.size,
+          blob.type,
+          language || 'auto'
+        );
+        const transcript = await transcribeWithFactory(blob, opts);
+        throwIfDisposed();
+        return transcript;
+      } catch (err) {
+        if (isTranscriptionCancelledError(err)) throw err;
+        lastErr = err;
+        const msg = err instanceof Error ? err.message : String(err);
+        if (!isTransientError(err)) {
+          composerLog('transcribe permanent failure attempt=%s/%d: %s', label, attempt, msg);
+          throw err;
+        }
+        if (attempt < STT_MAX_RETRIES) {
+          const delay = STT_RETRY_BASE_MS * Math.pow(2, attempt);
+          composerLog(
+            'transcribe transient failure attempt=%s/%d, retrying in %dms: %s',
+            label,
+            attempt,
+            delay,
+            msg
+          );
+          await new Promise(r => setTimeout(r, delay));
+          throwIfDisposed();
+        } else {
+          composerLog('transcribe exhausted retries attempt=%s/%d: %s', label, attempt, msg);
+        }
+      }
+    }
+    throw lastErr;
+  }
+
  /**
   * Send the recorder's native blob first (Opus-in-WebM ~3KB/sec) — Scribe
   * accepts it natively and it uploads ~30x faster than the 16kHz mono WAV
   * we used to transcode (~32KB/sec). If that ever fails (older STT
-   * provider behind a feature flag, codec mismatch, …), retry once with a
-   * re-encoded WAV so we don't regress correctness for the speed win.
+   * provider behind a feature flag, codec mismatch, …), re-encode to WAV
+   * and retry so we don't regress correctness for the speed win. Each path
+   * retries transient errors with exponential backoff.
   */
  async function transcribeWithFallback(blob: Blob): Promise<string> {
    const startedAt = Date.now();
-    const opts = language ? { language } : undefined;
    try {
-      composerLog(
-        'transcribe attempt=native bytes=%d mime=%s lang=%s',
-        blob.size,
-        blob.type,
-        language || 'auto'
-      );
-      const text = await transcribeWithFactory(blob, opts);
-      composerLog('transcribe ok attempt=native ms=%d', Math.round(Date.now() - startedAt));
+      const text = await transcribeWithRetry(blob, 'native');
+      composerLog('transcribe ok path=native ms=%d', Math.round(Date.now() - startedAt));
      return text;
    } catch (err) {
+      if (isTranscriptionCancelledError(err)) throw err;
      const msg = err instanceof Error ? err.message : String(err);
-      composerLog('transcribe failed attempt=native — falling back to wav: %s', msg);
+      // Permanent errors don't benefit from a WAV re-encode — bail early.
+      if (!isTransientError(err)) throw err;
+      composerLog('transcribe native path exhausted — falling back to wav: %s', msg);
      const reEncodeStart = Date.now();
      const wav = await encodeBlobToWav(blob);
      composerLog(
@@ -417,9 +570,9 @@ export function MicComposer({
        wav.size,
        Math.round(Date.now() - reEncodeStart)
      );
-      const text = await transcribeWithFactory(wav, opts);
+      const text = await transcribeWithRetry(wav, 'wav');
      composerLog(
-        'transcribe ok attempt=wav-fallback total_ms=%d',
+        'transcribe ok path=wav-fallback total_ms=%d',
        Math.round(Date.now() - startedAt)
      );
      return text;
@@ -433,7 +586,9 @@ export function MicComposer({
  const label = isBusy
    ? t('mic.transcribing')
    : isRecording
-      ? t('mic.tapToSend')
+      ? remainingSecs != null && remainingSecs <= 10
+        ? t('mic.tapToSendCountdown').replace('{seconds}', String(remainingSecs))
+        : t('mic.tapToSend')
      : disabled
        ? t('mic.waitingForAgent')
        : t('mic.tapAndSpeak');
@@ -366,7 +366,7 @@ describe('useHumanMascot state machine', () => {
    expect(result.current.face).toBe('thinking');
  });

-  it('listening does not override speaking', () => {
+  it('listening overrides streaming speech deltas', () => {
    const { result, rerender } = renderHook(
      ({ listening }: { listening: boolean }) => useHumanMascot({ listening }),
      { initialProps: { listening: false } }
@@ -375,7 +375,8 @@ describe('useHumanMascot state machine', () => {
      capturedListeners?.onTextDelta?.(fakeEvent({ round: 1, delta: 'hi' }));
    });
    rerender({ listening: true });
-    expect(result.current.face).toBe('speaking');
+    expect(result.current.face).toBe('listening');
+    expect(result.current.viseme).toEqual(VISEMES.REST);
  });
 });

@@ -519,6 +520,81 @@ describe('useHumanMascot TTS playback', () => {
    });
  });

+  it('stops in-flight TTS and shows listening when the microphone becomes active', async () => {
+    const fake = makeFakePlayback(1000);
+    const stopSpy = vi.spyOn(fake.handle, 'stop');
+    (synthesizeSpeech as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
+      audio_base64: 'AAA=',
+      audio_mime: 'audio/mpeg',
+      visemes: [{ viseme: 'aa', start_ms: 0, end_ms: 100 }],
+    });
+    (playBase64Audio as ReturnType<typeof vi.fn>).mockResolvedValueOnce(fake.handle);
+
+    const { result, rerender } = renderHook(
+      ({ listening }: { listening: boolean }) => useHumanMascot({ speakReplies: true, listening }),
+      { initialProps: { listening: false } }
+    );
+    await act(async () => {
+      capturedListeners?.onDone?.(fakeDone('hello'));
+      await Promise.resolve();
+      await Promise.resolve();
+      await Promise.resolve();
+    });
+    expect(result.current.face).toBe('speaking');
+
+    act(() => {
+      rerender({ listening: true });
+    });
+
+    expect(stopSpy).toHaveBeenCalledTimes(1);
+    expect(result.current.face).toBe('listening');
+    expect(result.current.viseme).toEqual(VISEMES.REST);
+  });
+
+  it('drops pending TTS synthesis when listening starts before playback', async () => {
+    type TestTtsPayload = {
+      audio_base64: string;
+      audio_mime: string;
+      visemes: { viseme: string; start_ms: number; end_ms: number }[];
+    };
+    let resolveSynth!: (value: TestTtsPayload) => void;
+    const pendingSynth = new Promise<TestTtsPayload>(resolve => {
+      resolveSynth = resolve;
+    });
+    (synthesizeSpeech as ReturnType<typeof vi.fn>).mockReturnValueOnce(pendingSynth);
+
+    const { result, rerender } = renderHook(
+      ({ listening }: { listening: boolean }) => useHumanMascot({ speakReplies: true, listening }),
+      { initialProps: { listening: false } }
+    );
+    await act(async () => {
+      capturedListeners?.onDone?.(fakeDone('hello'));
+      await Promise.resolve();
+      await Promise.resolve();
+    });
+    expect(result.current.face).toBe('thinking');
+
+    act(() => {
+      rerender({ listening: true });
+    });
+    expect(result.current.face).toBe('listening');
+
+    await act(async () => {
+      resolveSynth({
+        audio_base64: 'AAA=',
+        audio_mime: 'audio/mpeg',
+        visemes: [{ viseme: 'aa', start_ms: 0, end_ms: 100 }],
+      });
+      await Promise.resolve();
+      await Promise.resolve();
+      await Promise.resolve();
+    });
+
+    expect(playBase64Audio).not.toHaveBeenCalled();
+    expect(result.current.face).toBe('listening');
+    expect(result.current.viseme).toEqual(VISEMES.REST);
+  });
+
  it('does not surface an unhandledrejection when a newer turn cancels in-flight playback (#1472)', async () => {
    // Two back-to-back turns: the first reaches the `await playBase64Audio`
    // point and then a second onDone bumps the playback seq. When the first
@@ -146,6 +146,8 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
  const { speakReplies = false, listening = false } = options;
  const speakRef = useRef(speakReplies);
  speakRef.current = speakReplies;
+  const listeningRef = useRef(listening);
+  listeningRef.current = listening;

  // Effective mascot voice id: resolves the manual override, the
  // locale-default toggle, and the build-time fallback into a single
@@ -191,21 +193,25 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
    const unsub = subscribeChatEvents({
      onInferenceStart: () => {
        clearAckTimer();
+        mascotLog('voice-session transition → thinking (inference_start)');
        setFace('thinking');
      },
      onIterationStart: e => {
        // Subsequent iterations mean the agent is grinding through tool rounds.
        if (e.round > 1) {
          clearAckTimer();
+          mascotLog('voice-session transition → confused (iteration round=%d)', e.round);
          setFace('confused');
        }
      },
      onToolCall: () => {
        clearAckTimer();
+        mascotLog('voice-session transition → confused (tool_call)');
        setFace('confused');
      },
      onToolResult: e => {
        if (!e.success) {
+          mascotLog('voice-session transition → concerned (tool_result failed)');
          // Don't fully derail — let the next inference step take over.
          setFace('concerned');
        } else {
@@ -214,6 +220,10 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
      },
      onTextDelta: e => {
        // Pseudo-lipsync only kicks in if no real audio is playing.
+        if (listeningRef.current) {
+          mascotLog('voice-session text_delta suppressed — listening is active');
+          return;
+        }
        if (playbackRef.current) return;
        clearAckTimer();
        setFace('speaking');
@@ -221,6 +231,10 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
        lastDeltaAtRef.current = window.performance.now();
      },
      onDone: e => {
+        if (listeningRef.current) {
+          mascotLog('voice-session onDone suppressed — listening is active');
+          return;
+        }
        const ackFace = pickConversationAckFace(e) ?? 'happy';
        if (!speakRef.current || !e.full_response?.trim()) {
          // Soft acknowledgement beat instead of snapping back to idle.
@@ -231,6 +245,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
        void startTtsPlayback(e.full_response, ackFace).catch(() => {});
      },
      onError: () => {
+        mascotLog('voice-session transition → concerned (chat_error), cancelling in-flight TTS');
        // Bump seq to invalidate any in-flight startTtsPlayback awaiters.
        playbackSeqRef.current++;
        const orphan = playbackRef.current;
@@ -260,6 +275,29 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
    };
  }, []);

+  useEffect(() => {
+    if (!listening) return;
+    clearAckTimer();
+    mascotLog(
+      'voice-session listening interrupt — cancelling TTS (had playback=%s)',
+      !!playbackRef.current
+    );
+    // Treat mic-hot as an explicit interruption: stale synthesis/playback
+    // callbacks must not switch the mascot back to speaking after we listen.
+    playbackSeqRef.current++;
+    const orphan = playbackRef.current;
+    playbackRef.current = null;
+    if (orphan) {
+      orphan.stop();
+      orphan.ended.catch(swallowAudioStop);
+    }
+    visemeFramesRef.current = [];
+    visemeCursorRef.current = 0;
+    targetRef.current = VISEMES.REST;
+    lastDeltaAtRef.current = 0;
+    setFace('idle');
+  }, [listening]);
+
  async function startTtsPlayback(
    text: string,
    ackFace: ConversationAckFace = 'happy'
@@ -402,7 +440,8 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas

  // `listening` is an external override so callers wiring dictation state
  // can reflect mic-on without racing the chat event subscription.
-  const effectiveFace: MascotFace = listening && face !== 'speaking' ? 'listening' : face;
+  const effectiveFace: MascotFace = listening ? 'listening' : face;
+  const effectiveViseme: VisemeShape = listening ? VISEMES.REST : viseme;

-  return { face: effectiveFace, viseme };
+  return { face: effectiveFace, viseme: effectiveViseme };
 }
@@ -413,6 +413,7 @@ const ar2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'تكوين إعدادات فرز الذكاء الاصطناعي لمشغلات التكامل Composio',
  'mic.deviceSelector': 'جهاز الميكروفون',
+  'mic.tapToSendCountdown': 'انقر للإرسال ({seconds} ث)',
 };

 export default ar2;
@@ -261,6 +261,7 @@ const ar3: TranslationMap = {
  'memory.ingestingTitle': 'جارٍ استيعاب {title}',
  'mic.noAudioCaptured': 'لم يُلتقط أي صوت',
  'mic.noSpeechDetected': 'لم يُكتشف أي كلام',
+  'mic.lowConfidenceResult': 'تعذّر فهم الصوت بوضوح — يرجى المحاولة مرة أخرى',
  'mic.failedToStopRecording': 'فشل إيقاف التسجيل: {message}',
  'mic.transcriptionFailed': 'فشل النسخ: {message}',
  'reflections.kind.retrospective': 'مراجعة',
@@ -425,6 +425,7 @@ const bn2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'Composio ইন্টিগ্রেশন ট্রিগারের জন্য AI ট্রাইজ সেটিংস কনফিগার করুন',
  'mic.deviceSelector': 'মাইক্রোফোন ডিভাইস',
+  'mic.tapToSendCountdown': 'পাঠাতে ট্যাপ করুন ({seconds}স)',
 };

 export default bn2;
@@ -266,6 +266,7 @@ const bn3: TranslationMap = {
  'memory.ingestingTitle': '{title} ইনজেস্ট হচ্ছে',
  'mic.noAudioCaptured': 'কোনো অডিও ক্যাপচার হয়নি',
  'mic.noSpeechDetected': 'কোনো কথা শনাক্ত হয়নি',
+  'mic.lowConfidenceResult': 'অডিও স্পষ্টভাবে বোঝা যায়নি — আবার চেষ্টা করুন',
  'mic.failedToStopRecording': 'রেকর্ডিং বন্ধ করতে ব্যর্থ: {message}',
  'mic.transcriptionFailed': 'ট্রান্সক্রিপশন ব্যর্থ: {message}',
  'reflections.kind.retrospective': 'পূর্বদর্শন',
@@ -436,6 +436,7 @@ const de2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'Konfiguriere KI-Triage-Einstellungen für Composio-Integrationsauslöser',
  'mic.deviceSelector': 'Mikrofongerät',
+  'mic.tapToSendCountdown': 'Zum Senden tippen ({seconds}s)',
 };

 export default de2;
@@ -268,6 +268,7 @@ const de3: TranslationMap = {
  'memory.ingestingTitle': 'Einnahme von {title}',
  'mic.noAudioCaptured': 'Kein Ton aufgenommen',
  'mic.noSpeechDetected': 'Keine Sprache erkannt',
+  'mic.lowConfidenceResult': 'Audio konnte nicht klar verstanden werden — bitte erneut versuchen',
  'mic.failedToStopRecording': 'Aufzeichnung konnte nicht gestoppt werden: {message}',
  'mic.transcriptionFailed': 'Transkription fehlgeschlagen: {message}',
  'reflections.kind.retrospective': 'Retrospektive',
@@ -350,6 +350,7 @@ const en2: TranslationMap = {
  'mic.stopRecording': 'Stop recording and send',
  'mic.startRecording': 'Start recording',
  'mic.deviceSelector': 'Microphone device',
+  'mic.tapToSendCountdown': 'Tap to send ({seconds}s)',
  'token.usageLimitReached': 'Usage limit reached',
  'token.approachingLimit': 'Approaching limit',
  'token.planClickForDetails': 'plan - click for details',
@@ -266,6 +266,7 @@ const en3: TranslationMap = {
  'memory.ingestingTitle': 'Ingesting {title}',
  'mic.noAudioCaptured': 'No audio captured',
  'mic.noSpeechDetected': 'No speech detected',
+  'mic.lowConfidenceResult': 'Could not understand the audio clearly — please try again',
  'mic.failedToStopRecording': 'Failed to stop recording: {message}',
  'mic.transcriptionFailed': 'Transcription failed: {message}',
  'reflections.kind.retrospective': 'Retrospective',
@@ -435,6 +435,7 @@ const es2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'Configurar los ajustes de clasificación de IA para los activadores de integración Composio',
  'mic.deviceSelector': 'Dispositivo de micrófono',
+  'mic.tapToSendCountdown': 'Toca para enviar ({seconds}s)',
 };

 export default es2;
@@ -269,6 +269,7 @@ const es3: TranslationMap = {
  'memory.ingestingTitle': 'Ingiriendo {title}',
  'mic.noAudioCaptured': 'No se capturó audio',
  'mic.noSpeechDetected': 'No se detectó habla',
+  'mic.lowConfidenceResult': 'No se pudo entender el audio con claridad — intenta de nuevo',
  'mic.failedToStopRecording': 'No se pudo detener la grabación: {message}',
  'mic.transcriptionFailed': 'Transcripción fallida: {message}',
  'reflections.kind.retrospective': 'Retrospectiva',
@@ -437,6 +437,7 @@ const fr2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    "Configurez les paramètres de triage IA pour les déclencheurs d'intégration Composio",
  'mic.deviceSelector': 'Dispositif de microphone',
+  'mic.tapToSendCountdown': 'Appuie pour envoyer ({seconds}s)',
 };

 export default fr2;
@@ -270,6 +270,7 @@ const fr3: TranslationMap = {
  'memory.ingestingTitle': 'Ingestion de {title}',
  'mic.noAudioCaptured': 'Aucun audio capturé',
  'mic.noSpeechDetected': 'Aucune parole détectée',
+  'mic.lowConfidenceResult': "Impossible de comprendre l'audio clairement — réessaie",
  'mic.failedToStopRecording': "Échec de l'arrêt de l'enregistrement : {message}",
  'mic.transcriptionFailed': 'Échec de la transcription : {message}',
  'reflections.kind.retrospective': 'Rétrospective',
@@ -423,6 +423,7 @@ const hi2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'Composio एकीकरण ट्रिगर के लिए AI ट्राइएज सेटिंग्स कॉन्फ़िगर करें',
  'mic.deviceSelector': 'माइक्रोफोन डिवाइस',
+  'mic.tapToSendCountdown': 'भेजने के लिए टैप करें ({seconds}स)',
 };

 export default hi2;
@@ -266,6 +266,7 @@ const hi3: TranslationMap = {
  'memory.ingestingTitle': '{title} इन्जेस्ट हो रहा है',
  'mic.noAudioCaptured': 'कोई ऑडियो कैप्चर नहीं हुआ',
  'mic.noSpeechDetected': 'कोई बोलना नहीं मिला',
+  'mic.lowConfidenceResult': 'ऑडियो स्पष्ट रूप से समझ नहीं आया — कृपया पुनः प्रयास करें',
  'mic.failedToStopRecording': 'रिकॉर्डिंग रोकने में दिक्कत: {message}',
  'mic.transcriptionFailed': 'ट्रांसक्रिप्शन विफल: {message}',
  'reflections.kind.retrospective': 'रेट्रोस्पेक्टिव',
@@ -423,6 +423,7 @@ const id2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'Konfigurasikan pengaturan triase AI untuk pemicu integrasi Composio',
  'mic.deviceSelector': 'Perangkat mikrofon',
+  'mic.tapToSendCountdown': 'Ketuk untuk mengirim ({seconds}d)',
 };

 export default id2;
@@ -266,6 +266,7 @@ const id3: TranslationMap = {
  'memory.ingestingTitle': 'Mengingesti {title}',
  'mic.noAudioCaptured': 'Tidak ada audio tertangkap',
  'mic.noSpeechDetected': 'Tidak ada suara terdeteksi',
+  'mic.lowConfidenceResult': 'Tidak dapat memahami audio dengan jelas — silakan coba lagi',
  'mic.failedToStopRecording': 'Gagal menghentikan perekaman: {message}',
  'mic.transcriptionFailed': 'Transkripsi gagal: {message}',
  'reflections.kind.retrospective': 'Retrospektif',
@@ -431,6 +431,7 @@ const it2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'Configura le impostazioni di triage AI per i trigger di integrazione Composio',
  'mic.deviceSelector': 'Dispositivo microfono',
+  'mic.tapToSendCountdown': 'Tocca per inviare ({seconds}s)',
 };

 export default it2;
@@ -269,6 +269,7 @@ const it3: TranslationMap = {
  'memory.ingestingTitle': 'Ingestione di {title}',
  'mic.noAudioCaptured': 'Nessun audio catturato',
  'mic.noSpeechDetected': 'Nessun parlato rilevato',
+  'mic.lowConfidenceResult': "Impossibile comprendere l'audio chiaramente — riprova",
  'mic.failedToStopRecording': 'Impossibile fermare la registrazione: {message}',
  'mic.transcriptionFailed': 'Trascrizione fallita: {message}',
  'reflections.kind.retrospective': 'Retrospettiva',
@@ -382,6 +382,7 @@ const ko2: TranslationMap = {
  'localModel.ollamaServer.unreachable': '연결할 수 없음',
  'localModel.ollamaServer.validationError': '유효한 http:// 또는 https:// URL이어야 합니다.',
  'mic.deviceSelector': '마이크 장치',
+  'mic.tapToSendCountdown': '탭하여 보내기 ({seconds}초)',
  'devOptions.menuAi': 'AI 구성',
  'devOptions.menuAiDesc': '클라우드 공급자, 로컬 Ollama 모델 및 워크로드별 라우팅',
  'devOptions.menuScreenAware': '화면 인식',
@@ -264,6 +264,7 @@ const ko3: TranslationMap = {
  'memory.ingestingTitle': '{title} 수집 중',
  'mic.noAudioCaptured': '캡처된 오디오가 없습니다',
  'mic.noSpeechDetected': '음성이 감지되지 않았습니다',
+  'mic.lowConfidenceResult': '오디오를 명확하게 이해할 수 없습니다 — 다시 시도해 주세요',
  'mic.failedToStopRecording': '녹음을 중지하지 못했습니다: {message}',
  'mic.transcriptionFailed': '전사에 실패했습니다: {message}',
  'reflections.kind.retrospective': '회고',
@@ -435,6 +435,7 @@ const pt2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'Definir configurações de triagem de IA para gatilhos de integração Composio',
  'mic.deviceSelector': 'Dispositivo de microfone',
+  'mic.tapToSendCountdown': 'Toque para enviar ({seconds}s)',
 };

 export default pt2;
@@ -268,6 +268,7 @@ const pt3: TranslationMap = {
  'memory.ingestingTitle': 'Ingerindo {title}',
  'mic.noAudioCaptured': 'Nenhum áudio capturado',
  'mic.noSpeechDetected': 'Nenhuma fala detectada',
+  'mic.lowConfidenceResult': 'Não foi possível entender o áudio com clareza — tente novamente',
  'mic.failedToStopRecording': 'Falha ao parar a gravação: {message}',
  'mic.transcriptionFailed': 'Falha na transcrição: {message}',
  'reflections.kind.retrospective': 'Retrospectiva',
@@ -430,6 +430,7 @@ const ru2: TranslationMap = {
  'devOptions.menuComposioTriggersDesc':
    'Настройка параметров сортировки AI для триггеров интеграции Composio',
  'mic.deviceSelector': 'Микрофонное устройство',
+  'mic.tapToSendCountdown': 'Нажми для отправки ({seconds}с)',
 };

 export default ru2;
@@ -267,6 +267,7 @@ const ru3: TranslationMap = {
  'memory.ingestingTitle': 'Загрузка {title}',
  'mic.noAudioCaptured': 'Аудио не захвачено',
  'mic.noSpeechDetected': 'Речь не обнаружена',
+  'mic.lowConfidenceResult': 'Не удалось чётко распознать аудио — попробуйте ещё раз',
  'mic.failedToStopRecording': 'Не удалось остановить запись: {message}',
  'mic.transcriptionFailed': 'Ошибка транскрипции: {message}',
  'reflections.kind.retrospective': 'Ретроспектива',
@@ -399,6 +399,7 @@ const zhCN2: TranslationMap = {
  'devOptions.menuComposioTriggers': '集成触发器',
  'devOptions.menuComposioTriggersDesc': '为 Composio 集成触发器配置 AI 分级设置',
  'mic.deviceSelector': '麦克风装置',
+  'mic.tapToSendCountdown': '点击发送 ({seconds}秒)',
 };

 export default zhCN2;
@@ -255,6 +255,7 @@ const zhCN3: TranslationMap = {
  'memory.ingestingTitle': '正在摄取 {title}',
  'mic.noAudioCaptured': '未捕获到音频',
  'mic.noSpeechDetected': '未检测到语音',
+  'mic.lowConfidenceResult': '无法清楚地理解音频 — 请重试',
  'mic.failedToStopRecording': '停止录音失败: {message}',
  'mic.transcriptionFailed': '转录失败: {message}',
  'reflections.kind.retrospective': '回顾',
@@ -1650,6 +1650,7 @@ const en: TranslationMap = {
  'mic.stopRecording': 'Stop recording and send',
  'mic.startRecording': 'Start recording',
  'mic.deviceSelector': 'Microphone device',
+  'mic.tapToSendCountdown': 'Tap to send ({seconds}s)',

  // Token
  'token.usageLimitReached': 'Usage limit reached',
@@ -2019,6 +2020,7 @@ const en: TranslationMap = {
  // Mic: error messages
  'mic.noAudioCaptured': 'No audio captured',
  'mic.noSpeechDetected': 'No speech detected',
+  'mic.lowConfidenceResult': 'Could not understand the audio clearly — please try again',
  'mic.failedToStopRecording': 'Failed to stop recording: {message}',
  'mic.transcriptionFailed': 'Transcription failed: {message}',