mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-30 23:14:37 +00:00
feat(voice): polish mascot TTS/STT session states and lifecycle (#2916)
Co-authored-by: Taimoor <astikkosapparel009@gmail.com>
This commit is contained in:
co-authored by
Taimoor
parent
31263d1e20
commit
301ba10ce4
@@ -1,7 +1,12 @@
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { isLowConfidenceTranscript, MAX_RECORDING_MS, MicComposer } from './MicComposer';
|
||||
import {
|
||||
isLowConfidenceTranscript,
|
||||
MAX_RECORDING_MS,
|
||||
MicComposer,
|
||||
STT_MAX_RETRIES,
|
||||
} from './MicComposer';
|
||||
|
||||
// transcribeWithFactory + encodeBlobToWav are the network/heavy boundaries —
|
||||
// mock them here so we can drive the state machine without touching real APIs.
|
||||
@@ -709,6 +714,68 @@ describe('MicComposer', () => {
|
||||
fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
|
||||
await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('Hello, how are you?'));
|
||||
});
|
||||
|
||||
it('retry counter is monotone across native + WAV paths (no double-count)', async () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||
try {
|
||||
// 3 native failures + 2 WAV failures + 1 WAV success = 6 calls total.
|
||||
//
|
||||
// The double-count bug only manifests on the SECOND WAV retry callback:
|
||||
// fixed: snapshot nativeRetries=2 → handleRetry(2+2)=4 → "4 of 4"
|
||||
// buggy: read mutable cumulativeRetries=3 → handleRetry(3+2)=5 → "5 of 4"
|
||||
//
|
||||
// By the time WAV attempt 2 (call 6, the success) starts executing, React
|
||||
// has already flushed the setRetryCount update from WAV attempt 1's retry
|
||||
// callback (which fired synchronously before the 500ms backoff sleep that
|
||||
// immediately preceded call 6). We read the rendered <span> text inside
|
||||
// that mock to assert the actual value the component shows, not a literal.
|
||||
let retryLabelAtSuccessCall = '';
|
||||
transcribeWithFactoryMock
|
||||
.mockRejectedValueOnce(new Error('native 0'))
|
||||
.mockRejectedValueOnce(new Error('native 1'))
|
||||
.mockRejectedValueOnce(new Error('native 2'))
|
||||
.mockRejectedValueOnce(new Error('wav 0'))
|
||||
.mockRejectedValueOnce(new Error('wav 1'))
|
||||
.mockImplementationOnce(async () => {
|
||||
// WAV attempt 2 — the success call. At this point setRetryCount was
|
||||
// called synchronously before the 500ms backoff that preceded this call.
|
||||
// React flushed during that backoff (fake-timer advancement is act-wrapped),
|
||||
// so the DOM reflects the actual retryCount value: 4 (fixed) or 5 (buggy).
|
||||
const el = document.querySelector('span.text-xs');
|
||||
retryLabelAtSuccessCall = el?.textContent ?? '';
|
||||
return 'cross-path ok';
|
||||
});
|
||||
encodeBlobToWavMock.mockResolvedValueOnce(
|
||||
new Blob([new Uint8Array([0])], { type: 'audio/wav' })
|
||||
);
|
||||
const onSubmit = vi.fn();
|
||||
render(<MicComposer disabled={false} onSubmit={onSubmit} />);
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: /start recording/i }));
|
||||
await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled());
|
||||
await waitFor(() =>
|
||||
expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument()
|
||||
);
|
||||
fireEvent.click(screen.getByRole('button', { name: /stop recording and send/i }));
|
||||
|
||||
// Drive all 6 calls to completion by advancing through backoffs.
|
||||
// native: 500ms + 1000ms, WAV: 500ms + 1000ms.
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
|
||||
await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('cross-path ok'));
|
||||
expect(transcribeWithFactoryMock).toHaveBeenCalledTimes(6);
|
||||
|
||||
// The label seen inside call 6 must read "Retrying... (4 of 4)", not
|
||||
// "Retrying... (5 of 4)". If the snapshot fix is absent, cumulativeRetries
|
||||
// would be mutated to 3 after WAV retry 0, then 3+2=5 on WAV retry 1.
|
||||
expect(retryLabelAtSuccessCall).toMatch(/retrying.*4 of 4/i);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── isLowConfidenceTranscript unit tests ──────────────────────────────────
|
||||
@@ -752,3 +819,10 @@ describe('isLowConfidenceTranscript', () => {
|
||||
expect(isLowConfidenceTranscript('বাংলা')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('STT_MAX_RETRIES export', () => {
|
||||
it('is a positive integer', () => {
|
||||
expect(Number.isInteger(STT_MAX_RETRIES)).toBe(true);
|
||||
expect(STT_MAX_RETRIES).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -110,6 +110,12 @@ export interface MicComposerProps {
|
||||
|
||||
type RecordingState = 'idle' | 'recording' | 'transcribing';
|
||||
|
||||
/**
|
||||
* Exported so tests can assert on the retry ceiling without duplicating the
|
||||
* constant.
|
||||
*/
|
||||
export { STT_MAX_RETRIES };
|
||||
|
||||
/**
|
||||
* Tap-to-toggle mic composer for the mascot page. Captures audio via the
|
||||
* browser's `MediaRecorder`, hands the resulting Blob to the factory-
|
||||
@@ -133,6 +139,13 @@ export function MicComposer({
|
||||
}: MicComposerProps) {
|
||||
const { t } = useT();
|
||||
const [state, setState] = useState<RecordingState>('idle');
|
||||
// Tracks how many STT retry attempts have been made for the current
|
||||
// recording, so the label can change from "Transcribing…" to
|
||||
// "Retrying… (1 of 2)" while the user waits.
|
||||
const [retryCount, setRetryCount] = useState(0);
|
||||
// Monotonic session counter — lets log lines across startRecording /
|
||||
// stopRecording / finalizeRecording be correlated without a UUID.
|
||||
const sessionIdRef = useRef(0);
|
||||
const [devices, setDevices] = useState<AudioInputDevice[]>([]);
|
||||
const [selectedDeviceId, setSelectedDeviceId] = useState<string>('');
|
||||
const [deviceMenuOpen, setDeviceMenuOpen] = useState(false);
|
||||
@@ -413,9 +426,16 @@ export function MicComposer({
|
||||
recorderRef.current = recorder;
|
||||
recorder.start();
|
||||
setState('recording');
|
||||
setRetryCount(0);
|
||||
startInFlightRef.current = false;
|
||||
startRecordingTimer();
|
||||
composerLog('recording started mime=%s', recorder.mimeType || '(default)');
|
||||
const sessionId = ++sessionIdRef.current;
|
||||
composerLog(
|
||||
'[session:%d] recording started mime=%s device=%s',
|
||||
sessionId,
|
||||
recorder.mimeType || '(default)',
|
||||
selectedDeviceId || 'default'
|
||||
);
|
||||
}
|
||||
|
||||
function stopRecording() {
|
||||
@@ -423,6 +443,7 @@ export function MicComposer({
|
||||
if (!recorder || recorder.state === 'inactive') return;
|
||||
clearRecordingTimer();
|
||||
setState('transcribing');
|
||||
composerLog('[session:%d] recording stopped — transcribing', sessionIdRef.current);
|
||||
try {
|
||||
recorder.stop();
|
||||
} catch (err) {
|
||||
@@ -495,8 +516,15 @@ export function MicComposer({
|
||||
* Retry a transcription call up to `STT_MAX_RETRIES` times with
|
||||
* exponential backoff for transient errors. Permanent errors (stale
|
||||
* sidecar, empty blob) are rethrown immediately.
|
||||
*
|
||||
* `onRetry` is called before each retry attempt with the 1-based attempt
|
||||
* number so the UI can reflect progress (e.g. "Retrying… 1 of 2").
|
||||
*/
|
||||
async function transcribeWithRetry(blob: Blob, label: string): Promise<string> {
|
||||
async function transcribeWithRetry(
|
||||
blob: Blob,
|
||||
label: string,
|
||||
onRetry?: (attempt: number) => void
|
||||
): Promise<string> {
|
||||
const opts = language ? { language } : undefined;
|
||||
let lastErr: unknown;
|
||||
const throwIfDisposed = () => {
|
||||
@@ -506,7 +534,8 @@ export function MicComposer({
|
||||
throwIfDisposed();
|
||||
try {
|
||||
composerLog(
|
||||
'transcribe attempt=%s/%d bytes=%d mime=%s lang=%s',
|
||||
'[session:%d] transcribe attempt=%s/%d bytes=%d mime=%s lang=%s',
|
||||
sessionIdRef.current,
|
||||
label,
|
||||
attempt,
|
||||
blob.size,
|
||||
@@ -515,28 +544,48 @@ export function MicComposer({
|
||||
);
|
||||
const transcript = await transcribeWithFactory(blob, opts);
|
||||
throwIfDisposed();
|
||||
composerLog(
|
||||
'[session:%d] transcribe ok path=%s attempt=%d',
|
||||
sessionIdRef.current,
|
||||
label,
|
||||
attempt
|
||||
);
|
||||
return transcript;
|
||||
} catch (err) {
|
||||
if (isTranscriptionCancelledError(err)) throw err;
|
||||
lastErr = err;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (!isTransientError(err)) {
|
||||
composerLog('transcribe permanent failure attempt=%s/%d: %s', label, attempt, msg);
|
||||
composerLog(
|
||||
'[session:%d] transcribe permanent failure path=%s attempt=%d: %s',
|
||||
sessionIdRef.current,
|
||||
label,
|
||||
attempt,
|
||||
msg
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
if (attempt < STT_MAX_RETRIES) {
|
||||
const delay = STT_RETRY_BASE_MS * Math.pow(2, attempt);
|
||||
composerLog(
|
||||
'transcribe transient failure attempt=%s/%d, retrying in %dms: %s',
|
||||
'[session:%d] transcribe transient failure path=%s attempt=%d retrying in %dms: %s',
|
||||
sessionIdRef.current,
|
||||
label,
|
||||
attempt,
|
||||
delay,
|
||||
msg
|
||||
);
|
||||
onRetry?.(attempt + 1);
|
||||
await new Promise(r => setTimeout(r, delay));
|
||||
throwIfDisposed();
|
||||
} else {
|
||||
composerLog('transcribe exhausted retries attempt=%s/%d: %s', label, attempt, msg);
|
||||
composerLog(
|
||||
'[session:%d] transcribe exhausted retries path=%s attempt=%d: %s',
|
||||
sessionIdRef.current,
|
||||
label,
|
||||
attempt,
|
||||
msg
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -553,26 +602,49 @@ export function MicComposer({
|
||||
*/
|
||||
async function transcribeWithFallback(blob: Blob): Promise<string> {
|
||||
const startedAt = Date.now();
|
||||
// Track the cumulative retry number across both codec paths so the label
|
||||
// always shows "Retrying… N of M" relative to the overall attempt count.
|
||||
let cumulativeRetries = 0;
|
||||
const handleRetry = (attempt: number) => {
|
||||
cumulativeRetries = attempt;
|
||||
setRetryCount(cumulativeRetries);
|
||||
composerLog('[session:%d] stt retry overall=%d', sessionIdRef.current, cumulativeRetries);
|
||||
};
|
||||
try {
|
||||
const text = await transcribeWithRetry(blob, 'native');
|
||||
composerLog('transcribe ok path=native ms=%d', Math.round(Date.now() - startedAt));
|
||||
const text = await transcribeWithRetry(blob, 'native', handleRetry);
|
||||
composerLog(
|
||||
'[session:%d] transcribe ok path=native ms=%d',
|
||||
sessionIdRef.current,
|
||||
Math.round(Date.now() - startedAt)
|
||||
);
|
||||
return text;
|
||||
} catch (err) {
|
||||
if (isTranscriptionCancelledError(err)) throw err;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
// Permanent errors don't benefit from a WAV re-encode — bail early.
|
||||
if (!isTransientError(err)) throw err;
|
||||
composerLog('transcribe native path exhausted — falling back to wav: %s', msg);
|
||||
composerLog(
|
||||
'[session:%d] transcribe native path exhausted — falling back to wav: %s',
|
||||
sessionIdRef.current,
|
||||
msg
|
||||
);
|
||||
const reEncodeStart = Date.now();
|
||||
const wav = await encodeBlobToWav(blob);
|
||||
composerLog(
|
||||
'wav fallback bytes=%d encode_ms=%d',
|
||||
'[session:%d] wav fallback bytes=%d encode_ms=%d',
|
||||
sessionIdRef.current,
|
||||
wav.size,
|
||||
Math.round(Date.now() - reEncodeStart)
|
||||
);
|
||||
const text = await transcribeWithRetry(wav, 'wav');
|
||||
// Snapshot before the WAV path so the WAV callback doesn't compound
|
||||
// against a value that handleRetry already mutated on native retries.
|
||||
const nativeRetries = cumulativeRetries;
|
||||
const text = await transcribeWithRetry(wav, 'wav', attempt => {
|
||||
handleRetry(nativeRetries + attempt);
|
||||
});
|
||||
composerLog(
|
||||
'transcribe ok path=wav-fallback total_ms=%d',
|
||||
'[session:%d] transcribe ok path=wav-fallback total_ms=%d',
|
||||
sessionIdRef.current,
|
||||
Math.round(Date.now() - startedAt)
|
||||
);
|
||||
return text;
|
||||
@@ -584,7 +656,11 @@ export function MicComposer({
|
||||
const buttonDisabled = disabled || isBusy;
|
||||
|
||||
const label = isBusy
|
||||
? t('mic.transcribing')
|
||||
? retryCount > 0
|
||||
? t('mic.retryingTranscription')
|
||||
.replace('{attempt}', String(retryCount))
|
||||
.replace('{max}', String(STT_MAX_RETRIES * 2))
|
||||
: t('mic.transcribing')
|
||||
: isRecording
|
||||
? remainingSecs != null && remainingSecs <= 10
|
||||
? t('mic.tapToSendCountdown').replace('{seconds}', String(remainingSecs))
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
ACK_FACE_HOLD_MS,
|
||||
pickConversationAckFace,
|
||||
pickViseme,
|
||||
TTS_MAX_PLAYBACK_MS,
|
||||
useHumanMascot,
|
||||
} from './useHumanMascot';
|
||||
import { type PlaybackHandle, playBase64Audio } from './voice/audioPlayer';
|
||||
@@ -550,6 +551,40 @@ describe('useHumanMascot state machine', () => {
|
||||
expect(result.current.face).toBe('listening');
|
||||
expect(result.current.viseme).toEqual(VISEMES.REST);
|
||||
});
|
||||
|
||||
it('listening override transitions to listening face and cancels ack timer', () => {
|
||||
const { result, rerender } = renderHook(
|
||||
({ listening }: { listening: boolean }) => useHumanMascot({ speakReplies: false, listening }),
|
||||
{ initialProps: { listening: false } }
|
||||
);
|
||||
// Trigger a happy ack that starts the hold timer.
|
||||
act(() => {
|
||||
capturedListeners?.onDone?.(
|
||||
fakeEvent({
|
||||
full_response: 'Here you go.',
|
||||
reaction_emoji: null,
|
||||
rounds_used: 1,
|
||||
total_input_tokens: 1,
|
||||
total_output_tokens: 1,
|
||||
})
|
||||
);
|
||||
});
|
||||
expect(result.current.face).toBe('happy');
|
||||
// Mic activates before the hold timer expires.
|
||||
rerender({ listening: true });
|
||||
expect(result.current.face).toBe('listening');
|
||||
// The hold timer should have been cancelled — advancing past its deadline
|
||||
// must not flip the face back to idle.
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1);
|
||||
});
|
||||
expect(result.current.face).toBe('listening');
|
||||
});
|
||||
|
||||
it('TTS_MAX_PLAYBACK_MS is a positive number', () => {
|
||||
expect(TTS_MAX_PLAYBACK_MS).toBeGreaterThan(0);
|
||||
expect(typeof TTS_MAX_PLAYBACK_MS).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
describe('useHumanMascot TTS playback', () => {
|
||||
|
||||
@@ -6,7 +6,12 @@ import { type ChatSubagentDoneEvent, subscribeChatEvents } from '../../services/
|
||||
import { selectEffectiveMascotVoiceId } from '../../store/mascotSlice';
|
||||
import type { MascotFace } from './Mascot';
|
||||
import { lerpViseme, VISEMES, type VisemeShape } from './Mascot/visemes';
|
||||
import { type PlaybackHandle, playBase64Audio, swallowAudioStop } from './voice/audioPlayer';
|
||||
import {
|
||||
type PlaybackHandle,
|
||||
type PlaybackOptions,
|
||||
playBase64Audio,
|
||||
swallowAudioStop,
|
||||
} from './voice/audioPlayer';
|
||||
import {
|
||||
proceduralVisemes,
|
||||
synthesizeSpeech,
|
||||
@@ -20,6 +25,14 @@ const mascotLog = debug('human:mascot');
|
||||
/** ms the mouth holds the target viseme before decaying back to rest. */
|
||||
const VISEME_DECAY_MS = 180;
|
||||
|
||||
/**
|
||||
* Safety ceiling for a single TTS utterance. Runaway audio (network stall,
|
||||
* decoder that never emits `ended`) is auto-stopped at this limit so the
|
||||
* mascot never gets stuck in a permanent `speaking` pose.
|
||||
* 5 minutes comfortably covers any real reply; exported for tests.
|
||||
*/
|
||||
export const TTS_MAX_PLAYBACK_MS = 5 * 60 * 1_000;
|
||||
|
||||
/**
|
||||
* Heuristic — does this timeline contain at least one frame whose code maps
|
||||
* to a non-REST mouth shape? Used to detect the "backend shipped frames in
|
||||
@@ -336,9 +349,13 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
useEffect(() => {
|
||||
if (!listening) return;
|
||||
clearAckTimer();
|
||||
const ttsWasInFlight = playbackRef.current != null;
|
||||
mascotLog(
|
||||
'voice-session listening interrupt — cancelling TTS (had playback=%s)',
|
||||
!!playbackRef.current
|
||||
'voice-session listening-active tts-in-flight=%s — %s',
|
||||
ttsWasInFlight,
|
||||
ttsWasInFlight
|
||||
? 'user started recording while TTS was playing (interrupted)'
|
||||
: 'mic activated, no TTS to cancel'
|
||||
);
|
||||
// Treat mic-hot as an explicit interruption: stale synthesis/playback
|
||||
// callbacks must not switch the mascot back to speaking after we listen.
|
||||
@@ -413,7 +430,12 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas
|
||||
// the user-gesture chain that authorized speech stays intact. If we
|
||||
// awaited anything else between the user click and play(), CEF would
|
||||
// reject playback under its autoplay policy.
|
||||
const handle = await playBase64Audio(tts.audio_base64, tts.audio_mime ?? 'audio/mpeg');
|
||||
const ttsOptions: PlaybackOptions = { maxDurationMs: TTS_MAX_PLAYBACK_MS };
|
||||
const handle = await playBase64Audio(
|
||||
tts.audio_base64,
|
||||
tts.audio_mime ?? 'audio/mpeg',
|
||||
ttsOptions
|
||||
);
|
||||
if (!isStillCurrent()) {
|
||||
handle.stop();
|
||||
handle.ended.catch(swallowAudioStop);
|
||||
|
||||
@@ -150,6 +150,52 @@ describe('playBase64Audio', () => {
|
||||
// Idempotent — second stop() is a no-op.
|
||||
handle.stop();
|
||||
});
|
||||
|
||||
it('auto-stops after maxDurationMs and rejects ended with AudioStoppedError', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
installAudio(url => new FakeAudio(url));
|
||||
const handle = await playBase64Audio('AAA=', 'audio/mpeg', { maxDurationMs: 1000 });
|
||||
// Before the deadline, ended is still pending.
|
||||
let settled: 'resolved' | 'rejected' | null = null;
|
||||
handle.ended
|
||||
.then(() => {
|
||||
settled = 'resolved';
|
||||
})
|
||||
.catch(() => {
|
||||
settled = 'rejected';
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(settled).toBeNull();
|
||||
// Advance past the deadline.
|
||||
await vi.advanceTimersByTimeAsync(1010);
|
||||
expect(settled).toBe('rejected');
|
||||
const err = await handle.ended.catch(e => e);
|
||||
expect(err).toBeInstanceOf(AudioStoppedError);
|
||||
expect(handle.currentMs()).toBe(-1);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('maxDurationMs timer is cleared when audio ends naturally before the deadline', async () => {
|
||||
vi.useFakeTimers();
|
||||
const clearTimeoutSpy = vi.spyOn(window, 'clearTimeout');
|
||||
try {
|
||||
const created = installAudio(url => new FakeAudio(url));
|
||||
const handle = await playBase64Audio('AAA=', 'audio/mpeg', { maxDurationMs: 5000 });
|
||||
// Simulate natural end before the 5s deadline.
|
||||
created[0].emit('ended');
|
||||
await Promise.resolve();
|
||||
// clearTimeout must have been called to cancel the timer.
|
||||
expect(clearTimeoutSpy).toHaveBeenCalled();
|
||||
// ended resolves without error.
|
||||
await expect(handle.ended).resolves.toBeUndefined();
|
||||
} finally {
|
||||
clearTimeoutSpy.mockRestore();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('isAudioStopped / swallowAudioStop', () => {
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
* Lightweight base64 → playable HTMLAudio wrapper. We don't need WebAudio
|
||||
* graph here; the viseme scheduler reads `currentTime` directly.
|
||||
*/
|
||||
import debug from 'debug';
|
||||
|
||||
const audioLog = debug('human:audio-player');
|
||||
|
||||
/**
|
||||
* Sentinel thrown by the `ended` promise when `stop()` is called. Callers that
|
||||
@@ -49,10 +52,25 @@ export interface PlaybackHandle {
|
||||
ended: Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for {@link playBase64Audio}.
|
||||
*/
|
||||
export interface PlaybackOptions {
|
||||
/**
|
||||
* Auto-stop playback after this many milliseconds as a safety bound against
|
||||
* runaway TTS. If the audio finishes naturally before the deadline the timer
|
||||
* is cleared — the option has no effect on normal, reasonably-sized replies.
|
||||
* When omitted, no duration limit is applied.
|
||||
*/
|
||||
maxDurationMs?: number;
|
||||
}
|
||||
|
||||
export async function playBase64Audio(
|
||||
base64: string,
|
||||
mime: string = 'audio/mpeg'
|
||||
mime: string = 'audio/mpeg',
|
||||
options: PlaybackOptions = {}
|
||||
): Promise<PlaybackHandle> {
|
||||
const { maxDurationMs } = options;
|
||||
const bytes = Uint8Array.from(atob(base64), c => c.charCodeAt(0));
|
||||
const blob = new Blob([bytes], { type: mime });
|
||||
const url = URL.createObjectURL(blob);
|
||||
@@ -68,16 +86,26 @@ export async function playBase64Audio(
|
||||
rejectEnded = rej;
|
||||
});
|
||||
|
||||
// Max-duration safety timer — cleared on any terminal event so it never
|
||||
// fires after playback has already ended.
|
||||
let durationTimer: number | null = null;
|
||||
|
||||
const cleanup = () => {
|
||||
if (durationTimer != null) {
|
||||
window.clearTimeout(durationTimer);
|
||||
durationTimer = null;
|
||||
}
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
|
||||
audio.addEventListener('ended', () => {
|
||||
endedNaturally = true;
|
||||
audioLog('playback ended naturally mime=%s', mime);
|
||||
cleanup();
|
||||
resolveEnded();
|
||||
});
|
||||
audio.addEventListener('error', () => {
|
||||
audioLog('playback decoder error mime=%s', mime);
|
||||
cleanup();
|
||||
rejectEnded(new Error('audio playback error'));
|
||||
});
|
||||
@@ -95,6 +123,7 @@ export async function playBase64Audio(
|
||||
audio.addEventListener(
|
||||
'loadedmetadata',
|
||||
() => {
|
||||
audioLog('playback metadata ready duration=%ss mime=%s', audio.duration.toFixed(2), mime);
|
||||
resolveMetadata();
|
||||
},
|
||||
{ once: true }
|
||||
@@ -104,6 +133,14 @@ export async function playBase64Audio(
|
||||
// the text-length estimate path in that case.
|
||||
window.setTimeout(() => resolveMetadata(), 500);
|
||||
|
||||
const stop = () => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
audio.pause();
|
||||
cleanup();
|
||||
rejectEnded(new AudioStoppedError());
|
||||
};
|
||||
|
||||
try {
|
||||
await audio.play();
|
||||
} catch (err) {
|
||||
@@ -112,17 +149,29 @@ export async function playBase64Audio(
|
||||
throw err;
|
||||
}
|
||||
|
||||
audioLog('playback started mime=%s maxDurationMs=%s', mime, maxDurationMs ?? 'none');
|
||||
|
||||
// Arm the max-duration safety timer after `play()` succeeds. Fires only if
|
||||
// the audio hasn't ended naturally by then (e.g. very long TTS response or a
|
||||
// decoder that never emits `ended`).
|
||||
if (maxDurationMs != null) {
|
||||
durationTimer = window.setTimeout(() => {
|
||||
durationTimer = null;
|
||||
if (!stopped && !endedNaturally) {
|
||||
audioLog(
|
||||
'playback auto-stopped after maxDurationMs=%d — preventing runaway TTS',
|
||||
maxDurationMs
|
||||
);
|
||||
stop();
|
||||
}
|
||||
}, maxDurationMs);
|
||||
}
|
||||
|
||||
return {
|
||||
currentMs: () => (endedNaturally || stopped ? -1 : audio.currentTime * 1000),
|
||||
durationMs: () => (Number.isFinite(audio.duration) ? audio.duration * 1000 : 0),
|
||||
metadataReady,
|
||||
stop: () => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
audio.pause();
|
||||
cleanup();
|
||||
rejectEnded(new AudioStoppedError());
|
||||
},
|
||||
stop,
|
||||
ended,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1631,6 +1631,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'تم رفض إذن الميكروفون',
|
||||
'mic.failedToStartRecorder': 'فشل تشغيل المسجّل',
|
||||
'mic.transcribing': 'جارٍ النسخ...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'انقر للإرسال',
|
||||
'mic.waitingForAgent': 'في انتظار الوكيل...',
|
||||
'mic.tapAndSpeak': 'انقر وتحدث',
|
||||
|
||||
@@ -1664,6 +1664,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'মাইক্রোফোন অনুমতি অস্বীকৃত',
|
||||
'mic.failedToStartRecorder': 'রেকর্ডার শুরু করতে ব্যর্থ',
|
||||
'mic.transcribing': 'ট্রান্সক্রাইব হচ্ছে...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'পাঠাতে ট্যাপ করুন',
|
||||
'mic.waitingForAgent': 'এজেন্টের জন্য অপেক্ষা করছে...',
|
||||
'mic.tapAndSpeak': 'ট্যাপ করুন ও কথা বলুন',
|
||||
|
||||
@@ -1708,6 +1708,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'Mikrofonberechtigung verweigert',
|
||||
'mic.failedToStartRecorder': 'Der Rekorder konnte nicht gestartet werden',
|
||||
'mic.transcribing': 'Transkribieren...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Zum Senden tippen',
|
||||
'mic.waitingForAgent': 'Warten auf Agent...',
|
||||
'mic.tapAndSpeak': 'Tippen und sprechen',
|
||||
|
||||
@@ -1790,6 +1790,7 @@ const en: TranslationMap = {
|
||||
'mic.permissionDenied': 'Microphone permission denied',
|
||||
'mic.failedToStartRecorder': 'Failed to start recorder',
|
||||
'mic.transcribing': 'Transcribing...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Tap to send',
|
||||
'mic.waitingForAgent': 'Waiting for agent...',
|
||||
'mic.tapAndSpeak': 'Tap and speak',
|
||||
|
||||
@@ -1700,6 +1700,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'Permiso de micrófono denegado',
|
||||
'mic.failedToStartRecorder': 'No se pudo iniciar la grabadora',
|
||||
'mic.transcribing': 'Transcribiendo...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Toca para enviar',
|
||||
'mic.waitingForAgent': 'Esperando al agente...',
|
||||
'mic.tapAndSpeak': 'Toca y habla',
|
||||
|
||||
@@ -1706,6 +1706,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'Permission microphone refusée',
|
||||
'mic.failedToStartRecorder': "Échec du démarrage de l'enregistreur",
|
||||
'mic.transcribing': 'Transcription…',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Appuie pour envoyer',
|
||||
'mic.waitingForAgent': "En attente de l'agent…",
|
||||
'mic.tapAndSpeak': 'Appuie et parle',
|
||||
|
||||
@@ -1664,6 +1664,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'माइक्रोफोन की अनुमति नहीं मिली',
|
||||
'mic.failedToStartRecorder': 'रिकॉर्डर शुरू नहीं हो पाया',
|
||||
'mic.transcribing': 'ट्रांसक्राइब हो रहा है...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'भेजने के लिए टैप करें',
|
||||
'mic.waitingForAgent': 'एजेंट का इंतज़ार है...',
|
||||
'mic.tapAndSpeak': 'टैप करें और बोलें',
|
||||
|
||||
@@ -1667,6 +1667,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'Izin mikrofon ditolak',
|
||||
'mic.failedToStartRecorder': 'Gagal memulai perekam',
|
||||
'mic.transcribing': 'Mentranskripsi...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Ketuk untuk mengirim',
|
||||
'mic.waitingForAgent': 'Menunggu agen...',
|
||||
'mic.tapAndSpeak': 'Ketuk dan bicara',
|
||||
|
||||
@@ -1692,6 +1692,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'Permesso microfono negato',
|
||||
'mic.failedToStartRecorder': 'Avvio del registratore fallito',
|
||||
'mic.transcribing': 'Trascrizione...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Tocca per inviare',
|
||||
'mic.waitingForAgent': "In attesa dell'agente...",
|
||||
'mic.tapAndSpeak': 'Tocca e parla',
|
||||
|
||||
@@ -1647,6 +1647,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': '마이크 권한이 거부되었습니다',
|
||||
'mic.failedToStartRecorder': '녹음기를 시작하지 못했습니다',
|
||||
'mic.transcribing': '전사 중...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': '탭하여 보내기',
|
||||
'mic.waitingForAgent': '에이전트를 기다리는 중...',
|
||||
'mic.tapAndSpeak': '탭하고 말하기',
|
||||
|
||||
@@ -1683,6 +1683,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'Odmowa dostępu do mikrofonu',
|
||||
'mic.failedToStartRecorder': 'Nie udało się uruchomić rejestratora',
|
||||
'mic.transcribing': 'Transkrypcja...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Dotknij, aby wysłać',
|
||||
'mic.waitingForAgent': 'Czekam na agenta...',
|
||||
'mic.tapAndSpeak': 'Dotknij i mów',
|
||||
|
||||
@@ -1700,6 +1700,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'Permissão de microfone negada',
|
||||
'mic.failedToStartRecorder': 'Falha ao iniciar o gravador',
|
||||
'mic.transcribing': 'Transcrevendo...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Toque para enviar',
|
||||
'mic.waitingForAgent': 'Aguardando agente...',
|
||||
'mic.tapAndSpeak': 'Toque e fale',
|
||||
|
||||
@@ -1677,6 +1677,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': 'Доступ к микрофону запрещён',
|
||||
'mic.failedToStartRecorder': 'Не удалось запустить запись',
|
||||
'mic.transcribing': 'Транскрипция...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': 'Нажми для отправки',
|
||||
'mic.waitingForAgent': 'Ожидание агента...',
|
||||
'mic.tapAndSpeak': 'Нажми и говори',
|
||||
|
||||
@@ -1576,6 +1576,7 @@ const messages: TranslationMap = {
|
||||
'mic.permissionDenied': '麦克风权限被拒绝',
|
||||
'mic.failedToStartRecorder': '启动录音失败',
|
||||
'mic.transcribing': '转录中...',
|
||||
'mic.retryingTranscription': 'Retrying... ({attempt} of {max})',
|
||||
'mic.tapToSend': '点击发送',
|
||||
'mic.waitingForAgent': '等待助手...',
|
||||
'mic.tapAndSpeak': '点击说话',
|
||||
|
||||
Reference in New Issue
Block a user