mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 10:52:15 +00:00
feat(speech): add frontend speech API client and useSpeech hook
Adds transcribeAudio() and fetchSpeechHealth() to the API client, and the useSpeech React hook for managing mic recording lifecycle. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
a85db10cf2
commit
2dcd05227a
@@ -20,3 +20,33 @@ export async function fetchServerInfo(): Promise<ServerInfo> {
|
||||
if (!res.ok) throw new Error(`Failed to fetch server info: ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export interface TranscriptionResult {
|
||||
text: string;
|
||||
language: string | null;
|
||||
confidence: number | null;
|
||||
duration_seconds: number;
|
||||
}
|
||||
|
||||
export interface SpeechHealth {
|
||||
available: boolean;
|
||||
backend?: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export async function transcribeAudio(audioBlob: Blob, filename = 'recording.webm'): Promise<TranscriptionResult> {
|
||||
const formData = new FormData();
|
||||
formData.append('file', audioBlob, filename);
|
||||
const res = await fetch(`${BASE}/v1/speech/transcribe`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
if (!res.ok) throw new Error(`Transcription failed: ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function fetchSpeechHealth(): Promise<SpeechHealth> {
|
||||
const res = await fetch(`${BASE}/v1/speech/health`);
|
||||
if (!res.ok) return { available: false };
|
||||
return res.json();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { useState, useCallback, useRef, useEffect } from 'react';
|
||||
import { transcribeAudio, fetchSpeechHealth } from '../api/client';
|
||||
|
||||
export type SpeechState = 'idle' | 'recording' | 'transcribing';
|
||||
|
||||
export function useSpeech() {
|
||||
const [state, setState] = useState<SpeechState>('idle');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [available, setAvailable] = useState(false);
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const chunksRef = useRef<Blob[]>([]);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
|
||||
// Check if speech backend is available on mount
|
||||
useEffect(() => {
|
||||
fetchSpeechHealth()
|
||||
.then((health) => setAvailable(health.available))
|
||||
.catch(() => setAvailable(false));
|
||||
}, []);
|
||||
|
||||
const startRecording = useCallback(async (): Promise<void> => {
|
||||
setError(null);
|
||||
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
setError('Microphone not supported in this browser');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
const recorder = new MediaRecorder(stream);
|
||||
chunksRef.current = [];
|
||||
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunksRef.current.push(e.data);
|
||||
};
|
||||
|
||||
recorder.start();
|
||||
mediaRecorderRef.current = recorder;
|
||||
setState('recording');
|
||||
} catch (err) {
|
||||
setError('Microphone access denied');
|
||||
setState('idle');
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stopRecording = useCallback(async (): Promise<string> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const recorder = mediaRecorderRef.current;
|
||||
if (!recorder || recorder.state !== 'recording') {
|
||||
reject(new Error('Not recording'));
|
||||
return;
|
||||
}
|
||||
|
||||
recorder.onstop = async () => {
|
||||
setState('transcribing');
|
||||
|
||||
// Stop all audio tracks
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
|
||||
const blob = new Blob(chunksRef.current, { type: recorder.mimeType || 'audio/webm' });
|
||||
chunksRef.current = [];
|
||||
|
||||
try {
|
||||
const result = await transcribeAudio(blob);
|
||||
setState('idle');
|
||||
resolve(result.text);
|
||||
} catch (err) {
|
||||
setState('idle');
|
||||
const msg = err instanceof Error ? err.message : 'Transcription failed';
|
||||
setError(msg);
|
||||
reject(err);
|
||||
}
|
||||
};
|
||||
|
||||
recorder.stop();
|
||||
});
|
||||
}, []);
|
||||
|
||||
return {
|
||||
state,
|
||||
error,
|
||||
available,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
isRecording: state === 'recording',
|
||||
isTranscribing: state === 'transcribing',
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user