mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-31 03:12:16 +00:00
Merge origin/main into gabebo-ui
Integrate speech-to-text (Phase 24) and eval trackers features. Resolve conflicts: add speech Tauri commands to lib.rs, merge MicButton + useSpeech into InputArea, consolidate speech API functions into lib/api.ts (removed old api/client.ts). Made-with: Cursor
This commit is contained in:
@@ -3,6 +3,8 @@ import { Send, Square, Paperclip } from 'lucide-react';
|
||||
import { useAppStore, generateId } from '../../lib/store';
|
||||
import { streamChat } from '../../lib/sse';
|
||||
import { fetchSavings } from '../../lib/api';
|
||||
import { MicButton } from './MicButton';
|
||||
import { useSpeech } from '../../hooks/useSpeech';
|
||||
import type { ChatMessage, ToolCallInfo, TokenUsage } from '../../types';
|
||||
|
||||
export function InputArea() {
|
||||
@@ -21,7 +23,23 @@ export function InputArea() {
|
||||
const setStreamState = useAppStore((s) => s.setStreamState);
|
||||
const resetStream = useAppStore((s) => s.resetStream);
|
||||
|
||||
// Auto-resize textarea
|
||||
const { state: speechState, available: speechAvailable, startRecording, stopRecording } = useSpeech();
|
||||
|
||||
const handleMicClick = useCallback(async () => {
|
||||
if (speechState === 'recording') {
|
||||
try {
|
||||
const text = await stopRecording();
|
||||
if (text) {
|
||||
setInput((prev) => (prev ? prev + ' ' + text : text));
|
||||
}
|
||||
} catch {
|
||||
// Error is captured in useSpeech
|
||||
}
|
||||
} else {
|
||||
await startRecording();
|
||||
}
|
||||
}, [speechState, startRecording, stopRecording]);
|
||||
|
||||
useEffect(() => {
|
||||
const el = textareaRef.current;
|
||||
if (!el) return;
|
||||
@@ -238,18 +256,27 @@ export function InputArea() {
|
||||
<Square size={16} />
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
onClick={sendMessage}
|
||||
disabled={!input.trim()}
|
||||
className="p-2 rounded-xl transition-colors shrink-0 cursor-pointer disabled:opacity-30 disabled:cursor-default"
|
||||
style={{
|
||||
background: input.trim() ? 'var(--color-accent)' : 'var(--color-bg-tertiary)',
|
||||
color: input.trim() ? 'white' : 'var(--color-text-tertiary)',
|
||||
}}
|
||||
title="Send message"
|
||||
>
|
||||
<Send size={16} />
|
||||
</button>
|
||||
<div className="flex items-center gap-1">
|
||||
{speechAvailable && (
|
||||
<MicButton
|
||||
state={speechState}
|
||||
onClick={handleMicClick}
|
||||
disabled={streamState.isStreaming}
|
||||
/>
|
||||
)}
|
||||
<button
|
||||
onClick={sendMessage}
|
||||
disabled={!input.trim()}
|
||||
className="p-2 rounded-xl transition-colors shrink-0 cursor-pointer disabled:opacity-30 disabled:cursor-default"
|
||||
style={{
|
||||
background: input.trim() ? 'var(--color-accent)' : 'var(--color-bg-tertiary)',
|
||||
color: input.trim() ? 'white' : 'var(--color-text-tertiary)',
|
||||
}}
|
||||
title="Send message"
|
||||
>
|
||||
<Send size={16} />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center justify-center mt-2 text-[11px]" style={{ color: 'var(--color-text-tertiary)' }}>
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import type { SpeechState } from '../../hooks/useSpeech';
|
||||
|
||||
interface MicButtonProps {
|
||||
state: SpeechState;
|
||||
onClick: () => void;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
export function MicButton({ state, onClick, disabled }: MicButtonProps) {
|
||||
const title =
|
||||
state === 'recording'
|
||||
? 'Stop recording'
|
||||
: state === 'transcribing'
|
||||
? 'Transcribing...'
|
||||
: 'Voice input';
|
||||
|
||||
return (
|
||||
<button
|
||||
className={`mic-btn ${state !== 'idle' ? `mic-${state}` : ''}`}
|
||||
onClick={onClick}
|
||||
disabled={disabled || state === 'transcribing'}
|
||||
title={title}
|
||||
style={{
|
||||
background: state === 'recording' ? '#e74c3c' : 'transparent',
|
||||
border: '1px solid var(--border, #555)',
|
||||
borderRadius: '8px',
|
||||
padding: '8px',
|
||||
cursor: disabled || state === 'transcribing' ? 'default' : 'pointer',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
minWidth: '36px',
|
||||
height: '36px',
|
||||
color: state === 'recording' ? '#fff' : 'var(--text, #cdd6f4)',
|
||||
opacity: disabled || state === 'transcribing' ? 0.5 : 1,
|
||||
animation: state === 'recording' ? 'pulse 1.5s ease-in-out infinite' : 'none',
|
||||
}}
|
||||
>
|
||||
{state === 'transcribing' ? (
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="currentColor">
|
||||
<circle cx="8" cy="8" r="6" fill="none" stroke="currentColor" strokeWidth="2" strokeDasharray="28" strokeDashoffset="10">
|
||||
<animateTransform attributeName="transform" type="rotate" from="0 8 8" to="360 8 8" dur="1s" repeatCount="indefinite" />
|
||||
</circle>
|
||||
</svg>
|
||||
) : (
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="currentColor">
|
||||
<path d="M5 3a3 3 0 0 1 6 0v5a3 3 0 0 1-6 0V3z" />
|
||||
<path d="M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z" />
|
||||
</svg>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { useState, useCallback, useRef, useEffect } from 'react';
|
||||
import { transcribeAudio, fetchSpeechHealth } from '../lib/api';
|
||||
|
||||
export type SpeechState = 'idle' | 'recording' | 'transcribing';
|
||||
|
||||
export function useSpeech() {
|
||||
const [state, setState] = useState<SpeechState>('idle');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [available, setAvailable] = useState(false);
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const chunksRef = useRef<Blob[]>([]);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
|
||||
// Check if speech backend is available on mount
|
||||
useEffect(() => {
|
||||
fetchSpeechHealth()
|
||||
.then((health) => setAvailable(health.available))
|
||||
.catch(() => setAvailable(false));
|
||||
}, []);
|
||||
|
||||
const startRecording = useCallback(async (): Promise<void> => {
|
||||
setError(null);
|
||||
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
setError('Microphone not supported in this browser');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
const recorder = new MediaRecorder(stream);
|
||||
chunksRef.current = [];
|
||||
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunksRef.current.push(e.data);
|
||||
};
|
||||
|
||||
recorder.start();
|
||||
mediaRecorderRef.current = recorder;
|
||||
setState('recording');
|
||||
} catch (err) {
|
||||
setError('Microphone access denied');
|
||||
setState('idle');
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stopRecording = useCallback(async (): Promise<string> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const recorder = mediaRecorderRef.current;
|
||||
if (!recorder || recorder.state !== 'recording') {
|
||||
reject(new Error('Not recording'));
|
||||
return;
|
||||
}
|
||||
|
||||
recorder.onstop = async () => {
|
||||
setState('transcribing');
|
||||
|
||||
// Stop all audio tracks
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
|
||||
const blob = new Blob(chunksRef.current, { type: recorder.mimeType || 'audio/webm' });
|
||||
chunksRef.current = [];
|
||||
|
||||
try {
|
||||
const result = await transcribeAudio(blob);
|
||||
setState('idle');
|
||||
resolve(result.text);
|
||||
} catch (err) {
|
||||
setState('idle');
|
||||
const msg = err instanceof Error ? err.message : 'Transcription failed';
|
||||
setError(msg);
|
||||
reject(err);
|
||||
}
|
||||
};
|
||||
|
||||
recorder.stop();
|
||||
});
|
||||
}, []);
|
||||
|
||||
return {
|
||||
state,
|
||||
error,
|
||||
available,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
isRecording: state === 'recording',
|
||||
isTranscribing: state === 'transcribing',
|
||||
};
|
||||
}
|
||||
@@ -125,3 +125,55 @@ export async function fetchTraces(limit: number = 50): Promise<unknown> {
|
||||
if (!res.ok) throw new Error(`Failed: ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Speech
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface TranscriptionResult {
|
||||
text: string;
|
||||
language: string | null;
|
||||
confidence: number | null;
|
||||
duration_seconds: number;
|
||||
}
|
||||
|
||||
export interface SpeechHealth {
|
||||
available: boolean;
|
||||
backend?: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export async function transcribeAudio(audioBlob: Blob, filename = 'recording.webm'): Promise<TranscriptionResult> {
|
||||
if (isTauri()) {
|
||||
try {
|
||||
const buffer = await audioBlob.arrayBuffer();
|
||||
return await tauriInvoke<TranscriptionResult>('transcribe_audio', {
|
||||
audioData: Array.from(new Uint8Array(buffer)),
|
||||
filename,
|
||||
});
|
||||
} catch {
|
||||
// Fall through to fetch
|
||||
}
|
||||
}
|
||||
const formData = new FormData();
|
||||
formData.append('file', audioBlob, filename);
|
||||
const res = await fetch(`${getBase()}/v1/speech/transcribe`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
if (!res.ok) throw new Error(`Transcription failed: ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function fetchSpeechHealth(): Promise<SpeechHealth> {
|
||||
if (isTauri()) {
|
||||
try {
|
||||
return await tauriInvoke<SpeechHealth>('speech_health');
|
||||
} catch {
|
||||
return { available: false };
|
||||
}
|
||||
}
|
||||
const res = await fetch(`${getBase()}/v1/speech/health`);
|
||||
if (!res.ok) return { available: false };
|
||||
return res.json();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user