Merge origin/main into gabebo-ui

Integrate speech-to-text (Phase 24) and eval trackers features.
Resolve conflicts: add speech Tauri commands to lib.rs, merge
MicButton + useSpeech into InputArea, consolidate speech API
functions into lib/api.ts (removed old api/client.ts).

Made-with: Cursor
This commit is contained in:
Gabriel Bo
2026-03-03 18:44:09 -08:00
49 changed files with 5093 additions and 97 deletions
+40 -13
View File
@@ -3,6 +3,8 @@ import { Send, Square, Paperclip } from 'lucide-react';
import { useAppStore, generateId } from '../../lib/store';
import { streamChat } from '../../lib/sse';
import { fetchSavings } from '../../lib/api';
import { MicButton } from './MicButton';
import { useSpeech } from '../../hooks/useSpeech';
import type { ChatMessage, ToolCallInfo, TokenUsage } from '../../types';
export function InputArea() {
@@ -21,7 +23,23 @@ export function InputArea() {
const setStreamState = useAppStore((s) => s.setStreamState);
const resetStream = useAppStore((s) => s.resetStream);
// Auto-resize textarea
const { state: speechState, available: speechAvailable, startRecording, stopRecording } = useSpeech();
const handleMicClick = useCallback(async () => {
if (speechState === 'recording') {
try {
const text = await stopRecording();
if (text) {
setInput((prev) => (prev ? prev + ' ' + text : text));
}
} catch {
// Error is captured in useSpeech
}
} else {
await startRecording();
}
}, [speechState, startRecording, stopRecording]);
useEffect(() => {
const el = textareaRef.current;
if (!el) return;
@@ -238,18 +256,27 @@ export function InputArea() {
<Square size={16} />
</button>
) : (
<button
onClick={sendMessage}
disabled={!input.trim()}
className="p-2 rounded-xl transition-colors shrink-0 cursor-pointer disabled:opacity-30 disabled:cursor-default"
style={{
background: input.trim() ? 'var(--color-accent)' : 'var(--color-bg-tertiary)',
color: input.trim() ? 'white' : 'var(--color-text-tertiary)',
}}
title="Send message"
>
<Send size={16} />
</button>
<div className="flex items-center gap-1">
{speechAvailable && (
<MicButton
state={speechState}
onClick={handleMicClick}
disabled={streamState.isStreaming}
/>
)}
<button
onClick={sendMessage}
disabled={!input.trim()}
className="p-2 rounded-xl transition-colors shrink-0 cursor-pointer disabled:opacity-30 disabled:cursor-default"
style={{
background: input.trim() ? 'var(--color-accent)' : 'var(--color-bg-tertiary)',
color: input.trim() ? 'white' : 'var(--color-text-tertiary)',
}}
title="Send message"
>
<Send size={16} />
</button>
</div>
)}
</div>
<div className="flex items-center justify-center mt-2 text-[11px]" style={{ color: 'var(--color-text-tertiary)' }}>
@@ -0,0 +1,53 @@
import type { SpeechState } from '../../hooks/useSpeech';
interface MicButtonProps {
state: SpeechState;
onClick: () => void;
disabled?: boolean;
}
export function MicButton({ state, onClick, disabled }: MicButtonProps) {
const title =
state === 'recording'
? 'Stop recording'
: state === 'transcribing'
? 'Transcribing...'
: 'Voice input';
return (
<button
className={`mic-btn ${state !== 'idle' ? `mic-${state}` : ''}`}
onClick={onClick}
disabled={disabled || state === 'transcribing'}
title={title}
style={{
background: state === 'recording' ? '#e74c3c' : 'transparent',
border: '1px solid var(--border, #555)',
borderRadius: '8px',
padding: '8px',
cursor: disabled || state === 'transcribing' ? 'default' : 'pointer',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
minWidth: '36px',
height: '36px',
color: state === 'recording' ? '#fff' : 'var(--text, #cdd6f4)',
opacity: disabled || state === 'transcribing' ? 0.5 : 1,
animation: state === 'recording' ? 'pulse 1.5s ease-in-out infinite' : 'none',
}}
>
{state === 'transcribing' ? (
<svg width="16" height="16" viewBox="0 0 16 16" fill="currentColor">
<circle cx="8" cy="8" r="6" fill="none" stroke="currentColor" strokeWidth="2" strokeDasharray="28" strokeDashoffset="10">
<animateTransform attributeName="transform" type="rotate" from="0 8 8" to="360 8 8" dur="1s" repeatCount="indefinite" />
</circle>
</svg>
) : (
<svg width="16" height="16" viewBox="0 0 16 16" fill="currentColor">
<path d="M5 3a3 3 0 0 1 6 0v5a3 3 0 0 1-6 0V3z" />
<path d="M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z" />
</svg>
)}
</button>
);
}
+92
View File
@@ -0,0 +1,92 @@
import { useState, useCallback, useRef, useEffect } from 'react';
import { transcribeAudio, fetchSpeechHealth } from '../lib/api';
export type SpeechState = 'idle' | 'recording' | 'transcribing';
export function useSpeech() {
const [state, setState] = useState<SpeechState>('idle');
const [error, setError] = useState<string | null>(null);
const [available, setAvailable] = useState(false);
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
const chunksRef = useRef<Blob[]>([]);
const streamRef = useRef<MediaStream | null>(null);
// Check if speech backend is available on mount
useEffect(() => {
fetchSpeechHealth()
.then((health) => setAvailable(health.available))
.catch(() => setAvailable(false));
}, []);
const startRecording = useCallback(async (): Promise<void> => {
setError(null);
if (!navigator.mediaDevices?.getUserMedia) {
setError('Microphone not supported in this browser');
return;
}
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
streamRef.current = stream;
const recorder = new MediaRecorder(stream);
chunksRef.current = [];
recorder.ondataavailable = (e) => {
if (e.data.size > 0) chunksRef.current.push(e.data);
};
recorder.start();
mediaRecorderRef.current = recorder;
setState('recording');
} catch (err) {
setError('Microphone access denied');
setState('idle');
}
}, []);
const stopRecording = useCallback(async (): Promise<string> => {
return new Promise((resolve, reject) => {
const recorder = mediaRecorderRef.current;
if (!recorder || recorder.state !== 'recording') {
reject(new Error('Not recording'));
return;
}
recorder.onstop = async () => {
setState('transcribing');
// Stop all audio tracks
streamRef.current?.getTracks().forEach((track) => track.stop());
streamRef.current = null;
const blob = new Blob(chunksRef.current, { type: recorder.mimeType || 'audio/webm' });
chunksRef.current = [];
try {
const result = await transcribeAudio(blob);
setState('idle');
resolve(result.text);
} catch (err) {
setState('idle');
const msg = err instanceof Error ? err.message : 'Transcription failed';
setError(msg);
reject(err);
}
};
recorder.stop();
});
}, []);
return {
state,
error,
available,
startRecording,
stopRecording,
isRecording: state === 'recording',
isTranscribing: state === 'transcribing',
};
}
+52
View File
@@ -125,3 +125,55 @@ export async function fetchTraces(limit: number = 50): Promise<unknown> {
if (!res.ok) throw new Error(`Failed: ${res.status}`);
return res.json();
}
// ---------------------------------------------------------------------------
// Speech
// ---------------------------------------------------------------------------
export interface TranscriptionResult {
text: string;
language: string | null;
confidence: number | null;
duration_seconds: number;
}
export interface SpeechHealth {
available: boolean;
backend?: string;
reason?: string;
}
export async function transcribeAudio(audioBlob: Blob, filename = 'recording.webm'): Promise<TranscriptionResult> {
if (isTauri()) {
try {
const buffer = await audioBlob.arrayBuffer();
return await tauriInvoke<TranscriptionResult>('transcribe_audio', {
audioData: Array.from(new Uint8Array(buffer)),
filename,
});
} catch {
// Fall through to fetch
}
}
const formData = new FormData();
formData.append('file', audioBlob, filename);
const res = await fetch(`${getBase()}/v1/speech/transcribe`, {
method: 'POST',
body: formData,
});
if (!res.ok) throw new Error(`Transcription failed: ${res.status}`);
return res.json();
}
export async function fetchSpeechHealth(): Promise<SpeechHealth> {
if (isTauri()) {
try {
return await tauriInvoke<SpeechHealth>('speech_health');
} catch {
return { available: false };
}
}
const res = await fetch(`${getBase()}/v1/speech/health`);
if (!res.ok) return { available: false };
return res.json();
}