diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 2f71dcd89..9de9b692f 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -1372,7 +1372,16 @@ pub fn run() { } }; if let Some(path) = fake_camera_arg { - args.push(("--use-fake-device-for-media-stream", None)); + // `--use-file-for-fake-video-capture` alone (CEF 146 / Chromium 128+) + // injects the Y4M as the video capture source without replacing the + // audio capture device. The old belt-and-suspenders flag + // `--use-fake-device-for-media-stream` is deliberately omitted here: + // it replaced ALL media capture devices — including audio — with fake + // ones, causing a sine-wave test tone (beeping) to be recorded instead + // of the real microphone whenever `getUserMedia({audio:true})` was + // called from the main app WebView (e.g. the mascot voice composer). + // `--use-fake-ui-for-media-stream` is kept so Meet's permission prompt + // is auto-granted without interrupting the join flow. args.push(("--use-fake-ui-for-media-stream", None)); args.push(("--use-file-for-fake-video-capture", Some(path))); } diff --git a/app/src/features/human/MicCloudComposer.test.tsx b/app/src/features/human/MicCloudComposer.test.tsx index 68357510f..040d73fc4 100644 --- a/app/src/features/human/MicCloudComposer.test.tsx +++ b/app/src/features/human/MicCloudComposer.test.tsx @@ -147,14 +147,45 @@ describe('MicCloudComposer', () => { expect(opts).toEqual({ language: 'es' }); }); - it('surfaces a permission-denied error via onError', async () => { - getUserMediaMock.mockRejectedValueOnce(new Error('NotAllowed')); + it('surfaces a permission-denied error via onError for NotAllowedError', async () => { + const err = Object.assign(new DOMException('', 'NotAllowedError')); + getUserMediaMock.mockRejectedValueOnce(err); const onError = vi.fn(); render(); fireEvent.click(screen.getByRole('button', { name: /start recording/i })); await waitFor(() => expect(onError).toHaveBeenCalledWith(expect.stringMatching(/permission/i))); }); + it('surfaces a device-unavailable error for OverconstrainedError', async () => { + const err = new DOMException('', 'OverconstrainedError'); + getUserMediaMock.mockRejectedValueOnce(err); + const onError = vi.fn(); + render(); + fireEvent.click(screen.getByRole('button', { name: /start recording/i })); + await waitFor(() => + expect(onError).toHaveBeenCalledWith(expect.stringMatching(/unavailable/i)) + ); + }); + + it('surfaces an in-use error for NotReadableError', async () => { + const err = new DOMException('', 'NotReadableError'); + getUserMediaMock.mockRejectedValueOnce(err); + const onError = vi.fn(); + render(); + fireEvent.click(screen.getByRole('button', { name: /start recording/i })); + await waitFor(() => expect(onError).toHaveBeenCalledWith(expect.stringMatching(/in use/i))); + }); + + it('surfaces a generic error for non-DOMException getUserMedia failures', async () => { + getUserMediaMock.mockRejectedValueOnce(new Error('some other error')); + const onError = vi.fn(); + render(); + fireEvent.click(screen.getByRole('button', { name: /start recording/i })); + await waitFor(() => + expect(onError).toHaveBeenCalledWith(expect.stringMatching(/microphone error/i)) + ); + }); + it('falls back to wav re-encode when the native attempt fails', async () => { transcribeCloudMock .mockRejectedValueOnce(new Error('codec not accepted')) @@ -288,4 +319,164 @@ describe('MicCloudComposer', () => { expect(removeSpy).toHaveBeenCalledWith('keydown', expect.any(Function)); removeSpy.mockRestore(); }); + + // ── Device selector (showDeviceSelector) ───────────────────────────────── + + it('enumerates devices on mount when showDeviceSelector is true', async () => { + const enumerateDevicesMock = vi.fn().mockResolvedValue([ + { kind: 'audioinput', deviceId: 'dev1', label: 'Built-in Mic' }, + { kind: 'audioinput', deviceId: 'dev2', label: 'USB Headset' }, + { kind: 'videoinput', deviceId: 'cam1', label: 'Camera' }, + ]); + Object.defineProperty(globalThis.navigator, 'mediaDevices', { + value: { getUserMedia: getUserMediaMock, enumerateDevices: enumerateDevicesMock }, + configurable: true, + writable: true, + }); + + render(); + + await waitFor(() => expect(enumerateDevicesMock).toHaveBeenCalled()); + expect(await screen.findByRole('combobox', { name: /microphone device/i })).toBeInTheDocument(); + expect(screen.getByText('Built-in Mic')).toBeInTheDocument(); + expect(screen.getByText('USB Headset')).toBeInTheDocument(); + // Video input must not appear + expect(screen.queryByText('Camera')).not.toBeInTheDocument(); + }); + + it('does not show the selector when showDeviceSelector is false (default)', async () => { + const enumerateDevicesMock = vi.fn().mockResolvedValue([ + { kind: 'audioinput', deviceId: 'dev1', label: 'Built-in Mic' }, + { kind: 'audioinput', deviceId: 'dev2', label: 'USB Headset' }, + ]); + Object.defineProperty(globalThis.navigator, 'mediaDevices', { + value: { getUserMedia: getUserMediaMock, enumerateDevices: enumerateDevicesMock }, + configurable: true, + writable: true, + }); + + render(); + + await waitFor(() => { + expect( + screen.queryByRole('combobox', { name: /microphone device/i }) + ).not.toBeInTheDocument(); + expect(enumerateDevicesMock).not.toHaveBeenCalled(); + }); + }); + + it('shows the selector disabled when only one device is available', async () => { + const enumerateDevicesMock = vi + .fn() + .mockResolvedValue([{ kind: 'audioinput', deviceId: 'dev1', label: 'Built-in Mic' }]); + Object.defineProperty(globalThis.navigator, 'mediaDevices', { + value: { getUserMedia: getUserMediaMock, enumerateDevices: enumerateDevicesMock }, + configurable: true, + writable: true, + }); + + render(); + + const select = await screen.findByRole('combobox', { name: /microphone device/i }); + expect(select).toBeInTheDocument(); + expect(select).toBeDisabled(); + }); + + it('falls back to "Microphone N" label when browser hides labels before permission', async () => { + const enumerateDevicesMock = vi.fn().mockResolvedValue([ + { kind: 'audioinput', deviceId: 'dev1', label: '' }, + { kind: 'audioinput', deviceId: 'dev2', label: '' }, + ]); + Object.defineProperty(globalThis.navigator, 'mediaDevices', { + value: { getUserMedia: getUserMediaMock, enumerateDevices: enumerateDevicesMock }, + configurable: true, + writable: true, + }); + + render(); + + await waitFor(() => expect(screen.queryByRole('combobox')).toBeInTheDocument()); + expect(screen.getByText('Microphone 1')).toBeInTheDocument(); + expect(screen.getByText('Microphone 2')).toBeInTheDocument(); + }); + + it('passes the selected deviceId as an exact constraint to getUserMedia', async () => { + const enumerateDevicesMock = vi.fn().mockResolvedValue([ + { kind: 'audioinput', deviceId: 'dev1', label: 'Built-in Mic' }, + { kind: 'audioinput', deviceId: 'dev2', label: 'USB Headset' }, + ]); + transcribeCloudMock.mockResolvedValueOnce('hello'); + Object.defineProperty(globalThis.navigator, 'mediaDevices', { + value: { getUserMedia: getUserMediaMock, enumerateDevices: enumerateDevicesMock }, + configurable: true, + writable: true, + }); + + render(); + + // Wait for the selector to appear and pick the second device + const select = await screen.findByRole('combobox', { name: /microphone device/i }); + fireEvent.change(select, { target: { value: 'dev2' } }); + + fireEvent.click(screen.getByRole('button', { name: /start recording/i })); + await waitFor(() => expect(getUserMediaMock).toHaveBeenCalled()); + + expect(getUserMediaMock).toHaveBeenCalledWith( + expect.objectContaining({ audio: expect.objectContaining({ deviceId: { exact: 'dev2' } }) }) + ); + }); + + it('refreshes device labels after getUserMedia permission is granted', async () => { + const enumerateDevicesMock = vi + .fn() + // First call (on mount): labels hidden + .mockResolvedValueOnce([ + { kind: 'audioinput', deviceId: 'dev1', label: '' }, + { kind: 'audioinput', deviceId: 'dev2', label: '' }, + ]) + // Second call (post-permission): real labels + .mockResolvedValueOnce([ + { kind: 'audioinput', deviceId: 'dev1', label: 'Built-in Mic' }, + { kind: 'audioinput', deviceId: 'dev2', label: 'USB Headset' }, + ]); + transcribeCloudMock.mockResolvedValueOnce('ok'); + Object.defineProperty(globalThis.navigator, 'mediaDevices', { + value: { getUserMedia: getUserMediaMock, enumerateDevices: enumerateDevicesMock }, + configurable: true, + writable: true, + }); + + render(); + + // Mount enumerate ran — labels are blank placeholders + await waitFor(() => expect(screen.queryByRole('combobox')).toBeInTheDocument()); + expect(screen.getByText('Microphone 1')).toBeInTheDocument(); + + // Start recording → triggers the post-permission refresh + fireEvent.click(screen.getByRole('button', { name: /start recording/i })); + await waitFor(() => + expect(screen.getByRole('button', { name: /stop recording and send/i })).toBeInTheDocument() + ); + + // After permission, real labels should now be visible + await waitFor(() => expect(screen.getByText('Built-in Mic')).toBeInTheDocument()); + expect(screen.getByText('USB Headset')).toBeInTheDocument(); + }); + + it('handles enumerateDevices throwing gracefully (no crash, selector hidden)', async () => { + const enumerateDevicesMock = vi.fn().mockRejectedValue(new Error('NotAllowed')); + Object.defineProperty(globalThis.navigator, 'mediaDevices', { + value: { getUserMedia: getUserMediaMock, enumerateDevices: enumerateDevicesMock }, + configurable: true, + writable: true, + }); + + render(); + + await waitFor(() => expect(enumerateDevicesMock).toHaveBeenCalled()); + // Selector requires >1 device; error yields 0 → selector stays hidden + expect(screen.queryByRole('combobox', { name: /microphone device/i })).not.toBeInTheDocument(); + // Composer still functional + expect(screen.getByText('Tap and speak')).toBeInTheDocument(); + }); }); diff --git a/app/src/features/human/MicCloudComposer.tsx b/app/src/features/human/MicCloudComposer.tsx index 1e4960c06..a0d379535 100644 --- a/app/src/features/human/MicCloudComposer.tsx +++ b/app/src/features/human/MicCloudComposer.tsx @@ -4,6 +4,12 @@ import { useEffect, useRef, useState } from 'react'; import { transcribeCloud } from './voice/sttClient'; import { encodeBlobToWav } from './voice/wavEncoder'; +/** Minimal descriptor for an audio input device. */ +interface AudioInputDevice { + deviceId: string; + label: string; +} + const composerLog = debug('human:mic-composer'); /** MIME types MediaRecorder will be asked to use, in priority order. @@ -34,6 +40,8 @@ export interface MicCloudComposerProps { * passing a hint is meaningfully more accurate than auto-detect on * short utterances. Set to empty string to let Scribe auto-detect. */ language?: string; + /** Show a microphone device selector beneath the button. Defaults to false. */ + showDeviceSelector?: boolean; } type RecordingState = 'idle' | 'recording' | 'transcribing'; @@ -52,8 +60,11 @@ export function MicCloudComposer({ onSubmit, onError, language = 'en', + showDeviceSelector = false, }: MicCloudComposerProps) { const [state, setState] = useState('idle'); + const [devices, setDevices] = useState([]); + const [selectedDeviceId, setSelectedDeviceId] = useState(''); const recorderRef = useRef(null); const streamRef = useRef(null); const chunksRef = useRef([]); @@ -85,6 +96,33 @@ export function MicCloudComposer({ }; }, []); + // Enumerate audio input devices when the selector is shown, and refresh the + // list after the user grants mic permission (labels are hidden until then). + useEffect(() => { + if (!showDeviceSelector) return; + async function loadDevices() { + if (!navigator.mediaDevices?.enumerateDevices) return; + try { + const all = await navigator.mediaDevices.enumerateDevices(); + const inputs = all + .filter(d => d.kind === 'audioinput') + .map((d, i) => ({ deviceId: d.deviceId, label: d.label || `Microphone ${i + 1}` })); + setDevices(inputs); + // Keep the selected device valid; fall back to default. + setSelectedDeviceId(prev => + inputs.some(d => d.deviceId === prev) ? prev : (inputs[0]?.deviceId ?? '') + ); + composerLog('enumerated %d audio inputs', inputs.length); + } catch (err) { + composerLog('enumerateDevices failed: %s', err); + } + } + void loadDevices(); + const onDeviceChange = () => void loadDevices(); + navigator.mediaDevices?.addEventListener?.('devicechange', onDeviceChange); + return () => navigator.mediaDevices?.removeEventListener?.('devicechange', onDeviceChange); + }, [showDeviceSelector]); + // Spacebar = tap-to-toggle (#1471). Scoped to whatever surface mounts // this composer — today only the Human agent page. The listener lives // on the window so the user doesn't have to click the mascot stage @@ -196,6 +234,7 @@ export function MicCloudComposer({ // transcription drops words in our flow) stream = await navigator.mediaDevices.getUserMedia({ audio: { + ...(selectedDeviceId ? { deviceId: { exact: selectedDeviceId } } : {}), channelCount: 1, sampleRate: 48000, echoCancellation: true, @@ -203,11 +242,32 @@ export function MicCloudComposer({ autoGainControl: true, }, }); + // After the first successful grant, refresh device labels (they are + // blank until the user has given permission). + if (showDeviceSelector) { + const all = await navigator.mediaDevices.enumerateDevices(); + const inputs = all + .filter(d => d.kind === 'audioinput') + .map((d, i) => ({ deviceId: d.deviceId, label: d.label || `Microphone ${i + 1}` })); + setDevices(inputs); + } } catch (err) { startInFlightRef.current = false; const msg = err instanceof Error ? err.message : String(err); composerLog('getUserMedia rejected: %s', msg); - onError?.(`Microphone permission denied: ${msg}`); + if (err instanceof DOMException) { + if (err.name === 'NotAllowedError' || err.name === 'SecurityError') { + onError?.(`Microphone permission denied: ${msg}`); + } else if (err.name === 'NotFoundError' || err.name === 'OverconstrainedError') { + onError?.('Selected microphone is unavailable — try a different device.'); + } else if (err.name === 'NotReadableError') { + onError?.('Microphone is in use by another application.'); + } else { + onError?.(`Microphone error: ${msg}`); + } + } else { + onError?.(`Microphone error: ${msg}`); + } return; } @@ -365,50 +425,66 @@ export function MicCloudComposer({ : 'Tap and speak'; return ( -
- - {label} + strokeWidth={1.8} + viewBox="0 0 24 24"> + + + )} + + {label} +
); } diff --git a/app/src/pages/Conversations.tsx b/app/src/pages/Conversations.tsx index 843f77d5e..5ff0f849e 100644 --- a/app/src/pages/Conversations.tsx +++ b/app/src/pages/Conversations.tsx @@ -1676,6 +1676,7 @@ const Conversations = ({ variant = 'page', composer = 'text' }: ConversationsPro disabled={composerInteractionBlocked || !selectedThreadId} onSubmit={text => handleSendMessage(text)} onError={message => setSendError(chatSendError('voice_transcription', message))} + showDeviceSelector /> ) : inputMode === 'text' ? (
diff --git a/src/openhuman/memory/store/unified/init.rs b/src/openhuman/memory/store/unified/init.rs index c42b86f8a..6601da612 100644 --- a/src/openhuman/memory/store/unified/init.rs +++ b/src/openhuman/memory/store/unified/init.rs @@ -137,8 +137,8 @@ impl UnifiedMemory { // We run the ALTER TABLE statements directly on `conn` before wrapping it in Arc. // SQL arrays are defined as constants in profile.rs to avoid duplication. { - use super::profile::{PHASE3_COLUMNS_SQL, PHASE3_INDEXES_SQL}; - for sql in PHASE3_COLUMNS_SQL.iter().chain(PHASE3_INDEXES_SQL.iter()) { + use super::profile::PHASE3_COLUMNS_SQL; + for sql in PHASE3_COLUMNS_SQL.iter() { match conn.execute(sql, []) { Ok(_) => tracing::debug!("[profile:init] applied: {sql}"), Err(e) => { diff --git a/src/openhuman/memory/store/unified/profile.rs b/src/openhuman/memory/store/unified/profile.rs index bd3984345..6904e3447 100644 --- a/src/openhuman/memory/store/unified/profile.rs +++ b/src/openhuman/memory/store/unified/profile.rs @@ -44,12 +44,6 @@ CREATE TABLE IF NOT EXISTS user_profile ( CREATE INDEX IF NOT EXISTS idx_profile_type ON user_profile(facet_type); - -CREATE INDEX IF NOT EXISTS idx_profile_state - ON user_profile(state); - -CREATE INDEX IF NOT EXISTS idx_profile_class - ON user_profile(class); "#; /// Phase 3 ALTER TABLE statements for adding new columns to existing databases. @@ -65,14 +59,6 @@ pub const PHASE3_COLUMNS_SQL: &[&str] = &[ "ALTER TABLE user_profile ADD COLUMN cue_families_json TEXT", ]; -/// Phase 3 index creation statements. Idempotent (IF NOT EXISTS). -/// -/// Shared between `migrate_profile_schema` and `init.rs`. -pub const PHASE3_INDEXES_SQL: &[&str] = &[ - "CREATE INDEX IF NOT EXISTS idx_profile_state ON user_profile(state)", - "CREATE INDEX IF NOT EXISTS idx_profile_class ON user_profile(class)", -]; - /// Idempotent schema migration for existing databases. /// /// New installs get the full schema from `PROFILE_INIT_SQL`. Existing databases @@ -100,13 +86,6 @@ pub fn migrate_profile_schema(conn: &Arc>) { } } } - - // Ensure the new indexes exist (idempotent — IF NOT EXISTS). - for sql in PHASE3_INDEXES_SQL { - if let Err(e) = conn.execute(sql, []) { - tracing::warn!("[profile] index creation failed (non-fatal): {sql}: {e}"); - } - } } // ── FacetState ───────────────────────────────────────────────────────────────