diff --git a/.github/Dockerfile b/.github/Dockerfile index 9ae630a8a..4d3d83618 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -5,6 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive # System deps for Tauri + mold linker + clang (for mold) RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ + cmake \ curl \ ca-certificates \ git \ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7e63b860..8ebc7bb5b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,6 +47,9 @@ jobs: if: steps.yarn-cache.outputs.cache-hit != 'true' run: yarn install --frozen-lockfile + - name: Install cmake (for whisper-rs) + run: apt-get update && apt-get install -y --no-install-recommends cmake && rm -rf /var/lib/apt/lists/* + - name: Build sidecar core binary run: cargo build --profile ci --target x86_64-unknown-linux-gnu --bin openhuman-core diff --git a/.github/workflows/release-packages.yml b/.github/workflows/release-packages.yml index 1bd9cae4c..ec99590fc 100644 --- a/.github/workflows/release-packages.yml +++ b/.github/workflows/release-packages.yml @@ -58,7 +58,7 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -y --no-install-recommends \ - pkg-config libssl-dev build-essential + pkg-config libssl-dev build-essential cmake - name: Build CLI binary and package tarball env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4ce77e4cd..2d5cd7ffa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -230,7 +230,7 @@ jobs: if: matrix.settings.platform == 'ubuntu-22.04' run: | sudo apt-get update - sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf + sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf cmake # Skip first 7 lines of Cargo.lock (workspace package version bumps) so the key tracks dependency changes only - name: Cargo.lock fingerprint (deps only) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59f918b04..90d0b42fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,6 +85,9 @@ jobs: app/src-tauri -> target cache-on-failure: true + - name: Install cmake (for whisper-rs) + run: apt-get update && apt-get install -y --no-install-recommends cmake && rm -rf /var/lib/apt/lists/* + - name: Check formatting (cargo fmt) run: cargo fmt --all -- --check diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index ce9e2ca95..365fdd064 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -77,6 +77,9 @@ jobs: . -> target cache-on-failure: true + - name: Install cmake (for whisper-rs) + run: apt-get update && apt-get install -y --no-install-recommends cmake && rm -rf /var/lib/apt/lists/* + - name: Check formatting (cargo fmt) run: cargo fmt --all -- --check diff --git a/Cargo.lock b/Cargo.lock index c244627e1..7e780fc38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -515,6 +515,26 @@ dependencies = [ "virtue", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.11.0", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.117", +] + [[package]] name = "bitfield" version = "0.19.4" @@ -725,6 +745,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cff-parser" version = "0.1.0" @@ -813,6 +842,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.9", +] + [[package]] name = "clap" version = "4.6.0" @@ -3305,6 +3345,16 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libloading" version = "0.9.0" @@ -4612,6 +4662,7 @@ dependencies = [ "wa-rs-tokio-transport", "wa-rs-ureq-http", "webpki-roots 1.0.6", + "whisper-rs", ] [[package]] @@ -4740,7 +4791,7 @@ version = "2.0.0-rc.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5df903c0d2c07b56950f1058104ab0c8557159f2741782223704de9be73c3c" dependencies = [ - "libloading", + "libloading 0.9.0", "ndarray", "ort-sys", "smallvec", @@ -8477,6 +8528,28 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "whisper-rs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2088172d00f936c348d6a72f488dc2660ab3f507263a195df308a3c2383229f6" +dependencies = [ + "whisper-rs-sys", +] + +[[package]] +name = "whisper-rs-sys" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6986c0fe081241d391f09b9a071fbcbb59720c3563628c3c829057cf69f2a56f" +dependencies = [ + "bindgen", + "cfg-if", + "cmake", + "fs_extra", + "semver", +] + [[package]] name = "whoami" version = "2.1.1" diff --git a/Cargo.toml b/Cargo.toml index 49ef87dfe..fd344194c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,7 @@ sentry = { version = "0.47.0", default-features = false, features = ["backtrace" tokio-stream = { version = "0.1.18", features = ["full"] } url = "2" socketioxide = { version = "0.15", features = ["extensions"] } +whisper-rs = "0.16" matrix-sdk = { version = "0.16", optional = true, default-features = false, features = ["e2e-encryption", "rustls-tls", "markdown"] } fantoccini = { version = "0.22.0", optional = true, default-features = false, features = ["rustls-tls"] } diff --git a/app/src/components/settings/panels/LocalModelPanel.tsx b/app/src/components/settings/panels/LocalModelPanel.tsx index c875a81b1..3cb412820 100644 --- a/app/src/components/settings/panels/LocalModelPanel.tsx +++ b/app/src/components/settings/panels/LocalModelPanel.tsx @@ -1129,9 +1129,12 @@ const LocalModelPanel = () => { {isTranscribeLoading ? 'Running...' : 'Run Transcription Test'} {transcribeOutput && ( -
+
Model: {transcribeOutput.model_id}
- 
{transcribeOutput.text}
+
+ Transcript: + 
{transcribeOutput.text}
+
)}
diff --git a/app/src/pages/Conversations.tsx b/app/src/pages/Conversations.tsx index d05aa6c73..db82bf14c 100644 --- a/app/src/pages/Conversations.tsx +++ b/app/src/pages/Conversations.tsx @@ -35,8 +35,9 @@ import { openhumanAutocompleteAccept, openhumanAutocompleteCurrent, openhumanLocalAiChat, - openhumanLocalAiTranscribeBytes, - openhumanLocalAiTts, + openhumanVoiceStatus, + openhumanVoiceTranscribeBytes, + openhumanVoiceTts, } from '../utils/tauriCommands'; const DEFAULT_THREAD_ID = 'default-thread'; @@ -282,6 +283,33 @@ const Conversations = () => { } }, [inputMode, isRecording]); + // Proactively check voice binary availability when switching to voice mode + useEffect(() => { + if (inputMode !== 'voice' || !rustChat) return; + let cancelled = false; + void (async () => { + try { + const resp = await openhumanVoiceStatus(); + if (cancelled) return; + const status = resp.result; + if (!status.stt_available) { + setVoiceStatus( + 'Speech-to-text unavailable: whisper-cli binary or STT model not found. Check Settings > Local Models.' + ); + } else { + setVoiceStatus('Ready — tap "Start Talking" to record.'); + } + } catch { + if (!cancelled) { + setVoiceStatus('Could not check voice availability.'); + } + } + })(); + return () => { + cancelled = true; + }; + }, [inputMode, rustChat]); + useEffect(() => { if (!rustChat || socketStatus !== 'connected') return; @@ -588,7 +616,15 @@ const Conversations = () => { const blob = new Blob(chunks, { type: mimeType || 'audio/webm' }); const audioBytes = Array.from(new Uint8Array(await blob.arrayBuffer())); const extension = getAudioExtension(mimeType || blob.type); - const result = await openhumanLocalAiTranscribeBytes(audioBytes, extension); + + // Build conversation context from recent messages for LLM cleanup. + const recentMessages = messages.slice(-10); + const context = + recentMessages.length > 0 + ? recentMessages.map(m => `${m.sender}: ${m.content}`).join('\n') + : undefined; + + const result = await openhumanVoiceTranscribeBytes(audioBytes, extension, context); const transcript = result.result.text.trim(); if (!transcript) { @@ -685,7 +721,7 @@ const Conversations = () => { void (async () => { try { - const ttsResult = await openhumanLocalAiTts(latestAgentMessage.content); + const ttsResult = await openhumanVoiceTts(latestAgentMessage.content); if (cancelled) return; const audioSrc = convertFileSrc(ttsResult.result.output_path); diff --git a/app/src/utils/tauriCommands.ts b/app/src/utils/tauriCommands.ts index b1cee307d..64df1704d 100644 --- a/app/src/utils/tauriCommands.ts +++ b/app/src/utils/tauriCommands.ts @@ -2057,3 +2057,75 @@ export async function runtimeSkillDataStats(skillId: string): Promise> { + return await callCoreRpc>({ + method: 'openhuman.voice_status', + params: {}, + }); +} + +export async function openhumanVoiceTranscribe( + audioPath: string, + context?: string, + skipCleanup?: boolean +): Promise> { + return await callCoreRpc>({ + method: 'openhuman.voice_transcribe', + params: { audio_path: audioPath, context, skip_cleanup: skipCleanup }, + }); +} + +export async function openhumanVoiceTranscribeBytes( + audioBytes: number[], + extension?: string, + context?: string, + skipCleanup?: boolean +): Promise> { + return await callCoreRpc>({ + method: 'openhuman.voice_transcribe_bytes', + params: { audio_bytes: audioBytes, extension, context, skip_cleanup: skipCleanup }, + }); +} + +export async function openhumanVoiceTts( + text: string, + outputPath?: string +): Promise> { + return await callCoreRpc>({ + method: 'openhuman.voice_tts', + params: { text, output_path: outputPath }, + }); +} diff --git a/app/test/e2e/specs/voice-mode.spec.ts b/app/test/e2e/specs/voice-mode.spec.ts new file mode 100644 index 000000000..82bf8b126 --- /dev/null +++ b/app/test/e2e/specs/voice-mode.spec.ts @@ -0,0 +1,206 @@ +// @ts-nocheck +/** + * E2E test: Voice mode integration + * + * Covers: + * - Navigating to conversations page + * - Switching to voice input mode + * - Voice status check fires and displays availability message + * - Voice input/reply mode toggle buttons render + * - Voice recording button renders in voice mode + * - Switching back to text mode restores text input + * + * The mock server runs on http://127.0.0.1:18473 + */ +import { waitForApp, waitForAppReady } from '../helpers/app-helpers'; +import { triggerAuthDeepLink } from '../helpers/deep-link-helpers'; +import { + clickText, + dumpAccessibilityTree, + textExists, + waitForWebView, + waitForWindowVisible, +} from '../helpers/element-helpers'; +import { clearRequestLog, getRequestLog, startMockServer, stopMockServer } from '../mock-server'; + +async function waitForRequest(method, urlFragment, timeout = 15_000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + const log = getRequestLog(); + const match = log.find(r => r.method === method && r.url.includes(urlFragment)); + if (match) return match; + await browser.pause(500); + } + return undefined; +} + +async function waitForTextToDisappear(text, timeout = 10_000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + if (!(await textExists(text))) return true; + await browser.pause(400); + } + return false; +} + +async function completeOnboardingIfVisible() { + if (await textExists('Skip for now')) { + await clickText('Skip for now', 10_000); + await waitForTextToDisappear('Skip for now', 8_000); + await browser.pause(1500); + } + + if (await textExists('Looks Amazing')) { + await clickText('Looks Amazing', 10_000); + await browser.pause(1500); + } else if (await textExists('Bring It On')) { + await clickText('Bring It On', 10_000); + await browser.pause(1500); + } + + if (await textExists('Got it')) { + await clickText('Got it', 10_000); + await browser.pause(1500); + } else if (await textExists('Continue')) { + await clickText('Continue', 10_000); + await browser.pause(1500); + } + + if (await textExists("Let's Go")) { + await clickText("Let's Go", 10_000); + } else if (await textExists("I'm Ready")) { + await clickText("I'm Ready", 10_000); + } +} + +async function waitForHome(timeout = 20_000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + if (await textExists('Message OpenHuman')) return true; + await browser.pause(700); + } + return false; +} + +async function waitForAnyText(candidates, timeout = 20_000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + for (const t of candidates) { + if (await textExists(t)) return t; + } + await browser.pause(600); + } + return null; +} + +describe('Voice mode integration', () => { + before(async () => { + await startMockServer(); + await waitForApp(); + clearRequestLog(); + }); + + after(async () => { + await stopMockServer(); + }); + + it('can switch to voice input mode, see status message, and switch back to text', async () => { + // --- Authenticate and reach conversations --- + await triggerAuthDeepLink('e2e-voice-token'); + await waitForWindowVisible(25_000); + await waitForWebView(15_000); + await waitForAppReady(15_000); + + const consume = await waitForRequest('POST', '/telegram/login-tokens/'); + expect(consume).toBeDefined(); + + await completeOnboardingIfVisible(); + + const onHome = await waitForHome(20_000); + if (!onHome) { + const tree = await dumpAccessibilityTree(); + console.log('[VoiceModeE2E] Home not reached. Tree:\n', tree.slice(0, 4000)); + } + expect(onHome).toBe(true); + + // --- Verify we see the text input area (default mode) --- + const hasTextInput = await waitForAnyText(['Message OpenHuman', 'Type a message'], 10_000); + expect(hasTextInput).not.toBeNull(); + + // --- Verify voice toggle buttons are visible --- + // The Input toggle group should show "Text" and "Voice" buttons + const hasInputLabel = await textExists('Input'); + expect(hasInputLabel).toBe(true); + + // --- Switch to voice input mode --- + // There are two "Voice" buttons (Input toggle and Reply toggle). + // We click the first one which is the Input mode toggle. + await clickText('Voice', 10_000); + await browser.pause(2_000); + + // --- Voice status check should fire --- + // Since whisper-cli is not installed in the E2E environment, + // we expect the unavailability message or the ready message. + const voiceStatusMessage = await waitForAnyText( + [ + 'Speech-to-text unavailable', + 'whisper-cli binary', + 'STT model not found', + 'Ready', + 'Start Talking', + 'Could not check voice availability', + ], + 15_000 + ); + + if (!voiceStatusMessage) { + const tree = await dumpAccessibilityTree(); + console.log('[VoiceModeE2E] No voice status message seen. Tree:\n', tree.slice(0, 5000)); + } + expect(voiceStatusMessage).not.toBeNull(); + + // --- Verify the voice recording button or unavailability message is visible --- + const hasVoiceButton = await waitForAnyText( + ['Start Talking', 'Transcribing', 'Stop & Send'], + 10_000 + ); + if (!hasVoiceButton) { + const hasStatus = await textExists('Speech-to-text unavailable'); + expect(hasStatus).toBe(true); + } + + // --- Switch back to text mode --- + // Click the "Text" button in the Input toggle group + await clickText('Text', 10_000); + await browser.pause(1_500); + + // --- Verify text input is restored --- + const textRestored = await waitForAnyText(['Message OpenHuman', 'Type a message'], 10_000); + expect(textRestored).not.toBeNull(); + }); + + it('shows reply mode toggle with text and voice options', async () => { + // Ensure conversations page is loaded (re-authenticate if state was lost). + const onConversations = await waitForAnyText( + ['Message OpenHuman', 'Type a message', 'Reply'], + 5_000 + ); + if (!onConversations) { + await triggerAuthDeepLink('e2e-voice-token'); + await waitForWindowVisible(25_000); + await waitForWebView(15_000); + await waitForAppReady(15_000); + await completeOnboardingIfVisible(); + await waitForHome(20_000); + } + + // The Reply toggle should be visible on the conversations page + const hasReplyLabel = await textExists('Reply'); + expect(hasReplyLabel).toBe(true); + + // Verify both reply mode options exist + // (There are multiple "Text" and "Voice" buttons — Input + Reply groups) + const hasText = await textExists('Text'); + expect(hasText).toBe(true); + }); +}); diff --git a/src/core/all.rs b/src/core/all.rs index c92dd0e12..89f1e49ff 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -65,6 +65,7 @@ fn build_registered_controllers() -> Vec { controllers.extend(crate::openhuman::memory::all_memory_registered_controllers()); controllers.extend(crate::openhuman::billing::all_billing_registered_controllers()); controllers.extend(crate::openhuman::team::all_team_registered_controllers()); + controllers.extend(crate::openhuman::voice::all_voice_registered_controllers()); controllers } @@ -95,6 +96,7 @@ fn build_declared_controller_schemas() -> Vec { schemas.extend(crate::openhuman::memory::all_memory_controller_schemas()); schemas.extend(crate::openhuman::billing::all_billing_controller_schemas()); schemas.extend(crate::openhuman::team::all_team_controller_schemas()); + schemas.extend(crate::openhuman::voice::all_voice_controller_schemas()); schemas } @@ -131,6 +133,7 @@ pub fn namespace_description(namespace: &str) -> Option<&'static str> { "memory" => Some("Document storage, vector search, key-value store, and knowledge graph."), "billing" => Some("Subscription plan, payment links, and credit top-up via the backend."), "team" => Some("Team member management, invites, and role changes via the backend."), + "voice" => Some("Speech-to-text and text-to-speech using local models."), _ => None, } } diff --git a/src/openhuman/config/schema/local_ai.rs b/src/openhuman/config/schema/local_ai.rs index 31492ccca..c3c655c9f 100644 --- a/src/openhuman/config/schema/local_ai.rs +++ b/src/openhuman/config/schema/local_ai.rs @@ -54,6 +54,14 @@ pub struct LocalAiConfig { /// Optional path to a manually-installed Ollama binary. #[serde(default)] pub ollama_binary_path: Option, + /// When true, load the whisper model in-process via whisper-rs instead of + /// shelling out to whisper-cli for each transcription call. + #[serde(default = "default_whisper_in_process")] + pub whisper_in_process: bool, + /// When true and Ollama is available, pass raw transcription through a + /// local LLM to fix grammar/punctuation using conversation context. + #[serde(default = "default_voice_llm_cleanup_enabled")] + pub voice_llm_cleanup_enabled: bool, } fn default_enabled() -> bool { @@ -149,6 +157,14 @@ fn default_max_suggestions() -> usize { 5 } +fn default_whisper_in_process() -> bool { + true +} + +fn default_voice_llm_cleanup_enabled() -> bool { + true +} + impl Default for LocalAiConfig { fn default() -> Self { Self { @@ -176,6 +192,8 @@ impl Default for LocalAiConfig { max_suggestions: default_max_suggestions(), selected_tier: None, ollama_binary_path: None, + whisper_in_process: default_whisper_in_process(), + voice_llm_cleanup_enabled: default_voice_llm_cleanup_enabled(), } } } diff --git a/src/openhuman/local_ai/mod.rs b/src/openhuman/local_ai/mod.rs index 350297813..b18b6d729 100644 --- a/src/openhuman/local_ai/mod.rs +++ b/src/openhuman/local_ai/mod.rs @@ -7,10 +7,10 @@ pub mod presets; mod schemas; mod install; -mod model_ids; +pub(crate) mod model_ids; mod ollama_api; mod parse; -mod paths; +pub(crate) mod paths; mod service; mod types; @@ -23,6 +23,7 @@ pub use schemas::{ all_controller_schemas as all_local_ai_controller_schemas, all_registered_controllers as all_local_ai_registered_controllers, }; +pub(crate) use service::whisper_engine; pub use service::LocalAiService; pub use types::{ LocalAiAssetStatus, LocalAiAssetsStatus, LocalAiDownloadProgressItem, LocalAiDownloadsProgress, diff --git a/src/openhuman/local_ai/service/bootstrap.rs b/src/openhuman/local_ai/service/bootstrap.rs index 22908970d..8ca7a3fa2 100644 --- a/src/openhuman/local_ai/service/bootstrap.rs +++ b/src/openhuman/local_ai/service/bootstrap.rs @@ -10,6 +10,7 @@ impl LocalAiService { let vision_model_id = model_ids::effective_vision_model_id(config); let embedding_model_id = model_ids::effective_embedding_model_id(config); Self { + whisper: super::whisper_engine::new_handle(), status: parking_lot::Mutex::new(LocalAiStatus { state: "idle".to_string(), model_id: model_id.clone(), @@ -158,6 +159,35 @@ impl LocalAiService { return; } + // Attempt to load whisper model in-process if configured (blocking I/O). + if config.local_ai.whisper_in_process { + if let Ok(model_path) = + crate::openhuman::local_ai::paths::resolve_stt_model_path(config) + { + let model = std::path::PathBuf::from(&model_path); + let handle = self.whisper.clone(); + let load_result = tokio::task::spawn_blocking(move || { + super::whisper_engine::load_engine(&handle, &model) + }) + .await; + match load_result { + Ok(Ok(())) => { + log::info!("[local_ai] whisper engine loaded in-process: {model_path}"); + } + Ok(Err(e)) => { + log::warn!( + "[local_ai] whisper in-process load failed, will fall back to CLI: {e}" + ); + } + Err(e) => { + log::warn!("[local_ai] whisper load task panicked: {e}"); + } + } + } else { + log::debug!("[local_ai] STT model not found, whisper in-process not loaded"); + } + } + let mut status = self.status.lock(); status.state = "ready".to_string(); status.vision_state = if config.local_ai.preload_vision_model { diff --git a/src/openhuman/local_ai/service/mod.rs b/src/openhuman/local_ai/service/mod.rs index 7508618d9..13263e8c2 100644 --- a/src/openhuman/local_ai/service/mod.rs +++ b/src/openhuman/local_ai/service/mod.rs @@ -6,6 +6,7 @@ mod ollama_admin; mod public_infer; mod speech; mod vision_embed; +pub(crate) mod whisper_engine; use crate::openhuman::local_ai::types::LocalAiStatus; use parking_lot::Mutex; @@ -15,4 +16,6 @@ pub struct LocalAiService { pub(crate) bootstrap_lock: tokio::sync::Mutex<()>, pub(crate) last_memory_summary_at: Mutex>, pub(crate) http: reqwest::Client, + /// In-process whisper.cpp context for low-latency STT. + pub(crate) whisper: whisper_engine::WhisperEngineHandle, } diff --git a/src/openhuman/local_ai/service/public_infer.rs b/src/openhuman/local_ai/service/public_infer.rs index 0d3cad0c7..325dd4f76 100644 --- a/src/openhuman/local_ai/service/public_infer.rs +++ b/src/openhuman/local_ai/service/public_infer.rs @@ -227,7 +227,7 @@ impl LocalAiService { } } - async fn inference( + pub(crate) async fn inference( &self, config: &Config, system: &str, diff --git a/src/openhuman/local_ai/service/speech.rs b/src/openhuman/local_ai/service/speech.rs index 2e36dabea..c2c8a0f52 100644 --- a/src/openhuman/local_ai/service/speech.rs +++ b/src/openhuman/local_ai/service/speech.rs @@ -1,5 +1,7 @@ use std::path::PathBuf; +use log::{debug, warn}; + use crate::openhuman::config::Config; use crate::openhuman::local_ai::model_ids; use crate::openhuman::local_ai::paths::{ @@ -8,8 +10,11 @@ use crate::openhuman::local_ai::paths::{ }; use crate::openhuman::local_ai::types::{LocalAiSpeechResult, LocalAiTtsResult}; +use super::whisper_engine; use super::LocalAiService; +const LOG_PREFIX: &str = "[speech]"; + impl LocalAiService { pub async fn transcribe( &self, @@ -19,6 +24,66 @@ impl LocalAiService { if !config.local_ai.enabled { return Err("local ai is disabled".to_string()); } + + // Try in-process whisper engine first (offloaded to a blocking thread). + if whisper_engine::is_loaded(&self.whisper) { + debug!("{LOG_PREFIX} using in-process whisper engine for {audio_path}"); + let handle = self.whisper.clone(); + let path = audio_path.to_string(); + let result = tokio::task::spawn_blocking(move || { + Self::transcribe_in_process_inner(&handle, &path) + }) + .await + .map_err(|e| format!("whisper task join error: {e}"))?; + match result { + Ok(text) => { + self.status.lock().stt_state = "ready".to_string(); + return Ok(LocalAiSpeechResult { + text, + model_id: model_ids::effective_stt_model_id(config), + }); + } + Err(e) => { + warn!("{LOG_PREFIX} in-process transcription failed, falling back to CLI: {e}"); + } + } + } + + // Fallback: subprocess per call (original behavior). + debug!("{LOG_PREFIX} using whisper-cli subprocess for {audio_path}"); + self.transcribe_subprocess(config, audio_path).await + } + + /// Transcribe using the in-process whisper-rs engine. Runs on a blocking + /// thread — takes the engine handle directly so it can be `Send`. + fn transcribe_in_process_inner( + handle: &whisper_engine::WhisperEngineHandle, + audio_path: &str, + ) -> Result { + let path = std::path::Path::new(audio_path); + let ext = path + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + + if ext == "wav" { + whisper_engine::transcribe_wav_file(handle, path, None) + } else { + warn!( + "{LOG_PREFIX} non-WAV input ({ext}), attempting WAV decode anyway \ + (may fail — use ffmpeg conversion for best results)" + ); + whisper_engine::transcribe_wav_file(handle, path, None) + } + } + + /// Original subprocess-based transcription via whisper-cli. + async fn transcribe_subprocess( + &self, + config: &Config, + audio_path: &str, + ) -> Result { let whisper_bin = resolve_whisper_binary().ok_or_else(|| { "whisper.cpp binary not found. Set WHISPER_BIN or install whisper-cli.".to_string() })?; diff --git a/src/openhuman/local_ai/service/whisper_engine.rs b/src/openhuman/local_ai/service/whisper_engine.rs new file mode 100644 index 000000000..c126a5441 --- /dev/null +++ b/src/openhuman/local_ai/service/whisper_engine.rs @@ -0,0 +1,332 @@ +//! In-process whisper.cpp inference via whisper-rs. +//! +//! Loads the GGML model once into a `WhisperContext` and reuses it across +//! transcription calls, eliminating the cold-start latency of spawning a +//! subprocess per request. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use log::{debug, info}; +use parking_lot::Mutex; +use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; + +const LOG_PREFIX: &str = "[whisper_engine]"; + +/// Wraps a loaded `WhisperContext` for reuse across transcription calls. +pub struct WhisperEngine { + context: WhisperContext, + model_path: PathBuf, +} + +/// Thread-safe handle to an optionally-loaded whisper engine. +pub type WhisperEngineHandle = Arc>>; + +/// Create a new empty engine handle. The engine is loaded lazily or during +/// bootstrap via [`load_engine`]. +pub fn new_handle() -> WhisperEngineHandle { + Arc::new(Mutex::new(None)) +} + +/// Attempt to load a whisper model into the engine. Returns an error string +/// if loading fails (e.g. model file missing, unsupported format). +pub fn load_engine(handle: &WhisperEngineHandle, model_path: &Path) -> Result<(), String> { + info!( + "{LOG_PREFIX} loading whisper model: {}", + model_path.display() + ); + + if !model_path.is_file() { + return Err(format!("whisper model not found: {}", model_path.display())); + } + + let params = WhisperContextParameters::default(); + let ctx = WhisperContext::new_with_params(model_path.to_str().unwrap_or(""), params) + .map_err(|e| format!("failed to load whisper model: {e}"))?; + + let engine = WhisperEngine { + context: ctx, + model_path: model_path.to_path_buf(), + }; + + *handle.lock() = Some(engine); + info!("{LOG_PREFIX} whisper model loaded successfully"); + Ok(()) +} + +/// Unload the whisper model from memory. +pub fn unload_engine(handle: &WhisperEngineHandle) { + let mut guard = handle.lock(); + if guard.is_some() { + *guard = None; + info!("{LOG_PREFIX} whisper model unloaded"); + } +} + +/// Returns true if a model is currently loaded. +pub fn is_loaded(handle: &WhisperEngineHandle) -> bool { + handle.lock().is_some() +} + +/// Returns the path of the currently loaded model, if any. +pub fn loaded_model_path(handle: &WhisperEngineHandle) -> Option { + handle.lock().as_ref().map(|e| e.model_path.clone()) +} + +/// Transcribe raw PCM audio (16 kHz, mono, f32 samples). +/// +/// Returns the concatenated transcript text or an error. +pub fn transcribe_pcm_f32( + handle: &WhisperEngineHandle, + audio_f32: &[f32], + language: Option<&str>, +) -> Result { + let mut guard = handle.lock(); + let engine = guard + .as_mut() + .ok_or_else(|| "whisper engine not loaded".to_string())?; + + debug!( + "{LOG_PREFIX} transcribing {} samples ({:.1}s of audio)", + audio_f32.len(), + audio_f32.len() as f64 / 16000.0 + ); + + let mut state = engine + .context + .create_state() + .map_err(|e| format!("failed to create whisper state: {e}"))?; + + let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 5 }); + + if let Some(lang) = language { + params.set_language(Some(lang)); + } else { + params.set_language(Some("en")); + } + + // Disable printing to stdout — we capture segments programmatically. + params.set_print_special(false); + params.set_print_progress(false); + params.set_print_realtime(false); + params.set_print_timestamps(false); + + // Use available CPU threads (capped at 4 to avoid over-subscription). + let n_threads = std::thread::available_parallelism() + .map(|n| n.get().min(4) as i32) + .unwrap_or(2); + params.set_n_threads(n_threads); + + state + .full(params, audio_f32) + .map_err(|e| format!("whisper inference failed: {e}"))?; + + let mut text = String::new(); + let mut segment_count = 0; + for segment in state.as_iter() { + match segment.to_str() { + Ok(segment_text) => text.push_str(segment_text), + Err(e) => { + debug!("{LOG_PREFIX} skipping segment: {e}"); + } + } + segment_count += 1; + } + + let trimmed = text.trim().to_string(); + debug!( + "{LOG_PREFIX} transcription complete: {} chars, {} segments", + trimmed.len(), + segment_count + ); + + Ok(trimmed) +} + +/// Transcribe raw PCM audio provided as 16-bit signed integers (16 kHz mono). +/// +/// Converts to f32 internally before running inference. +pub fn transcribe_pcm_i16( + handle: &WhisperEngineHandle, + audio_i16: &[i16], + language: Option<&str>, +) -> Result { + let mut audio_f32 = vec![0.0f32; audio_i16.len()]; + whisper_rs::convert_integer_to_float_audio(audio_i16, &mut audio_f32) + .map_err(|e| format!("audio conversion failed: {e}"))?; + transcribe_pcm_f32(handle, &audio_f32, language) +} + +/// Read a WAV file and transcribe it. The WAV must be 16 kHz mono PCM +/// (16-bit or 32-bit float). For other formats, convert to WAV first +/// (e.g. via ffmpeg). +pub fn transcribe_wav_file( + handle: &WhisperEngineHandle, + wav_path: &Path, + language: Option<&str>, +) -> Result { + debug!("{LOG_PREFIX} reading WAV file: {}", wav_path.display()); + + let raw_bytes = std::fs::read(wav_path).map_err(|e| format!("failed to read WAV file: {e}"))?; + + let audio_f32 = decode_wav_to_f32(&raw_bytes)?; + transcribe_pcm_f32(handle, &audio_f32, language) +} + +/// Minimal WAV decoder — extracts PCM samples as f32 from a standard +/// RIFF/WAVE file. Supports 16-bit integer and 32-bit float formats. +/// Resampling is NOT performed; the input should already be 16 kHz mono. +fn decode_wav_to_f32(data: &[u8]) -> Result, String> { + if data.len() < 44 { + return Err("WAV file too small".to_string()); + } + if &data[0..4] != b"RIFF" || &data[8..12] != b"WAVE" { + return Err("not a valid WAV file".to_string()); + } + + let mut pos = 12; + let mut fmt_found = false; + let mut audio_format: u16 = 0; + let mut num_channels: u16 = 0; + #[allow(unused_assignments)] + let mut sample_rate: u32 = 0; + let mut bits_per_sample: u16 = 0; + + while pos + 8 <= data.len() { + let chunk_id = &data[pos..pos + 4]; + let chunk_size = + u32::from_le_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]]) + as usize; + + if chunk_id == b"fmt " { + if chunk_size < 16 || pos + 8 + chunk_size > data.len() { + return Err("malformed fmt chunk".to_string()); + } + let fmt = &data[pos + 8..]; + audio_format = u16::from_le_bytes([fmt[0], fmt[1]]); + num_channels = u16::from_le_bytes([fmt[2], fmt[3]]); + sample_rate = u32::from_le_bytes([fmt[4], fmt[5], fmt[6], fmt[7]]); + bits_per_sample = u16::from_le_bytes([fmt[14], fmt[15]]); + fmt_found = true; + + if sample_rate != 16000 { + return Err(format!( + "unsupported sample rate {sample_rate} Hz, whisper requires 16000 Hz" + )); + } + if num_channels == 0 || num_channels > 2 { + return Err(format!( + "unsupported channel count {num_channels}, expected 1 (mono) or 2 (stereo)" + )); + } + } + + if chunk_id == b"data" && fmt_found { + let pcm_data = &data[pos + 8..pos + 8 + chunk_size.min(data.len() - pos - 8)]; + return convert_pcm_to_f32(pcm_data, audio_format, num_channels, bits_per_sample); + } + + pos += 8 + chunk_size; + if chunk_size % 2 != 0 { + pos += 1; + } + } + + Err("WAV file missing data chunk".to_string()) +} + +fn convert_pcm_to_f32( + pcm: &[u8], + audio_format: u16, + num_channels: u16, + bits_per_sample: u16, +) -> Result, String> { + match (audio_format, bits_per_sample) { + // PCM 16-bit + (1, 16) => { + let samples: Vec = pcm + .chunks_exact(2) + .map(|c| i16::from_le_bytes([c[0], c[1]])) + .collect(); + + let mono = if num_channels == 2 { + samples + .chunks_exact(2) + .map(|pair| ((pair[0] as i32 + pair[1] as i32) / 2) as i16) + .collect::>() + } else { + samples + }; + + Ok(mono.iter().map(|&s| s as f32 / 32768.0).collect()) + } + // IEEE float 32-bit + (3, 32) => { + let samples: Vec = pcm + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect(); + + if num_channels == 2 { + Ok(samples + .chunks_exact(2) + .map(|pair| (pair[0] + pair[1]) / 2.0) + .collect()) + } else { + Ok(samples) + } + } + _ => Err(format!( + "unsupported WAV format: audio_format={audio_format}, bits_per_sample={bits_per_sample}" + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn new_handle_starts_unloaded() { + let handle = new_handle(); + assert!(!is_loaded(&handle)); + assert!(loaded_model_path(&handle).is_none()); + } + + #[test] + fn load_engine_fails_for_missing_model() { + let handle = new_handle(); + let result = load_engine(&handle, Path::new("/nonexistent/model.bin")); + assert!(result.is_err()); + assert!(!is_loaded(&handle)); + } + + #[test] + fn transcribe_pcm_fails_when_not_loaded() { + let handle = new_handle(); + let audio = vec![0.0f32; 16000]; + let result = transcribe_pcm_f32(&handle, &audio, None); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("not loaded")); + } + + #[test] + fn decode_wav_rejects_too_small() { + let result = decode_wav_to_f32(&[0u8; 10]); + assert!(result.is_err()); + } + + #[test] + fn decode_wav_rejects_non_wav() { + let result = decode_wav_to_f32(&[0u8; 44]); + assert!(result.is_err()); + } + + #[test] + fn convert_i16_produces_correct_length() { + let handle = new_handle(); + let audio_i16 = vec![0i16; 100]; + let result = transcribe_pcm_i16(&handle, &audio_i16, None); + assert!(result.is_err()); // expected: engine not loaded + } +} diff --git a/src/openhuman/mod.rs b/src/openhuman/mod.rs index 184dd03bb..150a14190 100644 --- a/src/openhuman/mod.rs +++ b/src/openhuman/mod.rs @@ -37,5 +37,6 @@ pub mod skills; pub mod team; pub mod tools; pub mod util; +pub mod voice; pub mod webhooks; pub mod workspace; diff --git a/src/openhuman/voice/mod.rs b/src/openhuman/voice/mod.rs new file mode 100644 index 000000000..bf1e1be43 --- /dev/null +++ b/src/openhuman/voice/mod.rs @@ -0,0 +1,13 @@ +//! Voice domain — speech-to-text (whisper.cpp) and text-to-speech (piper). +//! +//! Provides RPC endpoints under the `openhuman.voice_*` namespace for +//! transcription, synthesis, and proactive availability checking. + +mod ops; +mod postprocess; +mod schemas; +mod types; + +pub use ops::*; +pub use schemas::{all_voice_controller_schemas, all_voice_registered_controllers, voice_schemas}; +pub use types::{VoiceSpeechResult, VoiceStatus, VoiceTtsResult}; diff --git a/src/openhuman/voice/ops.rs b/src/openhuman/voice/ops.rs new file mode 100644 index 000000000..9ed87b940 --- /dev/null +++ b/src/openhuman/voice/ops.rs @@ -0,0 +1,333 @@ +//! Voice domain business logic — STT (whisper.cpp) and TTS (piper). +//! +//! Each public function follows the `RpcOutcome` pattern used by other +//! domain modules (billing, health, etc.). + +use chrono::Utc; +use log::{debug, warn}; + +use crate::openhuman::config::Config; +use crate::openhuman::local_ai; +use crate::openhuman::local_ai::model_ids; +use crate::openhuman::local_ai::paths::{ + resolve_piper_binary, resolve_stt_model_path, resolve_tts_voice_path, resolve_whisper_binary, +}; +use crate::openhuman::local_ai::whisper_engine; +use crate::rpc::RpcOutcome; + +use super::postprocess; +use super::types::{VoiceSpeechResult, VoiceStatus, VoiceTtsResult}; + +const LOG_PREFIX: &str = "[voice]"; + +/// Check availability of STT/TTS binaries and models without executing them. +pub async fn voice_status(config: &Config) -> Result, String> { + debug!("{LOG_PREFIX} checking voice status"); + + let whisper_bin = resolve_whisper_binary(); + let piper_bin = resolve_piper_binary(); + let stt_model = resolve_stt_model_path(config).ok(); + let tts_voice = resolve_tts_voice_path(config).ok(); + + let service = local_ai::global(config); + let whisper_in_process = whisper_engine::is_loaded(&service.whisper); + + let stt_available = whisper_in_process || (whisper_bin.is_some() && stt_model.is_some()); + let tts_available = piper_bin.is_some() && tts_voice.is_some(); + + debug!( + "{LOG_PREFIX} stt_available={stt_available} tts_available={tts_available} \ + whisper_in_process={whisper_in_process} \ + whisper_bin={} piper_bin={} stt_model={} tts_voice={}", + safe_basename_path(&whisper_bin), + safe_basename_path(&piper_bin), + safe_basename_str(&stt_model), + safe_basename_str(&tts_voice), + ); + + let status = VoiceStatus { + stt_available, + tts_available, + stt_model_id: model_ids::effective_stt_model_id(config), + tts_voice_id: model_ids::effective_tts_voice_id(config), + whisper_binary: whisper_bin.map(|p| p.display().to_string()), + piper_binary: piper_bin.map(|p| p.display().to_string()), + stt_model_path: stt_model, + tts_voice_path: tts_voice, + whisper_in_process, + llm_cleanup_enabled: config.local_ai.voice_llm_cleanup_enabled, + }; + + Ok(RpcOutcome::single_log(status, "voice status checked")) +} + +/// Transcribe audio from a file path using whisper.cpp. +/// +/// If `context` is provided, the raw transcription is post-processed through +/// a local LLM to fix grammar and disambiguate words using conversation history. +pub async fn voice_transcribe( + config: &Config, + audio_path: &str, + context: Option<&str>, + skip_cleanup: bool, +) -> Result, String> { + debug!("{LOG_PREFIX} transcribing audio_path={audio_path}"); + + let service = local_ai::global(config); + let output = service + .transcribe(config, audio_path.trim()) + .await + .map_err(|e| e.to_string())?; + + let raw_text = output.text.clone(); + debug!( + "{LOG_PREFIX} transcription completed, text length={}", + raw_text.len() + ); + + let text = if skip_cleanup { + raw_text.clone() + } else { + postprocess::cleanup_transcription(config, &raw_text, context).await + }; + + Ok(RpcOutcome::single_log( + VoiceSpeechResult { + text, + raw_text, + model_id: output.model_id, + }, + "voice transcription completed", + )) +} + +/// Transcribe audio from raw bytes. Writes to a temp file, transcribes, cleans up. +/// +/// If `context` is provided, the raw transcription is post-processed through +/// a local LLM. +pub async fn voice_transcribe_bytes( + config: &Config, + audio_bytes: &[u8], + extension: Option, + context: Option<&str>, + skip_cleanup: bool, +) -> Result, String> { + let ext = normalize_extension(extension)?; + debug!( + "{LOG_PREFIX} transcribe_bytes size={} ext={ext}", + audio_bytes.len() + ); + + let service = local_ai::global(config); + + let voice_dir = std::env::temp_dir().join("openhuman_voice_input"); + tokio::fs::create_dir_all(&voice_dir) + .await + .map_err(|e| format!("failed to create voice input directory: {e}"))?; + + let filename = format!( + "voice-{}-{}.{}", + Utc::now().timestamp_millis(), + uuid::Uuid::new_v4(), + ext + ); + let file_path = voice_dir.join(filename); + tokio::fs::write(&file_path, audio_bytes) + .await + .map_err(|e| format!("failed to write audio file: {e}"))?; + + let output = service + .transcribe(config, file_path.to_string_lossy().as_ref()) + .await; + if let Err(e) = tokio::fs::remove_file(&file_path).await { + warn!( + "{LOG_PREFIX} failed to clean up temp audio file {}: {e}", + file_path.display() + ); + } + + let output = output.map_err(|e| e.to_string())?; + let raw_text = output.text.clone(); + + debug!( + "{LOG_PREFIX} transcribe_bytes completed, text length={}", + raw_text.len() + ); + + let text = if skip_cleanup { + raw_text.clone() + } else { + postprocess::cleanup_transcription(config, &raw_text, context).await + }; + + Ok(RpcOutcome::single_log( + VoiceSpeechResult { + text, + raw_text, + model_id: output.model_id, + }, + "voice transcription completed", + )) +} + +/// Synthesize speech from text using piper. +pub async fn voice_tts( + config: &Config, + text: &str, + output_path: Option<&str>, +) -> Result, String> { + debug!( + "{LOG_PREFIX} tts text_length={} output_path={:?}", + text.len(), + output_path + ); + + let service = local_ai::global(config); + let output = service + .tts(config, text.trim(), output_path) + .await + .map_err(|e| e.to_string())?; + + debug!("{LOG_PREFIX} tts completed, output={}", output.output_path); + + Ok(RpcOutcome::single_log( + VoiceTtsResult::from(output), + "voice tts completed", + )) +} + +/// Normalize an optional audio file extension. Returns a clean lowercase +/// alphanumeric extension string, defaulting to "webm". +pub(crate) fn normalize_extension(ext: Option) -> Result { + let normalized = ext + .unwrap_or_else(|| "webm".to_string()) + .trim() + .trim_start_matches('.') + .to_ascii_lowercase(); + + if normalized.is_empty() { + return Err("audio extension must not be empty".to_string()); + } + if !normalized.chars().all(|c| c.is_ascii_alphanumeric()) { + return Err(format!( + "invalid audio extension '{normalized}': must be alphanumeric" + )); + } + + Ok(normalized) +} + +/// Extract the file name from an `Option`, returning `""` if absent. +fn safe_basename_path(p: &Option) -> String { + p.as_ref() + .and_then(|pb| pb.file_name()) + .and_then(|n| n.to_str()) + .unwrap_or("") + .to_string() +} + +/// Extract the file name from an `Option` path, returning `""` if absent. +fn safe_basename_str(p: &Option) -> String { + p.as_ref() + .and_then(|s| std::path::Path::new(s).file_name()) + .and_then(|n| n.to_str()) + .unwrap_or("") + .to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_extension_defaults_to_webm() { + assert_eq!(normalize_extension(None).unwrap(), "webm"); + } + + #[test] + fn normalize_extension_strips_dot_and_lowercases() { + assert_eq!( + normalize_extension(Some(".WebM".to_string())).unwrap(), + "webm" + ); + assert_eq!(normalize_extension(Some("OGG".to_string())).unwrap(), "ogg"); + assert_eq!( + normalize_extension(Some(" .WAV ".to_string())).unwrap(), + "wav" + ); + } + + #[test] + fn normalize_extension_accepts_alphanumeric() { + assert_eq!(normalize_extension(Some("m4a".to_string())).unwrap(), "m4a"); + assert_eq!(normalize_extension(Some("mp3".to_string())).unwrap(), "mp3"); + } + + #[test] + fn normalize_extension_rejects_empty() { + assert!(normalize_extension(Some("".to_string())).is_err()); + assert!(normalize_extension(Some(" ".to_string())).is_err()); + assert!(normalize_extension(Some(".".to_string())).is_err()); + } + + #[test] + fn normalize_extension_rejects_invalid_chars() { + assert!(normalize_extension(Some("a/b".to_string())).is_err()); + assert!(normalize_extension(Some("web m".to_string())).is_err()); + assert!(normalize_extension(Some("a.b".to_string())).is_err()); + } + + #[tokio::test] + async fn voice_status_returns_without_error() { + let config = Config::default(); + let result = voice_status(&config).await; + assert!(result.is_ok()); + let status = result.unwrap().value; + assert!(!status.stt_model_id.is_empty()); + assert!(!status.tts_voice_id.is_empty()); + } + + /// RAII guard that restores an env var on drop, even on panic. + struct EnvGuard { + key: &'static str, + prev: Option, + } + impl EnvGuard { + fn set(key: &'static str, value: &str) -> Self { + let prev = std::env::var_os(key); + std::env::set_var(key, value); + Self { key, prev } + } + } + impl Drop for EnvGuard { + fn drop(&mut self) { + match &self.prev { + Some(v) => std::env::set_var(self.key, v), + None => std::env::remove_var(self.key), + } + } + } + + #[tokio::test] + async fn voice_status_detects_stub_binaries() { + let tmp = tempfile::tempdir().expect("tempdir"); + + let whisper_stub = tmp.path().join("whisper-cli"); + std::fs::write(&whisper_stub, b"#!/bin/sh\n").expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&whisper_stub, std::fs::Permissions::from_mode(0o755)) + .expect("chmod"); + } + + let _guard = EnvGuard::set("WHISPER_BIN", &whisper_stub.display().to_string()); + + let mut config = Config::default(); + config.workspace_dir = tmp.path().join("workspace"); + config.config_path = tmp.path().join("config.toml"); + + let result = voice_status(&config).await.unwrap(); + assert!(result.value.whisper_binary.is_some()); + } +} diff --git a/src/openhuman/voice/postprocess.rs b/src/openhuman/voice/postprocess.rs new file mode 100644 index 000000000..aca890afd --- /dev/null +++ b/src/openhuman/voice/postprocess.rs @@ -0,0 +1,111 @@ +//! LLM-based post-processing for voice transcription. +//! +//! Passes raw whisper output through a local LLM (Ollama) to clean up +//! grammar, punctuation, and filler words. Optionally uses conversation +//! context to disambiguate unclear words (names, technical terms). + +use log::{debug, warn}; + +use crate::openhuman::config::Config; +use crate::openhuman::local_ai; + +const LOG_PREFIX: &str = "[voice_postprocess]"; + +const CLEANUP_SYSTEM_PROMPT: &str = "\ +You clean up voice transcription text. Fix grammar, punctuation, and \ +remove filler words (um, uh, like). Keep the original meaning intact. \ +If conversation context is provided, use it to disambiguate unclear \ +words (names, technical terms). Return ONLY the corrected text, \ +nothing else."; + +/// Clean up raw transcription text using a local LLM. +/// +/// Returns the cleaned text on success, or the original raw text if the +/// LLM is unavailable or cleanup fails (graceful degradation). +pub async fn cleanup_transcription( + config: &Config, + raw_text: &str, + conversation_context: Option<&str>, +) -> String { + if raw_text.trim().is_empty() { + return raw_text.to_string(); + } + + if !config.local_ai.voice_llm_cleanup_enabled { + debug!("{LOG_PREFIX} LLM cleanup disabled in config"); + return raw_text.to_string(); + } + + debug!( + "{LOG_PREFIX} cleaning up transcription ({} chars, context={})", + raw_text.len(), + conversation_context.is_some() + ); + + let prompt = match conversation_context { + Some(ctx) if !ctx.trim().is_empty() => { + format!( + "Conversation context:\n{ctx}\n\n\ + Transcribed text to clean up:\n{raw_text}" + ) + } + _ => raw_text.to_string(), + }; + + let service = local_ai::global(config); + + let result: Result = service + .inference(config, CLEANUP_SYSTEM_PROMPT, &prompt, Some(512), true) + .await; + + match result { + Ok(ref cleaned_ref) => { + let cleaned = cleaned_ref.trim().to_string(); + if cleaned.is_empty() { + warn!("{LOG_PREFIX} LLM returned empty cleanup, using raw text"); + raw_text.to_string() + } else { + debug!( + "{LOG_PREFIX} cleanup complete: {} chars -> {} chars", + raw_text.len(), + cleaned.len() + ); + cleaned + } + } + Err(e) => { + warn!("{LOG_PREFIX} LLM cleanup failed, using raw text: {e}"); + raw_text.to_string() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_text_returns_unchanged() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let config = Config::default(); + let result = rt.block_on(cleanup_transcription(&config, "", None)); + assert_eq!(result, ""); + } + + #[test] + fn whitespace_only_returns_unchanged() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let config = Config::default(); + let result = rt.block_on(cleanup_transcription(&config, " ", None)); + assert_eq!(result, " "); + } + + #[test] + fn disabled_cleanup_returns_raw_text() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut config = Config::default(); + config.local_ai.voice_llm_cleanup_enabled = false; + let result = rt.block_on(cleanup_transcription(&config, "um hello uh world", None)); + assert_eq!(result, "um hello uh world"); + } +} diff --git a/src/openhuman/voice/schemas.rs b/src/openhuman/voice/schemas.rs new file mode 100644 index 000000000..1d1ee237f --- /dev/null +++ b/src/openhuman/voice/schemas.rs @@ -0,0 +1,341 @@ +//! Controller schemas and RPC handler dispatch for the voice domain. + +use serde::de::DeserializeOwned; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::config::rpc as config_rpc; +use crate::rpc::RpcOutcome; + +// --------------------------------------------------------------------------- +// Param structs +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +struct TranscribeParams { + audio_path: String, + /// Optional conversation context for LLM post-processing. + #[serde(default)] + context: Option, + /// Skip LLM cleanup and return raw whisper output. + #[serde(default)] + skip_cleanup: bool, +} + +#[derive(Debug, Deserialize)] +struct TranscribeBytesParams { + audio_bytes: Vec, + #[serde(default)] + extension: Option, + /// Optional conversation context for LLM post-processing. + #[serde(default)] + context: Option, + /// Skip LLM cleanup and return raw whisper output. + #[serde(default)] + skip_cleanup: bool, +} + +#[derive(Debug, Deserialize)] +struct TtsParams { + text: String, + #[serde(default)] + output_path: Option, +} + +// --------------------------------------------------------------------------- +// Schema + registry exports +// --------------------------------------------------------------------------- + +pub fn all_voice_controller_schemas() -> Vec { + vec![ + voice_schemas("voice_status"), + voice_schemas("voice_transcribe"), + voice_schemas("voice_transcribe_bytes"), + voice_schemas("voice_tts"), + ] +} + +pub fn all_voice_registered_controllers() -> Vec { + vec![ + RegisteredController { + schema: voice_schemas("voice_status"), + handler: handle_voice_status, + }, + RegisteredController { + schema: voice_schemas("voice_transcribe"), + handler: handle_voice_transcribe, + }, + RegisteredController { + schema: voice_schemas("voice_transcribe_bytes"), + handler: handle_voice_transcribe_bytes, + }, + RegisteredController { + schema: voice_schemas("voice_tts"), + handler: handle_voice_tts, + }, + ] +} + +pub fn voice_schemas(function: &str) -> ControllerSchema { + match function { + "voice_status" => ControllerSchema { + namespace: "voice", + function: "status", + description: "Check availability of STT/TTS binaries and models.", + inputs: vec![], + outputs: vec![json_output("status", "Voice availability status.")], + }, + "voice_transcribe" => ControllerSchema { + namespace: "voice", + function: "transcribe", + description: + "Transcribe audio from a file path using whisper.cpp, with optional LLM cleanup.", + inputs: vec![ + required_string("audio_path", "Path to the audio file."), + optional_string("context", "Conversation context for LLM post-processing."), + optional_bool( + "skip_cleanup", + "Skip LLM cleanup, return raw whisper output.", + ), + ], + outputs: vec![json_output( + "speech", + "Transcription result with text and raw_text.", + )], + }, + "voice_transcribe_bytes" => ControllerSchema { + namespace: "voice", + function: "transcribe_bytes", + description: + "Transcribe audio from raw bytes using whisper.cpp, with optional LLM cleanup.", + inputs: vec![ + FieldSchema { + name: "audio_bytes", + ty: TypeSchema::Bytes, + comment: "Raw audio bytes.", + required: true, + }, + optional_string("extension", "Audio file extension (default: webm)."), + optional_string("context", "Conversation context for LLM post-processing."), + optional_bool( + "skip_cleanup", + "Skip LLM cleanup, return raw whisper output.", + ), + ], + outputs: vec![json_output( + "speech", + "Transcription result with text and raw_text.", + )], + }, + "voice_tts" => ControllerSchema { + namespace: "voice", + function: "tts", + description: "Synthesize speech from text using piper.", + inputs: vec![ + required_string("text", "Text to synthesize."), + optional_string("output_path", "Optional output file path."), + ], + outputs: vec![json_output("tts", "TTS result with output path.")], + }, + _ => ControllerSchema { + namespace: "voice", + function: "unknown", + description: "Unknown voice controller.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "error", + ty: TypeSchema::String, + comment: "Lookup error details.", + required: true, + }], + }, + } +} + +// --------------------------------------------------------------------------- +// Handlers +// --------------------------------------------------------------------------- + +fn handle_voice_status(_params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + to_json(crate::openhuman::voice::voice_status(&config).await?) + }) +} + +fn handle_voice_transcribe(params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + let p = deserialize_params::(params)?; + to_json( + crate::openhuman::voice::voice_transcribe( + &config, + &p.audio_path, + p.context.as_deref(), + p.skip_cleanup, + ) + .await?, + ) + }) +} + +fn handle_voice_transcribe_bytes(params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + let p = deserialize_params::(params)?; + to_json( + crate::openhuman::voice::voice_transcribe_bytes( + &config, + &p.audio_bytes, + p.extension, + p.context.as_deref(), + p.skip_cleanup, + ) + .await?, + ) + }) +} + +fn handle_voice_tts(params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + let p = deserialize_params::(params)?; + to_json( + crate::openhuman::voice::voice_tts(&config, &p.text, p.output_path.as_deref()).await?, + ) + }) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn to_json(outcome: RpcOutcome) -> Result { + let json_val = + serde_json::to_value(outcome.value).map_err(|e| format!("serialize error: {e}"))?; + Ok(json_val) +} + +fn deserialize_params(params: Map) -> Result { + serde_json::from_value(Value::Object(params)).map_err(|e| format!("invalid params: {e}")) +} + +fn required_string(name: &'static str, comment: &'static str) -> FieldSchema { + FieldSchema { + name, + ty: TypeSchema::String, + comment, + required: true, + } +} + +fn optional_string(name: &'static str, comment: &'static str) -> FieldSchema { + FieldSchema { + name, + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment, + required: false, + } +} + +fn optional_bool(name: &'static str, comment: &'static str) -> FieldSchema { + FieldSchema { + name, + ty: TypeSchema::Option(Box::new(TypeSchema::Bool)), + comment, + required: false, + } +} + +fn json_output(name: &'static str, comment: &'static str) -> FieldSchema { + FieldSchema { + name, + ty: TypeSchema::Json, + comment, + required: true, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schema_names_are_stable() { + let s = voice_schemas("voice_status"); + assert_eq!(s.namespace, "voice"); + assert_eq!(s.function, "status"); + + let s = voice_schemas("voice_transcribe"); + assert_eq!(s.namespace, "voice"); + assert_eq!(s.function, "transcribe"); + + let s = voice_schemas("voice_transcribe_bytes"); + assert_eq!(s.namespace, "voice"); + assert_eq!(s.function, "transcribe_bytes"); + + let s = voice_schemas("voice_tts"); + assert_eq!(s.namespace, "voice"); + assert_eq!(s.function, "tts"); + } + + #[test] + fn controller_lists_match_lengths() { + assert_eq!( + all_voice_controller_schemas().len(), + all_voice_registered_controllers().len() + ); + } + + #[test] + fn status_schema_has_no_inputs() { + let s = voice_schemas("voice_status"); + assert!(s.inputs.is_empty()); + } + + #[test] + fn transcribe_schema_requires_audio_path() { + let s = voice_schemas("voice_transcribe"); + assert!(s + .inputs + .iter() + .any(|i| i.name == "audio_path" && i.required)); + } + + #[test] + fn transcribe_bytes_schema_requires_audio_bytes() { + let s = voice_schemas("voice_transcribe_bytes"); + assert!(s + .inputs + .iter() + .any(|i| i.name == "audio_bytes" && i.required)); + } + + #[test] + fn transcribe_bytes_schema_has_optional_extension() { + let s = voice_schemas("voice_transcribe_bytes"); + let ext = s.inputs.iter().find(|i| i.name == "extension").unwrap(); + assert!(!ext.required); + } + + #[test] + fn tts_schema_requires_text() { + let s = voice_schemas("voice_tts"); + assert!(s.inputs.iter().any(|i| i.name == "text" && i.required)); + } + + #[test] + fn tts_schema_has_optional_output_path() { + let s = voice_schemas("voice_tts"); + let output_path = s.inputs.iter().find(|i| i.name == "output_path").unwrap(); + assert!(!output_path.required); + } + + #[test] + fn unknown_schema_returns_fallback() { + let s = voice_schemas("voice_nonexistent"); + assert_eq!(s.function, "unknown"); + } +} diff --git a/src/openhuman/voice/types.rs b/src/openhuman/voice/types.rs new file mode 100644 index 000000000..59760459d --- /dev/null +++ b/src/openhuman/voice/types.rs @@ -0,0 +1,147 @@ +//! Serializable DTOs for voice domain RPC responses. + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::local_ai::{LocalAiSpeechResult, LocalAiTtsResult}; + +/// Result of a speech-to-text transcription. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VoiceSpeechResult { + /// Final text — cleaned by LLM post-processing when available, + /// otherwise identical to `raw_text`. + pub text: String, + /// Raw whisper output before LLM cleanup. + pub raw_text: String, + pub model_id: String, +} + +/// Result of a text-to-speech synthesis. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VoiceTtsResult { + pub output_path: String, + pub voice_id: String, +} + +/// Proactive availability check for STT/TTS binaries and models. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VoiceStatus { + pub stt_available: bool, + pub tts_available: bool, + pub stt_model_id: String, + pub tts_voice_id: String, + pub whisper_binary: Option, + pub piper_binary: Option, + pub stt_model_path: Option, + pub tts_voice_path: Option, + /// Whether the whisper model is loaded in-process (low-latency mode). + pub whisper_in_process: bool, + /// Whether LLM post-processing is enabled for transcription cleanup. + pub llm_cleanup_enabled: bool, +} + +impl From for VoiceSpeechResult { + fn from(r: LocalAiSpeechResult) -> Self { + Self { + text: r.text.clone(), + raw_text: r.text, + model_id: r.model_id, + } + } +} + +impl From for VoiceTtsResult { + fn from(r: LocalAiTtsResult) -> Self { + Self { + output_path: r.output_path, + voice_id: r.voice_id, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn voice_speech_result_serializes_correctly() { + let r = VoiceSpeechResult { + text: "hello world".into(), + raw_text: "hello world um".into(), + model_id: "ggml-tiny-q5_1.bin".into(), + }; + let v = serde_json::to_value(&r).unwrap(); + assert_eq!(v["text"], "hello world"); + assert_eq!(v["raw_text"], "hello world um"); + assert_eq!(v["model_id"], "ggml-tiny-q5_1.bin"); + } + + #[test] + fn voice_tts_result_serializes_correctly() { + let r = VoiceTtsResult { + output_path: "/tmp/out.wav".into(), + voice_id: "en_US-lessac-medium".into(), + }; + let v = serde_json::to_value(&r).unwrap(); + assert_eq!(v["output_path"], "/tmp/out.wav"); + assert_eq!(v["voice_id"], "en_US-lessac-medium"); + } + + #[test] + fn voice_status_serializes_correctly() { + let s = VoiceStatus { + stt_available: true, + tts_available: false, + stt_model_id: "tiny.bin".into(), + tts_voice_id: "en_US-lessac-medium".into(), + whisper_binary: Some("/usr/local/bin/whisper-cli".into()), + piper_binary: None, + stt_model_path: Some("/models/stt/tiny.bin".into()), + tts_voice_path: None, + whisper_in_process: true, + llm_cleanup_enabled: true, + }; + let v = serde_json::to_value(&s).unwrap(); + assert_eq!(v["stt_available"], true); + assert_eq!(v["tts_available"], false); + assert!(v["piper_binary"].is_null()); + assert_eq!(v["whisper_in_process"], true); + assert_eq!(v["llm_cleanup_enabled"], true); + } + + #[test] + fn from_local_ai_speech_result() { + let local = LocalAiSpeechResult { + text: "test".into(), + model_id: "tiny".into(), + }; + let voice: VoiceSpeechResult = local.into(); + assert_eq!(voice.text, "test"); + assert_eq!(voice.raw_text, "test"); + assert_eq!(voice.model_id, "tiny"); + } + + #[test] + fn from_local_ai_tts_result() { + let local = LocalAiTtsResult { + output_path: "/out.wav".into(), + voice_id: "voice1".into(), + }; + let voice: VoiceTtsResult = local.into(); + assert_eq!(voice.output_path, "/out.wav"); + assert_eq!(voice.voice_id, "voice1"); + } + + #[test] + fn serde_round_trip_speech_result() { + let original = VoiceSpeechResult { + text: "round trip".into(), + raw_text: "round trip uh".into(), + model_id: "model".into(), + }; + let json = serde_json::to_string(&original).unwrap(); + let decoded: VoiceSpeechResult = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded.text, original.text); + assert_eq!(decoded.raw_text, original.raw_text); + assert_eq!(decoded.model_id, original.model_id); + } +} diff --git a/tests/json_rpc_e2e.rs b/tests/json_rpc_e2e.rs index fd188563b..ca32388d0 100644 --- a/tests/json_rpc_e2e.rs +++ b/tests/json_rpc_e2e.rs @@ -1556,3 +1556,64 @@ async fn team_rpc_e2e() { mock_join.abort(); rpc_join.abort(); } + +#[tokio::test] +async fn voice_status_returns_availability() { + let _env_lock = json_rpc_e2e_env_lock(); + let tmp = tempdir().expect("tempdir"); + let home = tmp.path(); + let openhuman_home = home.join(".openhuman"); + + let _home_guard = EnvVarGuard::set_to_path("HOME", home); + let _workspace_guard = EnvVarGuard::unset("OPENHUMAN_WORKSPACE"); + let _backend_url_guard = EnvVarGuard::unset("BACKEND_URL"); + let _vite_backend_guard = EnvVarGuard::unset("VITE_BACKEND_URL"); + let _whisper_guard = EnvVarGuard::unset("WHISPER_BIN"); + let _piper_guard = EnvVarGuard::unset("PIPER_BIN"); + + let (mock_addr, mock_join) = serve_on_ephemeral(mock_upstream_router()).await; + let mock_origin = format!("http://{}", mock_addr); + write_min_config(&openhuman_home, &mock_origin); + + let (rpc_addr, rpc_join) = serve_on_ephemeral(build_core_http_router(false)).await; + let rpc_base = format!("http://{}", rpc_addr); + + tokio::time::sleep(Duration::from_millis(100)).await; + + // voice_status does not require auth — it only checks filesystem availability + let status = post_json_rpc(&rpc_base, 1, "openhuman.voice_status", json!({})).await; + let result = assert_no_jsonrpc_error(&status, "voice_status"); + + // Without whisper/piper installed in the test env, both should be unavailable + assert!( + result.get("stt_available").is_some(), + "expected stt_available field: {result}" + ); + assert!( + result.get("tts_available").is_some(), + "expected tts_available field: {result}" + ); + assert!( + result.get("stt_model_id").is_some(), + "expected stt_model_id field: {result}" + ); + assert!( + result.get("tts_voice_id").is_some(), + "expected tts_voice_id field: {result}" + ); + + // Verify that without binaries, availability is false + assert_eq!( + result.get("stt_available").and_then(Value::as_bool), + Some(false), + "stt should be unavailable without whisper binary" + ); + assert_eq!( + result.get("tts_available").and_then(Value::as_bool), + Some(false), + "tts should be unavailable without piper binary" + ); + + mock_join.abort(); + rpc_join.abort(); +}