From b1e82f67b7f6b2b808ac7d5f6cf4f70ef19eeb9a Mon Sep 17 00:00:00 2001 From: Mega Mind <146339422+M3gA-Mind@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:58:48 +0530 Subject: [PATCH] feat(voice): expose always-on toggle + test-build logging for the always-on flow (#3619) --- .../components/settings/panels/VoicePanel.tsx | 9 +-- src/openhuman/config/schemas/controllers.rs | 2 +- src/openhuman/voice/always_on.rs | 72 +++++++++++++++++-- 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/app/src/components/settings/panels/VoicePanel.tsx b/app/src/components/settings/panels/VoicePanel.tsx index f6059a460..4fc295a96 100644 --- a/app/src/components/settings/panels/VoicePanel.tsx +++ b/app/src/components/settings/panels/VoicePanel.tsx @@ -18,7 +18,6 @@ import { type VoiceProviderView, type VoiceSettings, } from '../../../services/api/voiceSettingsApi'; -import { IS_DEV_LIKE } from '../../../utils/config'; import { openhumanGetVoiceServerSettings, openhumanUpdateVoiceServerSettings, @@ -97,11 +96,6 @@ interface VoicePanelProps { embedded?: boolean; } -/** Always-on listening toggle is hidden in production for now, but shown in - * dev/debug builds so the feature can be exercised. Set unconditionally to - * `true` to expose it everywhere. See docs/voice-system-actions.md. */ -const SHOW_ALWAYS_ON_TOGGLE = IS_DEV_LIKE; - const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => { const { t } = useT(); const { navigateBack, navigateToSettings, breadcrumbs } = useSettingsNavigation(); @@ -504,8 +498,7 @@ const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => {
{/* ─── Always-on listening (Phase 2) ──────────────────────────── */} - {/* Gated on SHOW_ALWAYS_ON_TOGGLE — shown in dev/debug builds, hidden in prod. */} - {SHOW_ALWAYS_ON_TOGGLE && settings && ( + {settings && ( ) -> Controller // silently wouldn't apply until the next launch. match config_rpc::load_config_with_timeout().await { Ok(config) => { - log::debug!("[config][rpc] voice settings saved; applying live always-on state"); + log::info!("[config][rpc] voice settings saved; applying live always-on state"); crate::openhuman::voice::always_on::start_if_enabled(&config).await; } Err(error) => { diff --git a/src/openhuman/voice/always_on.rs b/src/openhuman/voice/always_on.rs index 4b956b2b8..c55c09c20 100644 --- a/src/openhuman/voice/always_on.rs +++ b/src/openhuman/voice/always_on.rs @@ -248,12 +248,27 @@ pub async fn start_if_enabled(app_config: &Config) { // Privacy hook: pause capture while the screen is locked. spawn_lock_watcher(); + let onset_threshold = vad.onset_threshold; tokio::spawn(async move { let mut seg = VadSegmenter::new(vad); let mut pending: Vec = Vec::new(); let mut utterance: Vec = Vec::new(); + // Test-build diagnostics: confirm audio actually flows from the mic and + // surface live input levels vs the onset threshold (every ~5s) so the VAD + // can be tuned per mic/room without guessing. Levels are loudness, not PII. + let mut first_chunk_logged = false; + let mut level_peak: f32 = 0.0; + let mut level_frames: u32 = 0; + let mut last_level_log = std::time::Instant::now(); while let Some(chunk) = rx.recv().await { + if !first_chunk_logged { + first_chunk_logged = true; + log::info!( + "{LOG_PREFIX} first audio chunk received from mic (samples={}) — capture pipeline live", + chunk.len() + ); + } // Drop audio and abandon any in-flight utterance while paused // (screen locked) or toggled off — nothing is captured or sent. if PAUSED.load(Ordering::Relaxed) || !ENABLED.load(Ordering::Relaxed) { @@ -268,8 +283,26 @@ pub async fn start_if_enabled(app_config: &Config) { while pending.len() >= FRAME_SAMPLES { let frame: Vec = pending.drain(..FRAME_SAMPLES).collect(); let rms = chunk_rms(&frame); + level_peak = level_peak.max(rms); + level_frames += 1; + if last_level_log.elapsed() >= std::time::Duration::from_secs(5) { + log::info!( + "{LOG_PREFIX} mic level peak_rms={level_peak:.4} onset={onset_threshold:.4} frames={level_frames} ({})", + if level_peak >= onset_threshold { + "speech would trigger" + } else { + "below onset — lower vad_onset_threshold or check mic gain" + } + ); + level_peak = 0.0; + level_frames = 0; + last_level_log = std::time::Instant::now(); + } match seg.push_frame(rms, FRAME_MS) { Some(VadEvent::SpeechStart) => { + log::info!( + "{LOG_PREFIX} speech onset rms={rms:.4} (onset={onset_threshold:.4})" + ); utterance.clear(); utterance.extend_from_slice(&frame); notch_status("Listening", 2500); // pill: capturing speech @@ -328,6 +361,7 @@ fn notch_status(status: &str, ttl_ms: u32) { /// hotkey dictation uses. async fn transcribe_and_deliver(config: &Config, samples_16k: Vec) { use base64::Engine as _; + let sample_count = samples_16k.len(); let wav = match encode_wav_16k(&samples_16k) { Ok(w) => w, Err(e) => { @@ -340,6 +374,12 @@ async fn transcribe_and_deliver(config: &Config, samples_16k: Vec) { // honors the user's choice instead of forcing local whisper. let provider_name = crate::openhuman::voice::effective_stt_provider(config); let model = crate::openhuman::voice::DEFAULT_WHISPER_MODEL.to_string(); + // Which STT backend is doing the work matters when diagnosing slow/failed + // transcription across machines (local whisper download state vs cloud). + log::info!( + "{LOG_PREFIX} transcribing utterance: provider={provider_name} model={model} samples={sample_count} wav_bytes={}", + wav.len() + ); let provider = match crate::openhuman::voice::create_stt_provider(&provider_name, &model, config) { Ok(p) => p, @@ -349,6 +389,7 @@ async fn transcribe_and_deliver(config: &Config, samples_16k: Vec) { } }; let audio_b64 = base64::engine::general_purpose::STANDARD.encode(&wav); + let stt_started = std::time::Instant::now(); // Force English transcription. Auto-detect was rendering the English wake // word "Hey Tiny" in Hindi/Bengali/etc. script ("हे टाइनी"), which could never // match the Latin wake word. The wake word + commands here are English. @@ -364,8 +405,13 @@ async fn transcribe_and_deliver(config: &Config, samples_16k: Vec) { { Ok(outcome) => { let text = outcome.value.text.trim().to_string(); + log::info!( + "{LOG_PREFIX} transcription ok in {}ms (provider={provider_name}, chars={})", + stt_started.elapsed().as_millis(), + text.len() + ); if text.is_empty() { - log::debug!("{LOG_PREFIX} empty transcript dropped"); + log::info!("{LOG_PREFIX} empty transcript dropped"); return; } // Wake-word gate: only act on utterances addressed to the agent @@ -387,7 +433,10 @@ async fn transcribe_and_deliver(config: &Config, samples_16k: Vec) { } } } - Err(e) => log::warn!("{LOG_PREFIX} transcription failed ({provider_name}): {e}"), + Err(e) => log::warn!( + "{LOG_PREFIX} transcription failed ({provider_name}) after {}ms: {e}", + stt_started.elapsed().as_millis() + ), } } @@ -613,14 +662,24 @@ fn capture_on_thread( use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use cpal::{SampleFormat, StreamConfig}; - if matches!(detect_microphone_permission(), PermissionState::Denied) { + // Surface the mic permission state explicitly — a denied/Unknown state is the + // most common reason always-on "does nothing" and it differs per OS (macOS TCC + // prompt, Windows privacy settings), so log it on every test build. + let permission = detect_microphone_permission(); + log::info!("{LOG_PREFIX} microphone permission: {permission:?}"); + if matches!(permission, PermissionState::Denied) { + log::error!("{LOG_PREFIX} microphone permission denied — always-on cannot capture audio"); return Err("microphone permission denied".to_string()); } let host = cpal::default_host(); + log::info!("{LOG_PREFIX} audio host: {:?}", host.id()); let device = host .default_input_device() .ok_or_else(|| "no default audio input device".to_string())?; + let device_name = device + .name() + .unwrap_or_else(|e| format!("")); let supported = device .default_input_config() .map_err(|e| format!("no default input config: {e}"))?; @@ -628,8 +687,11 @@ fn capture_on_thread( let channels = supported.channels() as usize; let sample_format = supported.sample_format(); let stream_config: StreamConfig = supported.into(); + // Name + source rate/channels/format vary across M-chip, Intel, and Windows + // mics; capturing them makes a "wrong device" or "unsupported format" failure + // obvious from the log alone. We resample everything to 16 kHz mono downstream. log::info!( - "{LOG_PREFIX} capture device ready rate={source_rate} channels={channels} format={sample_format:?}" + "{LOG_PREFIX} capture device ready name='{device_name}' rate={source_rate}->{TARGET_SAMPLE_RATE} channels={channels} format={sample_format:?}" ); // Forward one resampled-to-16k mono chunk per callback. @@ -704,7 +766,7 @@ fn spawn_lock_watcher() { }); #[cfg(not(target_os = "macos"))] { - log::debug!("{LOG_PREFIX} screen-lock watcher unavailable on this platform"); + log::info!("{LOG_PREFIX} screen-lock watcher unavailable on this platform"); } }