feat(voice): sync overlay orb with chat voice button state (#487) (#490)

* feat(voice): sync overlay orb with chat voice button state (#487)

The overlay orb already reacts to hotkey-based dictation via Socket.IO
events, but the chat "Start Talking" button used local React state only.
Add a new RPC method `openhuman.overlay_stt_notify` that the chat button
calls at each voice state transition, which publishes to the existing
DICTATION_BUS / TRANSCRIPTION_BUS broadcast channels — so the overlay
reflects recording/transcribing/idle from both input paths with zero
changes to the Socket.IO bridge or overlay event handlers.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix(voice): address CI formatting and CodeRabbit review feedback

- Run cargo fmt and prettier to fix formatting violations
- Use typed enum OverlaySttState instead of raw String for state param
  (serde rejects invalid states at deserialization, eliminating the
  unknown state branch)
- Require `text` field for transcription_done state (return error if
  missing instead of silently ignoring)
- Replace raw transcript logging with metadata-only (has_text, text_len)
  to avoid logging sensitive user speech content
- Use "Voice input active" aria-label (covers recording + linger phases)
- Convert notifyOverlaySttState to arrow function with async/await per
  repo TS conventions

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
oxoxDev
2026-04-10 18:25:08 +05:30
committed by GitHub
co-authored by Claude Opus 4.6
parent 3a2e026346
commit 0cd0f7a670
6 changed files with 144 additions and 2 deletions
Generated
+1 -1
View File
@@ -5120,7 +5120,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "openhuman"
version = "0.52.0"
version = "0.52.2"
dependencies = [
"aes-gcm",
"anyhow",
+7 -1
View File
@@ -405,7 +405,13 @@ export default function OverlayApp() {
<div className="relative">
<button
type="button"
aria-label="Overlay orb"
aria-label={
mode === 'stt'
? 'Voice input active'
: mode === 'attention'
? 'Attention message'
: 'OpenHuman overlay'
}
onClick={goIdle}
onMouseEnter={() => {
setIsHovered(true);
+8
View File
@@ -34,6 +34,7 @@ import {
import type { ThreadMessage } from '../types/thread';
import {
isTauri,
notifyOverlaySttState,
openhumanAutocompleteAccept,
openhumanAutocompleteCurrent,
openhumanLocalAiAnalyzeSentiment,
@@ -762,6 +763,7 @@ const Conversations = () => {
const chunks = audioChunksRef.current;
audioChunksRef.current = [];
if (chunks.length === 0) {
notifyOverlaySttState('cancelled');
setVoiceStatus('No audio captured. Try again.');
return;
}
@@ -784,13 +786,16 @@ const Conversations = () => {
const transcript = result.text.trim();
if (!transcript) {
notifyOverlaySttState('cancelled');
setVoiceStatus('No speech detected. Try again.');
return;
}
notifyOverlaySttState('transcription_done', transcript);
setVoiceStatus(`Heard: ${transcript}`);
await handleSendMessage(transcript);
} catch (err) {
notifyOverlaySttState('error');
const message = err instanceof Error ? err.message : String(err);
setSendError(chatSendError('voice_transcription', `Voice transcription failed: ${message}`));
setVoiceStatus(null);
@@ -839,6 +844,7 @@ const Conversations = () => {
}
};
recorder.onerror = () => {
notifyOverlaySttState('error');
setIsRecording(false);
mediaStreamRef.current?.getTracks().forEach(track => track.stop());
mediaStreamRef.current = null;
@@ -853,7 +859,9 @@ const Conversations = () => {
setSendError(null);
setIsRecording(true);
recorder.start();
notifyOverlaySttState('recording_started');
} catch (err) {
notifyOverlaySttState('error');
const message = err instanceof Error ? err.message : String(err);
setSendError(chatSendError('microphone_access', `Microphone access failed: ${message}`));
setVoiceStatus(null);
+17
View File
@@ -171,6 +171,23 @@ export async function registerDictationHotkey(shortcut: string): Promise<void> {
console.debug('[dictation] registerDictationHotkey: done');
}
/**
* Notify the overlay of a voice/STT state change from the chat prompt button.
* Fire-and-forget — errors are logged but never propagated.
*/
export const notifyOverlaySttState = (
state: 'recording_started' | 'transcription_done' | 'cancelled' | 'error',
text?: string
): void => {
void (async () => {
try {
await callCoreRpc({ method: 'openhuman.overlay_stt_notify', params: { state, text } });
} catch (err: unknown) {
console.debug('[overlay_stt_notify] fire-and-forget error:', err);
}
})();
};
/**
* Unregister the global dictation hotkey if one is active.
*/
+21
View File
@@ -199,4 +199,25 @@ mod tests {
fn subscribe_returns_receiver() {
let _rx = subscribe_dictation_events();
}
#[test]
fn publish_dictation_event_reaches_subscriber() {
let mut rx = subscribe_dictation_events();
publish_dictation_event(DictationEvent {
event_type: "pressed".to_string(),
hotkey: "chat_button".to_string(),
activation_mode: "toggle".to_string(),
});
let evt = rx.try_recv().expect("should receive dictation event");
assert_eq!(evt.event_type, "pressed");
assert_eq!(evt.hotkey, "chat_button");
}
#[test]
fn publish_transcription_reaches_subscriber() {
let mut rx = subscribe_transcription_results();
publish_transcription("hello world".to_string());
let text = rx.try_recv().expect("should receive transcription");
assert_eq!(text, "hello world");
}
}
+90
View File
@@ -44,6 +44,24 @@ struct TtsParams {
output_path: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "snake_case")]
enum OverlaySttState {
RecordingStarted,
TranscriptionDone,
Cancelled,
Error,
}
#[derive(Debug, Deserialize)]
struct OverlaySttNotifyParams {
/// Voice state transition.
state: OverlaySttState,
/// Transcribed text (required when state is "transcription_done").
#[serde(default)]
text: Option<String>,
}
// ---------------------------------------------------------------------------
// Schema + registry exports
// ---------------------------------------------------------------------------
@@ -57,6 +75,7 @@ pub fn all_voice_controller_schemas() -> Vec<ControllerSchema> {
voice_schemas("voice_server_start"),
voice_schemas("voice_server_stop"),
voice_schemas("voice_server_status"),
voice_schemas("overlay_stt_notify"),
]
}
@@ -90,6 +109,10 @@ pub fn all_voice_registered_controllers() -> Vec<RegisteredController> {
schema: voice_schemas("voice_server_status"),
handler: handle_voice_server_status,
},
RegisteredController {
schema: voice_schemas("overlay_stt_notify"),
handler: handle_overlay_stt_notify,
},
]
}
@@ -183,6 +206,23 @@ pub fn voice_schemas(function: &str) -> ControllerSchema {
inputs: vec![],
outputs: vec![json_output("status", "Current voice server status.")],
},
"overlay_stt_notify" => ControllerSchema {
namespace: "voice",
function: "overlay_stt_notify",
description:
"Notify the overlay of a voice/STT state change from the chat prompt button.",
inputs: vec![
required_string(
"state",
"State transition: recording_started, transcription_done, cancelled, error.",
),
optional_string(
"text",
"Transcribed text (when state is transcription_done).",
),
],
outputs: vec![json_output("result", "Notification acknowledgement.")],
},
_ => ControllerSchema {
namespace: "voice",
function: "unknown",
@@ -375,6 +415,52 @@ fn handle_voice_server_status(_params: Map<String, Value>) -> ControllerFuture {
})
}
fn handle_overlay_stt_notify(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
let p = deserialize_params::<OverlaySttNotifyParams>(params)?;
log::debug!(
"[overlay_stt_notify] state={:?}, has_text={}, text_len={}",
p.state,
p.text.is_some(),
p.text.as_deref().map_or(0, |t| t.len())
);
use crate::openhuman::voice::dictation_listener::{
publish_dictation_event, publish_transcription, DictationEvent,
};
match p.state {
OverlaySttState::RecordingStarted => {
publish_dictation_event(DictationEvent {
event_type: "pressed".to_string(),
hotkey: "chat_button".to_string(),
activation_mode: "toggle".to_string(),
});
}
OverlaySttState::TranscriptionDone => {
let text = p.text.ok_or_else(|| {
"invalid params: `text` is required for transcription_done".to_string()
})?;
publish_transcription(text);
publish_dictation_event(DictationEvent {
event_type: "released".to_string(),
hotkey: "chat_button".to_string(),
activation_mode: "toggle".to_string(),
});
}
OverlaySttState::Cancelled | OverlaySttState::Error => {
publish_dictation_event(DictationEvent {
event_type: "released".to_string(),
hotkey: "chat_button".to_string(),
activation_mode: "toggle".to_string(),
});
}
}
Ok(serde_json::json!({ "ok": true }))
})
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
@@ -446,6 +532,10 @@ mod tests {
let s = voice_schemas("voice_tts");
assert_eq!(s.namespace, "voice");
assert_eq!(s.function, "tts");
let s = voice_schemas("overlay_stt_notify");
assert_eq!(s.namespace, "voice");
assert_eq!(s.function, "overlay_stt_notify");
}
#[test]