mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
* feat(voice): sync overlay orb with chat voice button state (#487) The overlay orb already reacts to hotkey-based dictation via Socket.IO events, but the chat "Start Talking" button used local React state only. Add a new RPC method `openhuman.overlay_stt_notify` that the chat button calls at each voice state transition, which publishes to the existing DICTATION_BUS / TRANSCRIPTION_BUS broadcast channels — so the overlay reflects recording/transcribing/idle from both input paths with zero changes to the Socket.IO bridge or overlay event handlers. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(voice): address CI formatting and CodeRabbit review feedback - Run cargo fmt and prettier to fix formatting violations - Use typed enum OverlaySttState instead of raw String for state param (serde rejects invalid states at deserialization, eliminating the unknown state branch) - Require `text` field for transcription_done state (return error if missing instead of silently ignoring) - Replace raw transcript logging with metadata-only (has_text, text_len) to avoid logging sensitive user speech content - Use "Voice input active" aria-label (covers recording + linger phases) - Convert notifyOverlaySttState to arrow function with async/await per repo TS conventions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
3a2e026346
commit
0cd0f7a670
Generated
+1
-1
@@ -5120,7 +5120,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||
|
||||
[[package]]
|
||||
name = "openhuman"
|
||||
version = "0.52.0"
|
||||
version = "0.52.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
|
||||
@@ -405,7 +405,13 @@ export default function OverlayApp() {
|
||||
<div className="relative">
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Overlay orb"
|
||||
aria-label={
|
||||
mode === 'stt'
|
||||
? 'Voice input active'
|
||||
: mode === 'attention'
|
||||
? 'Attention message'
|
||||
: 'OpenHuman overlay'
|
||||
}
|
||||
onClick={goIdle}
|
||||
onMouseEnter={() => {
|
||||
setIsHovered(true);
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
import type { ThreadMessage } from '../types/thread';
|
||||
import {
|
||||
isTauri,
|
||||
notifyOverlaySttState,
|
||||
openhumanAutocompleteAccept,
|
||||
openhumanAutocompleteCurrent,
|
||||
openhumanLocalAiAnalyzeSentiment,
|
||||
@@ -762,6 +763,7 @@ const Conversations = () => {
|
||||
const chunks = audioChunksRef.current;
|
||||
audioChunksRef.current = [];
|
||||
if (chunks.length === 0) {
|
||||
notifyOverlaySttState('cancelled');
|
||||
setVoiceStatus('No audio captured. Try again.');
|
||||
return;
|
||||
}
|
||||
@@ -784,13 +786,16 @@ const Conversations = () => {
|
||||
const transcript = result.text.trim();
|
||||
|
||||
if (!transcript) {
|
||||
notifyOverlaySttState('cancelled');
|
||||
setVoiceStatus('No speech detected. Try again.');
|
||||
return;
|
||||
}
|
||||
|
||||
notifyOverlaySttState('transcription_done', transcript);
|
||||
setVoiceStatus(`Heard: ${transcript}`);
|
||||
await handleSendMessage(transcript);
|
||||
} catch (err) {
|
||||
notifyOverlaySttState('error');
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
setSendError(chatSendError('voice_transcription', `Voice transcription failed: ${message}`));
|
||||
setVoiceStatus(null);
|
||||
@@ -839,6 +844,7 @@ const Conversations = () => {
|
||||
}
|
||||
};
|
||||
recorder.onerror = () => {
|
||||
notifyOverlaySttState('error');
|
||||
setIsRecording(false);
|
||||
mediaStreamRef.current?.getTracks().forEach(track => track.stop());
|
||||
mediaStreamRef.current = null;
|
||||
@@ -853,7 +859,9 @@ const Conversations = () => {
|
||||
setSendError(null);
|
||||
setIsRecording(true);
|
||||
recorder.start();
|
||||
notifyOverlaySttState('recording_started');
|
||||
} catch (err) {
|
||||
notifyOverlaySttState('error');
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
setSendError(chatSendError('microphone_access', `Microphone access failed: ${message}`));
|
||||
setVoiceStatus(null);
|
||||
|
||||
@@ -171,6 +171,23 @@ export async function registerDictationHotkey(shortcut: string): Promise<void> {
|
||||
console.debug('[dictation] registerDictationHotkey: done');
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify the overlay of a voice/STT state change from the chat prompt button.
|
||||
* Fire-and-forget — errors are logged but never propagated.
|
||||
*/
|
||||
export const notifyOverlaySttState = (
|
||||
state: 'recording_started' | 'transcription_done' | 'cancelled' | 'error',
|
||||
text?: string
|
||||
): void => {
|
||||
void (async () => {
|
||||
try {
|
||||
await callCoreRpc({ method: 'openhuman.overlay_stt_notify', params: { state, text } });
|
||||
} catch (err: unknown) {
|
||||
console.debug('[overlay_stt_notify] fire-and-forget error:', err);
|
||||
}
|
||||
})();
|
||||
};
|
||||
|
||||
/**
|
||||
* Unregister the global dictation hotkey if one is active.
|
||||
*/
|
||||
|
||||
@@ -199,4 +199,25 @@ mod tests {
|
||||
fn subscribe_returns_receiver() {
|
||||
let _rx = subscribe_dictation_events();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publish_dictation_event_reaches_subscriber() {
|
||||
let mut rx = subscribe_dictation_events();
|
||||
publish_dictation_event(DictationEvent {
|
||||
event_type: "pressed".to_string(),
|
||||
hotkey: "chat_button".to_string(),
|
||||
activation_mode: "toggle".to_string(),
|
||||
});
|
||||
let evt = rx.try_recv().expect("should receive dictation event");
|
||||
assert_eq!(evt.event_type, "pressed");
|
||||
assert_eq!(evt.hotkey, "chat_button");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publish_transcription_reaches_subscriber() {
|
||||
let mut rx = subscribe_transcription_results();
|
||||
publish_transcription("hello world".to_string());
|
||||
let text = rx.try_recv().expect("should receive transcription");
|
||||
assert_eq!(text, "hello world");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,24 @@ struct TtsParams {
|
||||
output_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum OverlaySttState {
|
||||
RecordingStarted,
|
||||
TranscriptionDone,
|
||||
Cancelled,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct OverlaySttNotifyParams {
|
||||
/// Voice state transition.
|
||||
state: OverlaySttState,
|
||||
/// Transcribed text (required when state is "transcription_done").
|
||||
#[serde(default)]
|
||||
text: Option<String>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Schema + registry exports
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -57,6 +75,7 @@ pub fn all_voice_controller_schemas() -> Vec<ControllerSchema> {
|
||||
voice_schemas("voice_server_start"),
|
||||
voice_schemas("voice_server_stop"),
|
||||
voice_schemas("voice_server_status"),
|
||||
voice_schemas("overlay_stt_notify"),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -90,6 +109,10 @@ pub fn all_voice_registered_controllers() -> Vec<RegisteredController> {
|
||||
schema: voice_schemas("voice_server_status"),
|
||||
handler: handle_voice_server_status,
|
||||
},
|
||||
RegisteredController {
|
||||
schema: voice_schemas("overlay_stt_notify"),
|
||||
handler: handle_overlay_stt_notify,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
@@ -183,6 +206,23 @@ pub fn voice_schemas(function: &str) -> ControllerSchema {
|
||||
inputs: vec![],
|
||||
outputs: vec![json_output("status", "Current voice server status.")],
|
||||
},
|
||||
"overlay_stt_notify" => ControllerSchema {
|
||||
namespace: "voice",
|
||||
function: "overlay_stt_notify",
|
||||
description:
|
||||
"Notify the overlay of a voice/STT state change from the chat prompt button.",
|
||||
inputs: vec![
|
||||
required_string(
|
||||
"state",
|
||||
"State transition: recording_started, transcription_done, cancelled, error.",
|
||||
),
|
||||
optional_string(
|
||||
"text",
|
||||
"Transcribed text (when state is transcription_done).",
|
||||
),
|
||||
],
|
||||
outputs: vec![json_output("result", "Notification acknowledgement.")],
|
||||
},
|
||||
_ => ControllerSchema {
|
||||
namespace: "voice",
|
||||
function: "unknown",
|
||||
@@ -375,6 +415,52 @@ fn handle_voice_server_status(_params: Map<String, Value>) -> ControllerFuture {
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_overlay_stt_notify(params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move {
|
||||
let p = deserialize_params::<OverlaySttNotifyParams>(params)?;
|
||||
log::debug!(
|
||||
"[overlay_stt_notify] state={:?}, has_text={}, text_len={}",
|
||||
p.state,
|
||||
p.text.is_some(),
|
||||
p.text.as_deref().map_or(0, |t| t.len())
|
||||
);
|
||||
|
||||
use crate::openhuman::voice::dictation_listener::{
|
||||
publish_dictation_event, publish_transcription, DictationEvent,
|
||||
};
|
||||
|
||||
match p.state {
|
||||
OverlaySttState::RecordingStarted => {
|
||||
publish_dictation_event(DictationEvent {
|
||||
event_type: "pressed".to_string(),
|
||||
hotkey: "chat_button".to_string(),
|
||||
activation_mode: "toggle".to_string(),
|
||||
});
|
||||
}
|
||||
OverlaySttState::TranscriptionDone => {
|
||||
let text = p.text.ok_or_else(|| {
|
||||
"invalid params: `text` is required for transcription_done".to_string()
|
||||
})?;
|
||||
publish_transcription(text);
|
||||
publish_dictation_event(DictationEvent {
|
||||
event_type: "released".to_string(),
|
||||
hotkey: "chat_button".to_string(),
|
||||
activation_mode: "toggle".to_string(),
|
||||
});
|
||||
}
|
||||
OverlaySttState::Cancelled | OverlaySttState::Error => {
|
||||
publish_dictation_event(DictationEvent {
|
||||
event_type: "released".to_string(),
|
||||
hotkey: "chat_button".to_string(),
|
||||
activation_mode: "toggle".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(serde_json::json!({ "ok": true }))
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -446,6 +532,10 @@ mod tests {
|
||||
let s = voice_schemas("voice_tts");
|
||||
assert_eq!(s.namespace, "voice");
|
||||
assert_eq!(s.function, "tts");
|
||||
|
||||
let s = voice_schemas("overlay_stt_notify");
|
||||
assert_eq!(s.namespace, "voice");
|
||||
assert_eq!(s.function, "overlay_stt_notify");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user