From 8acd34d2a9feca7d808306c2ea674e583b9b6e00 Mon Sep 17 00:00:00 2001 From: YellowSnnowmann <167776381+YellowSnnowmann@users.noreply.github.com> Date: Thu, 2 Apr 2026 20:54:46 +0530 Subject: [PATCH] Feat/add dictation (#278) * feat(dictation): implement voice dictation feature with overlay and settings panel - Added DictationOverlay component for real-time speech-to-text functionality, including recording, transcribing, and error handling. - Introduced useDictation hook to manage audio recording and transcription processes. - Created DictationPanel for configuring dictation settings, including hotkey registration and floating launcher preferences. - Updated SettingsHome to include a navigation option for the new dictation feature. - Integrated dictation state management with Redux, allowing for persistent settings and status checks. - Enhanced Tauri commands for registering global hotkeys and managing dictation state. This update provides a comprehensive voice dictation experience, enabling users to transcribe speech to text using local AI. * fix(dictation): update DictationOverlay to conditionally render based on floating launcher state - Reintroduced the useAppDispatch and useAppSelector hooks for state management. - Added showFloatingLauncher to the state selection, ensuring the overlay only renders when the floating launcher is active. - Simplified the return statement for position calculations in the drag handler. - Improved code readability by formatting JSX elements for better clarity. This update enhances the user experience by ensuring the dictation overlay behaves correctly based on the application's state. * feat(settings): add dictation route and panel to settings navigation - Updated the SettingsRoute type to include 'dictation' as a new route. - Integrated DictationPanel into the Settings page for user configuration. - Enhanced dictation state management by adding showFloatingLauncher to the dictationSlice, allowing for better control of the dictation overlay. This update improves the settings navigation and user experience by providing access to dictation features directly from the settings menu. * feat(dictation): integrate DictationOverlay and enhance dictation settings - Added DictationOverlay component to the main App for improved user interaction with dictation features. - Updated DictationPanel to include a new preference for showing a floating launcher, enhancing user control over dictation functionality. - Enhanced state management by incorporating showFloatingLauncher into the dictationSlice, allowing for better configuration of the dictation experience. This update improves the overall user experience by providing direct access to dictation features and customizable settings. * feat(dictation): enhance DictationOverlay and Conversations for improved text insertion - Added functionality to insert text into editable elements from the DictationOverlay, improving user interaction with dictation features. - Implemented event listener in Conversations to handle custom dictation insert events, allowing seamless integration of transcribed text into the input field. - Updated state management to ensure the correct editable target is used for text insertion, enhancing overall user experience. This update streamlines the dictation process, making it more intuitive and responsive to user actions. * fix(dictation): refine STT availability logic in dictationSlice - Updated the logic for determining STT availability to ensure it only considers the model file as available when both the model file exists and there is a method to run inference (either the in-process engine is loaded or a whisper binary is present). - This change prevents misleading user experiences by avoiding the display of the overlay when the necessary components for transcription are not available. This update enhances the reliability of the dictation feature by providing clearer conditions for STT availability. * refactor(dictation): streamline position management in DictationOverlay - Removed redundant state management for the position of the dictation overlay, consolidating logic to initialize and reset the position based on the current status. - Introduced a new `resetLauncherPosition` function to simplify resetting the overlay's position when necessary. - Updated event handling to ensure the overlay's position is reset appropriately after text insertion actions. This update enhances the clarity and efficiency of the DictationOverlay component, improving user experience during dictation interactions. * refactor(dictation): improve hotkey registration and unregistration process - Streamlined the logic for registering and unregistering dictation hotkeys, ensuring that old shortcuts are properly managed before new ones are registered. - Introduced rollback mechanisms to restore previous shortcuts in case of registration failures, enhancing reliability. - Simplified error handling and logging for better clarity during the hotkey management process. This update enhances the robustness of the dictation feature by ensuring a smoother transition between hotkey states. * refactor: delete unncessesary pr md file * refactor(dictation): enhance dictation functionality and error handling compatibility. * refactor(dictation): improve code formatting and readability across components - Enhanced formatting in DictationOverlay for better clarity in asynchronous action handling. - Streamlined text extraction logic in useDictation for improved readability. - Consolidated model directory setting in DictationPanel to a single line for simplicity. - Improved logging consistency in tauriCommands and speech service files. These changes enhance the maintainability and readability of the dictation-related components. * refactor(dictation): manage DictationOverlay position based on status changes - Introduced a reference to track the previous status of the dictation overlay. - Updated the effect to reset the overlay's position when transitioning from 'idle' to any other status, enhancing user experience during dictation sessions. This change improves the responsiveness of the DictationOverlay component to status changes, ensuring a smoother interaction for users. * refactor(tests): update MemoryWorkspace tests to specify span selector - Modified test assertions in MemoryWorkspace.test.tsx to include a selector for 'span' elements when checking for text presence. - This change enhances the specificity of the tests, ensuring they accurately target the intended elements in the rendered component. These updates improve the reliability of the MemoryWorkspace component tests. --- PR_DESCRIPTION.md | 102 ----- app/src-tauri/Cargo.lock | 69 ++- app/src-tauri/Cargo.toml | 1 + app/src-tauri/src/lib.rs | 158 ++++++- app/src/App.tsx | 2 + .../components/dictation/DictationOverlay.tsx | 373 +++++++++++++++ app/src/components/dictation/useDictation.ts | 430 ++++++++++++++++++ .../__tests__/MemoryWorkspace.test.tsx | 8 +- app/src/components/settings/SettingsHome.tsx | 17 + .../settings/hooks/useSettingsNavigation.ts | 4 +- .../settings/panels/DictationPanel.tsx | 288 ++++++++++++ .../panels/ScreenIntelligencePanel.tsx | 5 +- app/src/pages/Conversations.tsx | 26 ++ app/src/pages/Settings.tsx | 2 + app/src/store/dictationSlice.ts | 138 ++++++ app/src/store/index.ts | 8 + app/src/utils/tauriCommands.ts | 47 ++ src/openhuman/local_ai/service/bootstrap.rs | 7 +- src/openhuman/local_ai/service/mod.rs | 1 + src/openhuman/local_ai/service/speech.rs | 27 ++ 20 files changed, 1600 insertions(+), 113 deletions(-) delete mode 100644 PR_DESCRIPTION.md create mode 100644 app/src/components/dictation/DictationOverlay.tsx create mode 100644 app/src/components/dictation/useDictation.ts create mode 100644 app/src/components/settings/panels/DictationPanel.tsx create mode 100644 app/src/store/dictationSlice.ts diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md deleted file mode 100644 index 1620cac09..000000000 --- a/PR_DESCRIPTION.md +++ /dev/null @@ -1,102 +0,0 @@ -## Summary - -- Fix stacked socket listeners causing duplicate AI message bubbles on reconnect -- Route chat through local Ollama model when active — zero cloud tokens on the local path -- Multi-bubble delivery: assistant replies split into 2–5 natural chat bubbles with typing pauses -- Contextual emoji guidance in system prompts — replaces overuse with intentional, human-like usage -- 402 unit tests passing - -## Problem - -**Duplicate messages** — `subscribeChatEvents` was declared `async` with no actual `await` inside. The cleanup function returned by `.then()` lands in a microtask; React's synchronous cleanup had already run by then, so socket listeners were never removed. Each reconnect stacked another listener set, causing `chat:done` to fire N times and produce N copies of every reply. - -**Responses felt like AI slop** — replies arrived as a single wall of text. No typing feel, no natural pacing. - -**Ollama was wired for utilities but not chat** — `local_ai_status`, `suggest_questions`, TTS, and Whisper all used Ollama already, but the main conversation loop always hit the cloud socket regardless of whether a local model was running. - -**Generic emoji overuse** — the SOUL prompt said "Enthusiastic / Witty / Genuinely human" and the only emoji rule was "Minimal — match user's style". The model interpreted this as license to add 😄🔥✨ on every message. - -## Solution - -### 1 — Socket listener race condition - -`subscribeChatEvents` no longer uses `async` — the cleanup fn is returned synchronously so React can always call it. The `useEffect` stores it with `const cleanup = subscribeChatEvents(...)` and returns it directly. - -### 2 — Rust: `openhuman.local_ai_chat` RPC - -- `ollama_api.rs` — `OllamaChatMessage` / `OllamaChatRequest` / `OllamaChatResponse` for `/api/chat` -- `service/public_infer.rs` — `LocalAiService::chat_with_history()`: sends full message history to Ollama, updates latency/TPS status -- `ops.rs` — `local_ai_chat` async op -- `schemas.rs` — registered controller: schema, handler, param deserialization - -### 3 — Frontend: local chat gate + multi-bubble delivery - -When `isLocalModelActive` is true, `handleSendMessage` takes the local path: - -``` -User sends message - │ - ├── isLocalModelActive = true - │ openhumanLocalAiChat(history) ──► Ollama /api/chat - │ deliverLocalResponse() (no socket, zero cloud tokens) - │ segmentMessage() → 2–5 bubbles - │ dispatch each bubble with typing pause (500–1 400 ms) - │ - └── isLocalModelActive = false - chatSend() ──► socket chat:start ──► backend ──► cloud API - (unchanged) -``` - -Socket-connected guard is skipped on the local path so offline use works. - -### 4 — Emoji rules in system prompts - -`SOUL.md` adds an explicit **Emoji Rules** section: - -- Hard cap: one emoji max per message (none is always fine) -- Contextual, not decorative — must reinforce the specific content -- Never as a sentence opener; only at the end of a clause -- Skip entirely in errors, warnings, technical content, lists, or replies > 3 sentences -- Mirror the user's own emoji usage -- Concrete bad/good examples for model calibration - -`BOOTSTRAP.md` Communication Preferences entry updated to reference these rules. - -## Files Changed - -| File | Change | -|---|---| -| `app/src/services/chatService.ts` | Remove `async` from `subscribeChatEvents`; synchronous cleanup | -| `app/src/pages/Conversations.tsx` | Local chat gate, `deliverLocalResponse`, updated socket effect | -| `app/src/hooks/useLocalModelStatus.ts` | Polls `local_ai_status` every 12 s; returns `true` when `state === "ready"` | -| `app/src/utils/messageSegmentation.ts` | `segmentMessage()` + `getSegmentDelay()` | -| `app/src/utils/tauriCommands.ts` | `openhumanLocalAiChat()` RPC wrapper | -| `app/src/store/threadSlice.ts` | `addReaction` reducer; `activeThreadId` tracking | -| `src/openhuman/local_ai/ollama_api.rs` | Ollama `/api/chat` types | -| `src/openhuman/local_ai/service/public_infer.rs` | `chat_with_history()` method | -| `src/openhuman/local_ai/ops.rs` | `local_ai_chat` op | -| `src/openhuman/local_ai/schemas.rs` | Controller registration | -| `src/openhuman/agent/prompts/SOUL.md` | Emoji Rules section | -| `src/openhuman/agent/prompts/BOOTSTRAP.md` | Updated emoji preference line | - -## Submission Checklist - -- [x] **Bug fix** — duplicate message root cause identified and fixed -- [x] **Local-only gate** — cloud path entirely unchanged; no increase in billed token usage -- [x] **Rust compiles** — `cargo check --manifest-path Cargo.toml` clean -- [x] **TypeScript** — `tsc --noEmit` 0 errors -- [x] **Tests** — 402 passing (`yarn test`); new files: `messageSegmentation.test.ts` (14 tests), `localChatGating.test.ts` (9 tests) -- [x] **Commits** — 4 atomic commits, one per concern - -## Test validation - -```bash -cargo check --manifest-path Cargo.toml -cd app && yarn test -yarn tsc --noEmit -``` - -## Related - -- Follow-up: server-side reaction sync for multi-user channel modes -- Follow-up: streaming Ollama responses (currently non-streaming for simplicity) diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 22d5c3a4c..b4af2bb3c 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "OpenHuman" -version = "0.50.3" +version = "0.51.2" dependencies = [ "env_logger", "log", @@ -12,6 +12,7 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-deep-link", + "tauri-plugin-global-shortcut", "tauri-plugin-opener", "tokio", ] @@ -1360,6 +1361,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix", + "windows-link 0.2.1", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1492,6 +1503,24 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "global-hotkey" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9247516746aa8e53411a0db9b62b0e24efbcf6a76e0ba73e5a91b512ddabed7" +dependencies = [ + "crossbeam-channel", + "keyboard-types", + "objc2", + "objc2-app-kit", + "once_cell", + "serde", + "thiserror 2.0.18", + "windows-sys 0.59.0", + "x11rb", + "xkeysym", +] + [[package]] name = "gobject-sys" version = "0.18.0" @@ -3969,6 +3998,21 @@ dependencies = [ "windows-result 0.3.4", ] +[[package]] +name = "tauri-plugin-global-shortcut" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "424af23c7e88d05e4a1a6fc2c7be077912f8c76bd7900fd50aa2b7cbf5a2c405" +dependencies = [ + "global-hotkey", + "log", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "thiserror 2.0.18", +] + [[package]] name = "tauri-plugin-opener" version = "2.5.3" @@ -5506,6 +5550,29 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "yoke" version = "0.8.1" diff --git a/app/src-tauri/Cargo.toml b/app/src-tauri/Cargo.toml index e6600e128..5edca64b8 100644 --- a/app/src-tauri/Cargo.toml +++ b/app/src-tauri/Cargo.toml @@ -28,6 +28,7 @@ serde_json = "1" # Tauri core and plugins tauri = { version = "2.10", features = ["macos-private-api", "tray-icon"] } tauri-plugin-deep-link = "2.0.0" +tauri-plugin-global-shortcut = "2" tauri-plugin-opener = "2" serde_json = "1" tokio = { version = "1", features = ["rt-multi-thread", "process", "sync", "time", "net"] } diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 780633926..b04a2268d 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -3,15 +3,49 @@ compile_error!("src-tauri host is desktop-only. Non-desktop targets are not supp mod core_process; +use std::sync::Mutex; + use tauri::{ menu::{Menu, MenuItem}, tray::{MouseButton, MouseButtonState, TrayIconBuilder, TrayIconEvent}, - AppHandle, Manager, RunEvent, + AppHandle, Emitter, Manager, RunEvent, }; +use tauri_plugin_global_shortcut::{GlobalShortcutExt, ShortcutState}; #[cfg(any(windows, target_os = "linux"))] use tauri_plugin_deep_link::DeepLinkExt; +/// Tracks the currently registered dictation hotkey string so we can unregister it later. +struct DictationHotkeyState(Mutex>); + +fn expand_dictation_shortcuts(shortcut: &str) -> Vec { + let trimmed = shortcut.trim(); + if trimmed.is_empty() { + return vec![]; + } + + #[cfg(target_os = "macos")] + { + if trimmed.contains("CmdOrCtrl") { + let cmd_variant = trimmed.replace("CmdOrCtrl", "Cmd"); + let ctrl_variant = trimmed.replace("CmdOrCtrl", "Ctrl"); + if cmd_variant == ctrl_variant { + return vec![cmd_variant]; + } + return vec![cmd_variant, ctrl_variant]; + } + } + + #[cfg(not(target_os = "macos"))] + { + if trimmed.contains("CmdOrCtrl") { + return vec![trimmed.replace("CmdOrCtrl", "Ctrl")]; + } + } + + vec![trimmed.to_string()] +} + #[tauri::command] fn core_rpc_url() -> String { std::env::var("OPENHUMAN_CORE_RPC_URL") @@ -106,6 +140,122 @@ async fn restart_core_process( state.inner().restart().await } +/// Register (or re-register) the global dictation toggle hotkey. +/// Emits `dictation://toggle` to all webviews when the shortcut is pressed. +#[tauri::command] +async fn register_dictation_hotkey( + app: AppHandle, + shortcut: String, +) -> Result<(), String> { + log::info!("[dictation] register_dictation_hotkey: shortcut={shortcut}"); + + let old_shortcuts = { + let state = app.state::(); + let guard = state.0.lock().unwrap(); + guard.clone() + }; + + let expanded_shortcuts = expand_dictation_shortcuts(&shortcut); + if expanded_shortcuts.is_empty() { + return Err("Shortcut cannot be empty".to_string()); + } + log::info!( + "[dictation] expanded shortcuts: {}", + expanded_shortcuts.join(", ") + ); + + let register_shortcut = |shortcut_variant: &str| -> Result<(), String> { + let app_clone = app.clone(); + app.global_shortcut() + .on_shortcut(shortcut_variant, move |_app, _sc, event| { + if event.state == ShortcutState::Pressed { + log::debug!("[dictation] hotkey pressed — emitting dictation://toggle"); + if let Err(e) = app_clone.emit("dictation://toggle", ()) { + log::warn!("[dictation] emit failed: {e}"); + } + } + }) + .map_err(|e| format!("Failed to register shortcut '{shortcut_variant}': {e}")) + }; + + let mut unregistered_old: Vec = Vec::new(); + for old in &old_shortcuts { + log::debug!("[dictation] unregistering previous shortcut: {old}"); + if let Err(e) = app.global_shortcut().unregister(old.as_str()) { + for restored in &unregistered_old { + if let Err(restore_err) = register_shortcut(restored.as_str()) { + log::warn!( + "[dictation] rollback failed while restoring old shortcut '{restored}': {restore_err}" + ); + } + } + return Err(format!("Failed to unregister previous shortcut '{old}': {e}")); + } + unregistered_old.push(old.clone()); + } + + let mut newly_registered: Vec = Vec::new(); + for shortcut_variant in &expanded_shortcuts { + if let Err(err) = register_shortcut(shortcut_variant.as_str()) { + log::error!("[dictation] failed to register shortcut '{shortcut_variant}': {err}"); + for registered in &newly_registered { + if let Err(unregister_err) = app.global_shortcut().unregister(registered.as_str()) { + log::warn!( + "[dictation] rollback failed while unregistering '{registered}': {unregister_err}" + ); + } + } + for old in &old_shortcuts { + if let Err(restore_err) = register_shortcut(old.as_str()) { + log::warn!( + "[dictation] rollback failed while restoring old shortcut '{old}': {restore_err}" + ); + } + } + return Err(err); + } + newly_registered.push(shortcut_variant.clone()); + } + + // Persist all newly registered shortcuts. + { + let state = app.state::(); + let mut guard = state.0.lock().unwrap(); + *guard = expanded_shortcuts.clone(); + } + + log::info!( + "[dictation] shortcuts registered: {}", + expanded_shortcuts.join(", ") + ); + Ok(()) +} + +/// Unregister the global dictation hotkey (if any). +#[tauri::command] +async fn unregister_dictation_hotkey(app: AppHandle) -> Result<(), String> { + log::info!("[dictation] unregister_dictation_hotkey: called"); + let state = app.state::(); + let mut guard = state.0.lock().unwrap(); + if guard.is_empty() { + log::debug!("[dictation] no shortcut registered — nothing to unregister"); + } else { + let old_shortcuts = guard.clone(); + guard.clear(); + for old in old_shortcuts { + log::debug!("[dictation] unregistering shortcut: {old}"); + app.global_shortcut() + .unregister(old.as_str()) + .map_err(|e| { + log::warn!("[dictation] failed to unregister '{old}': {e}"); + format!("Failed to unregister shortcut '{old}': {e}") + })?; + log::info!("[dictation] shortcut unregistered: {old}"); + } + } + Ok(()) +} + fn is_daemon_mode() -> bool { std::env::args().any(|arg| arg == "daemon" || arg == "--daemon") } @@ -186,6 +336,8 @@ pub fn run() { tauri::Builder::default() .plugin(tauri_plugin_opener::init()) .plugin(tauri_plugin_deep_link::init()) + .plugin(tauri_plugin_global_shortcut::Builder::new().build()) + .manage(DictationHotkeyState(Mutex::new(Vec::new()))) .setup(move |app| { #[cfg(any(windows, target_os = "linux"))] { @@ -235,7 +387,9 @@ pub fn run() { service_start_direct, service_stop_direct, service_status_direct, - service_uninstall_direct + service_uninstall_direct, + register_dictation_hotkey, + unregister_dictation_hotkey ]) .build(tauri::generate_context!()) .expect("error while building tauri application") diff --git a/app/src/App.tsx b/app/src/App.tsx index ea6cf4278..2eb7bdf21 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -5,6 +5,7 @@ import { PersistGate } from 'redux-persist/integration/react'; import AppRoutes from './AppRoutes'; import ServiceBlockingGate from './components/daemon/ServiceBlockingGate'; +import DictationOverlay from './components/dictation/DictationOverlay'; import ErrorFallbackScreen from './components/ErrorFallbackScreen'; import LocalAIDownloadSnackbar from './components/LocalAIDownloadSnackbar'; import MiniSidebar from './components/MiniSidebar'; @@ -56,6 +57,7 @@ function App() { + diff --git a/app/src/components/dictation/DictationOverlay.tsx b/app/src/components/dictation/DictationOverlay.tsx new file mode 100644 index 000000000..75db55c89 --- /dev/null +++ b/app/src/components/dictation/DictationOverlay.tsx @@ -0,0 +1,373 @@ +import { useEffect, useRef, useState } from 'react'; + +import { resetDictation } from '../../store/dictationSlice'; +import { useAppDispatch, useAppSelector } from '../../store/hooks'; +import { openhumanAccessibilityInputAction } from '../../utils/tauriCommands'; +import { useDictation } from './useDictation'; + +const STATUS_COLORS: Record = { + idle: 'bg-stone-800', + recording: 'bg-red-600 animate-pulse', + transcribing: 'bg-amber-600', + ready: 'bg-primary-600', + error: 'bg-coral-600', +}; + +const STATUS_LABELS: Record = { + idle: 'Idle', + recording: 'Recording...', + transcribing: 'Transcribing...', + ready: 'Ready', + error: 'Error', +}; + +type EditableTarget = HTMLInputElement | HTMLTextAreaElement | HTMLElement; + +const isTextInput = (el: Element): el is HTMLInputElement | HTMLTextAreaElement => + el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement; + +const isEditableElement = (el: Element | null): el is EditableTarget => + !!el && (isTextInput(el) || (el instanceof HTMLElement && el.isContentEditable)); + +const insertIntoEditableTarget = (target: EditableTarget, text: string): boolean => { + if (isTextInput(target)) { + target.focus(); + const hasSelection = + typeof target.selectionStart === 'number' && typeof target.selectionEnd === 'number'; + if (hasSelection) { + target.setRangeText(text, target.selectionStart ?? 0, target.selectionEnd ?? 0, 'end'); + } else { + target.value = `${target.value}${text}`; + } + target.dispatchEvent(new Event('input', { bubbles: true })); + return true; + } + + if (target instanceof HTMLElement && target.isContentEditable) { + target.focus(); + const selection = window.getSelection(); + if (selection && selection.rangeCount > 0) { + const range = selection.getRangeAt(0); + range.deleteContents(); + range.insertNode(document.createTextNode(text)); + range.collapse(false); + selection.removeAllRanges(); + selection.addRange(range); + } else { + target.append(document.createTextNode(text)); + } + target.dispatchEvent(new Event('input', { bubbles: true })); + return true; + } + + return false; +}; + +const DictationOverlay = () => { + const dispatch = useAppDispatch(); + const { status, transcript, error, hotkey, showFloatingLauncher } = useAppSelector( + s => s.dictation + ); + const { startRecording, stopRecording, dismiss } = useDictation(); + const panelRef = useRef(null); + const dragRef = useRef<{ pointerId: number; offsetX: number; offsetY: number } | null>(null); + const lastEditableRef = useRef(null); + const previousStatusRef = useRef(status); + + const getActiveDefaultPosition = () => { + if (typeof window === 'undefined') return { x: 24, y: 24 }; + return { + x: Math.max(12, Math.round(window.innerWidth / 2 - 200)), + y: Math.max(12, window.innerHeight - 240), + }; + }; + + const getIdleDefaultPosition = () => { + if (typeof window === 'undefined') return { x: 24, y: 24 }; + const idleWidth = 250; + const idleHeight = 62; + return { + x: Math.max(12, window.innerWidth - idleWidth - 24), + y: Math.max(12, window.innerHeight - idleHeight - 24), + }; + }; + + const [position, setPosition] = useState<{ x: number; y: number }>(() => + status === 'idle' ? getIdleDefaultPosition() : getActiveDefaultPosition() + ); + + const resetLauncherPosition = () => { + setPosition(getIdleDefaultPosition()); + }; + + const clampToViewport = (x: number, y: number) => { + const panelWidth = panelRef.current?.offsetWidth ?? 360; + const panelHeight = panelRef.current?.offsetHeight ?? 220; + const maxX = Math.max(12, window.innerWidth - panelWidth - 12); + const maxY = Math.max(12, window.innerHeight - panelHeight - 12); + return { x: Math.max(12, Math.min(x, maxX)), y: Math.max(12, Math.min(y, maxY)) }; + }; + + // Keyboard shortcut: Escape to dismiss + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (e.key === 'Escape') { + console.debug('[dictation] Escape pressed — dismissing overlay'); + dismiss(); + } + }; + window.addEventListener('keydown', handler); + return () => window.removeEventListener('keydown', handler); + }, [dismiss]); + + useEffect(() => { + const onFocusIn = (event: FocusEvent) => { + const target = event.target as Element | null; + if (!isEditableElement(target)) return; + // Ignore overlay-owned elements so we keep the app context target. + if (panelRef.current?.contains(target)) return; + lastEditableRef.current = target; + }; + window.addEventListener('focusin', onFocusIn); + return () => window.removeEventListener('focusin', onFocusIn); + }, []); + + useEffect(() => { + const onResize = () => { + setPosition(prev => { + const base = + prev ?? (status === 'idle' ? getIdleDefaultPosition() : getActiveDefaultPosition()); + return clampToViewport(base.x, base.y); + }); + }; + window.addEventListener('resize', onResize); + return () => window.removeEventListener('resize', onResize); + }, [status]); + + useEffect(() => { + const previousStatus = previousStatusRef.current; + previousStatusRef.current = status; + if (previousStatus === 'idle' && status !== 'idle') { + setPosition(getActiveDefaultPosition()); + } + }, [status]); + + const handleDragStart = (e: React.PointerEvent) => { + if (e.button !== 0) return; + if ((e.target as HTMLElement).closest('button')) return; + if (!panelRef.current) return; + const rect = panelRef.current.getBoundingClientRect(); + dragRef.current = { + pointerId: e.pointerId, + offsetX: e.clientX - rect.left, + offsetY: e.clientY - rect.top, + }; + e.currentTarget.setPointerCapture(e.pointerId); + setPosition({ x: rect.left, y: rect.top }); + }; + + const handleDragMove = (e: React.PointerEvent) => { + const drag = dragRef.current; + if (!drag || drag.pointerId !== e.pointerId) return; + e.preventDefault(); + const next = clampToViewport(e.clientX - drag.offsetX, e.clientY - drag.offsetY); + setPosition(next); + }; + + const handleDragEnd = (e: React.PointerEvent) => { + const drag = dragRef.current; + if (!drag || drag.pointerId !== e.pointerId) return; + dragRef.current = null; + if (e.currentTarget.hasPointerCapture(e.pointerId)) { + e.currentTarget.releasePointerCapture(e.pointerId); + } + }; + + const handleInsert = async () => { + if (!transcript) return; + console.debug('[dictation] inserting transcript into active context'); + const insertEvent = new CustomEvent<{ text: string }>('dictation://insert-text', { + detail: { text: transcript }, + cancelable: true, + }); + window.dispatchEvent(insertEvent); + if (insertEvent.defaultPrevented) { + console.debug('[dictation] transcript handled by app context listener'); + resetLauncherPosition(); + dispatch(resetDictation()); + return; + } + + const active = document.activeElement; + const preferredTarget = + (isEditableElement(active) && !panelRef.current?.contains(active) ? active : null) ?? + lastEditableRef.current; + + if (preferredTarget && insertIntoEditableTarget(preferredTarget, transcript)) { + console.debug('[dictation] inserted transcript into DOM editable target'); + resetLauncherPosition(); + dispatch(resetDictation()); + return; + } + + console.debug('[dictation] no DOM target found, trying accessibility action'); + try { + const response = await openhumanAccessibilityInputAction({ + action: 'type', + text: transcript, + }); + const accepted = response.result.accepted === true; + const blocked = response.result.blocked === true; + if (accepted && !blocked) { + resetLauncherPosition(); + dispatch(resetDictation()); + return; + } + + console.debug( + '[dictation] accessibility insert not accepted (accepted=%s blocked=%s), falling back to clipboard', + accepted, + blocked + ); + await navigator.clipboard.writeText(transcript).catch(() => {}); + resetLauncherPosition(); + dispatch(resetDictation()); + } catch { + // Fallback to clipboard + console.debug('[dictation] accessibility insert failed, falling back to clipboard'); + await navigator.clipboard.writeText(transcript).catch(() => {}); + resetLauncherPosition(); + dispatch(resetDictation()); + } + }; + + const handleCopy = async () => { + if (!transcript) return; + console.debug('[dictation] copying transcript to clipboard'); + await navigator.clipboard.writeText(transcript).catch(() => {}); + resetLauncherPosition(); + dispatch(resetDictation()); + }; + + if (status === 'idle') { + if (!showFloatingLauncher) return null; + return ( +
+
+
+ ⋮⋮ +
+ +
+
+ ); + } + + return ( +
+
+ {/* Header */} +
+
+ + {STATUS_LABELS[status] ?? 'Dictation'} + +
+ +
+ + {/* Transcript */} + {transcript && ( +
+

{transcript}

+
+ )} + + {/* Error */} + {error && ( +
+

{error}

+
+ )} + + {/* Controls */} +
+ {(status === 'error' || status === 'ready') && ( + + )} + + {status === 'recording' && ( + + )} + + {transcript && ( + <> + + + + )} +
+ + {/* Hotkey hint */} +
+ {hotkey} to toggle · Esc to dismiss +
+
+
+ ); +}; + +export default DictationOverlay; diff --git a/app/src/components/dictation/useDictation.ts b/app/src/components/dictation/useDictation.ts new file mode 100644 index 000000000..3a032fc0b --- /dev/null +++ b/app/src/components/dictation/useDictation.ts @@ -0,0 +1,430 @@ +import { listen } from '@tauri-apps/api/event'; +import { useCallback, useEffect, useRef, useState } from 'react'; + +import { callCoreRpc } from '../../services/coreRpcClient'; +import { resetDictation, setError, setStatus, setTranscript } from '../../store/dictationSlice'; +import { useAppDispatch, useAppSelector } from '../../store/hooks'; +import { registerDictationHotkey } from '../../utils/tauriCommands'; + +const TARGET_SAMPLE_RATE = 16000; + +interface TranscribeResult { + text: string; + raw_text: string; + model_id: string; +} + +interface ParsedHotkey { + key: string; + cmdOrCtrl: boolean; + cmd: boolean; + ctrl: boolean; + shift: boolean; + alt: boolean; + super: boolean; +} + +function parseHotkey(shortcut: string): ParsedHotkey | null { + const tokens = shortcut + .split('+') + .map(t => t.trim().toLowerCase()) + .filter(Boolean); + if (tokens.length === 0) return null; + + const parsed: ParsedHotkey = { + key: '', + cmdOrCtrl: false, + cmd: false, + ctrl: false, + shift: false, + alt: false, + super: false, + }; + + for (const token of tokens) { + switch (token) { + case 'cmdorctrl': + case 'commandorcontrol': + parsed.cmdOrCtrl = true; + break; + case 'cmd': + case 'command': + case 'meta': + parsed.cmd = true; + break; + case 'ctrl': + case 'control': + parsed.ctrl = true; + break; + case 'shift': + parsed.shift = true; + break; + case 'alt': + case 'option': + parsed.alt = true; + break; + case 'super': + parsed.super = true; + break; + default: + parsed.key = token; + break; + } + } + + return parsed.key ? parsed : null; +} + +function matchesHotkeyEvent(event: KeyboardEvent, parsed: ParsedHotkey): boolean { + const key = event.key.toLowerCase(); + if (key !== parsed.key.toLowerCase()) return false; + if (parsed.shift !== event.shiftKey) return false; + if (parsed.alt !== event.altKey) return false; + + if (parsed.cmdOrCtrl && !(event.metaKey || event.ctrlKey)) return false; + if (parsed.cmd && !event.metaKey) return false; + if (parsed.ctrl && !event.ctrlKey) return false; + if (parsed.super && !event.metaKey) return false; + + return true; +} + +function floatTo16BitPCM(output: DataView, offset: number, input: Float32Array) { + for (let i = 0; i < input.length; i += 1, offset += 2) { + const sample = Math.max(-1, Math.min(1, input[i])); + output.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); + } +} + +function encodeWavMono16k(samples: Float32Array, sampleRate: number): Uint8Array { + const bytesPerSample = 2; + const blockAlign = bytesPerSample; + const byteRate = sampleRate * blockAlign; + const dataSize = samples.length * bytesPerSample; + const buffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(buffer); + + const writeString = (offset: number, value: string) => { + for (let i = 0; i < value.length; i += 1) { + view.setUint8(offset + i, value.charCodeAt(i)); + } + }; + + writeString(0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeString(8, 'WAVE'); + writeString(12, 'fmt '); + view.setUint32(16, 16, true); // PCM header size + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // mono + view.setUint32(24, sampleRate, true); + view.setUint32(28, byteRate, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, 16, true); // 16-bit + writeString(36, 'data'); + view.setUint32(40, dataSize, true); + floatTo16BitPCM(view, 44, samples); + + return new Uint8Array(buffer); +} + +async function toMono16k(audioBuffer: AudioBuffer): Promise { + const channels = audioBuffer.numberOfChannels; + const inputLength = audioBuffer.length; + + // Downmix to mono by averaging channels. + const mono = new Float32Array(inputLength); + for (let c = 0; c < channels; c += 1) { + const channelData = audioBuffer.getChannelData(c); + for (let i = 0; i < inputLength; i += 1) { + mono[i] += channelData[i] / channels; + } + } + + if (audioBuffer.sampleRate === TARGET_SAMPLE_RATE) { + return mono; + } + + // Resample to 16k via OfflineAudioContext for whisper-rs compatibility. + const targetLength = Math.max( + 1, + Math.round((mono.length * TARGET_SAMPLE_RATE) / audioBuffer.sampleRate) + ); + const offline = new OfflineAudioContext(1, targetLength, TARGET_SAMPLE_RATE); + const sourceBuffer = offline.createBuffer(1, mono.length, audioBuffer.sampleRate); + sourceBuffer.copyToChannel(mono, 0); + const source = offline.createBufferSource(); + source.buffer = sourceBuffer; + source.connect(offline.destination); + source.start(); + const rendered = await offline.startRendering(); + return rendered.getChannelData(0).slice(); +} + +async function convertBlobToWavBytes(blob: Blob): Promise { + const arrayBuffer = await blob.arrayBuffer(); + const audioContext = new AudioContext(); + try { + const decoded = await audioContext.decodeAudioData(arrayBuffer.slice(0)); + const mono16k = await toMono16k(decoded); + const wav = encodeWavMono16k(mono16k, TARGET_SAMPLE_RATE); + console.debug( + '[dictation] converted audio to wav bytes=%d sampleRate=%d', + wav.length, + TARGET_SAMPLE_RATE + ); + return Array.from(wav); + } finally { + await audioContext.close(); + } +} + +export function useDictation() { + const dispatch = useAppDispatch(); + const { status, hotkey } = useAppSelector(s => s.dictation); + const mediaRecorderRef = useRef(null); + const chunksRef = useRef([]); + const toggleRef = useRef<() => void>(() => {}); + const sessionIdRef = useRef(0); + const [isSupported, setIsSupported] = useState(false); + + useEffect(() => { + setIsSupported( + typeof navigator !== 'undefined' && + 'mediaDevices' in navigator && + 'getUserMedia' in navigator.mediaDevices + ); + }, []); + + const startRecording = useCallback(async () => { + if (status === 'recording') return; + const sessionId = sessionIdRef.current + 1; + sessionIdRef.current = sessionId; + dispatch(setStatus('recording')); + dispatch(setError(null)); + chunksRef.current = []; + + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') + ? 'audio/webm;codecs=opus' + : MediaRecorder.isTypeSupported('audio/webm') + ? 'audio/webm' + : 'audio/ogg'; + + const recorder = new MediaRecorder(stream, { mimeType }); + mediaRecorderRef.current = recorder; + + recorder.ondataavailable = e => { + if (e.data.size > 0) { + chunksRef.current.push(e.data); + } + }; + + recorder.onstop = async () => { + // Release mic + stream.getTracks().forEach(t => t.stop()); + if (sessionIdRef.current !== sessionId) { + console.debug('[dictation] ignoring stale onstop callback session=%d', sessionId); + return; + } + + const blob = new Blob(chunksRef.current, { type: mimeType }); + if (blob.size === 0) { + dispatch(setError('No audio recorded')); + return; + } + + dispatch(setStatus('transcribing')); + console.debug('[dictation] transcribing blob size=%d mimeType=%s', blob.size, mimeType); + + try { + let bytes: number[]; + let ext: string; + try { + bytes = await convertBlobToWavBytes(blob); + ext = 'wav'; + } catch (conversionErr) { + // Fallback for environments where decode/resample is unavailable. + console.warn( + '[dictation] wav conversion failed, falling back to raw blob path', + conversionErr + ); + const buffer = await blob.arrayBuffer(); + bytes = Array.from(new Uint8Array(buffer)); + const lowerMimeType = mimeType.toLowerCase(); + if (lowerMimeType.includes('ogg') || lowerMimeType.includes('webm')) { + ext = 'ogg'; + } else if (lowerMimeType.includes('wav')) { + ext = 'wav'; + } else if (lowerMimeType.includes('mpeg') || lowerMimeType.includes('mp3')) { + ext = 'mp3'; + } else if ( + lowerMimeType.includes('mp4') || + lowerMimeType.includes('m4a') || + lowerMimeType.includes('aac') + ) { + ext = 'm4a'; + } else if (lowerMimeType.includes('flac')) { + ext = 'flac'; + } else { + throw new Error( + `Unsupported audio container for dictation fallback path: ${mimeType || 'unknown'}` + ); + } + } + + console.debug( + '[dictation] calling voice_transcribe_bytes ext=%s bytes=%d', + ext, + bytes.length + ); + const response = await callCoreRpc< + { result?: TranscribeResult; logs?: string[] } | TranscribeResult + >({ + method: 'openhuman.voice_transcribe_bytes', + params: { audio_bytes: bytes, extension: ext, skip_cleanup: false }, + }); + if (sessionIdRef.current !== sessionId) { + console.debug( + '[dictation] ignoring stale transcription response session=%d', + sessionId + ); + return; + } + + const text = + response && typeof response === 'object' && 'result' in response + ? (response.result?.text?.trim() ?? '') + : ((response as TranscribeResult | null)?.text?.trim() ?? ''); + console.debug('[dictation] transcription result: %s', text || '(empty)'); + if (text) { + dispatch(setTranscript(text)); + } else { + dispatch(setError('No speech detected')); + } + } catch (err) { + if (sessionIdRef.current !== sessionId) { + console.debug('[dictation] ignoring stale transcription error session=%d', sessionId); + return; + } + const msg = err instanceof Error ? err.message : 'Transcription failed'; + console.error('[dictation] transcription error:', msg, err); + dispatch(setError(msg)); + } + }; + + console.debug('[dictation] starting MediaRecorder mimeType=%s', mimeType); + recorder.start(100); // collect data every 100 ms + } catch (err) { + const msg = err instanceof Error ? err.message : 'Microphone access denied'; + console.error('[dictation] getUserMedia error:', msg, err); + dispatch(setError(msg)); + } + }, [status, dispatch]); + + const stopRecording = useCallback(() => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') { + console.debug('[dictation] stopping MediaRecorder'); + mediaRecorderRef.current.stop(); + mediaRecorderRef.current = null; + } + }, []); + + const toggle = useCallback(() => { + console.debug('[dictation] toggle called status=%s', status); + if (status === 'recording') { + stopRecording(); + } else if (status === 'idle' || status === 'ready' || status === 'error') { + void startRecording(); + } + }, [status, startRecording, stopRecording]); + + const dismiss = useCallback(() => { + sessionIdRef.current += 1; + stopRecording(); + dispatch(resetDictation()); + }, [stopRecording, dispatch]); + + useEffect(() => { + toggleRef.current = toggle; + }, [toggle]); + + // Re-register persisted hotkey on startup / change. + useEffect(() => { + let disposed = false; + let retryTimer: ReturnType | null = null; + + void registerDictationHotkey(hotkey).catch(err => { + if (disposed) return; + console.warn('[dictation] auto register hotkey failed:', err); + retryTimer = setTimeout(() => { + if (disposed) return; + void registerDictationHotkey(hotkey).catch(retryErr => { + if (disposed) return; + console.warn('[dictation] hotkey retry failed:', retryErr); + }); + }, 2000); + }); + + return () => { + disposed = true; + if (retryTimer) { + clearTimeout(retryTimer); + } + }; + }, [hotkey]); + + // Listen for global hotkey event from Tauri + useEffect(() => { + let disposed = false; + let unlisten: (() => void) | null = null; + + listen('dictation://toggle', () => { + console.debug('[dictation] received dictation://toggle event'); + toggleRef.current(); + }) + .then(fn => { + if (disposed) { + fn(); + return; + } + unlisten = fn; + }) + .catch(err => { + console.warn('[dictation] failed to listen for toggle event:', err); + }); + + return () => { + disposed = true; + unlisten?.(); + }; + }, []); + + // In-app fallback hotkey handler when global registration is unavailable. + useEffect(() => { + const parsed = parseHotkey(hotkey); + if (!parsed) return; + + const onKeyDown = (event: KeyboardEvent) => { + if (event.repeat) return; + if (!matchesHotkeyEvent(event, parsed)) return; + // Ignore focused editable fields to avoid hijacking typing shortcuts. + const active = document.activeElement as HTMLElement | null; + const isEditable = + active instanceof HTMLInputElement || + active instanceof HTMLTextAreaElement || + !!active?.isContentEditable; + if (isEditable) return; + + console.debug('[dictation] in-app hotkey matched, toggling'); + event.preventDefault(); + toggleRef.current(); + }; + + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [hotkey]); + + return { status, isSupported, startRecording, stopRecording, toggle, dismiss }; +} diff --git a/app/src/components/intelligence/__tests__/MemoryWorkspace.test.tsx b/app/src/components/intelligence/__tests__/MemoryWorkspace.test.tsx index c0ed6a59c..e77e58e2d 100644 --- a/app/src/components/intelligence/__tests__/MemoryWorkspace.test.tsx +++ b/app/src/components/intelligence/__tests__/MemoryWorkspace.test.tsx @@ -71,12 +71,12 @@ describe('MemoryWorkspace', () => { renderWithProviders(); await waitFor(() => { - expect(screen.getByText('Alice')).toBeInTheDocument(); + expect(screen.getByText('Alice', { selector: 'span' })).toBeInTheDocument(); }); - expect(screen.getByText('AUTHORED')).toBeInTheDocument(); - expect(screen.getByText('Bob')).toBeInTheDocument(); - expect(screen.getByText('REVIEWED')).toBeInTheDocument(); + expect(screen.getByText('AUTHORED', { selector: 'span' })).toBeInTheDocument(); + expect(screen.getByText('Bob', { selector: 'span' })).toBeInTheDocument(); + expect(screen.getByText('REVIEWED', { selector: 'span' })).toBeInTheDocument(); // "Paper A" appears in both graph relations and documents list, // so just verify at least one instance is present expect(screen.getAllByText('Paper A').length).toBeGreaterThanOrEqual(1); diff --git a/app/src/components/settings/SettingsHome.tsx b/app/src/components/settings/SettingsHome.tsx index 90c401310..6483de617 100644 --- a/app/src/components/settings/SettingsHome.tsx +++ b/app/src/components/settings/SettingsHome.tsx @@ -281,6 +281,23 @@ const SettingsHome = () => { // onClick: handleViewEncryptionKey, // dangerous: false, // }, + { + id: 'dictation', + title: 'Voice Dictation', + description: 'Transcribe speech to text using local AI', + icon: ( + + + + ), + onClick: () => navigateToSettings('dictation'), + dangerous: false, + }, { id: 'local-model', title: 'Local AI Model', diff --git a/app/src/components/settings/hooks/useSettingsNavigation.ts b/app/src/components/settings/hooks/useSettingsNavigation.ts index f187c2b33..bf6e1aa22 100644 --- a/app/src/components/settings/hooks/useSettingsNavigation.ts +++ b/app/src/components/settings/hooks/useSettingsNavigation.ts @@ -21,7 +21,8 @@ export type SettingsRoute = | 'ai' | 'local-model' | 'tauri-commands' - | 'memory-debug'; + | 'memory-debug' + | 'dictation'; interface SettingsNavigationHook { currentRoute: SettingsRoute; @@ -62,6 +63,7 @@ export const useSettingsNavigation = (): SettingsNavigationHook => { if (path.includes('/settings/local-model')) return 'local-model'; if (path.includes('/settings/tauri-commands')) return 'tauri-commands'; if (path.includes('/settings/memory-debug')) return 'memory-debug'; + if (path.includes('/settings/dictation')) return 'dictation'; return 'home'; }; diff --git a/app/src/components/settings/panels/DictationPanel.tsx b/app/src/components/settings/panels/DictationPanel.tsx new file mode 100644 index 000000000..898e4ba8b --- /dev/null +++ b/app/src/components/settings/panels/DictationPanel.tsx @@ -0,0 +1,288 @@ +import { useEffect, useState } from 'react'; + +import { + checkDictationAvailability, + setHotkey, + setShowFloatingLauncher, +} from '../../../store/dictationSlice'; +import { useAppDispatch, useAppSelector } from '../../../store/hooks'; +import { isTauri, openhumanGetConfig, registerDictationHotkey } from '../../../utils/tauriCommands'; +import SettingsBackButton from '../components/SettingsBackButton'; +import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; + +const DictationPanel = () => { + const dispatch = useAppDispatch(); + const { navigateBack } = useSettingsNavigation(); + const { hotkey, showFloatingLauncher, voiceStatus, statusCheckError, isCheckingStatus } = + useAppSelector(s => s.dictation); + const [hotkeyInput, setHotkeyInput] = useState(hotkey); + const [isSavingHotkey, setIsSavingHotkey] = useState(false); + const [hotkeyError, setHotkeyError] = useState(null); + const [hotkeySuccess, setHotkeySuccess] = useState(false); + const [sttModelDirectory, setSttModelDirectory] = useState(null); + + useEffect(() => { + console.debug('[dictation-panel] mounting — checking availability'); + void dispatch(checkDictationAvailability()); + }, [dispatch]); + + useEffect(() => { + let disposed = false; + if (!isTauri()) return; + void openhumanGetConfig() + .then(response => { + if (disposed) return; + const workspaceDir = response.result.workspace_dir?.trim(); + if (!workspaceDir) return; + const separator = workspaceDir.includes('\\') ? '\\' : '/'; + const normalizedRoot = workspaceDir.replace(/[\\/]+$/, ''); + setSttModelDirectory([normalizedRoot, 'models', 'local-ai', 'stt'].join(separator)); + }) + .catch(err => { + console.debug('[dictation-panel] failed to resolve model directory from config', err); + }); + return () => { + disposed = true; + }; + }, []); + + // Keep local input in sync if hotkey changes externally + useEffect(() => { + setHotkeyInput(hotkey); + }, [hotkey]); + + const handleSaveHotkey = async () => { + const trimmed = hotkeyInput.trim(); + if (!trimmed) { + setHotkeyError('Hotkey cannot be empty'); + return; + } + + // Validate that the shortcut contains at least one recognized modifier and one key token. + const MODIFIERS = [ + 'cmdorctrl', + 'commandorcontrol', + 'ctrl', + 'control', + 'cmd', + 'command', + 'alt', + 'option', + 'shift', + 'super', + 'meta', + ]; + const tokens = trimmed + .split('+') + .map(t => t.trim().toLowerCase()) + .filter(Boolean); + const hasModifier = tokens.some(t => MODIFIERS.includes(t)); + const hasKey = tokens.some(t => !MODIFIERS.includes(t)); + if (!hasModifier || !hasKey) { + setHotkeyError( + 'Invalid format. Use e.g. CmdOrCtrl+Shift+D — must include a modifier and a key.' + ); + return; + } + + setIsSavingHotkey(true); + setHotkeyError(null); + setHotkeySuccess(false); + + console.debug('[dictation-panel] saving hotkey: %s', trimmed); + + try { + if (isTauri()) { + await registerDictationHotkey(trimmed); + } + dispatch(setHotkey(trimmed)); + setHotkeySuccess(true); + setTimeout(() => setHotkeySuccess(false), 2000); + } catch (err) { + const msg = err instanceof Error ? err.message : 'Failed to register hotkey'; + console.error('[dictation-panel] hotkey error:', msg); + setHotkeyError(msg); + } finally { + setIsSavingHotkey(false); + } + }; + + const statusLabel = () => { + if (isCheckingStatus) return 'Checking...'; + if (statusCheckError) return `Check failed: ${statusCheckError}`; + if (!voiceStatus) return 'Not checked'; + if (voiceStatus.stt_available) return 'Ready (model loaded)'; + if (voiceStatus.stt_model_path) return 'Model found — backend unavailable'; + return 'Model not found'; + }; + + const statusColor = () => { + if (isCheckingStatus) return 'bg-stone-500 animate-pulse'; + if (statusCheckError) return 'bg-amber-400'; + if (!voiceStatus) return 'bg-stone-500'; + if (voiceStatus.stt_available) return 'bg-green-400'; + if (voiceStatus.stt_model_path) return 'bg-amber-400'; + return 'bg-red-400'; + }; + + return ( +
+ + +
+
+

Voice Dictation

+

+ Transcribe speech to text using your microphone and local AI. +

+
+ + {/* STT Engine Status */} +
+
+
+

Speech-to-Text Engine

+

{statusLabel()}

+
+
+ +
+
+
+ + {/* Detailed status rows */} + {voiceStatus && ( +
+ + + + +
+ )} +
+ + {/* Model not found guidance */} + {voiceStatus && !voiceStatus.stt_model_path && !isCheckingStatus && ( +
+

+ Model file {voiceStatus.stt_model_id} was not + found. Go to Settings → Local AI Model to + download it, or place the file at{' '} + + {(sttModelDirectory ?? '/models/local-ai/stt') + + (sttModelDirectory?.includes('\\') ? '\\' : '/') + + voiceStatus.stt_model_id} + +

+
+ )} + + {/* Global Hotkey */} +
+
+

Global Hotkey

+

+ Press anywhere to start / stop dictation +

+
+
+ setHotkeyInput(e.target.value)} + placeholder="e.g. CmdOrCtrl+Shift+D" + className="flex-1 bg-stone-700/60 border border-stone-600/50 rounded-lg px-3 py-2 text-sm text-white placeholder-stone-500 focus:outline-none focus:ring-1 focus:ring-primary-500" + /> + +
+ {hotkeyError &&

{hotkeyError}

} +

+ Modifiers: CmdOrCtrl, Alt, Shift,{' '} + Super (also accepts CommandOrControl) +

+
+ + {/* Floating launcher preference */} +
+
+
+

Always show floating Start button

+

+ If disabled, dictation starts via hotkey only while idle. +

+
+ +
+
+ + {/* How to use */} +
+

How to use

+
    +
  1. Press the global hotkey (or click Record in the overlay)
    2. +
  2. Speak clearly into your microphone
    3. +
  3. Press the hotkey again (or click Stop) to finish
    4. +
  4. Wait a moment for transcription
    5. +
  5. Click Insert to type into the focused field, or Copy to clipboard
    6. +
+
+
+
+ ); +}; + +interface StatusRowProps { + label: string; + value: string; + ok: boolean; + muted?: boolean; +} + +const StatusRow = ({ label, value, ok, muted }: StatusRowProps) => ( +
+ + {label} + {value} +
+); + +export default DictationPanel; diff --git a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx index ecb7c2fde..d7f41f427 100644 --- a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx +++ b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx @@ -127,7 +127,8 @@ const ScreenIntelligencePanel = () => { const startDisabled = isStartingSession || isLoading || - !status?.platform_supported || + !status || + !status.platform_supported || status.session.active || status.permissions.accessibility !== 'granted'; const stopDisabled = isStoppingSession || !status?.session.active; @@ -436,7 +437,7 @@ const ScreenIntelligencePanel = () => { - {!status?.platform_supported && ( + {status !== null && !status.platform_supported && (
Screen Intelligence V1 is currently supported on macOS only.
diff --git a/app/src/pages/Conversations.tsx b/app/src/pages/Conversations.tsx index fe3a65d20..a8743875e 100644 --- a/app/src/pages/Conversations.tsx +++ b/app/src/pages/Conversations.tsx @@ -138,6 +138,7 @@ const Conversations = () => { const [isLoadingBudget, setIsLoadingBudget] = useState(false); const messagesEndRef = useRef(null); + const textInputRef = useRef(null); const mediaRecorderRef = useRef(null); const mediaStreamRef = useRef(null); const audioChunksRef = useRef([]); @@ -222,6 +223,30 @@ const Conversations = () => { } }, [messages]); + useEffect(() => { + const onDictationInsert = (event: Event) => { + const customEvent = event as CustomEvent<{ text?: string }>; + const text = customEvent.detail?.text?.trim(); + if (!text) return; + + customEvent.preventDefault(); + setInputMode('text'); + setInputValue(prev => { + const base = prev.trim(); + if (!base) return text; + return `${base}${base.endsWith(' ') ? '' : ' '}${text}`; + }); + + window.requestAnimationFrame(() => { + textInputRef.current?.focus(); + }); + }; + + window.addEventListener('dictation://insert-text', onDictationInsert as EventListener); + return () => + window.removeEventListener('dictation://insert-text', onDictationInsert as EventListener); + }, []); + useEffect(() => { if (sendError && inputValue.length > 0) { setSendError(null); @@ -1312,6 +1337,7 @@ const Conversations = () => { {inlineCompletionSuffix}