diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md deleted file mode 100644 index 1620cac09..000000000 --- a/PR_DESCRIPTION.md +++ /dev/null @@ -1,102 +0,0 @@ -## Summary - -- Fix stacked socket listeners causing duplicate AI message bubbles on reconnect -- Route chat through local Ollama model when active — zero cloud tokens on the local path -- Multi-bubble delivery: assistant replies split into 2–5 natural chat bubbles with typing pauses -- Contextual emoji guidance in system prompts — replaces overuse with intentional, human-like usage -- 402 unit tests passing - -## Problem - -**Duplicate messages** — `subscribeChatEvents` was declared `async` with no actual `await` inside. The cleanup function returned by `.then()` lands in a microtask; React's synchronous cleanup had already run by then, so socket listeners were never removed. Each reconnect stacked another listener set, causing `chat:done` to fire N times and produce N copies of every reply. - -**Responses felt like AI slop** — replies arrived as a single wall of text. No typing feel, no natural pacing. - -**Ollama was wired for utilities but not chat** — `local_ai_status`, `suggest_questions`, TTS, and Whisper all used Ollama already, but the main conversation loop always hit the cloud socket regardless of whether a local model was running. - -**Generic emoji overuse** — the SOUL prompt said "Enthusiastic / Witty / Genuinely human" and the only emoji rule was "Minimal — match user's style". The model interpreted this as license to add 😄🔥✨ on every message. - -## Solution - -### 1 — Socket listener race condition - -`subscribeChatEvents` no longer uses `async` — the cleanup fn is returned synchronously so React can always call it. The `useEffect` stores it with `const cleanup = subscribeChatEvents(...)` and returns it directly. - -### 2 — Rust: `openhuman.local_ai_chat` RPC - -- `ollama_api.rs` — `OllamaChatMessage` / `OllamaChatRequest` / `OllamaChatResponse` for `/api/chat` -- `service/public_infer.rs` — `LocalAiService::chat_with_history()`: sends full message history to Ollama, updates latency/TPS status -- `ops.rs` — `local_ai_chat` async op -- `schemas.rs` — registered controller: schema, handler, param deserialization - -### 3 — Frontend: local chat gate + multi-bubble delivery - -When `isLocalModelActive` is true, `handleSendMessage` takes the local path: - -``` -User sends message - │ - ├── isLocalModelActive = true - │ openhumanLocalAiChat(history) ──► Ollama /api/chat - │ deliverLocalResponse() (no socket, zero cloud tokens) - │ segmentMessage() → 2–5 bubbles - │ dispatch each bubble with typing pause (500–1 400 ms) - │ - └── isLocalModelActive = false - chatSend() ──► socket chat:start ──► backend ──► cloud API - (unchanged) -``` - -Socket-connected guard is skipped on the local path so offline use works. - -### 4 — Emoji rules in system prompts - -`SOUL.md` adds an explicit **Emoji Rules** section: - -- Hard cap: one emoji max per message (none is always fine) -- Contextual, not decorative — must reinforce the specific content -- Never as a sentence opener; only at the end of a clause -- Skip entirely in errors, warnings, technical content, lists, or replies > 3 sentences -- Mirror the user's own emoji usage -- Concrete bad/good examples for model calibration - -`BOOTSTRAP.md` Communication Preferences entry updated to reference these rules. - -## Files Changed - -| File | Change | -|---|---| -| `app/src/services/chatService.ts` | Remove `async` from `subscribeChatEvents`; synchronous cleanup | -| `app/src/pages/Conversations.tsx` | Local chat gate, `deliverLocalResponse`, updated socket effect | -| `app/src/hooks/useLocalModelStatus.ts` | Polls `local_ai_status` every 12 s; returns `true` when `state === "ready"` | -| `app/src/utils/messageSegmentation.ts` | `segmentMessage()` + `getSegmentDelay()` | -| `app/src/utils/tauriCommands.ts` | `openhumanLocalAiChat()` RPC wrapper | -| `app/src/store/threadSlice.ts` | `addReaction` reducer; `activeThreadId` tracking | -| `src/openhuman/local_ai/ollama_api.rs` | Ollama `/api/chat` types | -| `src/openhuman/local_ai/service/public_infer.rs` | `chat_with_history()` method | -| `src/openhuman/local_ai/ops.rs` | `local_ai_chat` op | -| `src/openhuman/local_ai/schemas.rs` | Controller registration | -| `src/openhuman/agent/prompts/SOUL.md` | Emoji Rules section | -| `src/openhuman/agent/prompts/BOOTSTRAP.md` | Updated emoji preference line | - -## Submission Checklist - -- [x] **Bug fix** — duplicate message root cause identified and fixed -- [x] **Local-only gate** — cloud path entirely unchanged; no increase in billed token usage -- [x] **Rust compiles** — `cargo check --manifest-path Cargo.toml` clean -- [x] **TypeScript** — `tsc --noEmit` 0 errors -- [x] **Tests** — 402 passing (`yarn test`); new files: `messageSegmentation.test.ts` (14 tests), `localChatGating.test.ts` (9 tests) -- [x] **Commits** — 4 atomic commits, one per concern - -## Test validation - -```bash -cargo check --manifest-path Cargo.toml -cd app && yarn test -yarn tsc --noEmit -``` - -## Related - -- Follow-up: server-side reaction sync for multi-user channel modes -- Follow-up: streaming Ollama responses (currently non-streaming for simplicity) diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 22d5c3a4c..b4af2bb3c 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "OpenHuman" -version = "0.50.3" +version = "0.51.2" dependencies = [ "env_logger", "log", @@ -12,6 +12,7 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-deep-link", + "tauri-plugin-global-shortcut", "tauri-plugin-opener", "tokio", ] @@ -1360,6 +1361,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix", + "windows-link 0.2.1", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1492,6 +1503,24 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "global-hotkey" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9247516746aa8e53411a0db9b62b0e24efbcf6a76e0ba73e5a91b512ddabed7" +dependencies = [ + "crossbeam-channel", + "keyboard-types", + "objc2", + "objc2-app-kit", + "once_cell", + "serde", + "thiserror 2.0.18", + "windows-sys 0.59.0", + "x11rb", + "xkeysym", +] + [[package]] name = "gobject-sys" version = "0.18.0" @@ -3969,6 +3998,21 @@ dependencies = [ "windows-result 0.3.4", ] +[[package]] +name = "tauri-plugin-global-shortcut" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "424af23c7e88d05e4a1a6fc2c7be077912f8c76bd7900fd50aa2b7cbf5a2c405" +dependencies = [ + "global-hotkey", + "log", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "thiserror 2.0.18", +] + [[package]] name = "tauri-plugin-opener" version = "2.5.3" @@ -5506,6 +5550,29 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "yoke" version = "0.8.1" diff --git a/app/src-tauri/Cargo.toml b/app/src-tauri/Cargo.toml index e6600e128..5edca64b8 100644 --- a/app/src-tauri/Cargo.toml +++ b/app/src-tauri/Cargo.toml @@ -28,6 +28,7 @@ serde_json = "1" # Tauri core and plugins tauri = { version = "2.10", features = ["macos-private-api", "tray-icon"] } tauri-plugin-deep-link = "2.0.0" +tauri-plugin-global-shortcut = "2" tauri-plugin-opener = "2" serde_json = "1" tokio = { version = "1", features = ["rt-multi-thread", "process", "sync", "time", "net"] } diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 780633926..b04a2268d 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -3,15 +3,49 @@ compile_error!("src-tauri host is desktop-only. Non-desktop targets are not supp mod core_process; +use std::sync::Mutex; + use tauri::{ menu::{Menu, MenuItem}, tray::{MouseButton, MouseButtonState, TrayIconBuilder, TrayIconEvent}, - AppHandle, Manager, RunEvent, + AppHandle, Emitter, Manager, RunEvent, }; +use tauri_plugin_global_shortcut::{GlobalShortcutExt, ShortcutState}; #[cfg(any(windows, target_os = "linux"))] use tauri_plugin_deep_link::DeepLinkExt; +/// Tracks the currently registered dictation hotkey string so we can unregister it later. +struct DictationHotkeyState(Mutex>); + +fn expand_dictation_shortcuts(shortcut: &str) -> Vec { + let trimmed = shortcut.trim(); + if trimmed.is_empty() { + return vec![]; + } + + #[cfg(target_os = "macos")] + { + if trimmed.contains("CmdOrCtrl") { + let cmd_variant = trimmed.replace("CmdOrCtrl", "Cmd"); + let ctrl_variant = trimmed.replace("CmdOrCtrl", "Ctrl"); + if cmd_variant == ctrl_variant { + return vec![cmd_variant]; + } + return vec![cmd_variant, ctrl_variant]; + } + } + + #[cfg(not(target_os = "macos"))] + { + if trimmed.contains("CmdOrCtrl") { + return vec![trimmed.replace("CmdOrCtrl", "Ctrl")]; + } + } + + vec![trimmed.to_string()] +} + #[tauri::command] fn core_rpc_url() -> String { std::env::var("OPENHUMAN_CORE_RPC_URL") @@ -106,6 +140,122 @@ async fn restart_core_process( state.inner().restart().await } +/// Register (or re-register) the global dictation toggle hotkey. +/// Emits `dictation://toggle` to all webviews when the shortcut is pressed. +#[tauri::command] +async fn register_dictation_hotkey( + app: AppHandle, + shortcut: String, +) -> Result<(), String> { + log::info!("[dictation] register_dictation_hotkey: shortcut={shortcut}"); + + let old_shortcuts = { + let state = app.state::(); + let guard = state.0.lock().unwrap(); + guard.clone() + }; + + let expanded_shortcuts = expand_dictation_shortcuts(&shortcut); + if expanded_shortcuts.is_empty() { + return Err("Shortcut cannot be empty".to_string()); + } + log::info!( + "[dictation] expanded shortcuts: {}", + expanded_shortcuts.join(", ") + ); + + let register_shortcut = |shortcut_variant: &str| -> Result<(), String> { + let app_clone = app.clone(); + app.global_shortcut() + .on_shortcut(shortcut_variant, move |_app, _sc, event| { + if event.state == ShortcutState::Pressed { + log::debug!("[dictation] hotkey pressed — emitting dictation://toggle"); + if let Err(e) = app_clone.emit("dictation://toggle", ()) { + log::warn!("[dictation] emit failed: {e}"); + } + } + }) + .map_err(|e| format!("Failed to register shortcut '{shortcut_variant}': {e}")) + }; + + let mut unregistered_old: Vec = Vec::new(); + for old in &old_shortcuts { + log::debug!("[dictation] unregistering previous shortcut: {old}"); + if let Err(e) = app.global_shortcut().unregister(old.as_str()) { + for restored in &unregistered_old { + if let Err(restore_err) = register_shortcut(restored.as_str()) { + log::warn!( + "[dictation] rollback failed while restoring old shortcut '{restored}': {restore_err}" + ); + } + } + return Err(format!("Failed to unregister previous shortcut '{old}': {e}")); + } + unregistered_old.push(old.clone()); + } + + let mut newly_registered: Vec = Vec::new(); + for shortcut_variant in &expanded_shortcuts { + if let Err(err) = register_shortcut(shortcut_variant.as_str()) { + log::error!("[dictation] failed to register shortcut '{shortcut_variant}': {err}"); + for registered in &newly_registered { + if let Err(unregister_err) = app.global_shortcut().unregister(registered.as_str()) { + log::warn!( + "[dictation] rollback failed while unregistering '{registered}': {unregister_err}" + ); + } + } + for old in &old_shortcuts { + if let Err(restore_err) = register_shortcut(old.as_str()) { + log::warn!( + "[dictation] rollback failed while restoring old shortcut '{old}': {restore_err}" + ); + } + } + return Err(err); + } + newly_registered.push(shortcut_variant.clone()); + } + + // Persist all newly registered shortcuts. + { + let state = app.state::(); + let mut guard = state.0.lock().unwrap(); + *guard = expanded_shortcuts.clone(); + } + + log::info!( + "[dictation] shortcuts registered: {}", + expanded_shortcuts.join(", ") + ); + Ok(()) +} + +/// Unregister the global dictation hotkey (if any). +#[tauri::command] +async fn unregister_dictation_hotkey(app: AppHandle) -> Result<(), String> { + log::info!("[dictation] unregister_dictation_hotkey: called"); + let state = app.state::(); + let mut guard = state.0.lock().unwrap(); + if guard.is_empty() { + log::debug!("[dictation] no shortcut registered — nothing to unregister"); + } else { + let old_shortcuts = guard.clone(); + guard.clear(); + for old in old_shortcuts { + log::debug!("[dictation] unregistering shortcut: {old}"); + app.global_shortcut() + .unregister(old.as_str()) + .map_err(|e| { + log::warn!("[dictation] failed to unregister '{old}': {e}"); + format!("Failed to unregister shortcut '{old}': {e}") + })?; + log::info!("[dictation] shortcut unregistered: {old}"); + } + } + Ok(()) +} + fn is_daemon_mode() -> bool { std::env::args().any(|arg| arg == "daemon" || arg == "--daemon") } @@ -186,6 +336,8 @@ pub fn run() { tauri::Builder::default() .plugin(tauri_plugin_opener::init()) .plugin(tauri_plugin_deep_link::init()) + .plugin(tauri_plugin_global_shortcut::Builder::new().build()) + .manage(DictationHotkeyState(Mutex::new(Vec::new()))) .setup(move |app| { #[cfg(any(windows, target_os = "linux"))] { @@ -235,7 +387,9 @@ pub fn run() { service_start_direct, service_stop_direct, service_status_direct, - service_uninstall_direct + service_uninstall_direct, + register_dictation_hotkey, + unregister_dictation_hotkey ]) .build(tauri::generate_context!()) .expect("error while building tauri application") diff --git a/app/src/App.tsx b/app/src/App.tsx index ea6cf4278..2eb7bdf21 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -5,6 +5,7 @@ import { PersistGate } from 'redux-persist/integration/react'; import AppRoutes from './AppRoutes'; import ServiceBlockingGate from './components/daemon/ServiceBlockingGate'; +import DictationOverlay from './components/dictation/DictationOverlay'; import ErrorFallbackScreen from './components/ErrorFallbackScreen'; import LocalAIDownloadSnackbar from './components/LocalAIDownloadSnackbar'; import MiniSidebar from './components/MiniSidebar'; @@ -56,6 +57,7 @@ function App() { + diff --git a/app/src/components/dictation/DictationOverlay.tsx b/app/src/components/dictation/DictationOverlay.tsx new file mode 100644 index 000000000..75db55c89 --- /dev/null +++ b/app/src/components/dictation/DictationOverlay.tsx @@ -0,0 +1,373 @@ +import { useEffect, useRef, useState } from 'react'; + +import { resetDictation } from '../../store/dictationSlice'; +import { useAppDispatch, useAppSelector } from '../../store/hooks'; +import { openhumanAccessibilityInputAction } from '../../utils/tauriCommands'; +import { useDictation } from './useDictation'; + +const STATUS_COLORS: Record = { + idle: 'bg-stone-800', + recording: 'bg-red-600 animate-pulse', + transcribing: 'bg-amber-600', + ready: 'bg-primary-600', + error: 'bg-coral-600', +}; + +const STATUS_LABELS: Record = { + idle: 'Idle', + recording: 'Recording...', + transcribing: 'Transcribing...', + ready: 'Ready', + error: 'Error', +}; + +type EditableTarget = HTMLInputElement | HTMLTextAreaElement | HTMLElement; + +const isTextInput = (el: Element): el is HTMLInputElement | HTMLTextAreaElement => + el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement; + +const isEditableElement = (el: Element | null): el is EditableTarget => + !!el && (isTextInput(el) || (el instanceof HTMLElement && el.isContentEditable)); + +const insertIntoEditableTarget = (target: EditableTarget, text: string): boolean => { + if (isTextInput(target)) { + target.focus(); + const hasSelection = + typeof target.selectionStart === 'number' && typeof target.selectionEnd === 'number'; + if (hasSelection) { + target.setRangeText(text, target.selectionStart ?? 0, target.selectionEnd ?? 0, 'end'); + } else { + target.value = `${target.value}${text}`; + } + target.dispatchEvent(new Event('input', { bubbles: true })); + return true; + } + + if (target instanceof HTMLElement && target.isContentEditable) { + target.focus(); + const selection = window.getSelection(); + if (selection && selection.rangeCount > 0) { + const range = selection.getRangeAt(0); + range.deleteContents(); + range.insertNode(document.createTextNode(text)); + range.collapse(false); + selection.removeAllRanges(); + selection.addRange(range); + } else { + target.append(document.createTextNode(text)); + } + target.dispatchEvent(new Event('input', { bubbles: true })); + return true; + } + + return false; +}; + +const DictationOverlay = () => { + const dispatch = useAppDispatch(); + const { status, transcript, error, hotkey, showFloatingLauncher } = useAppSelector( + s => s.dictation + ); + const { startRecording, stopRecording, dismiss } = useDictation(); + const panelRef = useRef(null); + const dragRef = useRef<{ pointerId: number; offsetX: number; offsetY: number } | null>(null); + const lastEditableRef = useRef(null); + const previousStatusRef = useRef(status); + + const getActiveDefaultPosition = () => { + if (typeof window === 'undefined') return { x: 24, y: 24 }; + return { + x: Math.max(12, Math.round(window.innerWidth / 2 - 200)), + y: Math.max(12, window.innerHeight - 240), + }; + }; + + const getIdleDefaultPosition = () => { + if (typeof window === 'undefined') return { x: 24, y: 24 }; + const idleWidth = 250; + const idleHeight = 62; + return { + x: Math.max(12, window.innerWidth - idleWidth - 24), + y: Math.max(12, window.innerHeight - idleHeight - 24), + }; + }; + + const [position, setPosition] = useState<{ x: number; y: number }>(() => + status === 'idle' ? getIdleDefaultPosition() : getActiveDefaultPosition() + ); + + const resetLauncherPosition = () => { + setPosition(getIdleDefaultPosition()); + }; + + const clampToViewport = (x: number, y: number) => { + const panelWidth = panelRef.current?.offsetWidth ?? 360; + const panelHeight = panelRef.current?.offsetHeight ?? 220; + const maxX = Math.max(12, window.innerWidth - panelWidth - 12); + const maxY = Math.max(12, window.innerHeight - panelHeight - 12); + return { x: Math.max(12, Math.min(x, maxX)), y: Math.max(12, Math.min(y, maxY)) }; + }; + + // Keyboard shortcut: Escape to dismiss + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (e.key === 'Escape') { + console.debug('[dictation] Escape pressed — dismissing overlay'); + dismiss(); + } + }; + window.addEventListener('keydown', handler); + return () => window.removeEventListener('keydown', handler); + }, [dismiss]); + + useEffect(() => { + const onFocusIn = (event: FocusEvent) => { + const target = event.target as Element | null; + if (!isEditableElement(target)) return; + // Ignore overlay-owned elements so we keep the app context target. + if (panelRef.current?.contains(target)) return; + lastEditableRef.current = target; + }; + window.addEventListener('focusin', onFocusIn); + return () => window.removeEventListener('focusin', onFocusIn); + }, []); + + useEffect(() => { + const onResize = () => { + setPosition(prev => { + const base = + prev ?? (status === 'idle' ? getIdleDefaultPosition() : getActiveDefaultPosition()); + return clampToViewport(base.x, base.y); + }); + }; + window.addEventListener('resize', onResize); + return () => window.removeEventListener('resize', onResize); + }, [status]); + + useEffect(() => { + const previousStatus = previousStatusRef.current; + previousStatusRef.current = status; + if (previousStatus === 'idle' && status !== 'idle') { + setPosition(getActiveDefaultPosition()); + } + }, [status]); + + const handleDragStart = (e: React.PointerEvent) => { + if (e.button !== 0) return; + if ((e.target as HTMLElement).closest('button')) return; + if (!panelRef.current) return; + const rect = panelRef.current.getBoundingClientRect(); + dragRef.current = { + pointerId: e.pointerId, + offsetX: e.clientX - rect.left, + offsetY: e.clientY - rect.top, + }; + e.currentTarget.setPointerCapture(e.pointerId); + setPosition({ x: rect.left, y: rect.top }); + }; + + const handleDragMove = (e: React.PointerEvent) => { + const drag = dragRef.current; + if (!drag || drag.pointerId !== e.pointerId) return; + e.preventDefault(); + const next = clampToViewport(e.clientX - drag.offsetX, e.clientY - drag.offsetY); + setPosition(next); + }; + + const handleDragEnd = (e: React.PointerEvent) => { + const drag = dragRef.current; + if (!drag || drag.pointerId !== e.pointerId) return; + dragRef.current = null; + if (e.currentTarget.hasPointerCapture(e.pointerId)) { + e.currentTarget.releasePointerCapture(e.pointerId); + } + }; + + const handleInsert = async () => { + if (!transcript) return; + console.debug('[dictation] inserting transcript into active context'); + const insertEvent = new CustomEvent<{ text: string }>('dictation://insert-text', { + detail: { text: transcript }, + cancelable: true, + }); + window.dispatchEvent(insertEvent); + if (insertEvent.defaultPrevented) { + console.debug('[dictation] transcript handled by app context listener'); + resetLauncherPosition(); + dispatch(resetDictation()); + return; + } + + const active = document.activeElement; + const preferredTarget = + (isEditableElement(active) && !panelRef.current?.contains(active) ? active : null) ?? + lastEditableRef.current; + + if (preferredTarget && insertIntoEditableTarget(preferredTarget, transcript)) { + console.debug('[dictation] inserted transcript into DOM editable target'); + resetLauncherPosition(); + dispatch(resetDictation()); + return; + } + + console.debug('[dictation] no DOM target found, trying accessibility action'); + try { + const response = await openhumanAccessibilityInputAction({ + action: 'type', + text: transcript, + }); + const accepted = response.result.accepted === true; + const blocked = response.result.blocked === true; + if (accepted && !blocked) { + resetLauncherPosition(); + dispatch(resetDictation()); + return; + } + + console.debug( + '[dictation] accessibility insert not accepted (accepted=%s blocked=%s), falling back to clipboard', + accepted, + blocked + ); + await navigator.clipboard.writeText(transcript).catch(() => {}); + resetLauncherPosition(); + dispatch(resetDictation()); + } catch { + // Fallback to clipboard + console.debug('[dictation] accessibility insert failed, falling back to clipboard'); + await navigator.clipboard.writeText(transcript).catch(() => {}); + resetLauncherPosition(); + dispatch(resetDictation()); + } + }; + + const handleCopy = async () => { + if (!transcript) return; + console.debug('[dictation] copying transcript to clipboard'); + await navigator.clipboard.writeText(transcript).catch(() => {}); + resetLauncherPosition(); + dispatch(resetDictation()); + }; + + if (status === 'idle') { + if (!showFloatingLauncher) return null; + return ( +
+
+
+ ⋮⋮ +
+ +
+
+ ); + } + + return ( +
+
+ {/* Header */} +
+
+ + {STATUS_LABELS[status] ?? 'Dictation'} + +
+ +
+ + {/* Transcript */} + {transcript && ( +
+

{transcript}

+
+ )} + + {/* Error */} + {error && ( +
+

{error}

+
+ )} + + {/* Controls */} +
+ {(status === 'error' || status === 'ready') && ( + + )} + + {status === 'recording' && ( + + )} + + {transcript && ( + <> + + + + )} +
+ + {/* Hotkey hint */} +
+ {hotkey} to toggle · Esc to dismiss +
+
+
+ ); +}; + +export default DictationOverlay; diff --git a/app/src/components/dictation/useDictation.ts b/app/src/components/dictation/useDictation.ts new file mode 100644 index 000000000..3a032fc0b --- /dev/null +++ b/app/src/components/dictation/useDictation.ts @@ -0,0 +1,430 @@ +import { listen } from '@tauri-apps/api/event'; +import { useCallback, useEffect, useRef, useState } from 'react'; + +import { callCoreRpc } from '../../services/coreRpcClient'; +import { resetDictation, setError, setStatus, setTranscript } from '../../store/dictationSlice'; +import { useAppDispatch, useAppSelector } from '../../store/hooks'; +import { registerDictationHotkey } from '../../utils/tauriCommands'; + +const TARGET_SAMPLE_RATE = 16000; + +interface TranscribeResult { + text: string; + raw_text: string; + model_id: string; +} + +interface ParsedHotkey { + key: string; + cmdOrCtrl: boolean; + cmd: boolean; + ctrl: boolean; + shift: boolean; + alt: boolean; + super: boolean; +} + +function parseHotkey(shortcut: string): ParsedHotkey | null { + const tokens = shortcut + .split('+') + .map(t => t.trim().toLowerCase()) + .filter(Boolean); + if (tokens.length === 0) return null; + + const parsed: ParsedHotkey = { + key: '', + cmdOrCtrl: false, + cmd: false, + ctrl: false, + shift: false, + alt: false, + super: false, + }; + + for (const token of tokens) { + switch (token) { + case 'cmdorctrl': + case 'commandorcontrol': + parsed.cmdOrCtrl = true; + break; + case 'cmd': + case 'command': + case 'meta': + parsed.cmd = true; + break; + case 'ctrl': + case 'control': + parsed.ctrl = true; + break; + case 'shift': + parsed.shift = true; + break; + case 'alt': + case 'option': + parsed.alt = true; + break; + case 'super': + parsed.super = true; + break; + default: + parsed.key = token; + break; + } + } + + return parsed.key ? parsed : null; +} + +function matchesHotkeyEvent(event: KeyboardEvent, parsed: ParsedHotkey): boolean { + const key = event.key.toLowerCase(); + if (key !== parsed.key.toLowerCase()) return false; + if (parsed.shift !== event.shiftKey) return false; + if (parsed.alt !== event.altKey) return false; + + if (parsed.cmdOrCtrl && !(event.metaKey || event.ctrlKey)) return false; + if (parsed.cmd && !event.metaKey) return false; + if (parsed.ctrl && !event.ctrlKey) return false; + if (parsed.super && !event.metaKey) return false; + + return true; +} + +function floatTo16BitPCM(output: DataView, offset: number, input: Float32Array) { + for (let i = 0; i < input.length; i += 1, offset += 2) { + const sample = Math.max(-1, Math.min(1, input[i])); + output.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); + } +} + +function encodeWavMono16k(samples: Float32Array, sampleRate: number): Uint8Array { + const bytesPerSample = 2; + const blockAlign = bytesPerSample; + const byteRate = sampleRate * blockAlign; + const dataSize = samples.length * bytesPerSample; + const buffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(buffer); + + const writeString = (offset: number, value: string) => { + for (let i = 0; i < value.length; i += 1) { + view.setUint8(offset + i, value.charCodeAt(i)); + } + }; + + writeString(0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeString(8, 'WAVE'); + writeString(12, 'fmt '); + view.setUint32(16, 16, true); // PCM header size + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // mono + view.setUint32(24, sampleRate, true); + view.setUint32(28, byteRate, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, 16, true); // 16-bit + writeString(36, 'data'); + view.setUint32(40, dataSize, true); + floatTo16BitPCM(view, 44, samples); + + return new Uint8Array(buffer); +} + +async function toMono16k(audioBuffer: AudioBuffer): Promise { + const channels = audioBuffer.numberOfChannels; + const inputLength = audioBuffer.length; + + // Downmix to mono by averaging channels. + const mono = new Float32Array(inputLength); + for (let c = 0; c < channels; c += 1) { + const channelData = audioBuffer.getChannelData(c); + for (let i = 0; i < inputLength; i += 1) { + mono[i] += channelData[i] / channels; + } + } + + if (audioBuffer.sampleRate === TARGET_SAMPLE_RATE) { + return mono; + } + + // Resample to 16k via OfflineAudioContext for whisper-rs compatibility. + const targetLength = Math.max( + 1, + Math.round((mono.length * TARGET_SAMPLE_RATE) / audioBuffer.sampleRate) + ); + const offline = new OfflineAudioContext(1, targetLength, TARGET_SAMPLE_RATE); + const sourceBuffer = offline.createBuffer(1, mono.length, audioBuffer.sampleRate); + sourceBuffer.copyToChannel(mono, 0); + const source = offline.createBufferSource(); + source.buffer = sourceBuffer; + source.connect(offline.destination); + source.start(); + const rendered = await offline.startRendering(); + return rendered.getChannelData(0).slice(); +} + +async function convertBlobToWavBytes(blob: Blob): Promise { + const arrayBuffer = await blob.arrayBuffer(); + const audioContext = new AudioContext(); + try { + const decoded = await audioContext.decodeAudioData(arrayBuffer.slice(0)); + const mono16k = await toMono16k(decoded); + const wav = encodeWavMono16k(mono16k, TARGET_SAMPLE_RATE); + console.debug( + '[dictation] converted audio to wav bytes=%d sampleRate=%d', + wav.length, + TARGET_SAMPLE_RATE + ); + return Array.from(wav); + } finally { + await audioContext.close(); + } +} + +export function useDictation() { + const dispatch = useAppDispatch(); + const { status, hotkey } = useAppSelector(s => s.dictation); + const mediaRecorderRef = useRef(null); + const chunksRef = useRef([]); + const toggleRef = useRef<() => void>(() => {}); + const sessionIdRef = useRef(0); + const [isSupported, setIsSupported] = useState(false); + + useEffect(() => { + setIsSupported( + typeof navigator !== 'undefined' && + 'mediaDevices' in navigator && + 'getUserMedia' in navigator.mediaDevices + ); + }, []); + + const startRecording = useCallback(async () => { + if (status === 'recording') return; + const sessionId = sessionIdRef.current + 1; + sessionIdRef.current = sessionId; + dispatch(setStatus('recording')); + dispatch(setError(null)); + chunksRef.current = []; + + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') + ? 'audio/webm;codecs=opus' + : MediaRecorder.isTypeSupported('audio/webm') + ? 'audio/webm' + : 'audio/ogg'; + + const recorder = new MediaRecorder(stream, { mimeType }); + mediaRecorderRef.current = recorder; + + recorder.ondataavailable = e => { + if (e.data.size > 0) { + chunksRef.current.push(e.data); + } + }; + + recorder.onstop = async () => { + // Release mic + stream.getTracks().forEach(t => t.stop()); + if (sessionIdRef.current !== sessionId) { + console.debug('[dictation] ignoring stale onstop callback session=%d', sessionId); + return; + } + + const blob = new Blob(chunksRef.current, { type: mimeType }); + if (blob.size === 0) { + dispatch(setError('No audio recorded')); + return; + } + + dispatch(setStatus('transcribing')); + console.debug('[dictation] transcribing blob size=%d mimeType=%s', blob.size, mimeType); + + try { + let bytes: number[]; + let ext: string; + try { + bytes = await convertBlobToWavBytes(blob); + ext = 'wav'; + } catch (conversionErr) { + // Fallback for environments where decode/resample is unavailable. + console.warn( + '[dictation] wav conversion failed, falling back to raw blob path', + conversionErr + ); + const buffer = await blob.arrayBuffer(); + bytes = Array.from(new Uint8Array(buffer)); + const lowerMimeType = mimeType.toLowerCase(); + if (lowerMimeType.includes('ogg') || lowerMimeType.includes('webm')) { + ext = 'ogg'; + } else if (lowerMimeType.includes('wav')) { + ext = 'wav'; + } else if (lowerMimeType.includes('mpeg') || lowerMimeType.includes('mp3')) { + ext = 'mp3'; + } else if ( + lowerMimeType.includes('mp4') || + lowerMimeType.includes('m4a') || + lowerMimeType.includes('aac') + ) { + ext = 'm4a'; + } else if (lowerMimeType.includes('flac')) { + ext = 'flac'; + } else { + throw new Error( + `Unsupported audio container for dictation fallback path: ${mimeType || 'unknown'}` + ); + } + } + + console.debug( + '[dictation] calling voice_transcribe_bytes ext=%s bytes=%d', + ext, + bytes.length + ); + const response = await callCoreRpc< + { result?: TranscribeResult; logs?: string[] } | TranscribeResult + >({ + method: 'openhuman.voice_transcribe_bytes', + params: { audio_bytes: bytes, extension: ext, skip_cleanup: false }, + }); + if (sessionIdRef.current !== sessionId) { + console.debug( + '[dictation] ignoring stale transcription response session=%d', + sessionId + ); + return; + } + + const text = + response && typeof response === 'object' && 'result' in response + ? (response.result?.text?.trim() ?? '') + : ((response as TranscribeResult | null)?.text?.trim() ?? ''); + console.debug('[dictation] transcription result: %s', text || '(empty)'); + if (text) { + dispatch(setTranscript(text)); + } else { + dispatch(setError('No speech detected')); + } + } catch (err) { + if (sessionIdRef.current !== sessionId) { + console.debug('[dictation] ignoring stale transcription error session=%d', sessionId); + return; + } + const msg = err instanceof Error ? err.message : 'Transcription failed'; + console.error('[dictation] transcription error:', msg, err); + dispatch(setError(msg)); + } + }; + + console.debug('[dictation] starting MediaRecorder mimeType=%s', mimeType); + recorder.start(100); // collect data every 100 ms + } catch (err) { + const msg = err instanceof Error ? err.message : 'Microphone access denied'; + console.error('[dictation] getUserMedia error:', msg, err); + dispatch(setError(msg)); + } + }, [status, dispatch]); + + const stopRecording = useCallback(() => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') { + console.debug('[dictation] stopping MediaRecorder'); + mediaRecorderRef.current.stop(); + mediaRecorderRef.current = null; + } + }, []); + + const toggle = useCallback(() => { + console.debug('[dictation] toggle called status=%s', status); + if (status === 'recording') { + stopRecording(); + } else if (status === 'idle' || status === 'ready' || status === 'error') { + void startRecording(); + } + }, [status, startRecording, stopRecording]); + + const dismiss = useCallback(() => { + sessionIdRef.current += 1; + stopRecording(); + dispatch(resetDictation()); + }, [stopRecording, dispatch]); + + useEffect(() => { + toggleRef.current = toggle; + }, [toggle]); + + // Re-register persisted hotkey on startup / change. + useEffect(() => { + let disposed = false; + let retryTimer: ReturnType | null = null; + + void registerDictationHotkey(hotkey).catch(err => { + if (disposed) return; + console.warn('[dictation] auto register hotkey failed:', err); + retryTimer = setTimeout(() => { + if (disposed) return; + void registerDictationHotkey(hotkey).catch(retryErr => { + if (disposed) return; + console.warn('[dictation] hotkey retry failed:', retryErr); + }); + }, 2000); + }); + + return () => { + disposed = true; + if (retryTimer) { + clearTimeout(retryTimer); + } + }; + }, [hotkey]); + + // Listen for global hotkey event from Tauri + useEffect(() => { + let disposed = false; + let unlisten: (() => void) | null = null; + + listen('dictation://toggle', () => { + console.debug('[dictation] received dictation://toggle event'); + toggleRef.current(); + }) + .then(fn => { + if (disposed) { + fn(); + return; + } + unlisten = fn; + }) + .catch(err => { + console.warn('[dictation] failed to listen for toggle event:', err); + }); + + return () => { + disposed = true; + unlisten?.(); + }; + }, []); + + // In-app fallback hotkey handler when global registration is unavailable. + useEffect(() => { + const parsed = parseHotkey(hotkey); + if (!parsed) return; + + const onKeyDown = (event: KeyboardEvent) => { + if (event.repeat) return; + if (!matchesHotkeyEvent(event, parsed)) return; + // Ignore focused editable fields to avoid hijacking typing shortcuts. + const active = document.activeElement as HTMLElement | null; + const isEditable = + active instanceof HTMLInputElement || + active instanceof HTMLTextAreaElement || + !!active?.isContentEditable; + if (isEditable) return; + + console.debug('[dictation] in-app hotkey matched, toggling'); + event.preventDefault(); + toggleRef.current(); + }; + + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [hotkey]); + + return { status, isSupported, startRecording, stopRecording, toggle, dismiss }; +} diff --git a/app/src/components/intelligence/__tests__/MemoryWorkspace.test.tsx b/app/src/components/intelligence/__tests__/MemoryWorkspace.test.tsx index c0ed6a59c..e77e58e2d 100644 --- a/app/src/components/intelligence/__tests__/MemoryWorkspace.test.tsx +++ b/app/src/components/intelligence/__tests__/MemoryWorkspace.test.tsx @@ -71,12 +71,12 @@ describe('MemoryWorkspace', () => { renderWithProviders(); await waitFor(() => { - expect(screen.getByText('Alice')).toBeInTheDocument(); + expect(screen.getByText('Alice', { selector: 'span' })).toBeInTheDocument(); }); - expect(screen.getByText('AUTHORED')).toBeInTheDocument(); - expect(screen.getByText('Bob')).toBeInTheDocument(); - expect(screen.getByText('REVIEWED')).toBeInTheDocument(); + expect(screen.getByText('AUTHORED', { selector: 'span' })).toBeInTheDocument(); + expect(screen.getByText('Bob', { selector: 'span' })).toBeInTheDocument(); + expect(screen.getByText('REVIEWED', { selector: 'span' })).toBeInTheDocument(); // "Paper A" appears in both graph relations and documents list, // so just verify at least one instance is present expect(screen.getAllByText('Paper A').length).toBeGreaterThanOrEqual(1); diff --git a/app/src/components/settings/SettingsHome.tsx b/app/src/components/settings/SettingsHome.tsx index 90c401310..6483de617 100644 --- a/app/src/components/settings/SettingsHome.tsx +++ b/app/src/components/settings/SettingsHome.tsx @@ -281,6 +281,23 @@ const SettingsHome = () => { // onClick: handleViewEncryptionKey, // dangerous: false, // }, + { + id: 'dictation', + title: 'Voice Dictation', + description: 'Transcribe speech to text using local AI', + icon: ( + + + + ), + onClick: () => navigateToSettings('dictation'), + dangerous: false, + }, { id: 'local-model', title: 'Local AI Model', diff --git a/app/src/components/settings/hooks/useSettingsNavigation.ts b/app/src/components/settings/hooks/useSettingsNavigation.ts index f187c2b33..bf6e1aa22 100644 --- a/app/src/components/settings/hooks/useSettingsNavigation.ts +++ b/app/src/components/settings/hooks/useSettingsNavigation.ts @@ -21,7 +21,8 @@ export type SettingsRoute = | 'ai' | 'local-model' | 'tauri-commands' - | 'memory-debug'; + | 'memory-debug' + | 'dictation'; interface SettingsNavigationHook { currentRoute: SettingsRoute; @@ -62,6 +63,7 @@ export const useSettingsNavigation = (): SettingsNavigationHook => { if (path.includes('/settings/local-model')) return 'local-model'; if (path.includes('/settings/tauri-commands')) return 'tauri-commands'; if (path.includes('/settings/memory-debug')) return 'memory-debug'; + if (path.includes('/settings/dictation')) return 'dictation'; return 'home'; }; diff --git a/app/src/components/settings/panels/DictationPanel.tsx b/app/src/components/settings/panels/DictationPanel.tsx new file mode 100644 index 000000000..898e4ba8b --- /dev/null +++ b/app/src/components/settings/panels/DictationPanel.tsx @@ -0,0 +1,288 @@ +import { useEffect, useState } from 'react'; + +import { + checkDictationAvailability, + setHotkey, + setShowFloatingLauncher, +} from '../../../store/dictationSlice'; +import { useAppDispatch, useAppSelector } from '../../../store/hooks'; +import { isTauri, openhumanGetConfig, registerDictationHotkey } from '../../../utils/tauriCommands'; +import SettingsBackButton from '../components/SettingsBackButton'; +import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; + +const DictationPanel = () => { + const dispatch = useAppDispatch(); + const { navigateBack } = useSettingsNavigation(); + const { hotkey, showFloatingLauncher, voiceStatus, statusCheckError, isCheckingStatus } = + useAppSelector(s => s.dictation); + const [hotkeyInput, setHotkeyInput] = useState(hotkey); + const [isSavingHotkey, setIsSavingHotkey] = useState(false); + const [hotkeyError, setHotkeyError] = useState(null); + const [hotkeySuccess, setHotkeySuccess] = useState(false); + const [sttModelDirectory, setSttModelDirectory] = useState(null); + + useEffect(() => { + console.debug('[dictation-panel] mounting — checking availability'); + void dispatch(checkDictationAvailability()); + }, [dispatch]); + + useEffect(() => { + let disposed = false; + if (!isTauri()) return; + void openhumanGetConfig() + .then(response => { + if (disposed) return; + const workspaceDir = response.result.workspace_dir?.trim(); + if (!workspaceDir) return; + const separator = workspaceDir.includes('\\') ? '\\' : '/'; + const normalizedRoot = workspaceDir.replace(/[\\/]+$/, ''); + setSttModelDirectory([normalizedRoot, 'models', 'local-ai', 'stt'].join(separator)); + }) + .catch(err => { + console.debug('[dictation-panel] failed to resolve model directory from config', err); + }); + return () => { + disposed = true; + }; + }, []); + + // Keep local input in sync if hotkey changes externally + useEffect(() => { + setHotkeyInput(hotkey); + }, [hotkey]); + + const handleSaveHotkey = async () => { + const trimmed = hotkeyInput.trim(); + if (!trimmed) { + setHotkeyError('Hotkey cannot be empty'); + return; + } + + // Validate that the shortcut contains at least one recognized modifier and one key token. + const MODIFIERS = [ + 'cmdorctrl', + 'commandorcontrol', + 'ctrl', + 'control', + 'cmd', + 'command', + 'alt', + 'option', + 'shift', + 'super', + 'meta', + ]; + const tokens = trimmed + .split('+') + .map(t => t.trim().toLowerCase()) + .filter(Boolean); + const hasModifier = tokens.some(t => MODIFIERS.includes(t)); + const hasKey = tokens.some(t => !MODIFIERS.includes(t)); + if (!hasModifier || !hasKey) { + setHotkeyError( + 'Invalid format. Use e.g. CmdOrCtrl+Shift+D — must include a modifier and a key.' + ); + return; + } + + setIsSavingHotkey(true); + setHotkeyError(null); + setHotkeySuccess(false); + + console.debug('[dictation-panel] saving hotkey: %s', trimmed); + + try { + if (isTauri()) { + await registerDictationHotkey(trimmed); + } + dispatch(setHotkey(trimmed)); + setHotkeySuccess(true); + setTimeout(() => setHotkeySuccess(false), 2000); + } catch (err) { + const msg = err instanceof Error ? err.message : 'Failed to register hotkey'; + console.error('[dictation-panel] hotkey error:', msg); + setHotkeyError(msg); + } finally { + setIsSavingHotkey(false); + } + }; + + const statusLabel = () => { + if (isCheckingStatus) return 'Checking...'; + if (statusCheckError) return `Check failed: ${statusCheckError}`; + if (!voiceStatus) return 'Not checked'; + if (voiceStatus.stt_available) return 'Ready (model loaded)'; + if (voiceStatus.stt_model_path) return 'Model found — backend unavailable'; + return 'Model not found'; + }; + + const statusColor = () => { + if (isCheckingStatus) return 'bg-stone-500 animate-pulse'; + if (statusCheckError) return 'bg-amber-400'; + if (!voiceStatus) return 'bg-stone-500'; + if (voiceStatus.stt_available) return 'bg-green-400'; + if (voiceStatus.stt_model_path) return 'bg-amber-400'; + return 'bg-red-400'; + }; + + return ( +
+ + +
+
+

Voice Dictation

+

+ Transcribe speech to text using your microphone and local AI. +

+
+ + {/* STT Engine Status */} +
+
+
+

Speech-to-Text Engine

+

{statusLabel()}

+
+
+ +
+
+
+ + {/* Detailed status rows */} + {voiceStatus && ( +
+ + + + +
+ )} +
+ + {/* Model not found guidance */} + {voiceStatus && !voiceStatus.stt_model_path && !isCheckingStatus && ( +
+

+ Model file {voiceStatus.stt_model_id} was not + found. Go to Settings → Local AI Model to + download it, or place the file at{' '} + + {(sttModelDirectory ?? '/models/local-ai/stt') + + (sttModelDirectory?.includes('\\') ? '\\' : '/') + + voiceStatus.stt_model_id} + +

+
+ )} + + {/* Global Hotkey */} +
+
+

Global Hotkey

+

+ Press anywhere to start / stop dictation +

+
+
+ setHotkeyInput(e.target.value)} + placeholder="e.g. CmdOrCtrl+Shift+D" + className="flex-1 bg-stone-700/60 border border-stone-600/50 rounded-lg px-3 py-2 text-sm text-white placeholder-stone-500 focus:outline-none focus:ring-1 focus:ring-primary-500" + /> + +
+ {hotkeyError &&

{hotkeyError}

} +

+ Modifiers: CmdOrCtrl, Alt, Shift,{' '} + Super (also accepts CommandOrControl) +

+
+ + {/* Floating launcher preference */} +
+
+
+

Always show floating Start button

+

+ If disabled, dictation starts via hotkey only while idle. +

+
+ +
+
+ + {/* How to use */} +
+

How to use

+
    +
  1. Press the global hotkey (or click Record in the overlay)
    2. +
  2. Speak clearly into your microphone
    3. +
  3. Press the hotkey again (or click Stop) to finish
    4. +
  4. Wait a moment for transcription
    5. +
  5. Click Insert to type into the focused field, or Copy to clipboard
    6. +
+
+
+
+ ); +}; + +interface StatusRowProps { + label: string; + value: string; + ok: boolean; + muted?: boolean; +} + +const StatusRow = ({ label, value, ok, muted }: StatusRowProps) => ( +
+ + {label} + {value} +
+); + +export default DictationPanel; diff --git a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx index ecb7c2fde..d7f41f427 100644 --- a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx +++ b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx @@ -127,7 +127,8 @@ const ScreenIntelligencePanel = () => { const startDisabled = isStartingSession || isLoading || - !status?.platform_supported || + !status || + !status.platform_supported || status.session.active || status.permissions.accessibility !== 'granted'; const stopDisabled = isStoppingSession || !status?.session.active; @@ -436,7 +437,7 @@ const ScreenIntelligencePanel = () => { - {!status?.platform_supported && ( + {status !== null && !status.platform_supported && (
Screen Intelligence V1 is currently supported on macOS only.
diff --git a/app/src/pages/Conversations.tsx b/app/src/pages/Conversations.tsx index fe3a65d20..a8743875e 100644 --- a/app/src/pages/Conversations.tsx +++ b/app/src/pages/Conversations.tsx @@ -138,6 +138,7 @@ const Conversations = () => { const [isLoadingBudget, setIsLoadingBudget] = useState(false); const messagesEndRef = useRef(null); + const textInputRef = useRef(null); const mediaRecorderRef = useRef(null); const mediaStreamRef = useRef(null); const audioChunksRef = useRef([]); @@ -222,6 +223,30 @@ const Conversations = () => { } }, [messages]); + useEffect(() => { + const onDictationInsert = (event: Event) => { + const customEvent = event as CustomEvent<{ text?: string }>; + const text = customEvent.detail?.text?.trim(); + if (!text) return; + + customEvent.preventDefault(); + setInputMode('text'); + setInputValue(prev => { + const base = prev.trim(); + if (!base) return text; + return `${base}${base.endsWith(' ') ? '' : ' '}${text}`; + }); + + window.requestAnimationFrame(() => { + textInputRef.current?.focus(); + }); + }; + + window.addEventListener('dictation://insert-text', onDictationInsert as EventListener); + return () => + window.removeEventListener('dictation://insert-text', onDictationInsert as EventListener); + }, []); + useEffect(() => { if (sendError && inputValue.length > 0) { setSendError(null); @@ -1312,6 +1337,7 @@ const Conversations = () => { {inlineCompletionSuffix}