From 8627cee96055ecbcf4c8b399c72dd4cca7764e17 Mon Sep 17 00:00:00 2001 From: Mega Mind <146339422+M3gA-Mind@users.noreply.github.com> Date: Tue, 7 Apr 2026 09:41:27 +0530 Subject: [PATCH] Feat/overlay (#378) * feat(autocomplete): add overlay TTL configuration to AutocompletePanel - Introduced `overlay_ttl_ms` parameter to the Autocomplete configuration, allowing users to set the overlay display duration. - Updated AutocompletePanel to include a new input field for adjusting the overlay TTL in milliseconds. - Enhanced parsing and saving logic to handle the new configuration parameter. - Added corresponding tests to ensure functionality and validate the new overlay TTL feature. This update improves user control over the autocomplete overlay behavior, enhancing the overall user experience. * Refactor accessibility code for improved readability and consistency - Simplified log statements in `precompile_helper_background` for better clarity. - Reformatted `detect_input_monitoring_permission` check in `keys.rs` for enhanced readability. - Rearranged imports in `mod.rs` to maintain consistent structure. - Improved formatting of `ElementBounds` initialization across multiple test cases in `overlay.rs` and `types.rs` for better visual alignment. - Enhanced test context creation in `types.rs` for improved clarity. These changes enhance code maintainability and readability across the accessibility module. * fix(overlay): parent core RPC, voice toggle, and debug for #342 - Pass OPENHUMAN_OVERLAY_PARENT_RPC_URL from sidecar spawn and strip inherited OPENHUMAN_CORE_PORT so the overlay no longer fights for the parent listen port. - Overlay UI uses HTTP JSON-RPC to the parent sidecar for globe, debug, and voice STT so state matches the main app; add parentCoreRpc helper mirroring legacy method aliases. - Skip embedded JSON-RPC server when parent URL is set; use OPENHUMAN_OVERLAY_EMBEDDED_CORE_PORT (default 7799) for standalone dev. - Fix screen intelligence status method name; add voice_status polling, STT section, collapsible debug summary, and connection banner when core is unreachable. - Voice capture switch gates the mic with real STT availability from voice_status. Closes #342 Made-with: Cursor * fix: address CodeRabbit review on overlay/autocomplete (PR #378) - helper: correlate JSON-RPC replies with monotonic request ids; discard mismatched lines until deadline (fixes stale response after timeout). - helper: serialize Swift compile via HELPER_COMPILE_LOCK (precompile vs first use). - Overlay Tab hint: pass tab_hint from Rust from accept_with_tab; Swift hides hint when empty. - keys: re-check Input Monitoring on an interval when denied so grant without restart works. - engine: use non-zero confidence placeholder (0.75) until inline_complete returns scores. - overlay dedupe: suppress identical badge only within 400ms, not for process lifetime. Co-authored-by: Code review feedback Made-with: Cursor * fix(ci): resolve clippy and warning issues for Rust gates - Use match on anchor_bounds in autocomplete overlay (avoid unnecessary unwrap) - Drop unused test imports in registry_ops and rpc dispatch - Prefix unused notion_doc_id in subconscious integration test Made-with: Cursor * fix(overlay): improve parent RPC URL handling in App component - Updated the useEffect hook to manage the parent RPC URL more robustly by introducing a mounted flag to prevent state updates on unmounted components. - Added error handling to set the parent RPC URL to null in case of invocation failure, enhancing the reliability of the component's behavior. * feat(overlay): implement timeout handling for parent core RPC requests - Introduced a default timeout for parent core RPC requests, enhancing reliability by preventing indefinite waiting for responses. - Added an AbortController to manage request timeouts, throwing a specific error message when a timeout occurs. - Updated the `callParentCoreRpc` function to accept a customizable timeout parameter, improving flexibility for RPC calls. * fix(overlay): allow stopping active recording regardless of config state - Updated the main button handler in the App component to always permit stopping an active recording when the status is "listening", improving user experience and control over the recording process. - Removed redundant code that previously checked the status before stopping the recording, streamlining the logic. * feat(overlay): update Cargo.lock with new dependencies and versions - Added new packages including `alsa`, `alsa-sys`, `arboard`, `block`, `cocoa`, `core-foundation`, `core-graphics`, `coreaudio-rs`, `coreaudio-sys`, `cpal`, `crunchy`, and `dasp_sample` to enhance functionality and support for audio processing and system interactions. - Updated existing dependencies to their latest versions for improved performance and compatibility. - Modified the `show_overlay` function in `ops.rs` to include an additional parameter, enhancing the overlay display functionality. * feat(autocomplete): add overlay_ttl_ms parameter to Autocomplete interfaces - Introduced a new optional parameter `overlay_ttl_ms` to both `AutocompleteSetStyleParams` and `AutocompleteConfig` interfaces, allowing for customizable overlay timeout settings. - This enhancement improves the flexibility of the autocomplete feature by enabling developers to specify how long the overlay should remain visible. --------- Co-authored-by: Code review feedback Co-authored-by: Steven Enamakel --- .../settings/panels/AutocompletePanel.tsx | 22 + .../__tests__/AutocompletePanel.test.tsx | 1 + app/src/utils/tauriCommands/autocomplete.ts | 2 + overlay/src-tauri/Cargo.lock | 755 ++++++++++++++++-- overlay/src-tauri/src/lib.rs | 57 +- overlay/src-tauri/tauri.conf.json | 2 +- overlay/src/App.tsx | 252 +++++- overlay/src/parentCoreRpc.ts | 95 +++ src/core/jsonrpc.rs | 55 +- src/openhuman/accessibility/helper.rs | 298 ++++++- src/openhuman/accessibility/keys.rs | 67 ++ src/openhuman/accessibility/mod.rs | 1 + src/openhuman/accessibility/overlay.rs | 103 ++- src/openhuman/accessibility/paste.rs | 31 +- src/openhuman/accessibility/text_util.rs | 146 ++++ src/openhuman/accessibility/types.rs | 143 ++++ src/openhuman/autocomplete/core/engine.rs | 35 +- src/openhuman/autocomplete/core/overlay.rs | 169 +++- src/openhuman/autocomplete/core/text.rs | 132 +++ src/openhuman/autocomplete/core/types.rs | 1 + src/openhuman/config/schema/autocomplete.rs | 7 + src/openhuman/overlay/process.rs | 32 +- src/openhuman/skills/registry_ops.rs | 2 +- .../subconscious/integration_test.rs | 2 +- src/openhuman/text_input/ops.rs | 2 +- src/rpc/dispatch.rs | 2 - 26 files changed, 2194 insertions(+), 220 deletions(-) create mode 100644 overlay/src/parentCoreRpc.ts diff --git a/app/src/components/settings/panels/AutocompletePanel.tsx b/app/src/components/settings/panels/AutocompletePanel.tsx index d30339c88..d6734f26e 100644 --- a/app/src/components/settings/panels/AutocompletePanel.tsx +++ b/app/src/components/settings/panels/AutocompletePanel.tsx @@ -28,6 +28,7 @@ const DEFAULT_CONFIG: AutocompleteConfig = { style_examples: [], disabled_apps: [], accept_with_tab: true, + overlay_ttl_ms: 1100, }; const MAX_LOG_ENTRIES = 200; @@ -56,6 +57,10 @@ const parseAutocompleteConfig = (raw: unknown): AutocompleteConfig => { typeof value.accept_with_tab === 'boolean' ? value.accept_with_tab : DEFAULT_CONFIG.accept_with_tab, + overlay_ttl_ms: + typeof value.overlay_ttl_ms === 'number' + ? value.overlay_ttl_ms + : DEFAULT_CONFIG.overlay_ttl_ms, }; }; @@ -77,6 +82,7 @@ const AutocompletePanel = () => { DEFAULT_CONFIG.disabled_apps.join('\n') ); const [acceptWithTab, setAcceptWithTab] = useState(DEFAULT_CONFIG.accept_with_tab); + const [overlayTtlMs, setOverlayTtlMs] = useState(String(DEFAULT_CONFIG.overlay_ttl_ms)); const [contextOverride, setContextOverride] = useState(''); const [focusDebug, setFocusDebug] = useState(''); const [logs, setLogs] = useState([]); @@ -153,6 +159,7 @@ const AutocompletePanel = () => { setStyleExamplesText(config.style_examples.join('\n')); setDisabledAppsText(config.disabled_apps.join('\n')); setAcceptWithTab(config.accept_with_tab); + setOverlayTtlMs(String(config.overlay_ttl_ms)); } catch (err) { setError(err instanceof Error ? err.message : 'Failed to load autocomplete settings'); } finally { @@ -253,6 +260,7 @@ const AutocompletePanel = () => { appendUiLog('saving autocomplete settings'); const debounce = Number(debounceMs); const max = Number(maxChars); + const ttl = Number(overlayTtlMs); const response = await openhumanAutocompleteSetStyle({ enabled, debounce_ms: Number.isFinite(debounce) ? Math.min(Math.max(debounce, 50), 2000) : 120, @@ -268,6 +276,7 @@ const AutocompletePanel = () => { .map(entry => entry.trim()) .filter(Boolean), accept_with_tab: acceptWithTab, + overlay_ttl_ms: Number.isFinite(ttl) ? Math.min(Math.max(ttl, 300), 10000) : 1100, }); setEnabled(response.result.config.enabled); @@ -278,6 +287,7 @@ const AutocompletePanel = () => { setStyleExamplesText(response.result.config.style_examples.join('\n')); setDisabledAppsText(response.result.config.disabled_apps.join('\n')); setAcceptWithTab(response.result.config.accept_with_tab); + setOverlayTtlMs(String(response.result.config.overlay_ttl_ms)); appendLogs(response.logs); setMessage('Autocomplete settings saved.'); await refreshStatus(); @@ -507,6 +517,18 @@ const AutocompletePanel = () => { className="w-28 rounded border border-stone-300 bg-white px-2 py-1 text-xs text-stone-700" /> +