From 6410db1fad56992a9fab730c38651b8efb369d5d Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Thu, 9 Apr 2026 23:01:56 -0700 Subject: [PATCH] feat(thu-fullrun): overlay attention, skills sync, credits & settings refresh (#479) * Update Conversations component to enhance user messaging for budget limits. Changed the warning text for exhausted weekly inference budget to improve clarity and user experience. * feat(schemas): add new configuration option for vision model usage - Introduced a new optional boolean field `use_vision_model` in the schemas for enabling vision LLM for screenshot analysis. - Updated the screen intelligence schemas to include a required `consent` field for starting sessions, replacing the previous `sample_interval_ms` field. - Enhanced the `ttl_secs` field description for clarity and modified the `capture_policy` to `screen_monitoring` for better understanding of its purpose. * feat(CoreStateProvider): enhance state management with optimistic updates and error handling - Implemented optimistic local commits for `setAnalyticsEnabled` and `setOnboardingCompletedFlag` to provide instant UI feedback while ensuring state consistency through authoritative snapshot refreshes. - Added error handling for the `refresh` function calls in `setAnalyticsEnabled`, `setOnboardingCompletedFlag`, and `clearSession` to log failures, improving robustness in state management during user interactions. - Updated dependencies in the `useCallback` hooks to include `refresh`, ensuring proper state updates and synchronization with the core. * feat(paths): centralize runtime path resolution for user-scoped skills data - Introduced a new module `paths.rs` to handle the resolution of runtime paths for skills, ensuring that `skills_data` and `workspace` directories are scoped per user. - Updated `bootstrap_skill_runtime` and `bootstrap_skills_runtime` functions to utilize the new path resolution logic, improving consistency and clarity in directory management. - Enhanced error logging for directory creation failures to include the specific path that failed, aiding in debugging. - Added a new optional field `overlay_ttl_ms` in the autocomplete schemas to support overlay time-to-live configuration. * feat(ScreenIntelligencePanel): optimize config synchronization to prevent user edit clobbering - Introduced a reference to track the last synced configuration signature, ensuring that user edits are preserved during periodic updates from the CoreStateProvider. - Updated the effect to compare serialized configuration values, allowing for re-sync only when actual changes occur, enhancing user experience and preventing unintended data loss. * feat(SkillManager): implement initial sync after OAuth completion - Added functionality to trigger an initial data sync immediately after OAuth completion, ensuring users see fresh data without waiting for the next scheduled sync. - Updated comments to clarify the change in sync behavior due to recent modifications in the Rust core, which no longer auto-triggers sync on OAuth completion. * fix(UsageLimitModal, Conversations): enhance user messaging for budget limits - Updated warning messages in both UsageLimitModal and Conversations components to provide clearer information regarding weekly limits and reset times. - Improved clarity in user notifications to enhance overall experience when budget limits are reached. * refactor(SkillSetupModal): improve session mode handling for skill configuration - Updated the SkillSetupModal component to lock the mode at mount time, ensuring users remain in the setup wizard during their session even if the skill is marked as complete. - Simplified mode management by replacing the forceSetup state with a sessionMode state, allowing explicit mode switching while maintaining a consistent user experience. * feat(SkillManager): enhance setup flow for OAuth-based skills - Updated the `startSetup` method to handle OAuth-based skills more effectively by implementing a fallback to core RPC for skills without a frontend runtime. - Improved error handling to treat missing `onSetupStart` implementations as successful completion for pure OAuth skills, allowing the setup wizard to display the "Connected!" screen. - Added detailed logging for both local runtime and core RPC fallback scenarios to improve traceability during the setup process. * feat(Home): enhance local AI status handling and asset management - Introduced a new state for local AI assets, allowing for better tracking of model file readiness. - Updated the loading logic to fetch both local AI status and assets concurrently, improving performance and error handling. - Implemented a mechanism to hide the Local Model Runtime card once all models are fully downloaded, enhancing user experience. - Added comprehensive comments to clarify the logic behind model readiness checks based on asset states. * refactor(Credits): update credit balance structure and terminology - Renamed credit categories in the RewardsCouponSection and PayAsYouGoCard components for clarity, changing "General credits" to "Promo credits" and "Top-up credits" to "Team top-up." - Updated the credit balance API to reflect the new structure, replacing `balanceUsd` and `topUpBalanceUsd` with `promotionBalanceUsd` and `teamTopupUsd`. - Adjusted normalization logic in the credits API to accommodate the new credit balance fields. - Modified tests to ensure correct handling of the updated credit balance structure. * feat(Settings): reorganize billing settings and update descriptions - Added a new top-level billing section to the settings, promoting it out of the Account & Security category for better visibility. - Updated the description for the Account & Security section to remove billing references, focusing on recovery phrase, team management, and linked account access. - Adjusted the settings navigation to accommodate the new billing section, ensuring proper routing and user experience. * refactor(Config): change logging level from info to debug for environment overrides - Updated logging statements in the Config implementation to use debug level instead of info, reducing verbosity during runtime while maintaining necessary traceability for configuration loading. * feat(Overlay): implement overlay attention event handling and refactor overlay app structure - Introduced a new overlay module to manage attention events, allowing the core to publish messages to the overlay window. - Enhanced the OverlayApp component to handle dictation and attention events, improving user interaction with the overlay. - Refactored the overlay state management to support different modes (idle, stt, attention) and added auto-dismiss functionality for attention messages. - Removed the Browser Access Toggle from the Skills page, streamlining the UI and focusing on core functionalities. - Updated tests to reflect changes in the Skills component and removed unnecessary mocks related to browser access. * fix(OverlayBubbleChip): reset typewriter animation on new bubble identity - Updated the OverlayBubbleChip component to reset the typewriter animation correctly when a new bubble is displayed by using the `key` prop. - Refactored the cleanup logic in the useEffect hook to ensure proper interval management and state reset, enhancing the user experience with bubble transitions. * refactor(rest): streamline key_bytes_from_string function and improve readability - Simplified the condition for checking the ASCII key length and character restrictions in the key_bytes_from_string function. - Consolidated the import statements for base64 engines into a single line for better clarity. - Adjusted test data formatting for improved readability in the key_bytes_from_string_tests module. * enhance(logging): improve color detection logic for terminal output - Updated the color detection logic in the logging module to prioritize environment variables (`NO_COLOR`, `FORCE_COLOR`, `CLICOLOR_FORCE`) for better control over color output. - Added detailed comments explaining the color resolution order, enhancing code clarity and maintainability. * test(Home): add mock for openhumanLocalAiAssetsStatus in tests - Enhanced the Home and HomeBootstrapButtons test files by adding a mock implementation for openhumanLocalAiAssetsStatus, which resolves to an object with null result and empty logs. This improves the test setup for local AI asset status handling. * refactor(SkillSetupModal): improve session mode handling and loading state - Updated the SkillSetupModal component to ensure session mode is determined after the first snapshot resolution, preventing premature defaults to the setup wizard. - Introduced a loading state to display a message while waiting for the skill setup status, enhancing user experience during the modal's initial render. - Refactored the SkillManager to throw errors for real failures during setup, ensuring proper error handling and user feedback. * refactor(Config): simplify logging for invalid proxy scope values - Consolidated the logging statement for invalid OPENHUMAN_PROXY_SCOPE values into a single line, improving code readability while maintaining the warning functionality. --- app/src-tauri/src/core_process.rs | 20 ++ .../rewards/RewardsCouponSection.tsx | 8 +- .../__tests__/RewardsCouponSection.test.tsx | 14 +- app/src/components/settings/SettingsHome.tsx | 19 +- .../settings/hooks/useSettingsNavigation.ts | 5 +- .../panels/ScreenIntelligencePanel.tsx | 13 +- .../panels/billing/PayAsYouGoCard.tsx | 42 +-- app/src/components/skills/SkillSetupModal.tsx | 42 ++- app/src/components/upsell/UsageLimitModal.tsx | 2 +- app/src/lib/skills/manager.ts | 69 +++- app/src/overlay/OverlayApp.tsx | 312 +++++++++++++++--- app/src/pages/Conversations.tsx | 4 +- app/src/pages/Home.tsx | 45 ++- app/src/pages/Settings.tsx | 18 +- app/src/pages/Skills.tsx | 56 +--- app/src/pages/__tests__/Home.test.tsx | 1 + .../__tests__/HomeBootstrapButtons.test.tsx | 1 + .../Skills.third-party-gmail-sync.test.tsx | 11 - ...ls.third-party-notion-debug-tools.test.tsx | 11 - app/src/providers/CoreStateProvider.tsx | 29 +- .../services/api/__tests__/billingApi.test.ts | 4 +- app/src/services/api/creditsApi.ts | 33 +- app/src/utils/desktopDeepLinkListener.ts | 5 +- src/api/rest.rs | 114 ++++++- src/core/jsonrpc.rs | 119 ++++--- src/core/logging.rs | 17 +- src/core/skills_cli.rs | 28 +- src/core/socketio.rs | 39 ++- src/openhuman/about_app/catalog.rs | 9 - src/openhuman/autocomplete/schemas.rs | 6 + src/openhuman/config/schema/load.rs | 19 +- src/openhuman/config/schemas.rs | 4 + src/openhuman/mod.rs | 1 + src/openhuman/overlay/bus.rs | 75 +++++ src/openhuman/overlay/mod.rs | 26 ++ src/openhuman/overlay/types.rs | 73 ++++ src/openhuman/screen_intelligence/schemas.rs | 16 +- src/openhuman/skills/mod.rs | 1 + src/openhuman/skills/paths.rs | 173 ++++++++++ 39 files changed, 1151 insertions(+), 333 deletions(-) create mode 100644 src/openhuman/overlay/bus.rs create mode 100644 src/openhuman/overlay/mod.rs create mode 100644 src/openhuman/overlay/types.rs create mode 100644 src/openhuman/skills/paths.rs diff --git a/app/src-tauri/src/core_process.rs b/app/src-tauri/src/core_process.rs index 9be5b6be1..2b1834d7c 100644 --- a/app/src-tauri/src/core_process.rs +++ b/app/src-tauri/src/core_process.rs @@ -1,3 +1,4 @@ +use std::io::IsTerminal; use std::path::PathBuf; use std::sync::Arc; @@ -7,6 +8,23 @@ use tokio::sync::Mutex; use tokio::task::JoinHandle; use tokio::time::{timeout, Duration}; +/// Propagate ANSI color hints to the spawned core child. +/// +/// Core's tracing formatter auto-detects color via `stderr.is_terminal()`, +/// but when core runs as a grandchild under `yarn tauri dev` the inherited +/// stderr may not register as a TTY even though the ultimate terminal +/// supports ANSI. If the Tauri process itself is attached to a TTY we +/// forward `FORCE_COLOR=1` so core emits colored log lines; `NO_COLOR` +/// (user opt-out) always wins and short-circuits the propagation. +fn apply_core_color_env(cmd: &mut Command) { + if std::env::var_os("NO_COLOR").is_some() { + return; + } + if std::io::stderr().is_terminal() { + cmd.env("FORCE_COLOR", "1"); + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum CoreRunMode { InProcess, @@ -119,6 +137,7 @@ impl CoreProcessHandle { .arg(self.port.to_string()); cmd }; + apply_core_color_env(&mut cmd); let child = cmd .spawn() .map_err(|e| format!("failed to spawn core process: {e}"))?; @@ -156,6 +175,7 @@ impl CoreProcessHandle { cmd }; + apply_core_color_env(&mut cmd); let child = cmd .spawn() .map_err(|e| format!("failed to spawn core process: {e}"))?; diff --git a/app/src/components/rewards/RewardsCouponSection.tsx b/app/src/components/rewards/RewardsCouponSection.tsx index 3d8fee0fa..dfed4ad88 100644 --- a/app/src/components/rewards/RewardsCouponSection.tsx +++ b/app/src/components/rewards/RewardsCouponSection.tsx @@ -163,18 +163,18 @@ const RewardsCouponSection = () => {
- General credits + Promo credits
- {creditBalance ? formatUsd(creditBalance.balanceUsd) : loading ? '…' : '—'} + {creditBalance ? formatUsd(creditBalance.promotionBalanceUsd) : loading ? '…' : '—'}
- Top-up credits + Team top-up
- {creditBalance ? formatUsd(creditBalance.topUpBalanceUsd) : loading ? '…' : '—'} + {creditBalance ? formatUsd(creditBalance.teamTopupUsd) : loading ? '…' : '—'}
diff --git a/app/src/components/rewards/__tests__/RewardsCouponSection.test.tsx b/app/src/components/rewards/__tests__/RewardsCouponSection.test.tsx index 7cea75897..cd753e99a 100644 --- a/app/src/components/rewards/__tests__/RewardsCouponSection.test.tsx +++ b/app/src/components/rewards/__tests__/RewardsCouponSection.test.tsx @@ -28,8 +28,8 @@ describe('RewardsCouponSection', () => { it('loads balances and refreshes history after a successful redemption', async () => { mocks.mockCreditsApi.getBalance - .mockResolvedValueOnce({ balanceUsd: 3, topUpBalanceUsd: 1, topUpBaselineUsd: null }) - .mockResolvedValueOnce({ balanceUsd: 8, topUpBalanceUsd: 1, topUpBaselineUsd: null }); + .mockResolvedValueOnce({ promotionBalanceUsd: 3, teamTopupUsd: 1 }) + .mockResolvedValueOnce({ promotionBalanceUsd: 8, teamTopupUsd: 1 }); mocks.mockCreditsApi.getUserCoupons .mockResolvedValueOnce([]) .mockResolvedValueOnce([ @@ -70,11 +70,7 @@ describe('RewardsCouponSection', () => { }); it('shows backend redemption errors without clearing the existing state', async () => { - mocks.mockCreditsApi.getBalance.mockResolvedValue({ - balanceUsd: 3, - topUpBalanceUsd: 0, - topUpBaselineUsd: null, - }); + mocks.mockCreditsApi.getBalance.mockResolvedValue({ promotionBalanceUsd: 3, teamTopupUsd: 0 }); mocks.mockCreditsApi.getUserCoupons.mockResolvedValue([]); mocks.mockCreditsApi.redeemCoupon.mockRejectedValueOnce({ error: 'This coupon has already been used.', @@ -94,8 +90,8 @@ describe('RewardsCouponSection', () => { it('shows pending coupon copy and keeps the current balance until the reward is fulfilled', async () => { mocks.mockCreditsApi.getBalance - .mockResolvedValueOnce({ balanceUsd: 3, topUpBalanceUsd: 0, topUpBaselineUsd: null }) - .mockResolvedValueOnce({ balanceUsd: 3, topUpBalanceUsd: 0, topUpBaselineUsd: null }); + .mockResolvedValueOnce({ promotionBalanceUsd: 3, teamTopupUsd: 0 }) + .mockResolvedValueOnce({ promotionBalanceUsd: 3, teamTopupUsd: 0 }); mocks.mockCreditsApi.getUserCoupons .mockResolvedValueOnce([]) .mockResolvedValueOnce([ diff --git a/app/src/components/settings/SettingsHome.tsx b/app/src/components/settings/SettingsHome.tsx index f4f0dcc15..9e2d7bad2 100644 --- a/app/src/components/settings/SettingsHome.tsx +++ b/app/src/components/settings/SettingsHome.tsx @@ -79,7 +79,7 @@ const SettingsHome = () => { { id: 'account', title: 'Account & Security', - description: 'Billing, recovery phrase, team management, and linked account access', + description: 'Recovery phrase, team management, and linked account access', icon: ( @@ -88,6 +88,23 @@ const SettingsHome = () => { onClick: () => navigateToSettings('account'), dangerous: false, }, + { + id: 'billing', + title: 'Billing & Usage', + description: 'Subscription plan, pay-as-you-go credits, and payment methods', + icon: ( + + + + ), + onClick: () => navigateToSettings('billing'), + dangerous: false, + }, { id: 'automation', title: 'Automation & Channels', diff --git a/app/src/components/settings/hooks/useSettingsNavigation.ts b/app/src/components/settings/hooks/useSettingsNavigation.ts index 794a00ea7..f5a56f49c 100644 --- a/app/src/components/settings/hooks/useSettingsNavigation.ts +++ b/app/src/components/settings/hooks/useSettingsNavigation.ts @@ -155,8 +155,11 @@ export const useSettingsNavigation = (): SettingsNavigationHook => { case 'ai-tools': return [settingsCrumb]; - // Leaf panels under account + // Top-level billing leaf (promoted out of Account & Security) case 'billing': + return [settingsCrumb]; + + // Leaf panels under account case 'recovery-phrase': case 'team': case 'connections': diff --git a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx index 96a4bbe6d..610a820f0 100644 --- a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx +++ b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx @@ -1,4 +1,4 @@ -import { type ComponentProps, useEffect, useMemo, useState } from 'react'; +import { type ComponentProps, useEffect, useMemo, useRef, useState } from 'react'; import ScreenIntelligenceDebugPanel from '../../../components/intelligence/ScreenIntelligenceDebugPanel'; import { useScreenIntelligenceState } from '../../../features/screen-intelligence/useScreenIntelligenceState'; @@ -79,10 +79,21 @@ const ScreenIntelligencePanel = () => { const [isSavingConfig, setIsSavingConfig] = useState(false); const [configError, setConfigError] = useState(null); + // CoreStateProvider polls every 2s (CoreStateProvider.tsx POLL_MS), producing a + // new `status` object reference on every tick even when the underlying config is + // unchanged. Keying this effect on `status?.config` identity would therefore + // clobber in-progress user edits every 2 seconds. Compare the serialized value + // instead, so we only re-sync when the server config has actually changed. + const lastSyncedConfigSigRef = useRef(null); useEffect(() => { if (!status?.config) { return; } + const sig = JSON.stringify(status.config); + if (lastSyncedConfigSigRef.current === sig) { + return; + } + lastSyncedConfigSigRef.current = sig; setEnabled(status.config.enabled ?? false); setPolicyMode( status.config.policy_mode === 'whitelist_only' ? 'whitelist_only' : 'all_except_blacklist' diff --git a/app/src/components/settings/panels/billing/PayAsYouGoCard.tsx b/app/src/components/settings/panels/billing/PayAsYouGoCard.tsx index 7e23eac69..b7afab80e 100644 --- a/app/src/components/settings/panels/billing/PayAsYouGoCard.tsx +++ b/app/src/components/settings/panels/billing/PayAsYouGoCard.tsx @@ -20,10 +20,15 @@ const PayAsYouGoCard = ({ onTopUp, onBalanceRefresh, }: PayAsYouGoCardProps) => { - const promoCredits = creditBalance?.balanceUsd ?? 0; - const topUpCredits = creditBalance?.topUpBalanceUsd ?? 0; - const topUpBaseline = creditBalance?.topUpBaselineUsd ?? null; - const availableCredits = promoCredits + topUpCredits; + // Backend `GET /payments/credits/balance` returns + // { promotionBalanceUsd, teamTopupUsd } + // `promotionBalanceUsd` lives on the user document + // (`IUserUsage.promotionBalanceUsd`) and unifies signup bonus, coupons, + // and referral rewards. `teamTopupUsd` is the team-level paid top-up pool. + // Together they make the pay-as-you-go spendable balance. + const promoCredits = creditBalance?.promotionBalanceUsd ?? 0; + const teamTopupCredits = creditBalance?.teamTopupUsd ?? 0; + const availableCredits = promoCredits + teamTopupCredits; // Coupon state (local — no need to share with other sections) const [couponCode, setCouponCode] = useState(''); @@ -78,30 +83,11 @@ const PayAsYouGoCard = ({ Signup + promo credits ${promoCredits.toFixed(2)}
-
-
- Top-up credits - - ${topUpCredits.toFixed(2)} - {topUpBaseline != null && topUpBaseline > 0 && ( - / ${topUpBaseline.toFixed(2)} - )} - -
- {topUpBaseline != null && topUpBaseline > 0 && ( -
-
-
- )} +
+ Team top-up credits + + ${teamTopupCredits.toFixed(2)} +
) : isLoadingCredits ? ( diff --git a/app/src/components/skills/SkillSetupModal.tsx b/app/src/components/skills/SkillSetupModal.tsx index b79d4f795..d36ca431e 100644 --- a/app/src/components/skills/SkillSetupModal.tsx +++ b/app/src/components/skills/SkillSetupModal.tsx @@ -31,13 +31,29 @@ export default function SkillSetupModal({ const modalRef = useRef(null); const snap = useSkillSnapshot(skillId); const setupComplete = snap?.setup_complete ?? false; - // Track whether the user has explicitly chosen to reconfigure (setup mode) - // even though setup is already complete. - const [forceSetup, setForceSetup] = useState(false); - // Derive mode: show manage if setup is complete (or no setup needed), - // unless the user explicitly chose to reconfigure. - const mode = forceSetup ? "setup" : (!hasSetup || setupComplete ? "manage" : "setup"); - const setMode = (m: "manage" | "setup") => setForceSetup(m === "setup"); + // Lock the mode in once we have a concrete snapshot — `useSkillSnapshot` + // returns `null` on the first render while it fetches, so reading + // `setup_complete` at mount time would always see `false` and wrongly + // default an already-connected skill into the setup wizard. + // + // We keep `sessionMode` stable after the first resolution so that an + // OAuth flow that flips `setup_complete` to true mid-wizard does not + // yank the user out of the wizard's own "complete" success screen. + // The user can still switch modes explicitly via `setMode` below + // (e.g. SkillManagementPanel's "Reconfigure" button). + const [sessionMode, setSessionMode] = useState<"manage" | "setup" | null>( + () => (!hasSetup ? "manage" : null), + ); + + useEffect(() => { + if (sessionMode !== null) return; + // Wait for the first concrete snapshot before deciding. + if (snap === null) return; + setSessionMode(setupComplete ? "manage" : "setup"); + }, [sessionMode, snap, setupComplete]); + + const setMode = (m: "manage" | "setup") => setSessionMode(m); + const mode = sessionMode; // Handle escape key useEffect(() => { @@ -71,7 +87,11 @@ export default function SkillSetupModal({ }; const headerTitle = - mode === "manage" ? `Manage ${skillName}` : `Connect ${skillName}`; + mode === null + ? skillName + : mode === "manage" + ? `Manage ${skillName}` + : `Connect ${skillName}`; const modalContent = (
- {mode === "manage" ? ( + {mode === null ? ( +
+ Loading… +
+ ) : mode === "manage" ? ( { console.log("[SkillManager] startSetup", skillId); const runtime = this.runtimes.get(skillId); - if (!runtime) { - console.log("[SkillManager] runtime not found", skillId); - throw new Error(`Skill ${skillId} runtime not found`); + if (runtime) { + emitSkillStateChange(skillId); + console.log("[SkillManager] setup started (local runtime)", skillId); + return runtime.setupStart(); } - emitSkillStateChange(skillId); - console.log("[SkillManager] setup started", skillId); - return runtime.setupStart(); + // No frontend runtime — dispatch via core RPC pass-through. + // + // The core side returns `null` (not an error) when the skill has no + // `onSetupStart` handler — see `handle_js_call` in + // `src/openhuman/skills/qjs_skill_instance/js_handlers.rs`, which + // falls through to `return "null"` when the function is missing. + // So any exception thrown here is a *real* failure (skill not + // running, RPC transport error, JS exception in the handler, …) + // and must be propagated so the caller can surface it to the user + // instead of silently pretending setup succeeded. + try { + const result = (await callCoreRpc({ + method: "openhuman.skills_setup_start", + params: { skill_id: skillId }, + })) as { step?: SetupStep } | null; + emitSkillStateChange(skillId); + console.log( + "[SkillManager] setup started (core RPC fallback)", + skillId, + result, + ); + if (!result || !result.step) { + return null; + } + return result.step; + } catch (err) { + console.warn( + "[SkillManager] setup_start core fallback failed", + skillId, + err, + ); + emitSkillStateChange(skillId); + throw err; + } } /** @@ -344,6 +385,22 @@ class SkillManager { } } + // Kick off an initial sync so the user sees fresh data immediately + // after connecting, rather than waiting for the next cron tick. + // The Rust core no longer auto-triggers sync on oauth/complete + // (removed in commit 840b1d3c), so the frontend drives it here. + // Fire-and-forget: any failure is logged but must not block the + // OAuth completion flow. + try { + console.log(`[SkillManager] kicking initial sync after OAuth for '${skillId}'`); + await this.triggerSync(skillId); + } catch (syncErr) { + console.warn( + `[SkillManager] initial post-OAuth sync failed for '${skillId}':`, + syncErr, + ); + } + emitSkillStateChange(skillId); } diff --git a/app/src/overlay/OverlayApp.tsx b/app/src/overlay/OverlayApp.tsx index ee3b54e36..327c66748 100644 --- a/app/src/overlay/OverlayApp.tsx +++ b/app/src/overlay/OverlayApp.tsx @@ -1,12 +1,40 @@ +/** + * OverlayApp + * + * Standalone React root rendered inside the Tauri `overlay` window (see + * `app/src-tauri/tauri.conf.json`). The overlay lives in its own WebView + * and cannot share Redux state with the main window, so it reacts to + * signals from the Rust core over a dedicated, unauthenticated Socket.IO + * connection (same pattern as `useDictationHotkey`). + * + * The overlay activates in two cases: + * + * 1. **STT / dictation** — when the user presses the dictation hotkey. + * The core emits `dictation:toggle` with `{type: "pressed" | "released"}` + * and `dictation:transcription` with `{text}`. "Pressed" opens the + * overlay into STT mode; "released" (or the final transcription) + * dismisses it. + * + * 2. **Attention message** — when the core (subconscious loop, heartbeat, + * …) publishes an `OverlayAttentionEvent` via + * `openhuman::overlay::publish_attention(...)`. The bridge in + * `core::socketio` forwards this as an `overlay:attention` event. + * The bubble auto-dismisses after its ttl. + * + * There is **no** demo loop — the overlay is entirely event-driven. + */ +import { invoke, isTauri } from '@tauri-apps/api/core'; import { currentMonitor, getCurrentWindow, LogicalPosition, LogicalSize, } from '@tauri-apps/api/window'; -import { useEffect, useMemo, useRef, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { io, Socket } from 'socket.io-client'; import RotatingTetrahedronCanvas from '../components/RotatingTetrahedronCanvas'; +import { CORE_RPC_URL } from '../utils/config'; const OVERLAY_IDLE_WIDTH = 50; const OVERLAY_IDLE_HEIGHT = 50; @@ -15,19 +43,50 @@ const OVERLAY_ACTIVE_HEIGHT = 208; const OVERLAY_IDLE_MARGIN = 10; const OVERLAY_ACTIVE_MARGIN = 20; const OVERLAY_IDLE_OPACITY = 0.6; -const SCENARIO_THREE_TEXT = '"Noted. Need milk."'; -type OverlayStatus = 'idle' | 'active'; -type OverlayScenario = 1 | 2 | 3; +/** Default auto-dismiss for an attention bubble when no ttl is supplied. */ +const DEFAULT_ATTENTION_TTL_MS = 6000; +/** Grace period after STT `released` before returning to idle, giving the + * final transcription time to arrive and the user a moment to read it. */ +const STT_RELEASE_LINGER_MS = 1500; +/** Placeholder bubble text while waiting for the first transcription. */ +const STT_LISTENING_PLACEHOLDER = '"Listening…"'; + +// ── State model ────────────────────────────────────────────────────────── + +type OverlayMode = 'idle' | 'stt' | 'attention'; +type BubbleTone = 'neutral' | 'accent' | 'success'; interface OverlayBubble { id: string; text: string; - tone: 'neutral' | 'accent' | 'success'; + tone: BubbleTone; compact?: boolean; } -function bubbleToneClass(tone: OverlayBubble['tone']) { +// ── Socket payload types ───────────────────────────────────────────────── + +interface DictationTogglePayload { + type?: string; + hotkey?: string; + activation_mode?: string; +} + +interface DictationTranscriptionPayload { + text?: string; +} + +interface OverlayAttentionPayload { + id?: string; + message?: string; + tone?: BubbleTone; + ttl_ms?: number; + source?: string; +} + +// ── Helpers ────────────────────────────────────────────────────────────── + +function bubbleToneClass(tone: BubbleTone) { switch (tone) { case 'accent': return 'bg-blue-700 text-white'; @@ -38,7 +97,27 @@ function bubbleToneClass(tone: OverlayBubble['tone']) { } } +/** Resolve the core process base URL (without /rpc suffix) for Socket.IO. + * Mirrors `useDictationHotkey.resolveCoreSocketUrl`. */ +async function resolveCoreSocketUrl(): Promise { + let rpcUrl = CORE_RPC_URL; + if (isTauri()) { + try { + const url = await invoke('core_rpc_url'); + if (url) rpcUrl = String(url); + } catch { + // fall through to default + } + } + const trimmed = rpcUrl.trim().replace(/\/+$/, ''); + return trimmed.endsWith('/rpc') ? trimmed.slice(0, -4) : trimmed; +} + +// ── Bubble chip with typewriter animation ──────────────────────────────── + function OverlayBubbleChip({ bubble }: { bubble: OverlayBubble }) { + // Reset the typewriter on every new bubble identity via `key` at the + // call site — that avoids a cascading setState inside this effect. const [displayedText, setDisplayedText] = useState(''); const indexRef = useRef(0); @@ -46,27 +125,25 @@ function OverlayBubbleChip({ bubble }: { bubble: OverlayBubble }) { if (!bubble.text) { return () => { indexRef.current = 0; - setDisplayedText(''); }; } - const timeoutId = window.setInterval( + const intervalId = window.setInterval( () => { indexRef.current += 1; setDisplayedText(bubble.text.slice(0, indexRef.current)); if (indexRef.current >= bubble.text.length) { - window.clearInterval(timeoutId); + window.clearInterval(intervalId); } }, bubble.compact ? 28 : 32 ); return () => { - window.clearInterval(timeoutId); + window.clearInterval(intervalId); indexRef.current = 0; - setDisplayedText(''); }; - }, [bubble.compact, bubble.id, bubble.text]); + }, [bubble.compact, bubble.text]); return (
(1); + const [mode, setMode] = useState('idle'); + const [bubble, setBubble] = useState(null); const [isHovered, setIsHovered] = useState(false); - useEffect(() => { - const timeoutId = window.setTimeout(() => { - setScenario(current => { - if (current === 1) return 2; - if (current === 2) return 3; - return 1; + /** Timer that returns the overlay to idle after a ttl (attention) or a + * grace period (stt release). We clear it whenever the mode changes. */ + const dismissTimerRef = useRef(null); + + const clearDismissTimer = useCallback(() => { + if (dismissTimerRef.current !== null) { + window.clearTimeout(dismissTimerRef.current); + dismissTimerRef.current = null; + } + }, []); + + const scheduleDismiss = useCallback( + (ms: number) => { + clearDismissTimer(); + dismissTimerRef.current = window.setTimeout(() => { + console.debug('[overlay] auto-dismiss → idle'); + setMode('idle'); + setBubble(null); + dismissTimerRef.current = null; + }, ms); + }, + [clearDismissTimer] + ); + + const goIdle = useCallback(() => { + clearDismissTimer(); + setMode('idle'); + setBubble(null); + }, [clearDismissTimer]); + + // ── Dictation: pressed / released ────────────────────────────────────── + const handleDictationToggle = useCallback( + (payload: DictationTogglePayload) => { + const type = payload?.type ?? 'pressed'; + console.debug(`[overlay] dictation:toggle type=${type}`); + + if (type === 'pressed') { + clearDismissTimer(); + setMode('stt'); + setBubble({ + id: `stt-${Date.now()}`, + text: STT_LISTENING_PLACEHOLDER, + tone: 'accent', + compact: true, + }); + return; + } + + if (type === 'released') { + // Linger briefly so any final transcription arriving shortly after + // has a chance to land in the bubble before we go idle. + scheduleDismiss(STT_RELEASE_LINGER_MS); + } + }, + [clearDismissTimer, scheduleDismiss] + ); + + // ── Dictation: final transcription text ──────────────────────────────── + const handleDictationTranscription = useCallback( + (payload: DictationTranscriptionPayload) => { + const text = payload?.text?.trim(); + if (!text) return; + console.debug(`[overlay] dictation:transcription chars=${text.length}`); + + setMode('stt'); + setBubble({ + id: `stt-final-${Date.now()}`, + text: `"${text}"`, + tone: 'accent', + compact: true, }); - }, 5000); + // Show the result briefly then dismiss, regardless of hotkey state. + scheduleDismiss(STT_RELEASE_LINGER_MS); + }, + [scheduleDismiss] + ); + + // ── Attention from subconscious / core ───────────────────────────────── + const handleAttention = useCallback( + (payload: OverlayAttentionPayload) => { + const message = payload?.message?.trim(); + if (!message) { + console.debug('[overlay] attention event with empty message — ignoring'); + return; + } + console.debug( + `[overlay] attention source=${payload?.source ?? 'unknown'} tone=${payload?.tone ?? 'neutral'} chars=${message.length}` + ); + + const ttl = + typeof payload?.ttl_ms === 'number' && payload.ttl_ms > 0 + ? payload.ttl_ms + : DEFAULT_ATTENTION_TTL_MS; + + setMode('attention'); + setBubble({ + id: payload?.id ?? `attention-${Date.now()}`, + text: `"${message}"`, + // Match the Rust-side `OverlayAttentionTone::default()` (Neutral) + // so missing/legacy payloads render as the neutral slate bubble. + tone: payload?.tone ?? 'neutral', + }); + scheduleDismiss(ttl); + }, + [scheduleDismiss] + ); + + // ── Socket.IO subscription lifecycle ─────────────────────────────────── + useEffect(() => { + let socket: Socket | null = null; + let disposed = false; + + const connect = async () => { + try { + const baseUrl = await resolveCoreSocketUrl(); + if (disposed) return; + + console.debug(`[overlay] connecting to core socket at ${baseUrl}`); + socket = io(baseUrl, { + path: '/socket.io/', + transports: ['websocket', 'polling'], + reconnection: true, + reconnectionDelay: 2000, + reconnectionAttempts: Infinity, + forceNew: true, + }); + + socket.on('connect', () => { + console.debug('[overlay] socket connected', socket?.id); + }); + + socket.on('connect_error', (err: Error) => { + console.debug('[overlay] socket connect error:', err.message); + }); + + socket.on('disconnect', (reason: string) => { + console.debug('[overlay] socket disconnected:', reason); + }); + + // Core emits each event under both colon and underscore forms + // (see `emit_with_aliases` in `src/core/socketio.rs`). Subscribe + // only to the canonical colon-delimited form so each signal fires + // the handler exactly once. + socket.on('dictation:toggle', handleDictationToggle); + socket.on('dictation:transcription', handleDictationTranscription); + socket.on('overlay:attention', handleAttention); + + socket.connect(); + } catch (err) { + console.warn('[overlay] failed to open core socket', err); + } + }; + + void connect(); return () => { - window.clearTimeout(timeoutId); + disposed = true; + if (socket) { + socket.disconnect(); + socket = null; + } + clearDismissTimer(); }; - }, [scenario]); + }, [clearDismissTimer, handleAttention, handleDictationToggle, handleDictationTranscription]); - const status: OverlayStatus = scenario === 1 ? 'idle' : 'active'; + // ── Window framing: resize / reposition on mode change ──────────────── + const status: 'idle' | 'active' = mode === 'idle' ? 'idle' : 'active'; useEffect(() => { const appWindow = getCurrentWindow(); @@ -141,23 +373,8 @@ export default function OverlayApp() { void updateWindowFrame(); }, [status]); - const bubbles = useMemo(() => { - if (scenario === 1) { - return []; - } - - if (scenario === 2) { - return [ - { - id: 'assistant', - text: '"Hey I think your coffee is getting cold. Want me to get you a new one?"', - tone: 'accent', - }, - ]; - } - - return [{ id: 'stt', text: SCENARIO_THREE_TEXT, tone: 'accent' }]; - }, [scenario]); + // ── Render ──────────────────────────────────────────────────────────── + const bubbles = useMemo(() => (bubble ? [bubble] : []), [bubble]); const orbClassName = useMemo(() => { if (status === 'active') { @@ -177,9 +394,10 @@ export default function OverlayApp() { className={`relative flex select-none flex-col items-end ${status === 'active' ? 'gap-3' : 'gap-0'}`}>
- {bubbles.map(bubble => ( -
- + {bubbles.map(b => ( +
+ {/* key on the chip itself remounts the typewriter for each new bubble */} +
))}
@@ -187,10 +405,8 @@ export default function OverlayApp() {
diff --git a/app/src/pages/Home.tsx b/app/src/pages/Home.tsx index 45f9f4f0a..c3a8e5938 100644 --- a/app/src/pages/Home.tsx +++ b/app/src/pages/Home.tsx @@ -9,13 +9,20 @@ import { triggerLocalAiAssetBootstrap, } from '../utils/localAiBootstrap'; import { formatBytes, formatEta, progressFromStatus } from '../utils/localAiHelpers'; -import { isTauri, type LocalAiStatus, openhumanLocalAiStatus } from '../utils/tauriCommands'; +import { + isTauri, + type LocalAiAssetsStatus, + type LocalAiStatus, + openhumanLocalAiAssetsStatus, + openhumanLocalAiStatus, +} from '../utils/tauriCommands'; const Home = () => { const { user } = useUser(); const navigate = useNavigate(); const userName = user?.firstName || 'User'; const [localAiStatus, setLocalAiStatus] = useState(null); + const [localAiAssets, setLocalAiAssets] = useState(null); const [downloadBusy, setDownloadBusy] = useState(false); const [bootstrapMessage, setBootstrapMessage] = useState(''); const autoRetryDoneRef = useRef(false); @@ -73,9 +80,16 @@ const Home = () => { let mounted = true; const load = async () => { try { - const status = await openhumanLocalAiStatus(); + const [status, assets] = await Promise.all([ + openhumanLocalAiStatus(), + openhumanLocalAiAssetsStatus().catch(err => { + console.warn('[Home] failed to load local AI assets status:', err); + return null; + }), + ]); if (mounted) { setLocalAiStatus(status.result); + setLocalAiAssets(assets?.result ?? null); // Auto-retry bootstrap once if Ollama is degraded (install/server issue). if (status.result?.state === 'degraded' && !autoRetryDoneRef.current) { @@ -155,6 +169,29 @@ const Home = () => { }, []); const modelProgress = useMemo(() => progressFromStatus(localAiStatus), [localAiStatus]); + // Hide the Local Model Runtime card once every capability's model file is + // present on disk. We use `assets_status` (which inspects the filesystem) + // instead of the in-memory `LocalAiStatus` sub-states, because the latter + // stay at `idle` until a capability is first exercised — even when the + // underlying model has already been downloaded. + // + // A capability is considered "done" when its asset state is: + // - `ready` → model file exists on disk + // - `disabled` → not applicable for the selected preset + // - `ondemand` → vision preset intentionally defers download until first use + const allModelsDownloaded = useMemo(() => { + if (!localAiStatus || !localAiAssets) return false; + if (localAiStatus.state !== 'ready') return false; + const isDone = (state: string | undefined | null): boolean => + state === 'ready' || state === 'disabled' || state === 'ondemand'; + return ( + isDone(localAiAssets.chat?.state) && + isDone(localAiAssets.vision?.state) && + isDone(localAiAssets.embedding?.state) && + isDone(localAiAssets.stt?.state) && + isDone(localAiAssets.tts?.state) + ); + }, [localAiStatus, localAiAssets]); const isInstalling = localAiStatus?.state === 'installing'; const indeterminateDownload = isInstalling || @@ -233,8 +270,8 @@ const Home = () => {
- {/* Local AI card (desktop only) */} - {isTauri() && ( + {/* Local AI card (desktop only) — hidden once all models are fully downloaded */} + {isTauri() && !allModelsDownloaded && (
diff --git a/app/src/pages/Settings.tsx b/app/src/pages/Settings.tsx index 55de1c621..c4d6c80c2 100644 --- a/app/src/pages/Settings.tsx +++ b/app/src/pages/Settings.tsx @@ -27,22 +27,6 @@ import SettingsSectionPage from '../components/settings/SettingsSectionPage'; import { APP_VERSION } from '../utils/config'; const accountSettingsItems = [ - { - id: 'billing', - title: 'Billing & Usage', - description: 'Manage your subscription, credits, and payment methods', - route: 'billing', - icon: ( - - - - ), - }, { id: 'recovery-phrase', title: 'Recovery Phrase', @@ -260,7 +244,7 @@ const Settings = () => { element={ } diff --git a/app/src/pages/Skills.tsx b/app/src/pages/Skills.tsx index 8fe205491..649cce196 100644 --- a/app/src/pages/Skills.tsx +++ b/app/src/pages/Skills.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from 'react'; +import { useMemo, useState } from 'react'; import { useNavigate } from 'react-router-dom'; import ChannelSetupModal from '../components/channels/ChannelSetupModal'; @@ -13,7 +13,6 @@ import { installSkill } from '../lib/skills/skillsApi'; import { useAppSelector } from '../store/hooks'; import type { ChannelConnectionStatus, ChannelDefinition, ChannelType } from '../types/channels'; import { IS_DEV } from '../utils/config'; -import { openhumanGetRuntimeFlags, openhumanSetBrowserAllowAll } from '../utils/tauriCommands'; const CHANNEL_ICONS: Record = { telegram: '\u2708\uFE0F', @@ -133,57 +132,6 @@ interface SkillItem { skill?: SkillListEntry; } -// ─── Browser Access Toggle ───────────────────────────────────────────────────── - -function BrowserAccessToggle() { - const [browserAllowAll, setBrowserAllowAll] = useState(false); - const [browserBusy, setBrowserBusy] = useState(false); - - useEffect(() => { - (async () => { - try { - const res = await openhumanGetRuntimeFlags(); - setBrowserAllowAll(res.result.browser_allow_all); - } catch { - // Silently ignore — toggle defaults to false - } - })(); - }, []); - - const handleToggle = async () => { - const next = !browserAllowAll; - setBrowserBusy(true); - try { - const res = await openhumanSetBrowserAllowAll(next); - setBrowserAllowAll(res.result.browser_allow_all); - } catch { - // silently ignore - } finally { - setBrowserBusy(false); - } - }; - - return ( -
-
-

Browser Access

-

Allow the browser tool to visit any public domain

-
-