From acc6246e591a56609e2e0ae5cdba2270e8035df0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Thu, 9 Apr 2026 15:51:27 -0700 Subject: [PATCH] Refactor core-polled app state and screen intelligence status (#464) * refactor(accessibility): remove device control and predictive input features from accessibility settings - Updated accessibility-related components and tests to eliminate device control and predictive input features. - Adjusted AccessibilityPanel and ScreenIntelligencePanel to reflect the removal of these features. - Modified related tests to ensure consistency with the updated accessibility status structure. - Cleaned up accessibility session parameters and state management to focus solely on screen monitoring. * refactor(accessibility): streamline featureOverrides state initialization - Simplified the initialization of featureOverrides state in AccessibilityPanel and ScreenIntelligencePanel components for better readability. - Consolidated parameter definitions in startAccessibilitySession to enhance clarity and maintainability. - Removed unnecessary re-exports in the screen_intelligence engine module to clean up the codebase. * chore(dependencies): update OpenHuman to version 0.51.19 in Cargo.lock * chore(dependencies): update OpenHuman version to 0.51.19 in Cargo.lock * feat(restart): implement core process restart functionality - Added a new `SystemRestartRequested` event to the `DomainEvent` enum to handle restart requests. - Introduced a `RestartSubscriber` that listens for restart events and manages the process respawn. - Created a `service_restart` function to publish restart requests via the event bus. - Updated service schemas to include a new `restart` controller with parameters for source and reason. - Enhanced documentation to reflect changes in behavior and added necessary code comments. * feat(accessibility): add last restart summary to Screen Intelligence Panel - Introduced `lastRestartSummary` to the accessibility state and updated relevant components to display the last successful core restart information. - Modified `PermissionsSection` and `ScreenIntelligencePanel` to include the new summary. - Updated tests to validate the display of the last restart summary and ensure proper state management during core restarts. - Refactored accessibility slice to handle the new restart summary in state updates. * feat(core): enhance startup process with restart delay and subscriber registration - Added a call to apply startup restart delay from environment variables in `run_core_from_args`. - Updated the `bootstrap_skill_runtime` function to register a `RestartSubscriber` for handling restart requests, ensuring consistent respawn logic across triggers. - Introduced a new `core_process` field in the `AccessibilityEngine` to track the core process status, including its PID and start time. - Implemented a helper function to capture the core process start time using `OnceLock` for efficient initialization. * feat(screen-intelligence): refactor accessibility state management and UI components - Replaced direct Redux state access with a new `useScreenIntelligenceState` hook across multiple components, including `AccessibilityPanel`, `ScreenIntelligencePanel`, and their respective subcomponents. - Streamlined permission and session handling by consolidating related functions and removing unnecessary dispatch calls. - Updated tests to mock the new state management approach, ensuring consistent behavior and validation of UI elements. - Removed the `SessionAndVisionSection` component to simplify the structure and improve maintainability. - Introduced a new API file for screen intelligence to encapsulate related functionality and improve code organization. * refactor(tests): clean up and optimize test files for accessibility and screen intelligence panels - Removed redundant imports and streamlined the structure of test files for `AccessibilityPanel` and `ScreenIntelligencePanel`. - Consolidated core process state initialization in test mocks for better readability. - Updated dependency imports and ensured consistent mocking of state management hooks across tests. - Enhanced the `ScreenPermissionsStep` component by improving the dependency array in the useEffect hook for better performance. * refactor(store): remove unused authentication and user management code - Deleted the `UserProvider`, `authSlice`, `authSelectors`, `userSlice`, `teamSlice`, and related test files to streamline the codebase. - This cleanup enhances maintainability by removing legacy code that is no longer in use. - Updated the store configuration to reflect the removal of these slices and ensure proper state management. * refactor(webhooks): reorganize types and remove legacy state management - Moved `TunnelRegistration` and `WebhookActivityEntry` types to a new `types.ts` file for better organization. - Updated imports in `TunnelList` and `WebhookActivity` components to reference the new types location. - Refactored `useWebhooks` hook to eliminate Redux state management in favor of local state, enhancing performance and reducing complexity. - Removed unused `aiSlice`, `inviteSlice`, and `webhooksSlice` along with their associated tests to streamline the codebase. * refactor(daemon): migrate state management from Redux to a custom store - Introduced a new `store.ts` file to manage daemon state, replacing the previous Redux slice. - Updated components and hooks to utilize the new state management approach, enhancing performance and reducing complexity. - Removed the legacy `daemonSlice` and associated Redux logic, streamlining the codebase. - Adjusted imports in various components and hooks to reference the new store structure. * refactor(screen-intelligence): integrate core state management and enhance status handling - Replaced direct state management in `useScreenIntelligenceState` with a new core state approach, utilizing `useCoreState` for improved performance and consistency. - Updated status fetching and permission handling to leverage the core state snapshot, streamlining the logic and reducing redundant API calls. - Introduced a new `CoreRuntimeSnapshot` interface to encapsulate runtime statuses, including screen intelligence, local AI, autocomplete, and service states. - Adjusted related components and hooks to align with the new state management structure, enhancing maintainability and readability. - Updated tests to validate the new runtime state structure and ensure proper functionality across the application. * refactor(components): reorganize imports and streamline function formatting - Moved the import of `Tunnel` and `tunnelsApi` in `TunnelList.tsx` for better organization. - Reformatted function definitions in `store.ts`, `useDaemonHealth.ts`, `useDaemonLifecycle.ts`, `useWebhooks.ts` for improved readability. - Cleaned up the structure of test files in `coreRpcClient.test.ts` by consolidating object properties for clarity. - These changes enhance code maintainability and readability across the application. * test(screen-intelligence): fix duplicate hook imports * fix(tests): update ScreenIntelligenceDebugPanel test to use baseState for refresh status and vision calls * refactor(invites): simplify error message rendering in Invites component - Consolidated the conditional rendering of the load error message in the Invites component for improved readability. - This change enhances the clarity of the code without altering functionality. * refactor(daemon): streamline state management and function definitions - Removed the `healthTimeoutId` from the `DaemonUserState` interface and related functions to simplify state management. - Converted several functions in `store.ts` to arrow function syntax for consistency and improved readability. - Updated the `Invites` component to handle asynchronous loading and error states more effectively, ensuring that in-flight requests are properly managed. - Refactored the `CoreStateProvider` to enhance the refresh logic and prevent multiple simultaneous refreshes. - Introduced a new `register_domain_subscribers` function in `jsonrpc.rs` to centralize event bus subscriber registration, improving code organization and maintainability. * fix: add debug logging, atomic restart guard, and idempotent subscriber registration - CoreStateProvider: add namespaced debug logger for polling failure diagnostics - service/bus.rs: add AtomicBool gate to prevent duplicate restart spawns - service/bus.rs: use OnceLock for idempotent RestartSubscriber registration - Invites.tsx: add debug log in loadInviteCodes catch block * style: apply prettier formatting to CoreStateProvider * fix: sanitize error logging, serialize refresh, and demote restart logs - CoreStateProvider: sanitize error objects in poll failure logs to avoid leaking tokens/headers - CoreStateProvider: move in-flight guard into refresh() via shared promise so all callers (poll, updateLocalState, storeSessionToken) are serialized - CoreStateProvider: log refreshTeams errors instead of swallowing them - service/bus.rs: demote duplicate-restart log to debug, omit reason from log output to avoid free-form text emission * style: apply cargo fmt to service/bus.rs --- CLAUDE.md | 2 + Cargo.lock | 2 +- .../daemon/DaemonHealthIndicator.tsx | 2 +- .../components/daemon/DaemonHealthPanel.tsx | 2 +- .../ScreenIntelligenceDebugPanel.tsx | 66 ++- .../ScreenIntelligenceDebugPanel.test.tsx | 263 ++++------ .../settings/panels/AccessibilityPanel.tsx | 107 +--- .../panels/ScreenIntelligencePanel.tsx | 200 +++++--- .../__tests__/AccessibilityPanel.test.tsx | 156 +++--- .../ScreenIntelligencePanel.test.tsx | 248 ++++----- .../PermissionsSection.tsx | 33 +- .../SessionAndVisionSection.tsx | 133 ----- app/src/components/webhooks/TunnelList.tsx | 2 +- .../components/webhooks/WebhookActivity.tsx | 2 +- app/src/features/daemon/store.ts | 146 ++++++ app/src/features/screen-intelligence/api.ts | 156 ++++++ .../useScreenIntelligenceState.ts | 255 ++++++++++ app/src/features/webhooks/types.ts | 21 + app/src/hooks/useDaemonHealth.ts | 117 +++-- app/src/hooks/useDaemonLifecycle.ts | 64 +-- app/src/hooks/useIntelligenceStats.ts | 25 +- app/src/hooks/useScreenIntelligenceItems.ts | 18 +- app/src/hooks/useWebhooks.ts | 111 ++-- app/src/lib/coreState/store.ts | 13 + app/src/pages/Invites.tsx | 77 ++- .../steps/ScreenPermissionsStep.tsx | 37 +- .../__tests__/ScreenPermissionsStep.test.tsx | 199 ++++---- app/src/providers/CoreStateProvider.tsx | 120 +++-- app/src/providers/UserProvider.tsx | 79 --- .../services/__tests__/coreRpcClient.test.ts | 4 +- app/src/services/coreStateApi.ts | 10 + app/src/services/daemonHealthService.ts | 25 +- .../__tests__/accessibilitySlice.test.ts | 166 ------ app/src/store/__tests__/aiSlice.test.ts | 125 ----- app/src/store/__tests__/authSelectors.test.ts | 76 --- app/src/store/__tests__/authSlice.test.ts | 136 ----- .../store/__tests__/socketSelectors.test.ts | 1 + app/src/store/__tests__/teamSlice.test.ts | 151 ------ app/src/store/__tests__/userSlice.test.ts | 66 --- app/src/store/accessibilitySlice.ts | 389 -------------- app/src/store/aiSlice.ts | 90 ---- app/src/store/authSelectors.ts | 19 - app/src/store/authSlice.ts | 112 ---- app/src/store/daemonSlice.ts | 199 -------- app/src/store/index.ts | 37 +- app/src/store/intelligenceSlice.ts | 478 ------------------ app/src/store/inviteSlice.ts | 95 ---- app/src/store/teamSlice.ts | 117 ----- app/src/store/userSlice.ts | 66 --- app/src/store/webhooksSlice.ts | 93 ---- app/src/test/test-utils.tsx | 6 - app/src/utils/tauriCommands/accessibility.ts | 11 +- app/src/utils/tauriCommands/hardware.ts | 19 + src/core/jsonrpc.rs | 51 +- src/core/screen_intelligence_cli.rs | 6 +- src/lib.rs | 1 + src/openhuman/app_state/ops.rs | 80 ++- src/openhuman/app_state/schemas.rs | 2 +- src/openhuman/event_bus/events.rs | 10 + src/openhuman/screen_intelligence/engine.rs | 25 +- src/openhuman/screen_intelligence/input.rs | 14 +- src/openhuman/screen_intelligence/ops.rs | 2 +- src/openhuman/screen_intelligence/server.rs | 2 - src/openhuman/screen_intelligence/state.rs | 2 - src/openhuman/screen_intelligence/tests.rs | 6 - src/openhuman/screen_intelligence/types.rs | 13 +- src/openhuman/service/bus.rs | 102 ++++ src/openhuman/service/mod.rs | 4 + src/openhuman/service/ops.rs | 7 + src/openhuman/service/restart.rs | 210 ++++++++ src/openhuman/service/schemas.rs | 60 ++- tests/json_rpc_e2e.rs | 61 +++ 72 files changed, 2135 insertions(+), 3670 deletions(-) delete mode 100644 app/src/components/settings/panels/screen-intelligence/SessionAndVisionSection.tsx create mode 100644 app/src/features/daemon/store.ts create mode 100644 app/src/features/screen-intelligence/api.ts create mode 100644 app/src/features/screen-intelligence/useScreenIntelligenceState.ts create mode 100644 app/src/features/webhooks/types.ts delete mode 100644 app/src/providers/UserProvider.tsx delete mode 100644 app/src/store/__tests__/accessibilitySlice.test.ts delete mode 100644 app/src/store/__tests__/aiSlice.test.ts delete mode 100644 app/src/store/__tests__/authSelectors.test.ts delete mode 100644 app/src/store/__tests__/authSlice.test.ts delete mode 100644 app/src/store/__tests__/teamSlice.test.ts delete mode 100644 app/src/store/__tests__/userSlice.test.ts delete mode 100644 app/src/store/accessibilitySlice.ts delete mode 100644 app/src/store/aiSlice.ts delete mode 100644 app/src/store/authSelectors.ts delete mode 100644 app/src/store/authSlice.ts delete mode 100644 app/src/store/daemonSlice.ts delete mode 100644 app/src/store/intelligenceSlice.ts delete mode 100644 app/src/store/inviteSlice.ts delete mode 100644 app/src/store/teamSlice.ts delete mode 100644 app/src/store/userSlice.ts delete mode 100644 app/src/store/webhooksSlice.ts create mode 100644 src/openhuman/service/bus.rs create mode 100644 src/openhuman/service/restart.rs diff --git a/CLAUDE.md b/CLAUDE.md index 901a33dfa..5ced5faba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -265,6 +265,7 @@ Deep link plugin is registered where supported; behavior is platform-specific (s - **`src/openhuman/` module layout**: **New** functionality must live in a **dedicated subdirectory** (its own folder/module, e.g. `openhuman/my_domain/mod.rs` plus related files, or a new subfolder under an existing domain). Do **not** add new standalone `*.rs` files directly at `src/openhuman/` root; place new code in a module directory and declare it from `mod.rs` (or merge into an existing domain folder). - **Controller schema contract**: Shared controller metadata types live in **`src/core/mod.rs`** (`ControllerSchema`, `FieldSchema`, `TypeSchema`) and are consumed by adapters (RPC/CLI) in different ways. - **Domain schema files**: For each domain, define controller schema metadata in a dedicated module inside the domain folder (example: **`src/openhuman/cron/schemas.rs`**) and export from the domain `mod.rs`. +- **Controller-only exposure rule**: Expose domain functionality to **CLI and JSON-RPC through the controller registry** (`schemas.rs` + registered handlers). Do **not** add domain-specific branches or one-off transport logic in `src/core/cli.rs` or `src/core/jsonrpc.rs` just to expose a feature. - **Light `mod.rs` rule**: Keep domain `mod.rs` files light and export-focused. Put operational code in sibling files (example: `ops.rs`, `store.rs`, `schedule.rs`, `types.rs`), then re-export the public API from `mod.rs`. - **`core_server/`** — Transport only: Axum/HTTP, JSON-RPC envelope, CLI parsing, **dispatch** (`core_server::dispatch`) — **no** heavy business logic here. - **Layering**: Implementation in `openhuman::/`, controllers in `openhuman::/rpc.rs`, routes in `core_server/`. @@ -384,6 +385,7 @@ In the parent **OpenHuman** desktop app, **Tauri / Rust is a delivery vehicle**: - **Unix-style modules**: Prefer **individual modules** with a **single, sharp responsibility**—each should do one thing really well. Compose behavior through small, well-named units and clear boundaries instead of monolithic code. - **Tests before the next layer**: Ship **enough unit tests and coverage** for the behavior you are adding or changing **before** building additional features on top of it. Treat untested code as incomplete; do not accumulate depth on a shaky base. +- **Documentation with code**: New or changed behavior must ship with matching documentation. At minimum, add concise rustdoc / code comments where the flow is not obvious, and update `AGENTS.md`, architecture docs, or feature docs when repository rules or user-visible behavior change. --- diff --git a/Cargo.lock b/Cargo.lock index 790906528..14bb9a1f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5120,7 +5120,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openhuman" -version = "0.49.17" +version = "0.51.19" dependencies = [ "aes-gcm", "anyhow", diff --git a/app/src/components/daemon/DaemonHealthIndicator.tsx b/app/src/components/daemon/DaemonHealthIndicator.tsx index 72f36761b..ca59aeb75 100644 --- a/app/src/components/daemon/DaemonHealthIndicator.tsx +++ b/app/src/components/daemon/DaemonHealthIndicator.tsx @@ -6,8 +6,8 @@ */ import type React from 'react'; +import type { DaemonStatus } from '../../features/daemon/store'; import { formatRelativeTime, useDaemonHealth } from '../../hooks/useDaemonHealth'; -import type { DaemonStatus } from '../../store/daemonSlice'; interface Props { userId?: string; diff --git a/app/src/components/daemon/DaemonHealthPanel.tsx b/app/src/components/daemon/DaemonHealthPanel.tsx index 47cc8caa4..dc1525177 100644 --- a/app/src/components/daemon/DaemonHealthPanel.tsx +++ b/app/src/components/daemon/DaemonHealthPanel.tsx @@ -15,8 +15,8 @@ import { } from '@heroicons/react/24/outline'; import { useEffect, useState } from 'react'; +import type { ComponentHealth, DaemonStatus } from '../../features/daemon/store'; import { formatRelativeTime, useDaemonHealth } from '../../hooks/useDaemonHealth'; -import type { ComponentHealth, DaemonStatus } from '../../store/daemonSlice'; import { IS_DEV } from '../../utils/config'; import { isTauri, diff --git a/app/src/components/intelligence/ScreenIntelligenceDebugPanel.tsx b/app/src/components/intelligence/ScreenIntelligenceDebugPanel.tsx index 80fe0ffb0..700b1b259 100644 --- a/app/src/components/intelligence/ScreenIntelligenceDebugPanel.tsx +++ b/app/src/components/intelligence/ScreenIntelligenceDebugPanel.tsx @@ -1,11 +1,9 @@ -import { useCallback, useEffect } from 'react'; +import { useCallback } from 'react'; import { - fetchAccessibilityStatus, - fetchAccessibilityVisionRecent, - runCaptureTest, -} from '../../store/accessibilitySlice'; -import { useAppDispatch, useAppSelector } from '../../store/hooks'; + type ScreenIntelligenceState, + useScreenIntelligenceState, +} from '../../features/screen-intelligence/useScreenIntelligenceState'; const formatBytes = (bytes: number | null | undefined): string => { if (bytes == null) return '-'; @@ -14,24 +12,42 @@ const formatBytes = (bytes: number | null | undefined): string => { return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; }; -const ScreenIntelligenceDebugPanel = () => { - const dispatch = useAppDispatch(); - const { status, captureTestResult, isCaptureTestRunning, recentVisionSummaries, lastError } = - useAppSelector(state => state.accessibility); +interface ScreenIntelligenceDebugPanelProps { + state?: Pick< + ScreenIntelligenceState, + | 'status' + | 'captureTestResult' + | 'isCaptureTestRunning' + | 'recentVisionSummaries' + | 'lastError' + | 'refreshStatus' + | 'refreshVision' + | 'runCaptureTest' + >; +} - useEffect(() => { - void dispatch(fetchAccessibilityStatus()); - void dispatch(fetchAccessibilityVisionRecent(5)); - }, [dispatch]); +const ScreenIntelligenceDebugPanelContent = ({ + state: providedState, +}: Required>) => { + const { + status, + captureTestResult, + isCaptureTestRunning, + recentVisionSummaries, + lastError, + refreshStatus, + refreshVision, + runCaptureTest, + } = providedState; const handleCaptureTest = useCallback(() => { - void dispatch(runCaptureTest()); - }, [dispatch]); + void runCaptureTest(); + }, [runCaptureTest]); const handleRefreshStatus = useCallback(() => { - void dispatch(fetchAccessibilityStatus()); - void dispatch(fetchAccessibilityVisionRecent(5)); - }, [dispatch]); + void refreshStatus(); + void refreshVision(5); + }, [refreshStatus, refreshVision]); const permissions = status?.permissions; const session = status?.session; @@ -199,6 +215,18 @@ const ScreenIntelligenceDebugPanel = () => { ); }; +const OwnedScreenIntelligenceDebugPanel = () => { + const state = useScreenIntelligenceState({ loadVision: true, visionLimit: 5, pollMs: 2000 }); + return ; +}; + +const ScreenIntelligenceDebugPanel = ({ state }: ScreenIntelligenceDebugPanelProps) => { + if (state) { + return ; + } + return ; +}; + const PermissionDot = ({ label, value }: { label: string; value?: string }) => { const color = value === 'granted' ? 'bg-green-500' : value === 'denied' ? 'bg-red-500' : 'bg-stone-600'; diff --git a/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx b/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx index aa1462b51..cbefe7947 100644 --- a/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx +++ b/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx @@ -1,177 +1,112 @@ -import { configureStore } from '@reduxjs/toolkit'; -import { fireEvent, render, screen, waitFor } from '@testing-library/react'; -import { Provider } from 'react-redux'; +import { fireEvent, render, screen } from '@testing-library/react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import accessibilityReducer from '../../../store/accessibilitySlice'; -import authReducer from '../../../store/authSlice'; -import socketReducer from '../../../store/socketSlice'; -import teamReducer from '../../../store/teamSlice'; -import userReducer from '../../../store/userSlice'; import { - type AccessibilityStatus, - type AccessibilityVisionRecentResult, - type CaptureTestResult, - type CommandResponse, - openhumanAccessibilityInputAction, - openhumanAccessibilityRequestPermission, - openhumanAccessibilityRequestPermissions, - openhumanAccessibilityStartSession, - openhumanAccessibilityStatus, - openhumanAccessibilityStopSession, - openhumanAccessibilityVisionFlush, - openhumanAccessibilityVisionRecent, - openhumanScreenIntelligenceCaptureTest, - restartCoreProcess, -} from '../../../utils/tauriCommands'; + type ScreenIntelligenceState, + useScreenIntelligenceState, +} from '../../../features/screen-intelligence/useScreenIntelligenceState'; import ScreenIntelligenceDebugPanel from '../ScreenIntelligenceDebugPanel'; -vi.mock('../../../utils/tauriCommands', () => ({ - isTauri: vi.fn(() => true), - openhumanAccessibilityInputAction: vi.fn(), - openhumanAccessibilityRequestPermission: vi.fn(), - openhumanAccessibilityRequestPermissions: vi.fn(), - openhumanAccessibilityStartSession: vi.fn(), - openhumanAccessibilityStatus: vi.fn(), - openhumanAccessibilityStopSession: vi.fn(), - openhumanAccessibilityVisionFlush: vi.fn(), - openhumanAccessibilityVisionRecent: vi.fn(), - openhumanScreenIntelligenceCaptureTest: vi.fn(), - restartCoreProcess: vi.fn(), +vi.mock('../../../features/screen-intelligence/useScreenIntelligenceState', () => ({ + useScreenIntelligenceState: vi.fn(), })); -const sampleStatus: AccessibilityStatus = { - platform_supported: true, - permissions: { - screen_recording: 'granted', - accessibility: 'granted', - input_monitoring: 'granted', - }, - features: { screen_monitoring: true, device_control: true, predictive_input: true }, - session: { - active: false, - started_at_ms: null, - expires_at_ms: null, - remaining_ms: null, - ttl_secs: 300, - panic_hotkey: 'Cmd+Shift+.', - stop_reason: null, - frames_in_memory: 0, - last_capture_at_ms: null, - last_context: null, - vision_enabled: true, - vision_state: 'idle', - vision_queue_depth: 0, - last_vision_at_ms: null, - last_vision_summary: null, - }, - config: { - enabled: true, - capture_policy: 'hybrid', - policy_mode: 'all_except_blacklist', - baseline_fps: 1, - vision_enabled: true, - session_ttl_secs: 300, - panic_stop_hotkey: 'Cmd+Shift+.', - autocomplete_enabled: true, - use_vision_model: true, - keep_screenshots: false, - allowlist: [], - denylist: [], - }, - denylist: [], - is_context_blocked: false, -}; - -const emptyVisionResponse: CommandResponse = { - result: { summaries: [] }, - logs: [], -}; - -const createStore = () => - configureStore({ - reducer: { - auth: authReducer, - socket: socketReducer, - user: userReducer, - team: teamReducer, - accessibility: accessibilityReducer, +const baseState: ScreenIntelligenceState = { + status: { + platform_supported: true, + permissions: { + screen_recording: 'granted', + accessibility: 'granted', + input_monitoring: 'granted', }, - }); - -function renderPanel() { - const store = createStore(); - render( - - - - ); - return store; -} + features: { screen_monitoring: true }, + session: { + active: false, + started_at_ms: null, + expires_at_ms: null, + remaining_ms: null, + ttl_secs: 300, + panic_hotkey: 'Cmd+Shift+.', + stop_reason: null, + frames_in_memory: 0, + last_capture_at_ms: null, + last_context: null, + vision_enabled: true, + vision_state: 'idle', + vision_queue_depth: 0, + last_vision_at_ms: null, + last_vision_summary: null, + }, + config: { + enabled: true, + capture_policy: 'hybrid', + policy_mode: 'all_except_blacklist', + baseline_fps: 1, + vision_enabled: true, + session_ttl_secs: 300, + panic_stop_hotkey: 'Cmd+Shift+.', + autocomplete_enabled: true, + use_vision_model: true, + keep_screenshots: false, + allowlist: [], + denylist: [], + }, + denylist: [], + is_context_blocked: false, + }, + lastRestartSummary: null, + recentVisionSummaries: [], + captureTestResult: null, + isCaptureTestRunning: false, + isLoading: false, + isRequestingPermissions: false, + isRestartingCore: false, + isStartingSession: false, + isStoppingSession: false, + isLoadingVision: false, + isFlushingVision: false, + lastError: null, + refreshStatus: vi.fn().mockResolvedValue(null), + requestPermission: vi.fn().mockResolvedValue(null), + refreshPermissionsWithRestart: vi.fn().mockResolvedValue(null), + startSession: vi.fn().mockResolvedValue(null), + stopSession: vi.fn().mockResolvedValue(null), + refreshVision: vi.fn().mockResolvedValue([]), + flushVision: vi.fn().mockResolvedValue(undefined), + runCaptureTest: vi.fn().mockResolvedValue(undefined), + clearError: vi.fn(), +}; describe('ScreenIntelligenceDebugPanel', () => { beforeEach(() => { - vi.mocked(openhumanAccessibilityStatus).mockResolvedValue({ result: sampleStatus, logs: [] }); - vi.mocked(openhumanAccessibilityVisionRecent).mockResolvedValue(emptyVisionResponse); - vi.mocked(openhumanAccessibilityInputAction).mockResolvedValue({ - result: {} as never, - logs: [], - }); - vi.mocked(openhumanAccessibilityRequestPermission).mockResolvedValue({ - result: sampleStatus.permissions, - logs: [], - } as never); - vi.mocked(openhumanAccessibilityRequestPermissions).mockResolvedValue({ - result: sampleStatus.permissions, - logs: [], - } as never); - vi.mocked(openhumanAccessibilityStartSession).mockResolvedValue({ - result: sampleStatus.session, - logs: [], - } as never); - vi.mocked(openhumanAccessibilityStopSession).mockResolvedValue({ - result: sampleStatus.session, - logs: [], - } as never); - vi.mocked(openhumanAccessibilityVisionFlush).mockResolvedValue({ - result: { accepted: true, summary: null }, - logs: [], - } as never); - vi.mocked(restartCoreProcess).mockResolvedValue(undefined); + vi.clearAllMocks(); + vi.mocked(useScreenIntelligenceState).mockReturnValue(baseState); }); it('renders successful capture diagnostics and preview image', async () => { - const captureResult: CaptureTestResult = { - ok: true, - capture_mode: 'windowed', - context: { - app_name: 'Safari', - window_title: 'GitHub', - bounds_x: 10, - bounds_y: 20, - bounds_width: 1440, - bounds_height: 900, + const state: ScreenIntelligenceState = { + ...baseState, + captureTestResult: { + ok: true, + capture_mode: 'windowed', + context: { + app_name: 'Safari', + window_title: 'GitHub', + bounds_x: 10, + bounds_y: 20, + bounds_width: 1440, + bounds_height: 900, + }, + image_ref: 'data:image/png;base64,ZmFrZQ==', + bytes_estimate: 2048, + error: null, + timing_ms: 155, }, - image_ref: 'data:image/png;base64,ZmFrZQ==', - bytes_estimate: 2048, - error: null, - timing_ms: 155, }; - vi.mocked(openhumanScreenIntelligenceCaptureTest).mockResolvedValue({ - result: captureResult, - logs: [], - }); - renderPanel(); + render(); - await waitFor(() => { - expect(openhumanAccessibilityStatus).toHaveBeenCalled(); - expect(openhumanAccessibilityVisionRecent).toHaveBeenCalledWith(5); - }); - - fireEvent.click(screen.getByRole('button', { name: 'Test Capture' })); - - expect(await screen.findByText('Success')).toBeInTheDocument(); + expect(screen.getByText('Success')).toBeInTheDocument(); expect(screen.getByText('windowed')).toBeInTheDocument(); expect(screen.getByText('155ms')).toBeInTheDocument(); expect(screen.getByText('2.0 KB')).toBeInTheDocument(); @@ -183,8 +118,9 @@ describe('ScreenIntelligenceDebugPanel', () => { }); it('renders capture failures without breaking the diagnostics panel', async () => { - vi.mocked(openhumanScreenIntelligenceCaptureTest).mockResolvedValue({ - result: { + const state: ScreenIntelligenceState = { + ...baseState, + captureTestResult: { ok: false, capture_mode: 'fullscreen', context: null, @@ -193,18 +129,19 @@ describe('ScreenIntelligenceDebugPanel', () => { error: 'screen recording permission is not granted', timing_ms: 42, }, - logs: [], - }); + }; - renderPanel(); + render(); - fireEvent.click(screen.getByRole('button', { name: 'Test Capture' })); + fireEvent.click(screen.getByRole('button', { name: 'Refresh' })); - expect(await screen.findByText('Failed')).toBeInTheDocument(); + expect(screen.getByText('Failed')).toBeInTheDocument(); expect(screen.getByText('fullscreen')).toBeInTheDocument(); expect(screen.getByText('42ms')).toBeInTheDocument(); expect(screen.getByText('screen recording permission is not granted')).toBeInTheDocument(); expect(screen.queryByAltText('Capture test result')).not.toBeInTheDocument(); expect(screen.getByText('Permissions')).toBeInTheDocument(); + expect(baseState.refreshStatus).toHaveBeenCalledTimes(1); + expect(baseState.refreshVision).toHaveBeenCalledWith(5); }); }); diff --git a/app/src/components/settings/panels/AccessibilityPanel.tsx b/app/src/components/settings/panels/AccessibilityPanel.tsx index 0c5f2a013..172147d31 100644 --- a/app/src/components/settings/panels/AccessibilityPanel.tsx +++ b/app/src/components/settings/panels/AccessibilityPanel.tsx @@ -1,15 +1,6 @@ -import { useEffect, useMemo, useState } from 'react'; +import { useMemo, useState } from 'react'; -import { - fetchAccessibilityStatus, - fetchAccessibilityVisionRecent, - flushAccessibilityVision, - refreshPermissionsWithRestart, - requestAccessibilityPermission, - startAccessibilitySession, - stopAccessibilitySession, -} from '../../../store/accessibilitySlice'; -import { useAppDispatch, useAppSelector } from '../../../store/hooks'; +import { useScreenIntelligenceState } from '../../../features/screen-intelligence/useScreenIntelligenceState'; import SettingsHeader from '../components/SettingsHeader'; import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; @@ -46,7 +37,6 @@ const PermissionBadge = ({ label, value }: { label: string; value: string }) => const AccessibilityPanel = () => { const { navigateBack, breadcrumbs } = useSettingsNavigation(); - const dispatch = useAppDispatch(); const { status, isLoading, @@ -58,31 +48,15 @@ const AccessibilityPanel = () => { isFlushingVision, recentVisionSummaries, lastError, - } = useAppSelector(state => state.accessibility); - const [featureOverrides, setFeatureOverrides] = useState<{ - screen_monitoring?: boolean; - device_control?: boolean; - predictive_input?: boolean; - }>({}); - - useEffect(() => { - void dispatch(fetchAccessibilityStatus()); - }, [dispatch]); - - useEffect(() => { - if (!status?.session.active) { - return; - } - const intervalId = window.setInterval(() => { - void dispatch(fetchAccessibilityStatus()); - void dispatch(fetchAccessibilityVisionRecent(10)); - }, 1000); - return () => window.clearInterval(intervalId); - }, [dispatch, status?.session.active]); - - useEffect(() => { - void dispatch(fetchAccessibilityVisionRecent(10)); - }, [dispatch]); + requestPermission, + refreshPermissionsWithRestart, + refreshStatus, + refreshVision, + startSession, + stopSession, + flushVision, + } = useScreenIntelligenceState({ loadVision: true, visionLimit: 10, pollMs: 2000 }); + const [featureOverrides, setFeatureOverrides] = useState<{ screen_monitoring?: boolean }>({}); const anyPermissionDenied = status?.permissions.accessibility === 'denied' || @@ -90,9 +64,6 @@ const AccessibilityPanel = () => { const screenMonitoring = featureOverrides.screen_monitoring ?? status?.features.screen_monitoring ?? true; - const deviceControl = featureOverrides.device_control ?? status?.features.device_control ?? true; - const predictiveInput = - featureOverrides.predictive_input ?? status?.features.predictive_input ?? true; const remaining = useMemo( () => formatRemaining(status?.session.remaining_ms ?? null), @@ -152,14 +123,14 @@ const AccessibilityPanel = () => { - {isOpen && } + {isOpen && } ); }; const ScreenIntelligencePanel = () => { const { navigateBack, breadcrumbs } = useSettingsNavigation(); - const dispatch = useAppDispatch(); const { status, + lastRestartSummary, isLoading, isRequestingPermissions, isRestartingCore, @@ -56,12 +55,18 @@ const ScreenIntelligencePanel = () => { isFlushingVision, recentVisionSummaries, lastError, - } = useAppSelector(state => state.accessibility); - const [featureOverrides, setFeatureOverrides] = useState<{ - screen_monitoring?: boolean; - device_control?: boolean; - predictive_input?: boolean; - }>({}); + refreshStatus, + refreshVision, + startSession, + stopSession, + flushVision, + requestPermission, + refreshPermissionsWithRestart, + runCaptureTest, + captureTestResult, + isCaptureTestRunning, + } = useScreenIntelligenceState({ loadVision: true, visionLimit: 10, pollMs: 2000 }); + const [featureOverrides, setFeatureOverrides] = useState<{ screen_monitoring?: boolean }>({}); const [enabled, setEnabled] = useState(false); const [policyMode, setPolicyMode] = useState<'all_except_blacklist' | 'whitelist_only'>( 'all_except_blacklist' @@ -74,25 +79,6 @@ const ScreenIntelligencePanel = () => { const [isSavingConfig, setIsSavingConfig] = useState(false); const [configError, setConfigError] = useState(null); - useEffect(() => { - void dispatch(fetchAccessibilityStatus()); - }, [dispatch]); - - useEffect(() => { - if (!status?.session.active) { - return; - } - const intervalId = window.setInterval(() => { - void dispatch(fetchAccessibilityStatus()); - void dispatch(fetchAccessibilityVisionRecent(10)); - }, 1000); - return () => window.clearInterval(intervalId); - }, [dispatch, status?.session.active]); - - useEffect(() => { - void dispatch(fetchAccessibilityVisionRecent(10)); - }, [dispatch]); - useEffect(() => { if (!status?.config) { return; @@ -110,9 +96,6 @@ const ScreenIntelligencePanel = () => { const screenMonitoring = featureOverrides.screen_monitoring ?? status?.features.screen_monitoring ?? true; - const deviceControl = featureOverrides.device_control ?? status?.features.device_control ?? true; - const predictiveInput = - featureOverrides.predictive_input ?? status?.features.predictive_input ?? true; const remaining = useMemo( () => formatRemaining(status?.session.remaining_ms ?? null), @@ -154,7 +137,7 @@ const ScreenIntelligencePanel = () => { .map(v => v.trim()) .filter(Boolean), }); - await dispatch(fetchAccessibilityStatus()); + await refreshStatus(); } catch (error) { setConfigError(error instanceof Error ? error.message : 'Failed to save screen intelligence'); } finally { @@ -177,10 +160,14 @@ const ScreenIntelligencePanel = () => { accessibility={status?.permissions.accessibility ?? 'unknown'} inputMonitoring={status?.permissions.input_monitoring ?? 'unknown'} anyPermissionDenied={anyPermissionDenied ?? false} + lastRestartSummary={lastRestartSummary} permissionCheckProcessPath={status?.permission_check_process_path} isRequestingPermissions={isRequestingPermissions} isRestartingCore={isRestartingCore} isLoading={isLoading} + requestPermission={requestPermission} + refreshPermissionsWithRestart={refreshPermissionsWithRestart} + refreshStatus={refreshStatus} />
@@ -296,53 +283,100 @@ const ScreenIntelligencePanel = () => { } /> - - - -
- +

Session

+
+
Status: {status?.session.active ? 'Active' : 'Stopped'}
+
Remaining: {remaining}
+
Frames (ephemeral): {status?.session.frames_in_memory ?? 0}
+
Panic stop: {status?.session.panic_hotkey ?? 'Cmd+Shift+.'}
+
Vision: {status?.session.vision_state ?? 'idle'}
+
Vision queue: {status?.session.vision_queue_depth ?? 0}
+
+ Last vision:{' '} + {status?.session.last_vision_at_ms + ? new Date(status.session.last_vision_at_ms).toLocaleTimeString() + : 'n/a'} +
+
+ +
+ + + +
+ + +
+
+

Vision Summaries

+ +
+ + {recentVisionSummaries.length === 0 ? ( +
No summaries yet.
+ ) : ( +
+ {recentVisionSummaries.map(summary => ( +
+
+ {new Date(summary.captured_at_ms).toLocaleTimeString()} ·{' '} + {summary.app_name ?? 'Unknown App'} + {summary.window_title ? ` · ${summary.window_title}` : ''} +
+
{summary.actionable_notes}
+
+ ))} +
+ )} +
+ + - - - {status !== null && !status.platform_supported && (
Screen Intelligence V1 is currently supported on macOS only. diff --git a/app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx b/app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx index b01ec5512..5bc1435df 100644 --- a/app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx +++ b/app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx @@ -1,97 +1,91 @@ -import { configureStore } from '@reduxjs/toolkit'; import { render, screen } from '@testing-library/react'; -import { Provider } from 'react-redux'; import { MemoryRouter } from 'react-router-dom'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; -import accessibilityReducer from '../../../../store/accessibilitySlice'; -import authReducer from '../../../../store/authSlice'; -import socketReducer from '../../../../store/socketSlice'; -import teamReducer from '../../../../store/teamSlice'; -import userReducer from '../../../../store/userSlice'; -import type { AccessibilityStatus } from '../../../../utils/tauriCommands'; +import { + type ScreenIntelligenceState, + useScreenIntelligenceState, +} from '../../../../features/screen-intelligence/useScreenIntelligenceState'; import AccessibilityPanel from '../AccessibilityPanel'; -const status: AccessibilityStatus = { - platform_supported: true, - permissions: { - screen_recording: 'unknown', - accessibility: 'granted', - input_monitoring: 'unknown', - }, - features: { screen_monitoring: true, device_control: true, predictive_input: true }, - session: { - active: false, - started_at_ms: null, - expires_at_ms: null, - remaining_ms: null, - ttl_secs: 300, - panic_hotkey: 'Cmd+Shift+.', - stop_reason: null, - frames_in_memory: 0, - last_capture_at_ms: null, - last_context: null, - vision_enabled: true, - vision_state: 'idle', - vision_queue_depth: 0, - last_vision_at_ms: null, - last_vision_summary: null, - }, - config: { - enabled: true, - capture_policy: 'hybrid', - policy_mode: 'all_except_blacklist', - baseline_fps: 1, - vision_enabled: true, - session_ttl_secs: 300, - panic_stop_hotkey: 'Cmd+Shift+.', - autocomplete_enabled: true, - use_vision_model: true, - keep_screenshots: false, - allowlist: [], - denylist: ['wallet'], - }, - denylist: ['wallet'], - is_context_blocked: false, -}; +vi.mock('../../../../features/screen-intelligence/useScreenIntelligenceState', () => ({ + useScreenIntelligenceState: vi.fn(), +})); -const createStore = () => - configureStore({ - reducer: { - auth: authReducer, - socket: socketReducer, - user: userReducer, - team: teamReducer, - accessibility: accessibilityReducer, +const mockState: ScreenIntelligenceState = { + status: { + platform_supported: true, + permissions: { + screen_recording: 'unknown', + accessibility: 'granted', + input_monitoring: 'unknown', }, - preloadedState: { - accessibility: { - status, - recentVisionSummaries: [], - captureTestResult: null, - isCaptureTestRunning: false, - isLoading: false, - isRequestingPermissions: false, - isRestartingCore: false, - isStartingSession: false, - isStoppingSession: false, - isLoadingVision: false, - isFlushingVision: false, - lastError: null, - }, + features: { screen_monitoring: true }, + session: { + active: false, + started_at_ms: null, + expires_at_ms: null, + remaining_ms: null, + ttl_secs: 300, + panic_hotkey: 'Cmd+Shift+.', + stop_reason: null, + frames_in_memory: 0, + last_capture_at_ms: null, + last_context: null, + vision_enabled: true, + vision_state: 'idle', + vision_queue_depth: 0, + last_vision_at_ms: null, + last_vision_summary: null, }, - }); + config: { + enabled: true, + capture_policy: 'hybrid', + policy_mode: 'all_except_blacklist', + baseline_fps: 1, + vision_enabled: true, + session_ttl_secs: 300, + panic_stop_hotkey: 'Cmd+Shift+.', + autocomplete_enabled: true, + use_vision_model: true, + keep_screenshots: false, + allowlist: [], + denylist: ['wallet'], + }, + denylist: ['wallet'], + is_context_blocked: false, + }, + lastRestartSummary: null, + recentVisionSummaries: [], + captureTestResult: null, + isCaptureTestRunning: false, + isLoading: false, + isRequestingPermissions: false, + isRestartingCore: false, + isStartingSession: false, + isStoppingSession: false, + isLoadingVision: false, + isFlushingVision: false, + lastError: null, + refreshStatus: vi.fn(), + requestPermission: vi.fn(), + refreshPermissionsWithRestart: vi.fn(), + startSession: vi.fn(), + stopSession: vi.fn(), + refreshVision: vi.fn(), + flushVision: vi.fn(), + runCaptureTest: vi.fn(), + clearError: vi.fn(), +}; describe('AccessibilityPanel', () => { it('renders permission and session sections', () => { - const store = createStore(); + vi.mocked(useScreenIntelligenceState).mockReturnValue(mockState); render( - - - - - + + + ); expect(screen.getByText('Accessibility Automation')).toBeInTheDocument(); diff --git a/app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx b/app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx index cb30858d9..679aab169 100644 --- a/app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx +++ b/app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx @@ -1,119 +1,106 @@ -import { configureStore } from '@reduxjs/toolkit'; import { fireEvent, render, screen, waitFor } from '@testing-library/react'; -import { Provider } from 'react-redux'; import { MemoryRouter } from 'react-router-dom'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import accessibilityReducer from '../../../../store/accessibilitySlice'; -import authReducer from '../../../../store/authSlice'; -import socketReducer from '../../../../store/socketSlice'; -import teamReducer from '../../../../store/teamSlice'; -import userReducer from '../../../../store/userSlice'; import { - type AccessibilityStatus, - type AccessibilityVisionRecentResult, - type CommandResponse, + type ScreenIntelligenceState, + useScreenIntelligenceState, +} from '../../../../features/screen-intelligence/useScreenIntelligenceState'; +import { type ConfigSnapshot, isTauri, - openhumanAccessibilityInputAction, - openhumanAccessibilityRequestPermission, - openhumanAccessibilityRequestPermissions, - openhumanAccessibilityStartSession, - openhumanAccessibilityStatus, - openhumanAccessibilityStopSession, - openhumanAccessibilityVisionFlush, - openhumanAccessibilityVisionRecent, - openhumanScreenIntelligenceCaptureTest, openhumanUpdateScreenIntelligenceSettings, - restartCoreProcess, } from '../../../../utils/tauriCommands'; import ScreenIntelligencePanel from '../ScreenIntelligencePanel'; -vi.mock('../../../../utils/tauriCommands', () => ({ - isTauri: vi.fn(() => true), - openhumanAccessibilityInputAction: vi.fn(), - openhumanAccessibilityRequestPermission: vi.fn(), - openhumanAccessibilityRequestPermissions: vi.fn(), - openhumanAccessibilityStartSession: vi.fn(), - openhumanAccessibilityStatus: vi.fn(), - openhumanAccessibilityStopSession: vi.fn(), - openhumanAccessibilityVisionFlush: vi.fn(), - openhumanAccessibilityVisionRecent: vi.fn(), - openhumanScreenIntelligenceCaptureTest: vi.fn(), - openhumanUpdateScreenIntelligenceSettings: vi.fn(), - restartCoreProcess: vi.fn(), +vi.mock('../../../../features/screen-intelligence/useScreenIntelligenceState', () => ({ + useScreenIntelligenceState: vi.fn(), })); -const baseStatus: AccessibilityStatus = { - platform_supported: true, - permissions: { - screen_recording: 'granted', - accessibility: 'granted', - input_monitoring: 'unknown', - }, - features: { screen_monitoring: true, device_control: true, predictive_input: true }, - session: { - active: false, - started_at_ms: null, - expires_at_ms: null, - remaining_ms: null, - ttl_secs: 300, - panic_hotkey: 'Cmd+Shift+.', - stop_reason: null, - frames_in_memory: 0, - last_capture_at_ms: null, - last_context: null, - vision_enabled: true, - vision_state: 'idle', - vision_queue_depth: 0, - last_vision_at_ms: null, - last_vision_summary: null, - }, - config: { - enabled: false, - capture_policy: 'hybrid', - policy_mode: 'all_except_blacklist', - baseline_fps: 1, - vision_enabled: true, - session_ttl_secs: 300, - panic_stop_hotkey: 'Cmd+Shift+.', - autocomplete_enabled: true, - use_vision_model: true, - keep_screenshots: false, - allowlist: ['Code'], - denylist: ['1Password'], - }, - denylist: ['1Password'], - is_context_blocked: false, - permission_check_process_path: '/tmp/openhuman-core', -}; +vi.mock('../../../../utils/tauriCommands', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + isTauri: vi.fn(() => true), + openhumanUpdateScreenIntelligenceSettings: vi.fn(), + }; +}); -const emptyVisionResponse: CommandResponse = { - result: { summaries: [] }, - logs: [], -}; - -const createStore = () => - configureStore({ - reducer: { - auth: authReducer, - socket: socketReducer, - user: userReducer, - team: teamReducer, - accessibility: accessibilityReducer, +const baseState: ScreenIntelligenceState = { + status: { + platform_supported: true, + core_process: { pid: 4242, started_at_ms: 1712700000000 }, + permissions: { + screen_recording: 'granted', + accessibility: 'granted', + input_monitoring: 'unknown', }, - }); + features: { screen_monitoring: true }, + session: { + active: false, + started_at_ms: null, + expires_at_ms: null, + remaining_ms: null, + ttl_secs: 300, + panic_hotkey: 'Cmd+Shift+.', + stop_reason: null, + frames_in_memory: 0, + last_capture_at_ms: null, + last_context: null, + vision_enabled: true, + vision_state: 'idle', + vision_queue_depth: 0, + last_vision_at_ms: null, + last_vision_summary: null, + }, + config: { + enabled: false, + capture_policy: 'hybrid', + policy_mode: 'all_except_blacklist', + baseline_fps: 1, + vision_enabled: true, + session_ttl_secs: 300, + panic_stop_hotkey: 'Cmd+Shift+.', + autocomplete_enabled: true, + use_vision_model: true, + keep_screenshots: false, + allowlist: ['Code'], + denylist: ['1Password'], + }, + denylist: ['1Password'], + is_context_blocked: false, + permission_check_process_path: '/tmp/openhuman-core', + }, + lastRestartSummary: null, + recentVisionSummaries: [], + captureTestResult: null, + isCaptureTestRunning: false, + isLoading: false, + isRequestingPermissions: false, + isRestartingCore: false, + isStartingSession: false, + isStoppingSession: false, + isLoadingVision: false, + isFlushingVision: false, + lastError: null, + refreshStatus: vi.fn().mockResolvedValue(null), + requestPermission: vi.fn().mockResolvedValue(null), + refreshPermissionsWithRestart: vi.fn().mockResolvedValue(null), + startSession: vi.fn().mockResolvedValue(null), + stopSession: vi.fn().mockResolvedValue(null), + refreshVision: vi.fn().mockResolvedValue([]), + flushVision: vi.fn().mockResolvedValue(undefined), + runCaptureTest: vi.fn().mockResolvedValue(undefined), + clearError: vi.fn(), +}; -function renderPanel() { - const store = createStore(); +function renderPanel(state: ScreenIntelligenceState = baseState) { + vi.mocked(useScreenIntelligenceState).mockReturnValue(state); render( - - - - - + + + ); - return store; } function createDeferred() { @@ -126,50 +113,12 @@ function createDeferred() { describe('ScreenIntelligencePanel', () => { beforeEach(() => { + vi.clearAllMocks(); vi.mocked(isTauri).mockReturnValue(true); - vi.mocked(openhumanAccessibilityStatus).mockResolvedValue({ result: baseStatus, logs: [] }); - vi.mocked(openhumanAccessibilityVisionRecent).mockResolvedValue(emptyVisionResponse); - vi.mocked(openhumanAccessibilityInputAction).mockResolvedValue({ - result: {} as never, - logs: [], - }); - vi.mocked(openhumanAccessibilityRequestPermission).mockResolvedValue({ - result: baseStatus.permissions, - logs: [], - } as never); - vi.mocked(openhumanAccessibilityRequestPermissions).mockResolvedValue({ - result: baseStatus.permissions, - logs: [], - } as never); - vi.mocked(openhumanAccessibilityStartSession).mockResolvedValue({ - result: baseStatus.session, - logs: [], - } as never); - vi.mocked(openhumanAccessibilityStopSession).mockResolvedValue({ - result: baseStatus.session, - logs: [], - } as never); - vi.mocked(openhumanAccessibilityVisionFlush).mockResolvedValue({ - result: { accepted: true, summary: null }, - logs: [], - } as never); - vi.mocked(openhumanScreenIntelligenceCaptureTest).mockResolvedValue({ - result: { - ok: false, - capture_mode: 'fullscreen', - context: null, - image_ref: null, - bytes_estimate: null, - error: 'screen capture is unsupported on this platform', - timing_ms: 12, - }, - logs: [], - }); - vi.mocked(restartCoreProcess).mockResolvedValue(undefined); }); - it('saves screen intelligence settings and clears the saving state', async () => { - const deferred = createDeferred>(); + it('saves screen intelligence settings and refreshes core-backed status', async () => { + const deferred = createDeferred<{ result: ConfigSnapshot; logs: [] }>(); vi.mocked(openhumanUpdateScreenIntelligenceSettings).mockReturnValueOnce(deferred.promise); renderPanel(); @@ -206,13 +155,14 @@ describe('ScreenIntelligencePanel', () => { screen.getByRole('button', { name: 'Save Screen Intelligence Settings' }) ).toBeInTheDocument(); }); - expect(openhumanAccessibilityStatus).toHaveBeenCalledTimes(2); + expect(baseState.refreshStatus).toHaveBeenCalledTimes(1); }); it('shows permission restart guidance and unsupported-platform messaging', async () => { - vi.mocked(openhumanAccessibilityStatus).mockResolvedValueOnce({ - result: { - ...baseStatus, + renderPanel({ + ...baseState, + status: { + ...baseState.status!, platform_supported: false, permissions: { screen_recording: 'denied', @@ -220,11 +170,8 @@ describe('ScreenIntelligencePanel', () => { input_monitoring: 'unknown', }, }, - logs: [], }); - renderPanel(); - expect(await screen.findByText('Permissions')).toBeInTheDocument(); expect(screen.getByText(/After granting in System Settings, click/i)).toBeInTheDocument(); expect( @@ -234,4 +181,13 @@ describe('ScreenIntelligencePanel', () => { screen.getByText('Screen Intelligence V1 is currently supported on macOS only.') ).toBeInTheDocument(); }); + + it('shows the last successful restart summary', async () => { + renderPanel({ + ...baseState, + lastRestartSummary: 'Core restarted: PID 4000 at 9:00:00 AM -> PID 4242 at 9:01:00 AM.', + }); + + expect(await screen.findByText(/Core restarted: PID 4000/i)).toBeInTheDocument(); + }); }); diff --git a/app/src/components/settings/panels/screen-intelligence/PermissionsSection.tsx b/app/src/components/settings/panels/screen-intelligence/PermissionsSection.tsx index a6a03b0e8..02e1602bc 100644 --- a/app/src/components/settings/panels/screen-intelligence/PermissionsSection.tsx +++ b/app/src/components/settings/panels/screen-intelligence/PermissionsSection.tsx @@ -1,9 +1,4 @@ -import { - fetchAccessibilityStatus, - refreshPermissionsWithRestart, - requestAccessibilityPermission, -} from '../../../../store/accessibilitySlice'; -import { useAppDispatch } from '../../../../store/hooks'; +import type { AccessibilityPermissionKind } from '../../../../utils/tauriCommands'; interface PermissionsBadgeProps { label: string; @@ -33,10 +28,14 @@ interface PermissionsSectionProps { accessibility: string; inputMonitoring: string; anyPermissionDenied: boolean; + lastRestartSummary: string | null; permissionCheckProcessPath: string | null | undefined; isRequestingPermissions: boolean; isRestartingCore: boolean; isLoading: boolean; + requestPermission: (permission: AccessibilityPermissionKind) => Promise; + refreshPermissionsWithRestart: () => Promise; + refreshStatus: () => Promise; } const PermissionsSection = ({ @@ -44,13 +43,15 @@ const PermissionsSection = ({ accessibility, inputMonitoring, anyPermissionDenied, + lastRestartSummary, permissionCheckProcessPath, isRequestingPermissions, isRestartingCore, isLoading, + requestPermission, + refreshPermissionsWithRestart, + refreshStatus, }: PermissionsSectionProps) => { - const dispatch = useAppDispatch(); - return (

Permissions

@@ -75,23 +76,29 @@ const PermissionsSection = ({
)} + {lastRestartSummary ? ( +
+ {lastRestartSummary} +
+ ) : null} + - - - - - -
-
-

Vision Summaries

- -
- - {recentVisionSummaries.length === 0 ? ( -
No summaries yet.
- ) : ( -
- {recentVisionSummaries.map(summary => ( -
-
- {new Date(summary.captured_at_ms).toLocaleTimeString()} ·{' '} - {summary.app_name ?? 'Unknown App'} - {summary.window_title ? ` · ${summary.window_title}` : ''} -
-
{summary.actionable_notes}
-
- ))} -
- )} -
- - ); -}; - -export default SessionAndVisionSection; diff --git a/app/src/components/webhooks/TunnelList.tsx b/app/src/components/webhooks/TunnelList.tsx index d941aa6a0..9ba110a29 100644 --- a/app/src/components/webhooks/TunnelList.tsx +++ b/app/src/components/webhooks/TunnelList.tsx @@ -1,7 +1,7 @@ import { useState } from 'react'; +import type { TunnelRegistration } from '../../features/webhooks/types'; import { type Tunnel, tunnelsApi } from '../../services/api/tunnelsApi'; -import type { TunnelRegistration } from '../../store/webhooksSlice'; import { BACKEND_URL } from '../../utils/config'; interface TunnelListProps { diff --git a/app/src/components/webhooks/WebhookActivity.tsx b/app/src/components/webhooks/WebhookActivity.tsx index 6a1cc918c..283894ed9 100644 --- a/app/src/components/webhooks/WebhookActivity.tsx +++ b/app/src/components/webhooks/WebhookActivity.tsx @@ -1,4 +1,4 @@ -import type { WebhookActivityEntry } from '../../store/webhooksSlice'; +import type { WebhookActivityEntry } from '../../features/webhooks/types'; interface WebhookActivityProps { activity: WebhookActivityEntry[]; diff --git a/app/src/features/daemon/store.ts b/app/src/features/daemon/store.ts new file mode 100644 index 000000000..3e9f97f0a --- /dev/null +++ b/app/src/features/daemon/store.ts @@ -0,0 +1,146 @@ +import { useSyncExternalStore } from 'react'; + +export type DaemonStatus = 'starting' | 'stopping' | 'running' | 'error' | 'disconnected'; +export type ComponentStatus = 'ok' | 'error' | 'starting'; + +export interface ComponentHealth { + status: ComponentStatus; + updated_at: string; + last_ok?: string; + last_error?: string; + restart_count: number; +} + +export interface HealthSnapshot { + pid: number; + updated_at: string; + uptime_seconds: number; + components: Record; +} + +export interface DaemonUserState { + status: DaemonStatus; + healthSnapshot: HealthSnapshot | null; + components: { + gateway?: ComponentHealth; + channels?: ComponentHealth; + heartbeat?: ComponentHealth; + scheduler?: ComponentHealth; + }; + lastHealthUpdate: string | null; + connectionAttempts: number; + autoStartEnabled: boolean; + isRecovering: boolean; +} + +interface DaemonState { + byUser: Record; +} + +const initialUserState: DaemonUserState = { + status: 'disconnected', + healthSnapshot: null, + components: {}, + lastHealthUpdate: null, + connectionAttempts: 0, + autoStartEnabled: false, + isRecovering: false, +}; + +let daemonState: DaemonState = { byUser: {} }; +const listeners = new Set<() => void>(); + +const emitChange = (): void => { + for (const listener of listeners) { + listener(); + } +}; + +const currentUserState = (userId: string): DaemonUserState => + daemonState.byUser[userId] ?? initialUserState; + +const updateUserState = ( + userId: string, + updater: (current: DaemonUserState) => DaemonUserState +): void => { + daemonState = { + ...daemonState, + byUser: { ...daemonState.byUser, [userId]: updater(currentUserState(userId)) }, + }; + emitChange(); +}; + +export const subscribeDaemonStore = (listener: () => void): (() => void) => { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +}; + +export const getDaemonUserState = (userId?: string): DaemonUserState => + currentUserState(userId || '__pending__'); + +export const useDaemonUserState = (userId?: string): DaemonUserState => + useSyncExternalStore( + subscribeDaemonStore, + () => getDaemonUserState(userId), + () => getDaemonUserState(userId) + ); + +export const updateHealthSnapshot = (userId: string, healthSnapshot: HealthSnapshot): void => { + updateUserState(userId, current => { + const componentStatuses = Object.values(healthSnapshot.components).map( + component => component.status + ); + + let status = current.status; + if (componentStatuses.length === 0) { + status = 'disconnected'; + } else if (componentStatuses.every(componentStatus => componentStatus === 'ok')) { + status = 'running'; + } else if (componentStatuses.some(componentStatus => componentStatus === 'error')) { + status = 'error'; + } else if (componentStatuses.some(componentStatus => componentStatus === 'starting')) { + status = 'starting'; + } + + return { + ...current, + status, + healthSnapshot, + components: healthSnapshot.components, + lastHealthUpdate: new Date().toISOString(), + isRecovering: status === 'running' ? false : current.isRecovering, + connectionAttempts: status === 'running' ? 0 : current.connectionAttempts, + }; + }); +}; + +export const setDaemonStatus = (userId: string, status: DaemonStatus): void => { + updateUserState(userId, current => ({ + ...current, + status, + healthSnapshot: status === 'disconnected' ? null : current.healthSnapshot, + components: status === 'disconnected' ? {} : current.components, + lastHealthUpdate: status === 'disconnected' ? null : current.lastHealthUpdate, + })); +}; + +export const incrementConnectionAttempts = (userId: string): void => { + updateUserState(userId, current => ({ + ...current, + connectionAttempts: current.connectionAttempts + 1, + })); +}; + +export const resetConnectionAttempts = (userId: string): void => { + updateUserState(userId, current => ({ ...current, connectionAttempts: 0 })); +}; + +export const setAutoStartEnabled = (userId: string, enabled: boolean): void => { + updateUserState(userId, current => ({ ...current, autoStartEnabled: enabled })); +}; + +export const setIsRecovering = (userId: string, isRecovering: boolean): void => { + updateUserState(userId, current => ({ ...current, isRecovering })); +}; diff --git a/app/src/features/screen-intelligence/api.ts b/app/src/features/screen-intelligence/api.ts new file mode 100644 index 000000000..ca928f02a --- /dev/null +++ b/app/src/features/screen-intelligence/api.ts @@ -0,0 +1,156 @@ +import { + type AccessibilityPermissionKind, + type AccessibilityStartSessionParams, + type AccessibilityStatus, + openhumanAccessibilityRequestPermission, + openhumanAccessibilityStartSession, + openhumanAccessibilityStatus, + openhumanAccessibilityStopSession, + openhumanAccessibilityVisionFlush, + openhumanAccessibilityVisionRecent, + openhumanScreenIntelligenceCaptureTest, + openhumanServiceRestart, +} from '../../utils/tauriCommands'; + +const ACCESSIBILITY_ERROR_PREFIX = '[screen-intelligence]'; + +const extractError = (error: unknown, fallback: string): string => { + if (error instanceof Error && error.message.trim()) { + return error.message; + } + if (typeof error === 'string' && error.trim()) { + return error; + } + if (error && typeof error === 'object') { + const msg = (error as { message?: unknown }).message; + if (typeof msg === 'string' && msg.trim()) { + return msg; + } + } + return fallback; +}; + +const formatCoreIdentity = (status: AccessibilityStatus | null | undefined): string | null => { + const process = status?.core_process; + if (!process) { + return null; + } + const startedAt = Number.isFinite(process.started_at_ms) + ? new Date(process.started_at_ms).toLocaleTimeString() + : null; + return startedAt ? `PID ${process.pid} at ${startedAt}` : `PID ${process.pid}`; +}; + +export interface RefreshPermissionsResult { + status: AccessibilityStatus; + restartSummary: string; +} + +export async function fetchScreenIntelligenceStatus(): Promise { + const response = await openhumanAccessibilityStatus(); + return response.result; +} + +export async function requestScreenIntelligencePermission( + permission: AccessibilityPermissionKind +): Promise { + await openhumanAccessibilityRequestPermission(permission); + return await fetchScreenIntelligenceStatus(); +} + +export async function refreshScreenIntelligencePermissionsWithRestart( + previousStatus: AccessibilityStatus | null +): Promise { + try { + const previousProcess = previousStatus?.core_process; + console.debug( + `${ACCESSIBILITY_ERROR_PREFIX} refreshPermissionsWithRestart: requesting core self-restart` + ); + await openhumanServiceRestart('screen-intelligence-ui', 'refresh_permissions'); + console.debug( + `${ACCESSIBILITY_ERROR_PREFIX} refreshPermissionsWithRestart: waiting for sidecar ready` + ); + await new Promise(resolve => setTimeout(resolve, 400)); + console.debug( + `${ACCESSIBILITY_ERROR_PREFIX} refreshPermissionsWithRestart: fetching updated status` + ); + + for (let attempt = 1; attempt <= 5; attempt += 1) { + try { + const status = await fetchScreenIntelligenceStatus(); + console.debug( + `${ACCESSIBILITY_ERROR_PREFIX} refreshPermissionsWithRestart: done screen_recording=%s accessibility=%s input_monitoring=%s`, + status.permissions.screen_recording, + status.permissions.accessibility, + status.permissions.input_monitoring + ); + const currentProcess = status.core_process; + if ( + previousProcess && + currentProcess && + previousProcess.pid === currentProcess.pid && + previousProcess.started_at_ms === currentProcess.started_at_ms + ) { + throw new Error( + `Core restart command completed, but the same core instance is still serving requests (${formatCoreIdentity(status)}).` + ); + } + + const previousLabel = formatCoreIdentity(previousStatus); + const currentLabel = formatCoreIdentity(status); + const restartSummary = + previousLabel && currentLabel + ? `Core restarted: ${previousLabel} -> ${currentLabel}.` + : currentLabel + ? `Core restarted. Now serving from ${currentLabel}.` + : 'Core restarted and permissions refreshed.'; + + return { status, restartSummary }; + } catch (error) { + if (attempt === 5) { + throw error; + } + console.debug( + `${ACCESSIBILITY_ERROR_PREFIX} refreshPermissionsWithRestart: status fetch failed (attempt %s), retrying`, + attempt + ); + await new Promise(resolve => setTimeout(resolve, 350 * attempt)); + } + } + + throw new Error('Failed to fetch accessibility status after core restart'); + } catch (error) { + const message = extractError(error, 'Failed to restart core and refresh permissions'); + console.error(`${ACCESSIBILITY_ERROR_PREFIX} refreshPermissionsWithRestart: error`, message); + throw new Error(message); + } +} + +export async function startScreenIntelligenceSession( + params: AccessibilityStartSessionParams +): Promise { + await openhumanAccessibilityStartSession(params); + return await fetchScreenIntelligenceStatus(); +} + +export async function stopScreenIntelligenceSession(reason?: string): Promise { + await openhumanAccessibilityStopSession(reason ? { reason } : undefined); + return await fetchScreenIntelligenceStatus(); +} + +export async function fetchScreenIntelligenceVisionRecent(limit?: number) { + const response = await openhumanAccessibilityVisionRecent(limit); + return response.result.summaries; +} + +export async function flushScreenIntelligenceVision() { + const response = await openhumanAccessibilityVisionFlush(); + return response.result.summary; +} + +export async function runScreenIntelligenceCaptureTest() { + const response = await openhumanScreenIntelligenceCaptureTest(); + return response.result; +} + +export { extractError }; diff --git a/app/src/features/screen-intelligence/useScreenIntelligenceState.ts b/app/src/features/screen-intelligence/useScreenIntelligenceState.ts new file mode 100644 index 000000000..772866044 --- /dev/null +++ b/app/src/features/screen-intelligence/useScreenIntelligenceState.ts @@ -0,0 +1,255 @@ +import { useCallback, useEffect, useState } from 'react'; + +import { getCoreStateSnapshot } from '../../lib/coreState/store'; +import { useCoreState } from '../../providers/CoreStateProvider'; +import type { + AccessibilityPermissionKind, + AccessibilityStartSessionParams, + AccessibilityStatus, + AccessibilityVisionSummary, + CaptureTestResult, +} from '../../utils/tauriCommands'; +import { + extractError, + fetchScreenIntelligenceVisionRecent, + flushScreenIntelligenceVision, + refreshScreenIntelligencePermissionsWithRestart, + requestScreenIntelligencePermission, + runScreenIntelligenceCaptureTest, + startScreenIntelligenceSession, + stopScreenIntelligenceSession, +} from './api'; + +export interface ScreenIntelligenceState { + status: AccessibilityStatus | null; + lastRestartSummary: string | null; + recentVisionSummaries: AccessibilityVisionSummary[]; + captureTestResult: CaptureTestResult | null; + isCaptureTestRunning: boolean; + isLoading: boolean; + isRequestingPermissions: boolean; + isRestartingCore: boolean; + isStartingSession: boolean; + isStoppingSession: boolean; + isLoadingVision: boolean; + isFlushingVision: boolean; + lastError: string | null; + refreshStatus: () => Promise; + requestPermission: ( + permission: AccessibilityPermissionKind + ) => Promise; + refreshPermissionsWithRestart: () => Promise; + startSession: (params: AccessibilityStartSessionParams) => Promise; + stopSession: (reason?: string) => Promise; + refreshVision: (limit?: number) => Promise; + flushVision: () => Promise; + runCaptureTest: () => Promise; + clearError: () => void; +} + +export interface UseScreenIntelligenceStateOptions { + pollMs?: number; + visionLimit?: number; + loadVision?: boolean; +} + +export function useScreenIntelligenceState( + options: UseScreenIntelligenceStateOptions = {} +): ScreenIntelligenceState { + const { pollMs = 2000, visionLimit = 10, loadVision = false } = options; + const { refresh: refreshCoreState, snapshot } = useCoreState(); + const status = snapshot.runtime.screenIntelligence; + const [lastRestartSummary, setLastRestartSummary] = useState(null); + const [recentVisionSummaries, setRecentVisionSummaries] = useState( + [] + ); + const [captureTestResult, setCaptureTestResult] = useState(null); + const [isCaptureTestRunning, setIsCaptureTestRunning] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [isRequestingPermissions, setIsRequestingPermissions] = useState(false); + const [isRestartingCore, setIsRestartingCore] = useState(false); + const [isStartingSession, setIsStartingSession] = useState(false); + const [isStoppingSession, setIsStoppingSession] = useState(false); + const [isLoadingVision, setIsLoadingVision] = useState(false); + const [isFlushingVision, setIsFlushingVision] = useState(false); + const [lastError, setLastError] = useState(null); + + const latestScreenIntelligenceStatus = useCallback( + (): AccessibilityStatus | null => getCoreStateSnapshot().snapshot.runtime.screenIntelligence, + [] + ); + + const refreshStatus = useCallback(async () => { + setIsLoading(true); + setLastError(null); + try { + await refreshCoreState(); + return latestScreenIntelligenceStatus(); + } catch (error) { + setLastError(extractError(error, 'Failed to fetch accessibility status')); + return null; + } finally { + setIsLoading(false); + } + }, [latestScreenIntelligenceStatus, refreshCoreState]); + + const refreshVision = useCallback( + async (limit = visionLimit) => { + setIsLoadingVision(true); + try { + const summaries = await fetchScreenIntelligenceVisionRecent(limit); + setRecentVisionSummaries(summaries); + return summaries; + } catch (error) { + setLastError(extractError(error, 'Failed to fetch accessibility vision summaries')); + return []; + } finally { + setIsLoadingVision(false); + } + }, + [visionLimit] + ); + + const requestPermission = useCallback( + async (permission: AccessibilityPermissionKind) => { + setIsRequestingPermissions(true); + setLastError(null); + setLastRestartSummary(null); + try { + await requestScreenIntelligencePermission(permission); + await refreshCoreState(); + return latestScreenIntelligenceStatus(); + } catch (error) { + setLastError(extractError(error, 'Failed to request accessibility permission')); + return null; + } finally { + setIsRequestingPermissions(false); + } + }, + [latestScreenIntelligenceStatus, refreshCoreState] + ); + + const refreshPermissionsWithRestart = useCallback(async () => { + setIsRestartingCore(true); + setLastError(null); + setLastRestartSummary(null); + try { + const result = await refreshScreenIntelligencePermissionsWithRestart(status); + setLastRestartSummary(result.restartSummary); + await refreshCoreState(); + return latestScreenIntelligenceStatus() ?? result.status; + } catch (error) { + setLastError(extractError(error, 'Failed to restart core and refresh permissions')); + return null; + } finally { + setIsRestartingCore(false); + } + }, [latestScreenIntelligenceStatus, refreshCoreState, status]); + + const startSession = useCallback( + async (params: AccessibilityStartSessionParams) => { + setIsStartingSession(true); + setLastError(null); + try { + await startScreenIntelligenceSession(params); + await refreshCoreState(); + return latestScreenIntelligenceStatus(); + } catch (error) { + setLastError(extractError(error, 'Failed to start accessibility session')); + return null; + } finally { + setIsStartingSession(false); + } + }, + [latestScreenIntelligenceStatus, refreshCoreState] + ); + + const stopSession = useCallback( + async (reason?: string) => { + setIsStoppingSession(true); + setLastError(null); + try { + await stopScreenIntelligenceSession(reason); + await refreshCoreState(); + return latestScreenIntelligenceStatus(); + } catch (error) { + setLastError(extractError(error, 'Failed to stop accessibility session')); + return null; + } finally { + setIsStoppingSession(false); + } + }, + [latestScreenIntelligenceStatus, refreshCoreState] + ); + + const flushVision = useCallback(async () => { + setIsFlushingVision(true); + try { + const summary = await flushScreenIntelligenceVision(); + if (summary) { + setRecentVisionSummaries(current => [summary, ...current].slice(0, 30)); + } + } catch (error) { + setLastError(extractError(error, 'Failed to flush accessibility vision')); + } finally { + setIsFlushingVision(false); + } + }, []); + + const runCaptureTest = useCallback(async () => { + setIsCaptureTestRunning(true); + setCaptureTestResult(null); + setLastError(null); + try { + const result = await runScreenIntelligenceCaptureTest(); + setCaptureTestResult(result); + } catch (error) { + setLastError(extractError(error, 'Failed to run capture test')); + } finally { + setIsCaptureTestRunning(false); + } + }, []); + + useEffect(() => { + if (loadVision) { + void refreshVision(visionLimit); + } + }, [loadVision, refreshVision, visionLimit]); + + useEffect(() => { + if (!loadVision) { + return; + } + + const intervalId = window.setInterval(() => { + void refreshVision(visionLimit); + }, pollMs); + + return () => window.clearInterval(intervalId); + }, [loadVision, pollMs, refreshVision, visionLimit]); + + return { + status, + lastRestartSummary, + recentVisionSummaries, + captureTestResult, + isCaptureTestRunning, + isLoading, + isRequestingPermissions, + isRestartingCore, + isStartingSession, + isStoppingSession, + isLoadingVision, + isFlushingVision, + lastError, + refreshStatus, + requestPermission, + refreshPermissionsWithRestart, + startSession, + stopSession, + refreshVision, + flushVision, + runCaptureTest, + clearError: () => setLastError(null), + }; +} diff --git a/app/src/features/webhooks/types.ts b/app/src/features/webhooks/types.ts new file mode 100644 index 000000000..612edae7b --- /dev/null +++ b/app/src/features/webhooks/types.ts @@ -0,0 +1,21 @@ +import type { Tunnel } from '../../services/api/tunnelsApi'; + +export type { Tunnel }; + +export interface TunnelRegistration { + tunnel_uuid: string; + target_kind?: string; + skill_id: string; + tunnel_name: string | null; + backend_tunnel_id: string | null; +} + +export interface WebhookActivityEntry { + correlation_id: string; + tunnel_name: string; + method: string; + path: string; + status_code: number | null; + skill_id: string | null; + timestamp: number; +} diff --git a/app/src/hooks/useDaemonHealth.ts b/app/src/hooks/useDaemonHealth.ts index d1e13c510..1b015dcf5 100644 --- a/app/src/hooks/useDaemonHealth.ts +++ b/app/src/hooks/useDaemonHealth.ts @@ -8,18 +8,12 @@ import { useCallback, useEffect } from 'react'; import { resetConnectionAttempts, - selectDaemonComponents, - selectDaemonConnectionAttempts, - selectDaemonHealthSnapshot, - selectDaemonLastHealthUpdate, - selectDaemonStatus, - selectIsDaemonAutoStartEnabled, - selectIsDaemonRecovering, setAutoStartEnabled, setDaemonStatus, setIsRecovering, -} from '../store/daemonSlice'; -import { useAppDispatch, useAppSelector } from '../store/hooks'; + useDaemonUserState, +} from '../features/daemon/store'; +import { daemonHealthService } from '../services/daemonHealthService'; import { type CommandResponse, openhumanAgentServerStatus, @@ -30,30 +24,21 @@ import { } from '../utils/tauriCommands'; export const useDaemonHealth = (userId?: string) => { - const dispatch = useAppDispatch(); - - // Selectors - const status = useAppSelector(state => selectDaemonStatus(state, userId)); - const components = useAppSelector(state => selectDaemonComponents(state, userId)); - const healthSnapshot = useAppSelector(state => selectDaemonHealthSnapshot(state, userId)); - const lastUpdate = useAppSelector(state => selectDaemonLastHealthUpdate(state, userId)); - const isAutoStartEnabled = useAppSelector(state => selectIsDaemonAutoStartEnabled(state, userId)); - const connectionAttempts = useAppSelector(state => selectDaemonConnectionAttempts(state, userId)); - const isRecovering = useAppSelector(state => selectIsDaemonRecovering(state, userId)); + const daemonState = useDaemonUserState(userId); const uid = userId || '__pending__'; const probeAgentStatus = useCallback(async (): Promise => { try { const result = await openhumanAgentServerStatus(); const running = !!result?.result?.running; - dispatch(setDaemonStatus({ userId: uid, status: running ? 'running' : 'disconnected' })); + setDaemonStatus(uid, running ? 'running' : 'disconnected'); return running; } catch (error) { console.error('[useDaemonHealth] Failed to probe agent status:', error); - dispatch(setDaemonStatus({ userId: uid, status: 'disconnected' })); + setDaemonStatus(uid, 'disconnected'); return false; } - }, [dispatch, uid]); + }, [uid]); const waitForAgentStatus = useCallback( async (targetRunning: boolean, timeoutMs = 10000): Promise => { @@ -73,28 +58,28 @@ export const useDaemonHealth = (userId?: string) => { // Action creators const startDaemon = useCallback(async (): Promise | null> => { try { - dispatch(setDaemonStatus({ userId: uid, status: 'starting' })); + setDaemonStatus(uid, 'starting'); const result = await openhumanServiceStart(); const running = await waitForAgentStatus(true); if (running) { if (result?.result) { (result.result as { state?: string }).state = 'Running'; } - dispatch(resetConnectionAttempts({ userId: uid })); + resetConnectionAttempts(uid); } else { - dispatch(setDaemonStatus({ userId: uid, status: 'error' })); + setDaemonStatus(uid, 'error'); } return result; } catch (error) { console.error('[useDaemonHealth] Failed to start daemon:', error); - dispatch(setDaemonStatus({ userId: uid, status: 'error' })); + setDaemonStatus(uid, 'error'); return null; } - }, [dispatch, uid, waitForAgentStatus]); + }, [uid, waitForAgentStatus]); const stopDaemon = useCallback(async (): Promise | null> => { try { - dispatch(setDaemonStatus({ userId: uid, status: 'starting' })); + setDaemonStatus(uid, 'stopping'); const result = await openhumanServiceStop(); await waitForAgentStatus(false, 7000); return result; @@ -102,12 +87,12 @@ export const useDaemonHealth = (userId?: string) => { console.error('[useDaemonHealth] Failed to stop daemon:', error); return null; } - }, [dispatch, uid, waitForAgentStatus]); + }, [uid, waitForAgentStatus]); const restartDaemon = useCallback(async (): Promise => { try { - dispatch(setIsRecovering({ userId: uid, isRecovering: true })); - dispatch(setDaemonStatus({ userId: uid, status: 'starting' })); + setIsRecovering(uid, true); + setDaemonStatus(uid, 'starting'); // Stop first await openhumanServiceStop(); @@ -121,20 +106,20 @@ export const useDaemonHealth = (userId?: string) => { const success = await waitForAgentStatus(true, 12000); if (success) { - dispatch(resetConnectionAttempts({ userId: uid })); + resetConnectionAttempts(uid); } else { - dispatch(setDaemonStatus({ userId: uid, status: 'error' })); + setDaemonStatus(uid, 'error'); } - dispatch(setIsRecovering({ userId: uid, isRecovering: false })); + setIsRecovering(uid, false); return success; } catch (error) { console.error('[useDaemonHealth] Failed to restart daemon:', error); - dispatch(setIsRecovering({ userId: uid, isRecovering: false })); - dispatch(setDaemonStatus({ userId: uid, status: 'error' })); + setIsRecovering(uid, false); + setDaemonStatus(uid, 'error'); return false; } - }, [dispatch, uid, waitForAgentStatus]); + }, [uid, waitForAgentStatus]); const checkDaemonStatus = useCallback(async (): Promise | null> => { @@ -152,37 +137,61 @@ export const useDaemonHealth = (userId?: string) => { const setAutoStart = useCallback( (enabled: boolean) => { - dispatch(setAutoStartEnabled({ userId: userId || '__pending__', enabled })); + setAutoStartEnabled(userId || '__pending__', enabled); }, - [dispatch, userId] + [userId] ); // Derived state - const isHealthy = status === 'running'; - const hasErrors = status === 'error'; - const isConnected = status !== 'disconnected'; - const isStarting = status === 'starting'; + const isHealthy = daemonState.status === 'running'; + const hasErrors = daemonState.status === 'error'; + const isConnected = daemonState.status !== 'disconnected'; + const isStarting = daemonState.status === 'starting'; - const componentCount = Object.keys(components).length; - const healthyComponentCount = Object.values(components).filter(c => c.status === 'ok').length; - const errorComponentCount = Object.values(components).filter(c => c.status === 'error').length; + const componentCount = Object.keys(daemonState.components).length; + const healthyComponentCount = Object.values(daemonState.components).filter( + c => c.status === 'ok' + ).length; + const errorComponentCount = Object.values(daemonState.components).filter( + c => c.status === 'error' + ).length; // Get uptime in human readable format - const uptimeText = healthSnapshot ? formatUptime(healthSnapshot.uptime_seconds) : 'Unknown'; + const uptimeText = daemonState.healthSnapshot + ? formatUptime(daemonState.healthSnapshot.uptime_seconds) + : 'Unknown'; useEffect(() => { void probeAgentStatus(); }, [probeAgentStatus]); + useEffect(() => { + let cleanup: (() => void) | null = null; + let cancelled = false; + + void daemonHealthService.setupHealthListener().then(result => { + if (cancelled) { + result?.(); + } else { + cleanup = result; + } + }); + + return () => { + cancelled = true; + cleanup?.(); + }; + }, []); + return { // State - status, - components, - healthSnapshot, - lastUpdate, - isAutoStartEnabled, - connectionAttempts, - isRecovering, + status: daemonState.status, + components: daemonState.components, + healthSnapshot: daemonState.healthSnapshot, + lastUpdate: daemonState.lastHealthUpdate, + isAutoStartEnabled: daemonState.autoStartEnabled, + connectionAttempts: daemonState.connectionAttempts, + isRecovering: daemonState.isRecovering, // Derived state isHealthy, diff --git a/app/src/hooks/useDaemonLifecycle.ts b/app/src/hooks/useDaemonLifecycle.ts index 46d1afcc7..1419dc2e9 100644 --- a/app/src/hooks/useDaemonLifecycle.ts +++ b/app/src/hooks/useDaemonLifecycle.ts @@ -12,13 +12,9 @@ import { useCallback, useEffect, useRef } from 'react'; import { incrementConnectionAttempts, resetConnectionAttempts, - selectDaemonConnectionAttempts, - selectDaemonStatus, - selectIsDaemonAutoStartEnabled, - selectIsDaemonRecovering, setIsRecovering, -} from '../store/daemonSlice'; -import { useAppDispatch, useAppSelector } from '../store/hooks'; + useDaemonUserState, +} from '../features/daemon/store'; import { isTauri } from '../utils/tauriCommands'; import { useDaemonHealth } from './useDaemonHealth'; @@ -29,14 +25,14 @@ const MAX_RETRY_DELAY_MS = 30000; // 30 seconds const AUTO_START_DELAY_MS = 3000; // 3 seconds after app start export const useDaemonLifecycle = (userId?: string) => { - const dispatch = useAppDispatch(); const daemonHealth = useDaemonHealth(userId); + const daemonState = useDaemonUserState(userId); - // Selectors - const status = useAppSelector(state => selectDaemonStatus(state, userId)); - const isAutoStartEnabled = useAppSelector(state => selectIsDaemonAutoStartEnabled(state, userId)); - const connectionAttempts = useAppSelector(state => selectDaemonConnectionAttempts(state, userId)); - const isRecovering = useAppSelector(state => selectIsDaemonRecovering(state, userId)); + const status = daemonState.status; + const isAutoStartEnabled = daemonState.autoStartEnabled; + const connectionAttempts = daemonState.connectionAttempts; + const isRecovering = daemonState.isRecovering; + const uid = userId || '__pending__'; // Refs for cleanup const autoStartTimeoutRef = useRef | null>(null); @@ -60,32 +56,24 @@ export const useDaemonLifecycle = (userId?: string) => { console.log('[DaemonLifecycle] Attempting auto-start of daemon'); try { - dispatch(setIsRecovering({ userId: userId || '__pending__', isRecovering: true })); + setIsRecovering(uid, true); const result = await daemonHealth.startDaemon(); if (result?.result && result.result.state === 'Running') { console.log('[DaemonLifecycle] Auto-start successful'); - dispatch(resetConnectionAttempts({ userId: userId || '__pending__' })); + resetConnectionAttempts(uid); } else { console.warn('[DaemonLifecycle] Auto-start failed:', result); - dispatch(incrementConnectionAttempts({ userId: userId || '__pending__' })); + incrementConnectionAttempts(uid); } } catch (error) { console.error('[DaemonLifecycle] Auto-start error:', error); - dispatch(incrementConnectionAttempts({ userId: userId || '__pending__' })); + incrementConnectionAttempts(uid); } finally { - dispatch(setIsRecovering({ userId: userId || '__pending__', isRecovering: false })); + setIsRecovering(uid, false); } } - }, [ - isAutoStartEnabled, - status, - isRecovering, - connectionAttempts, - userId, - dispatch, - daemonHealth, - ]); + }, [isAutoStartEnabled, status, isRecovering, connectionAttempts, uid, daemonHealth]); // Retry connection with exponential backoff const scheduleRetry = useCallback(() => { @@ -118,14 +106,14 @@ export const useDaemonLifecycle = (userId?: string) => { if (!isMountedRef.current) return; try { - dispatch(setIsRecovering({ userId: userId || '__pending__', isRecovering: true })); - dispatch(incrementConnectionAttempts({ userId: userId || '__pending__' })); + setIsRecovering(uid, true); + incrementConnectionAttempts(uid); const result = await daemonHealth.startDaemon(); if (result?.result && result.result.state === 'Running') { console.log('[DaemonLifecycle] Retry successful'); - dispatch(resetConnectionAttempts({ userId: userId || '__pending__' })); + resetConnectionAttempts(uid); } else { console.warn('[DaemonLifecycle] Retry failed:', result); // Will trigger another retry via useEffect @@ -134,18 +122,10 @@ export const useDaemonLifecycle = (userId?: string) => { console.error('[DaemonLifecycle] Retry error:', error); // Will trigger another retry via useEffect } finally { - dispatch(setIsRecovering({ userId: userId || '__pending__', isRecovering: false })); + setIsRecovering(uid, false); } }, retryDelay); - }, [ - connectionAttempts, - status, - isRecovering, - calculateRetryDelay, - userId, - dispatch, - daemonHealth, - ]); + }, [connectionAttempts, status, isRecovering, calculateRetryDelay, uid, daemonHealth]); // Handle visibility change (background/foreground) const handleVisibilityChange = useCallback(() => { @@ -231,7 +211,7 @@ export const useDaemonLifecycle = (userId?: string) => { useEffect(() => { if (status === 'running' && connectionAttempts > 0) { console.log('[DaemonLifecycle] Daemon healthy - resetting connection attempts'); - dispatch(resetConnectionAttempts({ userId: userId || '__pending__' })); + resetConnectionAttempts(uid); // Clear retry timeout if running if (retryTimeoutRef.current) { @@ -239,7 +219,7 @@ export const useDaemonLifecycle = (userId?: string) => { retryTimeoutRef.current = null; } } - }, [status, connectionAttempts, userId, dispatch]); + }, [status, connectionAttempts, uid]); // Return lifecycle state and controls return { @@ -252,7 +232,7 @@ export const useDaemonLifecycle = (userId?: string) => { // Actions attemptAutoStart, resetRetries: () => { - dispatch(resetConnectionAttempts({ userId: userId || '__pending__' })); + resetConnectionAttempts(uid); if (retryTimeoutRef.current) { clearTimeout(retryTimeoutRef.current); retryTimeoutRef.current = null; diff --git a/app/src/hooks/useIntelligenceStats.ts b/app/src/hooks/useIntelligenceStats.ts index b2ebbcc13..4277630ff 100644 --- a/app/src/hooks/useIntelligenceStats.ts +++ b/app/src/hooks/useIntelligenceStats.ts @@ -1,10 +1,12 @@ import { useCallback, useEffect, useState } from 'react'; import { callCoreRpc } from '../services/coreRpcClient'; -import type { AIStatus } from '../store/aiSlice'; -import { useAppSelector } from '../store/hooks'; import { aiListMemoryFiles, type GraphRelation, memoryGraphQuery } from '../utils/tauriCommands'; +export type AIStatus = 'idle' | 'initializing' | 'ready' | 'error'; + +const POLL_MS = 5000; + interface SessionEntry { sessionId: string; updatedAt: number; @@ -46,7 +48,7 @@ function entityCountsFromRelations(relations: GraphRelation[]): Record state.ai.status); + const [aiStatus, setAiStatus] = useState('idle'); const [sessions, setSessions] = useState(null); const [memoryFiles, setMemoryFiles] = useState(null); const [entities, setEntities] = useState | null>(null); @@ -54,7 +56,9 @@ export function useIntelligenceStats(): IntelligenceStats { const [isLoading, setIsLoading] = useState(true); const fetchStats = useCallback(async () => { + setAiStatus('initializing'); setIsLoading(true); + let hasSuccess = false; // Fetch local stats (Tauri invoke) try { @@ -68,6 +72,7 @@ export function useIntelligenceStats(): IntelligenceStats { compactions: entries.reduce((sum, e) => sum + (e.compactionCount || 0), 0), memoryFlushes: entries.filter(e => e.memoryFlushAt).length, }); + hasSuccess = true; } catch { setSessions(null); } @@ -75,6 +80,7 @@ export function useIntelligenceStats(): IntelligenceStats { try { const files = await aiListMemoryFiles('memory'); setMemoryFiles(files.length); + hasSuccess = true; } catch { setMemoryFiles(null); } @@ -90,17 +96,26 @@ export function useIntelligenceStats(): IntelligenceStats { setEntities(null); setEntityError(false); } + hasSuccess = true; } catch { setEntities(null); setEntityError(true); } + setAiStatus(hasSuccess ? 'ready' : 'error'); setIsLoading(false); }, []); useEffect(() => { - fetchStats(); - }, [fetchStats, aiStatus]); + void fetchStats(); + const intervalId = window.setInterval(() => { + void fetchStats(); + }, POLL_MS); + + return () => { + window.clearInterval(intervalId); + }; + }, [fetchStats]); return { sessions, memoryFiles, entities, entityError, aiStatus, isLoading, refetch: fetchStats }; } diff --git a/app/src/hooks/useScreenIntelligenceItems.ts b/app/src/hooks/useScreenIntelligenceItems.ts index c1d9c5aea..694cf74d2 100644 --- a/app/src/hooks/useScreenIntelligenceItems.ts +++ b/app/src/hooks/useScreenIntelligenceItems.ts @@ -1,7 +1,6 @@ -import { useEffect, useMemo } from 'react'; +import { useMemo } from 'react'; -import { fetchAccessibilityVisionRecent } from '../store/accessibilitySlice'; -import { useAppDispatch, useAppSelector } from '../store/hooks'; +import { useScreenIntelligenceState } from '../features/screen-intelligence/useScreenIntelligenceState'; import type { ActionableItem, ActionableItemPriority } from '../types/intelligence'; function confidenceToPriority(confidence: number): ActionableItemPriority { @@ -11,12 +10,11 @@ function confidenceToPriority(confidence: number): ActionableItemPriority { } export function useScreenIntelligenceItems() { - const dispatch = useAppDispatch(); - const { recentVisionSummaries, isLoadingVision } = useAppSelector(state => state.accessibility); - - useEffect(() => { - void dispatch(fetchAccessibilityVisionRecent(20)); - }, [dispatch]); + const { recentVisionSummaries, isLoadingVision, refreshVision } = useScreenIntelligenceState({ + loadVision: true, + visionLimit: 20, + pollMs: 2000, + }); const items: ActionableItem[] = useMemo(() => { return recentVisionSummaries.map(summary => ({ @@ -33,5 +31,5 @@ export function useScreenIntelligenceItems() { })); }, [recentVisionSummaries]); - return { items, loading: isLoadingVision }; + return { items, loading: isLoadingVision, refresh: () => refreshVision(20) }; } diff --git a/app/src/hooks/useWebhooks.ts b/app/src/hooks/useWebhooks.ts index d882f2737..7272e807d 100644 --- a/app/src/hooks/useWebhooks.ts +++ b/app/src/hooks/useWebhooks.ts @@ -1,20 +1,10 @@ import debug from 'debug'; import { useCallback, useEffect, useRef, useState } from 'react'; +import type { Tunnel, TunnelRegistration, WebhookActivityEntry } from '../features/webhooks/types'; import { useCoreState } from '../providers/CoreStateProvider'; import { tunnelsApi } from '../services/api/tunnelsApi'; import { getCoreHttpBaseUrl } from '../services/coreRpcClient'; -import { useAppDispatch, useAppSelector } from '../store/hooks'; -import { - addActivity, - addTunnel, - removeTunnel, - setError, - setLoading, - setRegistrations, - setTunnels, - type WebhookActivityEntry, -} from '../store/webhooksSlice'; import { openhumanWebhooksListLogs, openhumanWebhooksListRegistrations, @@ -47,11 +37,12 @@ function logToActivity(entry: WebhookDebugLogEntry): WebhookActivityEntry { */ export function useWebhooks() { const { snapshot } = useCoreState(); - const dispatch = useAppDispatch(); - const { tunnels, registrations, activity, loading, error } = useAppSelector( - state => state.webhooks - ); const token = snapshot.sessionToken; + const [tunnels, setTunnels] = useState([]); + const [registrations, setRegistrations] = useState([]); + const [activity, setActivity] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); const [coreConnected, setCoreConnected] = useState(false); const eventSourceRef = useRef(null); @@ -62,13 +53,11 @@ export function useWebhooks() { openhumanWebhooksListRegistrations(), openhumanWebhooksListLogs(100), ]); - dispatch(setRegistrations(regsResponse.result.result.registrations)); + setRegistrations(regsResponse.result.result.registrations); // Seed activity from debug logs const logs = logsResponse.result.result.logs; - for (const entry of logs.reverse()) { - dispatch(addActivity(logToActivity(entry))); - } + setActivity(logs.reverse().map(logToActivity)); log( 'Loaded %d registrations, %d logs from core', regsResponse.result.result.registrations.length, @@ -80,21 +69,24 @@ export function useWebhooks() { err instanceof Error ? err.message : err ); } - }, [dispatch]); + }, []); // ── Fetch tunnels from backend API ─────────────────────────────────────── const fetchTunnels = useCallback(async () => { - dispatch(setLoading(true)); + setLoading(true); + setError(null); try { const data = await tunnelsApi.getTunnels(); - dispatch(setTunnels(data)); + setTunnels(data); log('Fetched %d tunnels', data.length); } catch (err) { const msg = err instanceof Error ? err.message : 'Failed to fetch tunnels'; - dispatch(setError(msg)); + setError(msg); log('Error fetching tunnels: %s', msg); + } finally { + setLoading(false); } - }, [dispatch]); + }, []); // ── Subscribe to SSE for real-time webhook events ──────────────────────── useEffect(() => { @@ -147,36 +139,18 @@ export function useWebhooks() { }, [token, fetchTunnels, loadCoreData]); // ── CRUD actions ───────────────────────────────────────────────────────── - const createTunnel = useCallback( - async (name: string, description?: string) => { - try { - const tunnel = await tunnelsApi.createTunnel({ name, description }); - dispatch(addTunnel(tunnel)); - log('Created tunnel: %s (%s)', tunnel.name, tunnel.uuid); - return tunnel; - } catch (err) { - const msg = err instanceof Error ? err.message : 'Failed to create tunnel'; - dispatch(setError(msg)); - throw err; - } - }, - [dispatch] - ); + const createTunnel = useCallback(async (name: string, description?: string) => { + const tunnel = await tunnelsApi.createTunnel({ name, description }); + setTunnels(current => [...current, tunnel]); + log('Created tunnel: %s (%s)', tunnel.name, tunnel.uuid); + return tunnel; + }, []); - const deleteTunnel = useCallback( - async (id: string) => { - try { - await tunnelsApi.deleteTunnel(id); - dispatch(removeTunnel(id)); - log('Deleted tunnel: %s', id); - } catch (err) { - const msg = err instanceof Error ? err.message : 'Failed to delete tunnel'; - dispatch(setError(msg)); - throw err; - } - }, - [dispatch] - ); + const deleteTunnel = useCallback(async (id: string) => { + await tunnelsApi.deleteTunnel(id); + setTunnels(current => current.filter(tunnel => tunnel.id !== id)); + log('Deleted tunnel: %s', id); + }, []); const refreshTunnels = useCallback(async () => { await fetchTunnels(); @@ -192,31 +166,28 @@ export function useWebhooks() { tunnelName, backendTunnelId ); - dispatch(setRegistrations(response.result.result.registrations)); + setRegistrations(response.result.result.registrations); log('Registered echo for tunnel %s', tunnelUuid); } catch (err) { const msg = err instanceof Error ? err.message : 'Failed to register echo'; - dispatch(setError(msg)); + setError(msg); throw err; } }, - [dispatch] + [] ); - const unregisterEcho = useCallback( - async (tunnelUuid: string) => { - try { - const response = await openhumanWebhooksUnregisterEcho(tunnelUuid); - dispatch(setRegistrations(response.result.result.registrations)); - log('Unregistered echo for tunnel %s', tunnelUuid); - } catch (err) { - const msg = err instanceof Error ? err.message : 'Failed to unregister echo'; - dispatch(setError(msg)); - throw err; - } - }, - [dispatch] - ); + const unregisterEcho = useCallback(async (tunnelUuid: string) => { + try { + const response = await openhumanWebhooksUnregisterEcho(tunnelUuid); + setRegistrations(response.result.result.registrations); + log('Unregistered echo for tunnel %s', tunnelUuid); + } catch (err) { + const msg = err instanceof Error ? err.message : 'Failed to unregister echo'; + setError(msg); + throw err; + } + }, []); return { tunnels, diff --git a/app/src/lib/coreState/store.ts b/app/src/lib/coreState/store.ts index 880596a6f..f861d6b7a 100644 --- a/app/src/lib/coreState/store.ts +++ b/app/src/lib/coreState/store.ts @@ -1,5 +1,9 @@ import type { User } from '../../types/api'; import type { TeamInvite, TeamMember, TeamWithRole } from '../../types/team'; +import type { AccessibilityStatus } from '../../utils/tauriCommands/accessibility'; +import type { AutocompleteStatus } from '../../utils/tauriCommands/autocomplete'; +import type { ServiceStatus } from '../../utils/tauriCommands/hardware'; +import type { LocalAiStatus } from '../../utils/tauriCommands/localAi'; export interface CoreOnboardingTasks { accessibilityPermissionGranted: boolean; @@ -16,6 +20,13 @@ export interface CoreLocalState { onboardingTasks: CoreOnboardingTasks | null; } +export interface CoreRuntimeSnapshot { + screenIntelligence: AccessibilityStatus | null; + localAi: LocalAiStatus | null; + autocomplete: AutocompleteStatus | null; + service: ServiceStatus | null; +} + export interface CoreAppSnapshot { auth: { isAuthenticated: boolean; @@ -28,6 +39,7 @@ export interface CoreAppSnapshot { onboardingCompleted: boolean; analyticsEnabled: boolean; localState: CoreLocalState; + runtime: CoreRuntimeSnapshot; } export interface CoreState { @@ -46,6 +58,7 @@ const emptySnapshot: CoreAppSnapshot = { onboardingCompleted: false, analyticsEnabled: false, localState: { encryptionKey: null, primaryWalletAddress: null, onboardingTasks: null }, + runtime: { screenIntelligence: null, localAi: null, autocomplete: null, service: null }, }; let currentState: CoreState = { diff --git a/app/src/pages/Invites.tsx b/app/src/pages/Invites.tsx index 006dc9e2f..ca8335558 100644 --- a/app/src/pages/Invites.tsx +++ b/app/src/pages/Invites.tsx @@ -1,10 +1,14 @@ -import { useEffect, useState } from 'react'; +import debugFactory from 'debug'; +import { useEffect, useRef, useState } from 'react'; import { useUser } from '../hooks/useUser'; -import { useAppDispatch, useAppSelector } from '../store/hooks'; -import { clearRedeemStatus, fetchInviteCodes, redeemCode } from '../store/inviteSlice'; +import { inviteApi } from '../services/api/inviteApi'; import type { InviteCode } from '../types/invite'; +const log = debugFactory('invites'); + +type RedeemStatus = 'idle' | 'loading' | 'success' | 'error'; + const CodeRow = ({ invite }: { invite: InviteCode }) => { const [copied, setCopied] = useState(false); const claimed = invite.currentUses >= invite.maxUses; @@ -70,26 +74,75 @@ const CodeRow = ({ invite }: { invite: InviteCode }) => { }; const Invites = () => { - const dispatch = useAppDispatch(); const { user, refetch: refetchUser } = useUser(); - const { codes, isLoading, redeemStatus, redeemError } = useAppSelector(state => state.invite); + const [codes, setCodes] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [redeemStatus, setRedeemStatus] = useState('idle'); + const [redeemError, setRedeemError] = useState(null); const [redeemInput, setRedeemInput] = useState(''); + const redeemTimeoutRef = useRef(null); + const loadRequestIdRef = useRef(0); const hasBeenInvited = !!user?.referral?.invitedBy; + const [loadError, setLoadError] = useState(null); + + const loadInviteCodes = async () => { + const requestId = ++loadRequestIdRef.current; + setIsLoading(true); + setLoadError(null); + try { + const data = await inviteApi.getMyInviteCodes(); + if (requestId !== loadRequestIdRef.current) return; + setCodes(data); + } catch (error) { + if (requestId !== loadRequestIdRef.current) return; + log('loadInviteCodes failed requestId=%d error=%O', requestId, error); + setLoadError(error instanceof Error ? error.message : 'Failed to load invite codes'); + } finally { + if (requestId === loadRequestIdRef.current) { + setIsLoading(false); + } + } + }; + useEffect(() => { - dispatch(fetchInviteCodes()); - }, [dispatch]); + void loadInviteCodes(); + return () => { + // Invalidate any in-flight loadInviteCodes requests + loadRequestIdRef.current += 1; + if (redeemTimeoutRef.current) { + clearTimeout(redeemTimeoutRef.current); + redeemTimeoutRef.current = null; + } + }; + }, []); const handleRedeem = async () => { const trimmed = redeemInput.trim(); if (!trimmed) return; - const result = await dispatch(redeemCode(trimmed)); - if (redeemCode.fulfilled.match(result)) { + setRedeemStatus('loading'); + setRedeemError(null); + + try { + await inviteApi.redeemInviteCode(trimmed); + await loadInviteCodes(); setRedeemInput(''); - refetchUser(); - setTimeout(() => dispatch(clearRedeemStatus()), 3000); + setRedeemStatus('success'); + if (redeemTimeoutRef.current) { + clearTimeout(redeemTimeoutRef.current); + } + redeemTimeoutRef.current = window.setTimeout(() => { + redeemTimeoutRef.current = null; + setRedeemStatus('idle'); + setRedeemError(null); + }, 3000); + // Refresh user in background — don't let failure override the successful redeem + refetchUser().catch(() => {}); + } catch (error) { + setRedeemStatus('error'); + setRedeemError(error instanceof Error ? error.message : 'Failed to redeem invite code'); } }; @@ -140,6 +193,8 @@ const Invites = () => {

+ {loadError &&

{loadError}

} + {isLoading ? (
{Array.from({ length: 5 }).map((_, i) => ( diff --git a/app/src/pages/onboarding/steps/ScreenPermissionsStep.tsx b/app/src/pages/onboarding/steps/ScreenPermissionsStep.tsx index 9e9e0ec10..1799ab657 100644 --- a/app/src/pages/onboarding/steps/ScreenPermissionsStep.tsx +++ b/app/src/pages/onboarding/steps/ScreenPermissionsStep.tsx @@ -1,11 +1,6 @@ import { useEffect, useState } from 'react'; -import { - fetchAccessibilityStatus, - refreshPermissionsWithRestart, - requestAccessibilityPermission, -} from '../../../store/accessibilitySlice'; -import { useAppDispatch, useAppSelector } from '../../../store/hooks'; +import { useScreenIntelligenceState } from '../../../features/screen-intelligence/useScreenIntelligenceState'; import OnboardingNextButton from '../components/OnboardingNextButton'; interface ScreenPermissionsStepProps { @@ -14,15 +9,17 @@ interface ScreenPermissionsStepProps { } const ScreenPermissionsStep = ({ onNext, onBack: _onBack }: ScreenPermissionsStepProps) => { - const dispatch = useAppDispatch(); - const { status, isLoading, isRequestingPermissions, isRestartingCore, lastError } = - useAppSelector(state => state.accessibility); + const { + status, + isLoading, + isRequestingPermissions, + isRestartingCore, + lastError, + requestPermission, + refreshPermissionsWithRestart, + } = useScreenIntelligenceState({ pollMs: 2000 }); const [shouldAutoRefreshOnReturn, setShouldAutoRefreshOnReturn] = useState(false); - useEffect(() => { - void dispatch(fetchAccessibilityStatus()); - }, [dispatch]); - const accessibilityPermission = status?.permissions.accessibility ?? 'unknown'; const isGranted = accessibilityPermission === 'granted'; @@ -42,7 +39,7 @@ const ScreenPermissionsStep = ({ onNext, onBack: _onBack }: ScreenPermissionsSte } setShouldAutoRefreshOnReturn(false); - void dispatch(refreshPermissionsWithRestart()); + void refreshPermissionsWithRestart(); }; window.addEventListener('focus', refreshAfterReturn); @@ -52,11 +49,17 @@ const ScreenPermissionsStep = ({ onNext, onBack: _onBack }: ScreenPermissionsSte window.removeEventListener('focus', refreshAfterReturn); document.removeEventListener('visibilitychange', refreshAfterReturn); }; - }, [dispatch, isGranted, isLoading, isRestartingCore, shouldAutoRefreshOnReturn]); + }, [ + isGranted, + isLoading, + isRestartingCore, + refreshPermissionsWithRestart, + shouldAutoRefreshOnReturn, + ]); const handleRequestPermissions = () => { setShouldAutoRefreshOnReturn(true); - void dispatch(requestAccessibilityPermission('accessibility')); + void requestPermission('accessibility'); }; return ( @@ -102,7 +105,7 @@ const ScreenPermissionsStep = ({ onNext, onBack: _onBack }: ScreenPermissionsSte