mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-29 22:23:01 +00:00
feat: improve overlay debug visibility and triggers (#326)
* feat: initialize OpenHuman overlay with log viewer and Tauri integration - Added a new overlay project for OpenHuman, featuring a transparent window with a log viewer. - Implemented core components including TitleBar, ModuleFilter, LogViewer, and StatusBar for enhanced user interaction. - Integrated Tauri for desktop application capabilities, allowing for real-time log monitoring and management. - Configured TypeScript, Tailwind CSS, and Vite for a modern development experience. - Included necessary files such as package.json, tsconfig.json, and README.md to guide setup and usage. * refactor: update async task spawning in Tauri integration - Replaced `tokio::spawn` with `tauri::async_runtime::spawn` for improved compatibility with Tauri's async runtime. - This change enhances the integration of the OpenHuman core server within the Tauri application, ensuring better performance and stability. * feat: implement click-through toggle in overlay title bar - Added a click-through toggle feature to the TitleBar component, allowing mouse events to pass through the overlay. - Updated the App component to manage the click-through state and handle its toggling. - Enhanced the MODULE_LABELS in types.ts to include additional known modules for filtering. - Modified Tauri backend to support the click-through functionality, ensuring proper state management and event handling. * feat: implement audio transcription functionality in overlay - Added audio recording and transcription capabilities to the overlay, allowing users to capture and transcribe speech. - Introduced functions for audio processing, including converting audio blobs to WAV format and handling audio streams. - Updated the App component to manage recording states and display transcription results. - Enhanced the user interface with a microphone icon indicating recording status. - Adjusted window dimensions and properties in the Tauri configuration for improved user experience. * feat: enhance transcript insertion and logging in overlay - Implemented a new function to insert transcribed text into the currently focused field of the active application, improving user experience. - Added detailed logging for overlay actions, including recording start/stop events and transcript insertion attempts, to aid in debugging and monitoring. - Updated the App component to utilize the new insertion functionality and log relevant information during transcription processes. * feat: implement macOS Globe/Fn key listener integration - Added a new Globe/Fn key listener feature for macOS, enabling the application to respond to Globe key events. - Implemented functions to start, poll, and stop the Globe listener, enhancing user interaction with the overlay. - Updated the App component to manage the listener's lifecycle and display relevant status messages. - Modified Tauri configuration to adjust overlay visibility settings for improved user experience. - Introduced a new module for Globe listener management, encapsulating related functionality and improving code organization. * feat: add macOS support for accessory activation policy and create Info.plist - Introduced a new Info.plist file to configure macOS application settings. - Implemented accessory activation policy for the overlay on macOS, enhancing user experience by allowing the app to run in the background without a dock icon. - Updated the application setup to include macOS-specific configurations for improved functionality. * feat: enhance overlay debug state and UI components - Introduced new interfaces for managing accessibility and autocomplete statuses, improving the structure of debug information. - Implemented a polling mechanism to refresh the overlay debug state, capturing accessibility and autocomplete data in real-time. - Updated the App component to display active application and window titles, along with autocomplete suggestions and phases. - Adjusted the overlay dimensions in Tauri configuration for better user experience and visibility. - Enhanced logging for debug snapshot updates, aiding in monitoring and troubleshooting. * refactor: improve code formatting and readability in accessibility and screen intelligence modules - Reformatted code in `globe.rs`, `ops.rs`, `schemas.rs`, and `types.rs` for better clarity and consistency. - Enhanced logging statements and function definitions to follow a more uniform style. - Updated imports in `types.rs` to maintain organization and improve accessibility module integration.
This commit is contained in:
Steven Enamakel
@@ -0,0 +1,712 @@
|
||||
import { invoke } from "@tauri-apps/api/core";
|
||||
import { getCurrentWindow } from "@tauri-apps/api/window";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
const TARGET_SAMPLE_RATE = 16000;
|
||||
|
||||
type OverlayStatus = "idle" | "listening" | "transcribing" | "ready" | "error";
|
||||
|
||||
interface TranscribeResult {
|
||||
text: string;
|
||||
raw_text: string;
|
||||
model_id: string;
|
||||
}
|
||||
|
||||
interface GlobeHotkeyStatus {
|
||||
supported: boolean;
|
||||
running: boolean;
|
||||
input_monitoring_permission: string;
|
||||
last_error: string | null;
|
||||
events_pending: number;
|
||||
}
|
||||
|
||||
interface GlobeHotkeyPollResult {
|
||||
status: GlobeHotkeyStatus;
|
||||
events: string[];
|
||||
}
|
||||
|
||||
interface AppContextInfo {
|
||||
app_name: string | null;
|
||||
window_title: string | null;
|
||||
}
|
||||
|
||||
interface AccessibilitySessionStatus {
|
||||
active: boolean;
|
||||
capture_count: number;
|
||||
frames_in_memory: number;
|
||||
last_capture_at_ms: number | null;
|
||||
last_context: string | null;
|
||||
last_window_title: string | null;
|
||||
vision_enabled: boolean;
|
||||
vision_state: string;
|
||||
vision_queue_depth: number;
|
||||
}
|
||||
|
||||
interface AccessibilityStatus {
|
||||
is_context_blocked: boolean;
|
||||
foreground_context: AppContextInfo | null;
|
||||
session: AccessibilitySessionStatus;
|
||||
}
|
||||
|
||||
interface AutocompleteSuggestion {
|
||||
value: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
interface AutocompleteStatus {
|
||||
platform_supported: boolean;
|
||||
enabled: boolean;
|
||||
running: boolean;
|
||||
phase: string;
|
||||
app_name: string | null;
|
||||
last_error: string | null;
|
||||
updated_at_ms: number | null;
|
||||
suggestion: AutocompleteSuggestion | null;
|
||||
}
|
||||
|
||||
interface OverlayDebugSnapshot {
|
||||
screen: AccessibilityStatus | null;
|
||||
autocomplete: AutocompleteStatus | null;
|
||||
updatedAt: number | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
function logOverlay(message: string, details?: unknown) {
|
||||
if (details) {
|
||||
console.debug(`[overlay] ${message}`, details);
|
||||
return;
|
||||
}
|
||||
console.debug(`[overlay] ${message}`);
|
||||
}
|
||||
|
||||
function formatTimestamp(timestampMs: number | null): string {
|
||||
if (!timestampMs) {
|
||||
return "none";
|
||||
}
|
||||
|
||||
try {
|
||||
return new Date(timestampMs).toLocaleTimeString([], {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
} catch {
|
||||
return String(timestampMs);
|
||||
}
|
||||
}
|
||||
|
||||
function floatTo16BitPCM(output: DataView, offset: number, input: Float32Array) {
|
||||
for (let i = 0; i < input.length; i += 1, offset += 2) {
|
||||
const sample = Math.max(-1, Math.min(1, input[i]));
|
||||
output.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
|
||||
}
|
||||
}
|
||||
|
||||
function encodeWavMono16k(samples: Float32Array, sampleRate: number): Uint8Array {
|
||||
const bytesPerSample = 2;
|
||||
const blockAlign = bytesPerSample;
|
||||
const byteRate = sampleRate * blockAlign;
|
||||
const dataSize = samples.length * bytesPerSample;
|
||||
const buffer = new ArrayBuffer(44 + dataSize);
|
||||
const view = new DataView(buffer);
|
||||
|
||||
const writeString = (offset: number, value: string) => {
|
||||
for (let i = 0; i < value.length; i += 1) {
|
||||
view.setUint8(offset + i, value.charCodeAt(i));
|
||||
}
|
||||
};
|
||||
|
||||
writeString(0, "RIFF");
|
||||
view.setUint32(4, 36 + dataSize, true);
|
||||
writeString(8, "WAVE");
|
||||
writeString(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, 1, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, byteRate, true);
|
||||
view.setUint16(32, blockAlign, true);
|
||||
view.setUint16(34, 16, true);
|
||||
writeString(36, "data");
|
||||
view.setUint32(40, dataSize, true);
|
||||
floatTo16BitPCM(view, 44, samples);
|
||||
|
||||
return new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
async function toMono16k(audioBuffer: AudioBuffer): Promise<Float32Array> {
|
||||
const channels = audioBuffer.numberOfChannels;
|
||||
const mono = new Float32Array(audioBuffer.length);
|
||||
|
||||
for (let c = 0; c < channels; c += 1) {
|
||||
const channelData = audioBuffer.getChannelData(c);
|
||||
for (let i = 0; i < audioBuffer.length; i += 1) {
|
||||
mono[i] += channelData[i] / channels;
|
||||
}
|
||||
}
|
||||
|
||||
if (audioBuffer.sampleRate === TARGET_SAMPLE_RATE) {
|
||||
return mono;
|
||||
}
|
||||
|
||||
const targetLength = Math.max(
|
||||
1,
|
||||
Math.round((mono.length * TARGET_SAMPLE_RATE) / audioBuffer.sampleRate),
|
||||
);
|
||||
const offline = new OfflineAudioContext(1, targetLength, TARGET_SAMPLE_RATE);
|
||||
const sourceBuffer = offline.createBuffer(1, mono.length, audioBuffer.sampleRate);
|
||||
sourceBuffer.copyToChannel(mono, 0);
|
||||
const source = offline.createBufferSource();
|
||||
source.buffer = sourceBuffer;
|
||||
source.connect(offline.destination);
|
||||
source.start();
|
||||
const rendered = await offline.startRendering();
|
||||
return rendered.getChannelData(0).slice();
|
||||
}
|
||||
|
||||
async function convertBlobToWavBytes(blob: Blob): Promise<number[]> {
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
const audioContext = new AudioContext();
|
||||
|
||||
try {
|
||||
const decoded = await audioContext.decodeAudioData(arrayBuffer.slice(0));
|
||||
const mono16k = await toMono16k(decoded);
|
||||
return Array.from(encodeWavMono16k(mono16k, TARGET_SAMPLE_RATE));
|
||||
} finally {
|
||||
await audioContext.close();
|
||||
}
|
||||
}
|
||||
|
||||
function MicrophoneIcon({ active }: { active: boolean }) {
|
||||
return (
|
||||
<svg
|
||||
aria-hidden="true"
|
||||
className={`h-9 w-9 transition-transform duration-200 ${active ? "scale-105" : ""}`}
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="1.8"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<rect x="9" y="3" width="6" height="11" rx="3" />
|
||||
<path d="M6 11a6 6 0 0 0 12 0" />
|
||||
<path d="M12 17v4" />
|
||||
<path d="M8.5 21h7" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
export function App() {
|
||||
const appWindow = getCurrentWindow();
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const chunksRef = useRef<Blob[]>([]);
|
||||
const sessionIdRef = useRef(0);
|
||||
const globePollInFlightRef = useRef(false);
|
||||
|
||||
const [status, setStatus] = useState<OverlayStatus>("idle");
|
||||
const [message, setMessage] = useState("Click to start listening");
|
||||
const [transcript, setTranscript] = useState("");
|
||||
const [debugSnapshot, setDebugSnapshot] = useState<OverlayDebugSnapshot>({
|
||||
screen: null,
|
||||
autocomplete: null,
|
||||
updatedAt: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
let disposed = false;
|
||||
|
||||
const showOverlayFallback = async (message: string) => {
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
logOverlay("globe listener unavailable", { message });
|
||||
setMessage(message);
|
||||
await appWindow.show().catch(() => {});
|
||||
};
|
||||
|
||||
const startGlobeListener = async () => {
|
||||
try {
|
||||
const result = await invoke<GlobeHotkeyStatus>("core_rpc", {
|
||||
method: "openhuman.screen_intelligence_globe_listener_start",
|
||||
params: {},
|
||||
});
|
||||
logOverlay("globe listener start result", result);
|
||||
|
||||
if (!result.supported) {
|
||||
await showOverlayFallback("Globe/Fn hotkey is only supported on macOS");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!result.running) {
|
||||
await showOverlayFallback(
|
||||
result.last_error ?? "Globe/Fn listener could not start. Check Input Monitoring.",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[overlay] failed to start globe listener", error);
|
||||
await showOverlayFallback("Failed to start Globe/Fn listener");
|
||||
}
|
||||
};
|
||||
|
||||
const pollGlobeListener = async () => {
|
||||
if (disposed || globePollInFlightRef.current) {
|
||||
return;
|
||||
}
|
||||
globePollInFlightRef.current = true;
|
||||
|
||||
try {
|
||||
const result = await invoke<GlobeHotkeyPollResult>("core_rpc", {
|
||||
method: "openhuman.screen_intelligence_globe_listener_poll",
|
||||
params: {},
|
||||
});
|
||||
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!result.status.running && result.status.last_error) {
|
||||
setMessage(result.status.last_error);
|
||||
}
|
||||
|
||||
if (result.events.includes("FN_UP")) {
|
||||
const visible = await appWindow.isVisible();
|
||||
logOverlay("received FN_UP", { visible });
|
||||
if (visible) {
|
||||
await appWindow.hide();
|
||||
} else {
|
||||
await appWindow.show();
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!disposed) {
|
||||
console.warn("[overlay] globe listener poll failed", error);
|
||||
}
|
||||
} finally {
|
||||
globePollInFlightRef.current = false;
|
||||
}
|
||||
};
|
||||
|
||||
void startGlobeListener();
|
||||
const intervalId = window.setInterval(() => {
|
||||
void pollGlobeListener();
|
||||
}, 175);
|
||||
|
||||
return () => {
|
||||
disposed = true;
|
||||
window.clearInterval(intervalId);
|
||||
void invoke("core_rpc", {
|
||||
method: "openhuman.screen_intelligence_globe_listener_stop",
|
||||
params: {},
|
||||
}).catch(() => {});
|
||||
};
|
||||
}, [appWindow]);
|
||||
|
||||
useEffect(() => {
|
||||
let disposed = false;
|
||||
let pollInFlight = false;
|
||||
|
||||
const pollDebugState = async () => {
|
||||
if (disposed || pollInFlight) {
|
||||
return;
|
||||
}
|
||||
pollInFlight = true;
|
||||
|
||||
try {
|
||||
const [screen, autocomplete] = await Promise.all([
|
||||
invoke<AccessibilityStatus>("core_rpc", {
|
||||
method: "openhuman.accessibility_status",
|
||||
params: {},
|
||||
}),
|
||||
invoke<AutocompleteStatus>("core_rpc", {
|
||||
method: "openhuman.autocomplete_status",
|
||||
params: {},
|
||||
}),
|
||||
]);
|
||||
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
|
||||
logOverlay("debug snapshot refreshed", {
|
||||
screenActive: screen.session.active,
|
||||
captureCount: screen.session.capture_count,
|
||||
autocompletePhase: autocomplete.phase,
|
||||
hasSuggestion: Boolean(autocomplete.suggestion?.value),
|
||||
});
|
||||
|
||||
setDebugSnapshot({
|
||||
screen,
|
||||
autocomplete,
|
||||
updatedAt: Date.now(),
|
||||
error: null,
|
||||
});
|
||||
} catch (error) {
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextError =
|
||||
error instanceof Error ? error.message : "Failed to refresh overlay debug state";
|
||||
console.warn("[overlay] debug snapshot poll failed", error);
|
||||
setDebugSnapshot((previous) => ({
|
||||
...previous,
|
||||
updatedAt: Date.now(),
|
||||
error: nextError,
|
||||
}));
|
||||
} finally {
|
||||
pollInFlight = false;
|
||||
}
|
||||
};
|
||||
|
||||
void pollDebugState();
|
||||
const intervalId = window.setInterval(() => {
|
||||
void pollDebugState();
|
||||
}, 900);
|
||||
|
||||
return () => {
|
||||
disposed = true;
|
||||
window.clearInterval(intervalId);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const insertTranscriptIntoFocusedField = useCallback(
|
||||
async (text: string) => {
|
||||
logOverlay("inserting transcript into focused field", { length: text.length });
|
||||
await appWindow.hide();
|
||||
await new Promise((resolve) => window.setTimeout(resolve, 120));
|
||||
|
||||
try {
|
||||
await invoke("insert_text_into_focused_field", { text });
|
||||
logOverlay("transcript inserted via accessibility helper");
|
||||
} catch (error) {
|
||||
console.warn("[overlay] accessibility insert failed, falling back to clipboard", error);
|
||||
await navigator.clipboard.writeText(text);
|
||||
}
|
||||
},
|
||||
[appWindow],
|
||||
);
|
||||
|
||||
const resetForNextCapture = useCallback(() => {
|
||||
setTranscript("");
|
||||
setMessage("Click to start listening");
|
||||
setStatus("idle");
|
||||
}, []);
|
||||
|
||||
const cleanupStream = useCallback(() => {
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
}, []);
|
||||
|
||||
const transcribeBlob = useCallback(
|
||||
async (blob: Blob, sessionId: number) => {
|
||||
try {
|
||||
const audioBytes = await convertBlobToWavBytes(blob);
|
||||
const result = await invoke<TranscribeResult>("core_rpc", {
|
||||
method: "openhuman.voice_transcribe_bytes",
|
||||
params: {
|
||||
audio_bytes: audioBytes,
|
||||
extension: "wav",
|
||||
skip_cleanup: false,
|
||||
},
|
||||
});
|
||||
|
||||
if (sessionIdRef.current !== sessionId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextTranscript = result.text.trim();
|
||||
if (!nextTranscript) {
|
||||
setTranscript("");
|
||||
setStatus("error");
|
||||
setMessage("No speech detected");
|
||||
return;
|
||||
}
|
||||
|
||||
setTranscript(nextTranscript);
|
||||
setStatus("ready");
|
||||
setMessage("Inserting text...");
|
||||
await insertTranscriptIntoFocusedField(nextTranscript);
|
||||
if (sessionIdRef.current !== sessionId) {
|
||||
return;
|
||||
}
|
||||
setMessage("Inserted into active field");
|
||||
} catch (error) {
|
||||
if (sessionIdRef.current !== sessionId) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.error("[overlay] transcription failed", error);
|
||||
setTranscript("");
|
||||
setStatus("error");
|
||||
setMessage(error instanceof Error ? error.message : "Transcription failed");
|
||||
}
|
||||
},
|
||||
[insertTranscriptIntoFocusedField],
|
||||
);
|
||||
|
||||
const stopRecording = useCallback(() => {
|
||||
if (!mediaRecorderRef.current || mediaRecorderRef.current.state === "inactive") {
|
||||
return;
|
||||
}
|
||||
|
||||
setStatus("transcribing");
|
||||
setMessage("Transcribing...");
|
||||
mediaRecorderRef.current.stop();
|
||||
mediaRecorderRef.current = null;
|
||||
}, []);
|
||||
|
||||
const startRecording = useCallback(async () => {
|
||||
const nextSessionId = sessionIdRef.current + 1;
|
||||
sessionIdRef.current = nextSessionId;
|
||||
setTranscript("");
|
||||
setStatus("listening");
|
||||
setMessage("Listening...");
|
||||
chunksRef.current = [];
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
const mimeType = MediaRecorder.isTypeSupported("audio/webm;codecs=opus")
|
||||
? "audio/webm;codecs=opus"
|
||||
: MediaRecorder.isTypeSupported("audio/webm")
|
||||
? "audio/webm"
|
||||
: "audio/ogg";
|
||||
|
||||
const recorder = new MediaRecorder(stream, { mimeType });
|
||||
mediaRecorderRef.current = recorder;
|
||||
|
||||
recorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
chunksRef.current.push(event.data);
|
||||
}
|
||||
};
|
||||
|
||||
recorder.onerror = (event) => {
|
||||
console.error("[overlay] media recorder error", event);
|
||||
cleanupStream();
|
||||
setStatus("error");
|
||||
setMessage("Microphone recording failed");
|
||||
};
|
||||
|
||||
recorder.onstop = () => {
|
||||
cleanupStream();
|
||||
const blob = new Blob(chunksRef.current, { type: mimeType });
|
||||
chunksRef.current = [];
|
||||
|
||||
if (blob.size === 0) {
|
||||
setStatus("error");
|
||||
setMessage("No audio recorded");
|
||||
return;
|
||||
}
|
||||
|
||||
logOverlay("recording stopped, starting transcription", {
|
||||
blobSize: blob.size,
|
||||
mimeType,
|
||||
});
|
||||
void transcribeBlob(blob, nextSessionId);
|
||||
};
|
||||
|
||||
logOverlay("recording started", { mimeType });
|
||||
recorder.start(100);
|
||||
} catch (error) {
|
||||
console.error("[overlay] getUserMedia failed", error);
|
||||
cleanupStream();
|
||||
setStatus("error");
|
||||
setMessage(error instanceof Error ? error.message : "Microphone access failed");
|
||||
}
|
||||
}, [cleanupStream, transcribeBlob]);
|
||||
|
||||
const handleMainButton = useCallback(() => {
|
||||
if (status === "listening") {
|
||||
logOverlay("main button toggled to stop listening");
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
|
||||
logOverlay("main button toggled to start listening", { priorStatus: status });
|
||||
void startRecording();
|
||||
}, [startRecording, status, stopRecording]);
|
||||
|
||||
const shellClassName = useMemo(() => {
|
||||
if (status === "listening") {
|
||||
return "from-red-500/90 via-rose-500/80 to-orange-400/85 text-white shadow-[0_0_64px_rgba(248,113,113,0.38)]";
|
||||
}
|
||||
if (status === "transcribing") {
|
||||
return "from-amber-400/90 via-orange-400/80 to-yellow-300/80 text-stone-950 shadow-[0_0_56px_rgba(251,191,36,0.34)]";
|
||||
}
|
||||
if (status === "error") {
|
||||
return "from-red-600/90 via-rose-700/80 to-stone-900/90 text-white shadow-[0_0_56px_rgba(190,24,93,0.35)]";
|
||||
}
|
||||
if (status === "ready") {
|
||||
return "from-emerald-400/90 via-teal-400/80 to-cyan-300/80 text-stone-950 shadow-[0_0_56px_rgba(45,212,191,0.34)]";
|
||||
}
|
||||
return "from-slate-900/92 via-slate-800/92 to-slate-700/92 text-white shadow-[0_0_48px_rgba(15,23,42,0.42)]";
|
||||
}, [status]);
|
||||
|
||||
const activeScreenApp =
|
||||
debugSnapshot.screen?.foreground_context?.app_name ??
|
||||
debugSnapshot.screen?.session.last_context ??
|
||||
"Unknown app";
|
||||
const activeScreenWindow =
|
||||
debugSnapshot.screen?.foreground_context?.window_title ??
|
||||
debugSnapshot.screen?.session.last_window_title ??
|
||||
"No active window title";
|
||||
const autocompleteSuggestion = debugSnapshot.autocomplete?.suggestion?.value?.trim() ?? "";
|
||||
const autocompletePhase = debugSnapshot.autocomplete?.phase ?? "unknown";
|
||||
const autocompleteRunning =
|
||||
debugSnapshot.autocomplete?.running && debugSnapshot.autocomplete?.enabled;
|
||||
|
||||
return (
|
||||
<div className="flex h-screen w-screen items-start justify-start bg-transparent p-3">
|
||||
<div className="relative select-none">
|
||||
{status === "listening" ? (
|
||||
<>
|
||||
<span className="pointer-events-none absolute inset-0 rounded-full border border-white/15 animate-ping" />
|
||||
<span className="pointer-events-none absolute -inset-3 rounded-full border border-red-300/30 blur-[2px]" />
|
||||
</>
|
||||
) : null}
|
||||
|
||||
<div
|
||||
className={`relative w-[348px] rounded-[32px] border border-white/15 bg-gradient-to-br p-3 backdrop-blur-xl transition-all duration-200 ${shellClassName}`}
|
||||
onMouseDown={(event) => {
|
||||
if (event.target instanceof HTMLElement && event.target.closest("button")) {
|
||||
return;
|
||||
}
|
||||
void appWindow.startDragging();
|
||||
}}
|
||||
>
|
||||
<div className="mb-3 flex items-center justify-between gap-2">
|
||||
<span className="rounded-full bg-black/15 px-2 py-1 text-[10px] font-semibold uppercase tracking-[0.24em]">
|
||||
Voice
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="h-7 w-7 rounded-full border border-white/15 bg-black/20 text-sm transition hover:bg-black/30"
|
||||
onClick={() => appWindow.hide()}
|
||||
aria-label="Hide overlay"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="flex items-start gap-3">
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleMainButton}
|
||||
className={`group relative flex h-[108px] w-[108px] shrink-0 items-center justify-center rounded-full border border-white/20 bg-black/20 transition duration-200 hover:bg-black/28 ${
|
||||
status === "listening" ? "scale-[1.02]" : ""
|
||||
}`}
|
||||
aria-label={status === "listening" ? "Stop listening" : "Start listening"}
|
||||
>
|
||||
<span className="absolute inset-3 rounded-full border border-white/12" />
|
||||
<MicrophoneIcon active={status === "listening"} />
|
||||
</button>
|
||||
|
||||
<div className="min-w-0 flex-1 pt-1">
|
||||
<p className="text-[11px] font-semibold uppercase tracking-[0.18em] opacity-80">
|
||||
{status}
|
||||
</p>
|
||||
<p className="mt-1 text-xs leading-4 opacity-90">{message}</p>
|
||||
|
||||
{transcript ? (
|
||||
<div className="mt-3 rounded-2xl border border-white/10 bg-black/15 px-3 py-2 text-[11px] leading-4 opacity-95">
|
||||
{transcript}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{(status === "ready" || status === "error") && !transcript ? (
|
||||
<button
|
||||
type="button"
|
||||
className="mt-3 w-full rounded-full border border-white/12 bg-black/15 px-3 py-2 text-[11px] font-medium transition hover:bg-black/25"
|
||||
onClick={resetForNextCapture}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-3 rounded-[24px] border border-white/10 bg-black/15 p-3">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<span className="text-[10px] font-semibold uppercase tracking-[0.22em] opacity-75">
|
||||
Debug
|
||||
</span>
|
||||
<span className="text-[10px] opacity-65">
|
||||
{debugSnapshot.updatedAt ? formatTimestamp(debugSnapshot.updatedAt) : "waiting"}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{debugSnapshot.error ? (
|
||||
<div className="mt-3 rounded-2xl border border-red-300/20 bg-red-950/20 px-3 py-2 text-[11px] leading-4 text-red-100">
|
||||
{debugSnapshot.error}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="mt-3 grid gap-3">
|
||||
<section className="rounded-2xl border border-white/8 bg-black/10 px-3 py-2">
|
||||
<div className="text-[10px] font-semibold uppercase tracking-[0.18em] opacity-70">
|
||||
Screen Intelligence
|
||||
</div>
|
||||
<div className="mt-2 space-y-1 text-[11px] leading-4 opacity-90">
|
||||
<p>Active screen: {activeScreenApp}</p>
|
||||
<p className="truncate">Window: {activeScreenWindow}</p>
|
||||
<p>
|
||||
Screenshots: {debugSnapshot.screen?.session.capture_count ?? 0} total,{" "}
|
||||
{debugSnapshot.screen?.session.frames_in_memory ?? 0} in memory
|
||||
</p>
|
||||
<p>
|
||||
Session: {debugSnapshot.screen?.session.active ? "active" : "idle"} | Vision:{" "}
|
||||
{debugSnapshot.screen?.session.vision_enabled ? "on" : "off"} /{" "}
|
||||
{debugSnapshot.screen?.session.vision_state ?? "idle"}
|
||||
</p>
|
||||
<p>
|
||||
Queue: {debugSnapshot.screen?.session.vision_queue_depth ?? 0} | Blocked:{" "}
|
||||
{debugSnapshot.screen?.is_context_blocked ? "yes" : "no"}
|
||||
</p>
|
||||
<p>
|
||||
Last capture:{" "}
|
||||
{formatTimestamp(debugSnapshot.screen?.session.last_capture_at_ms ?? null)}
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="rounded-2xl border border-white/8 bg-black/10 px-3 py-2">
|
||||
<div className="text-[10px] font-semibold uppercase tracking-[0.18em] opacity-70">
|
||||
Autocomplete
|
||||
</div>
|
||||
<div className="mt-2 space-y-1 text-[11px] leading-4 opacity-90">
|
||||
<p>
|
||||
Status: {autocompleteRunning ? "active" : "idle"} | Phase: {autocompletePhase}
|
||||
</p>
|
||||
<p>App: {debugSnapshot.autocomplete?.app_name ?? activeScreenApp}</p>
|
||||
<p>
|
||||
Processing:{" "}
|
||||
{autocompletePhase === "refreshing" || autocompletePhase === "processing"
|
||||
? "yes"
|
||||
: "no"}
|
||||
</p>
|
||||
<p>
|
||||
Suggestions:{" "}
|
||||
{autocompleteSuggestion ? "1 ready" : "none"}
|
||||
</p>
|
||||
<div className="rounded-xl border border-white/8 bg-black/10 px-2 py-2 text-[11px] leading-4">
|
||||
{autocompleteSuggestion || "No autocomplete suggestion available."}
|
||||
</div>
|
||||
{debugSnapshot.autocomplete?.last_error ? (
|
||||
<p className="text-red-100">
|
||||
Error: {debugSnapshot.autocomplete.last_error}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { useEffect, useMemo, useRef } from "react";
|
||||
import type { LogEntry } from "../types";
|
||||
import { LEVEL_COLORS } from "../types";
|
||||
|
||||
interface LogViewerProps {
|
||||
entries: LogEntry[];
|
||||
activeModule: string;
|
||||
levelFilter: string;
|
||||
}
|
||||
|
||||
/** Format ISO timestamp to HH:MM:SS.mmm */
|
||||
function formatTime(ts: string): string {
|
||||
try {
|
||||
const d = new Date(ts);
|
||||
const h = String(d.getHours()).padStart(2, "0");
|
||||
const m = String(d.getMinutes()).padStart(2, "0");
|
||||
const s = String(d.getSeconds()).padStart(2, "0");
|
||||
const ms = String(d.getMilliseconds()).padStart(3, "0");
|
||||
return `${h}:${m}:${s}.${ms}`;
|
||||
} catch {
|
||||
return ts.slice(11, 23);
|
||||
}
|
||||
}
|
||||
|
||||
const LEVEL_ORDER: Record<string, number> = {
|
||||
TRACE: 0,
|
||||
DEBUG: 1,
|
||||
INFO: 2,
|
||||
WARN: 3,
|
||||
ERROR: 4,
|
||||
FATAL: 5,
|
||||
};
|
||||
|
||||
/**
|
||||
* Virtualized-ish log viewer. Auto-scrolls to bottom as new entries arrive.
|
||||
* Filters by module and minimum log level.
|
||||
*/
|
||||
export function LogViewer({ entries, activeModule, levelFilter }: LogViewerProps) {
|
||||
const bottomRef = useRef<HTMLDivElement>(null);
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const isAtBottom = useRef(true);
|
||||
|
||||
const filtered = useMemo(() => {
|
||||
const minLevel = LEVEL_ORDER[levelFilter] ?? 0;
|
||||
return entries.filter((e) => {
|
||||
if (activeModule !== "all" && e.module !== activeModule) return false;
|
||||
const entryLevel = LEVEL_ORDER[e.level] ?? 0;
|
||||
return entryLevel >= minLevel;
|
||||
});
|
||||
}, [entries, activeModule, levelFilter]);
|
||||
|
||||
// Track scroll position to decide auto-scroll
|
||||
const handleScroll = () => {
|
||||
const el = containerRef.current;
|
||||
if (!el) return;
|
||||
const threshold = 40;
|
||||
isAtBottom.current = el.scrollHeight - el.scrollTop - el.clientHeight < threshold;
|
||||
};
|
||||
|
||||
// Auto-scroll when new entries arrive (only if already at bottom)
|
||||
useEffect(() => {
|
||||
if (isAtBottom.current) {
|
||||
bottomRef.current?.scrollIntoView({ behavior: "instant" });
|
||||
}
|
||||
}, [filtered.length]);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={containerRef}
|
||||
onScroll={handleScroll}
|
||||
className="flex-1 overflow-y-auto log-scroll font-mono text-[11px] leading-[18px] px-2 py-1 bg-gray-950/90"
|
||||
>
|
||||
{filtered.length === 0 && (
|
||||
<div className="text-white/20 text-center py-8 text-xs">
|
||||
No logs yet...
|
||||
</div>
|
||||
)}
|
||||
|
||||
{filtered.map((entry, i) => (
|
||||
<div key={i} className="flex gap-2 hover:bg-white/[0.03] px-1 rounded">
|
||||
<span className="text-white/25 shrink-0">{formatTime(entry.ts)}</span>
|
||||
<span className={`shrink-0 w-[42px] text-right ${LEVEL_COLORS[entry.level] ?? "text-white/40"}`}>
|
||||
{entry.level}
|
||||
</span>
|
||||
<span className="text-purple-400/60 shrink-0 w-[80px] truncate" title={entry.target}>
|
||||
{entry.module}
|
||||
</span>
|
||||
<span className="text-white/80 break-all">{entry.message}</span>
|
||||
</div>
|
||||
))}
|
||||
|
||||
<div ref={bottomRef} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { MODULE_LABELS } from "../types";
|
||||
|
||||
interface ModuleFilterProps {
|
||||
modules: Set<string>;
|
||||
activeModule: string;
|
||||
onSelect: (module: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Horizontal tab bar for filtering logs by module.
|
||||
* Shows "All" plus every module that has emitted at least one log.
|
||||
*/
|
||||
export function ModuleFilter({ modules, activeModule, onSelect }: ModuleFilterProps) {
|
||||
const tabs = ["all", ...Array.from(modules).sort()];
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-1 px-2 py-1.5 bg-gray-900/60 border-b border-white/5 overflow-x-auto shrink-0">
|
||||
{tabs.map((mod) => {
|
||||
const isActive = mod === activeModule;
|
||||
const label = MODULE_LABELS[mod] ?? mod;
|
||||
return (
|
||||
<button
|
||||
key={mod}
|
||||
onClick={() => onSelect(mod)}
|
||||
className={`px-2 py-0.5 rounded text-[10px] font-mono whitespace-nowrap transition-colors ${
|
||||
isActive
|
||||
? "bg-primary-500/30 text-primary-500 border border-primary-500/40"
|
||||
: "text-white/40 hover:text-white/60 hover:bg-white/5"
|
||||
}`}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
interface StatusBarProps {
|
||||
filteredInfo: string;
|
||||
levelFilter: string;
|
||||
onLevelChange: (level: string) => void;
|
||||
onClear: () => void;
|
||||
}
|
||||
|
||||
const LEVELS = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"];
|
||||
|
||||
/**
|
||||
* Bottom status bar showing entry count, level filter, and clear button.
|
||||
*/
|
||||
export function StatusBar({
|
||||
filteredInfo,
|
||||
levelFilter,
|
||||
onLevelChange,
|
||||
onClear,
|
||||
}: StatusBarProps) {
|
||||
return (
|
||||
<div className="flex items-center justify-between px-3 py-1 bg-gray-900/80 border-t border-white/5 shrink-0">
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-[10px] text-white/30 font-mono">
|
||||
{filteredInfo}
|
||||
</span>
|
||||
|
||||
<select
|
||||
value={levelFilter}
|
||||
onChange={(e) => onLevelChange(e.target.value)}
|
||||
className="text-[10px] bg-transparent text-white/50 border border-white/10 rounded px-1 py-0.5 cursor-pointer hover:border-white/20"
|
||||
>
|
||||
{LEVELS.map((l) => (
|
||||
<option key={l} value={l} className="bg-gray-900 text-white">
|
||||
{l}+
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<button
|
||||
onClick={onClear}
|
||||
className="text-[10px] text-white/30 hover:text-white/60 transition-colors"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { getCurrentWindow } from "@tauri-apps/api/window";
|
||||
import { invoke } from "@tauri-apps/api/core";
|
||||
|
||||
interface TitleBarProps {
|
||||
clickThrough: boolean;
|
||||
onClickThroughToggle: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Custom title bar for the frameless overlay window.
|
||||
* Supports dragging, window controls, and click-through toggle.
|
||||
*/
|
||||
export function TitleBar({ clickThrough, onClickThroughToggle }: TitleBarProps) {
|
||||
const appWindow = getCurrentWindow();
|
||||
|
||||
const handleClickThrough = async () => {
|
||||
const next = !clickThrough;
|
||||
try {
|
||||
await invoke("set_click_through", { enabled: next });
|
||||
onClickThroughToggle();
|
||||
} catch (e) {
|
||||
console.error("Failed to set click-through:", e);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
data-tauri-drag-region
|
||||
className="flex items-center justify-between h-8 px-3 bg-gray-900/80 backdrop-blur-md border-b border-white/5 cursor-move select-none shrink-0"
|
||||
>
|
||||
<span className="text-[11px] font-medium text-white/60 tracking-wide uppercase">
|
||||
OpenHuman
|
||||
</span>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
{/* Click-through toggle */}
|
||||
<button
|
||||
onClick={handleClickThrough}
|
||||
className={`text-[10px] px-1.5 py-0.5 rounded transition-colors ${
|
||||
clickThrough
|
||||
? "bg-blue-500/30 text-blue-400 border border-blue-500/40"
|
||||
: "text-white/30 hover:text-white/50"
|
||||
}`}
|
||||
title={clickThrough ? "Click-through ON (clicks pass through)" : "Click-through OFF"}
|
||||
>
|
||||
{clickThrough ? "CT" : "CT"}
|
||||
</button>
|
||||
|
||||
{/* Minimize */}
|
||||
<button
|
||||
onClick={() => appWindow.minimize()}
|
||||
className="w-3 h-3 rounded-full bg-amber-500/80 hover:bg-amber-400 transition-colors"
|
||||
title="Minimize"
|
||||
/>
|
||||
{/* Close */}
|
||||
<button
|
||||
onClick={() => appWindow.hide()}
|
||||
className="w-3 h-3 rounded-full bg-red-500/80 hover:bg-red-400 transition-colors"
|
||||
title="Hide"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { invoke } from "@tauri-apps/api/core";
|
||||
import { listen } from "@tauri-apps/api/event";
|
||||
import type { LogEntry } from "../types";
|
||||
|
||||
const MAX_ENTRIES = 5000;
|
||||
|
||||
/**
|
||||
* Subscribes to `core:log` Tauri events and manages the log buffer.
|
||||
* On mount, fetches buffered history so we don't miss startup logs.
|
||||
*/
|
||||
export function useLogs() {
|
||||
const [entries, setEntries] = useState<LogEntry[]>([]);
|
||||
const [modules, setModules] = useState<Set<string>>(new Set());
|
||||
const entriesRef = useRef<LogEntry[]>([]);
|
||||
|
||||
// Track seen modules for the filter UI
|
||||
const modulesRef = useRef<Set<string>>(new Set());
|
||||
|
||||
const addEntries = useCallback((newEntries: LogEntry[]) => {
|
||||
const current = entriesRef.current;
|
||||
const updated = [...current, ...newEntries];
|
||||
// Trim to max
|
||||
if (updated.length > MAX_ENTRIES) {
|
||||
updated.splice(0, updated.length - MAX_ENTRIES);
|
||||
}
|
||||
entriesRef.current = updated;
|
||||
setEntries(updated);
|
||||
|
||||
// Track modules
|
||||
let modulesChanged = false;
|
||||
for (const e of newEntries) {
|
||||
if (!modulesRef.current.has(e.module)) {
|
||||
modulesRef.current.add(e.module);
|
||||
modulesChanged = true;
|
||||
}
|
||||
}
|
||||
if (modulesChanged) {
|
||||
setModules(new Set(modulesRef.current));
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
// Fetch buffered history from Rust
|
||||
invoke<LogEntry[]>("get_log_history")
|
||||
.then((history) => {
|
||||
if (history.length > 0) {
|
||||
addEntries(history);
|
||||
}
|
||||
})
|
||||
.catch(console.error);
|
||||
|
||||
// Subscribe to live log events
|
||||
const unlisten = listen<LogEntry>("core:log", (event) => {
|
||||
addEntries([event.payload]);
|
||||
});
|
||||
|
||||
return () => {
|
||||
unlisten.then((fn) => fn());
|
||||
};
|
||||
}, [addEntries]);
|
||||
|
||||
const clear = useCallback(() => {
|
||||
entriesRef.current = [];
|
||||
setEntries([]);
|
||||
}, []);
|
||||
|
||||
return { entries, modules, clear };
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import React from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { App } from "./App";
|
||||
import "./styles.css";
|
||||
|
||||
ReactDOM.createRoot(document.getElementById("root")!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>
|
||||
);
|
||||
@@ -0,0 +1,33 @@
|
||||
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600&family=JetBrains+Mono:wght@300;400;500&display=swap');
|
||||
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
html, body, #root {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
height: 100%;
|
||||
background: transparent;
|
||||
overflow: hidden;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: "Inter", system-ui, sans-serif;
|
||||
}
|
||||
|
||||
/* Scrollbar styling for log viewer */
|
||||
.log-scroll::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
.log-scroll::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
.log-scroll::-webkit-scrollbar-thumb {
|
||||
background: rgba(255, 255, 255, 0.15);
|
||||
border-radius: 3px;
|
||||
}
|
||||
.log-scroll::-webkit-scrollbar-thumb:hover {
|
||||
background: rgba(255, 255, 255, 0.25);
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/** A single log entry from the Rust core, received via `core:log` event. */
|
||||
export interface LogEntry {
|
||||
ts: string;
|
||||
level: string;
|
||||
module: string;
|
||||
target: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
/** Known module names for filtering.
|
||||
* As new apps/domains emit logs, they auto-appear in the filter bar.
|
||||
* This map provides friendly labels for known modules. */
|
||||
export const MODULE_LABELS: Record<string, string> = {
|
||||
all: "All",
|
||||
// ── Core domains ──
|
||||
skills: "Skills",
|
||||
rpc: "RPC",
|
||||
core: "Core",
|
||||
core_server: "Server",
|
||||
config: "Config",
|
||||
cron: "Cron",
|
||||
memory: "Memory",
|
||||
channels: "Channels",
|
||||
overlay: "Overlay",
|
||||
about_app: "About",
|
||||
subconscious: "Subconscious",
|
||||
// ── Apps / subsystems ──
|
||||
screen_recorder: "Screen Rec",
|
||||
autocomplete: "Autocomplete",
|
||||
agent: "Agent",
|
||||
search: "Search",
|
||||
// ── Infra ──
|
||||
axum: "HTTP",
|
||||
tower_http: "HTTP",
|
||||
socketioxide: "Socket.IO",
|
||||
hyper: "Hyper",
|
||||
reqwest: "Reqwest",
|
||||
rusqlite: "SQLite",
|
||||
};
|
||||
|
||||
/** Level colors for the log viewer. */
|
||||
export const LEVEL_COLORS: Record<string, string> = {
|
||||
TRACE: "text-gray-500",
|
||||
DEBUG: "text-blue-400",
|
||||
INFO: "text-green-400",
|
||||
WARN: "text-amber-400",
|
||||
ERROR: "text-red-400",
|
||||
FATAL: "text-red-600",
|
||||
};
|
||||
Reference in New Issue
Block a user