mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-30 23:14:37 +00:00
feat: improve overlay debug visibility and triggers (#326)
* feat: initialize OpenHuman overlay with log viewer and Tauri integration - Added a new overlay project for OpenHuman, featuring a transparent window with a log viewer. - Implemented core components including TitleBar, ModuleFilter, LogViewer, and StatusBar for enhanced user interaction. - Integrated Tauri for desktop application capabilities, allowing for real-time log monitoring and management. - Configured TypeScript, Tailwind CSS, and Vite for a modern development experience. - Included necessary files such as package.json, tsconfig.json, and README.md to guide setup and usage. * refactor: update async task spawning in Tauri integration - Replaced `tokio::spawn` with `tauri::async_runtime::spawn` for improved compatibility with Tauri's async runtime. - This change enhances the integration of the OpenHuman core server within the Tauri application, ensuring better performance and stability. * feat: implement click-through toggle in overlay title bar - Added a click-through toggle feature to the TitleBar component, allowing mouse events to pass through the overlay. - Updated the App component to manage the click-through state and handle its toggling. - Enhanced the MODULE_LABELS in types.ts to include additional known modules for filtering. - Modified Tauri backend to support the click-through functionality, ensuring proper state management and event handling. * feat: implement audio transcription functionality in overlay - Added audio recording and transcription capabilities to the overlay, allowing users to capture and transcribe speech. - Introduced functions for audio processing, including converting audio blobs to WAV format and handling audio streams. - Updated the App component to manage recording states and display transcription results. - Enhanced the user interface with a microphone icon indicating recording status. - Adjusted window dimensions and properties in the Tauri configuration for improved user experience. * feat: enhance transcript insertion and logging in overlay - Implemented a new function to insert transcribed text into the currently focused field of the active application, improving user experience. - Added detailed logging for overlay actions, including recording start/stop events and transcript insertion attempts, to aid in debugging and monitoring. - Updated the App component to utilize the new insertion functionality and log relevant information during transcription processes. * feat: implement macOS Globe/Fn key listener integration - Added a new Globe/Fn key listener feature for macOS, enabling the application to respond to Globe key events. - Implemented functions to start, poll, and stop the Globe listener, enhancing user interaction with the overlay. - Updated the App component to manage the listener's lifecycle and display relevant status messages. - Modified Tauri configuration to adjust overlay visibility settings for improved user experience. - Introduced a new module for Globe listener management, encapsulating related functionality and improving code organization. * feat: add macOS support for accessory activation policy and create Info.plist - Introduced a new Info.plist file to configure macOS application settings. - Implemented accessory activation policy for the overlay on macOS, enhancing user experience by allowing the app to run in the background without a dock icon. - Updated the application setup to include macOS-specific configurations for improved functionality. * feat: enhance overlay debug state and UI components - Introduced new interfaces for managing accessibility and autocomplete statuses, improving the structure of debug information. - Implemented a polling mechanism to refresh the overlay debug state, capturing accessibility and autocomplete data in real-time. - Updated the App component to display active application and window titles, along with autocomplete suggestions and phases. - Adjusted the overlay dimensions in Tauri configuration for better user experience and visibility. - Enhanced logging for debug snapshot updates, aiding in monitoring and troubleshooting. * refactor: improve code formatting and readability in accessibility and screen intelligence modules - Reformatted code in `globe.rs`, `ops.rs`, `schemas.rs`, and `types.rs` for better clarity and consistency. - Enhanced logging statements and function definitions to follow a more uniform style. - Updated imports in `types.rs` to maintain organization and improve accessibility module integration.
This commit is contained in:
Steven Enamakel
@@ -0,0 +1,144 @@
|
||||
mod log_bridge;
|
||||
|
||||
use log_bridge::{LogBuffer, LogEntry, TauriLogLayer};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use tauri::Manager;
|
||||
#[cfg(target_os = "macos")]
|
||||
use tauri::ActivationPolicy;
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
/// Tauri state holding the log ring buffer and click-through toggle.
|
||||
struct OverlayState {
|
||||
log_buffer: Arc<LogBuffer>,
|
||||
click_through: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
// ── Tauri commands ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Return all buffered log entries (for initial load / reconnect).
|
||||
#[tauri::command]
|
||||
fn get_log_history(state: tauri::State<'_, OverlayState>) -> Vec<LogEntry> {
|
||||
state.log_buffer.snapshot()
|
||||
}
|
||||
|
||||
/// Toggle click-through mode. When enabled, mouse events pass through
|
||||
/// the overlay to the window underneath.
|
||||
#[tauri::command]
|
||||
fn set_click_through(
|
||||
window: tauri::WebviewWindow,
|
||||
state: tauri::State<'_, OverlayState>,
|
||||
enabled: bool,
|
||||
) -> Result<(), String> {
|
||||
state.click_through.store(enabled, Ordering::Relaxed);
|
||||
window
|
||||
.set_ignore_cursor_events(enabled)
|
||||
.map_err(|e| e.to_string())?;
|
||||
log::debug!("[overlay] click-through set to {}", enabled);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forward an RPC call to openhuman_core's dispatch in-process.
|
||||
/// Uses the same invoke_method path as the HTTP JSON-RPC server.
|
||||
#[tauri::command]
|
||||
async fn core_rpc(method: String, params: serde_json::Value) -> Result<serde_json::Value, String> {
|
||||
log::debug!("[overlay] core_rpc: method={}", method);
|
||||
let state = openhuman_core::core::jsonrpc::default_state();
|
||||
openhuman_core::core::jsonrpc::invoke_method(state, &method, params).await
|
||||
}
|
||||
|
||||
/// Insert text into the currently focused field in the previously active app.
|
||||
#[tauri::command]
|
||||
fn insert_text_into_focused_field(text: String) -> Result<(), String> {
|
||||
log::debug!(
|
||||
"[overlay] insert_text_into_focused_field len={}",
|
||||
text.chars().count()
|
||||
);
|
||||
openhuman_core::openhuman::accessibility::apply_text_to_focused_field(&text)
|
||||
}
|
||||
|
||||
// ── App entry ───────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||
pub fn run() {
|
||||
// Shared state
|
||||
let log_buffer = Arc::new(LogBuffer::new(5000));
|
||||
let click_through = Arc::new(AtomicBool::new(false));
|
||||
|
||||
tauri::Builder::default()
|
||||
.plugin(tauri_plugin_opener::init())
|
||||
.manage(OverlayState {
|
||||
log_buffer: log_buffer.clone(),
|
||||
click_through: click_through.clone(),
|
||||
})
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
get_log_history,
|
||||
set_click_through,
|
||||
core_rpc,
|
||||
insert_text_into_focused_field,
|
||||
])
|
||||
.setup(move |app| {
|
||||
let app_handle = app.handle().clone();
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
app.set_activation_policy(ActivationPolicy::Accessory);
|
||||
log::debug!("[overlay] macOS: activation policy set to accessory");
|
||||
}
|
||||
|
||||
// ── Tracing subscriber with Tauri bridge layer ──────────────
|
||||
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| {
|
||||
EnvFilter::new(
|
||||
"debug,hyper=info,reqwest=info,tungstenite=info,tokio_tungstenite=info",
|
||||
)
|
||||
});
|
||||
|
||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||
.with_target(true)
|
||||
.with_ansi(true);
|
||||
|
||||
let tauri_layer = TauriLogLayer::new(app_handle.clone(), log_buffer.clone());
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(fmt_layer)
|
||||
.with(tauri_layer)
|
||||
.init();
|
||||
|
||||
// Bridge `log` crate macros into tracing
|
||||
tracing_log::LogTracer::init().ok();
|
||||
|
||||
log::info!("[overlay] overlay process started, tracing bridge active");
|
||||
|
||||
// ── Start openhuman_core JSON-RPC server in-process ─────────
|
||||
// Use port 7799 to avoid conflicts with a standalone core on 7788.
|
||||
// Override with OPENHUMAN_CORE_PORT env var.
|
||||
tauri::async_runtime::spawn(async move {
|
||||
let port = std::env::var("OPENHUMAN_CORE_PORT")
|
||||
.ok()
|
||||
.and_then(|p| p.parse::<u16>().ok())
|
||||
.unwrap_or(7799);
|
||||
log::info!("[overlay] starting openhuman_core server on 127.0.0.1:{}...", port);
|
||||
match openhuman_core::core::jsonrpc::run_server(None, Some(port), true).await {
|
||||
Ok(()) => log::info!("[overlay] core server shut down cleanly"),
|
||||
Err(e) => log::error!("[overlay] core server error: {}", e),
|
||||
}
|
||||
});
|
||||
|
||||
// ── macOS: floating panel + visible on all workspaces ───────
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
if let Some(window) = app.get_webview_window("overlay") {
|
||||
window.set_always_on_top(true).ok();
|
||||
window.set_visible_on_all_workspaces(true).ok();
|
||||
log::debug!("[overlay] macOS: set always-on-top + visible-on-all-workspaces");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.run(tauri::generate_context!())
|
||||
.expect("error while running overlay");
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
//! Captures `tracing` logs from openhuman_core and forwards them as Tauri events.
|
||||
//!
|
||||
//! Each log entry is emitted as a `core:log` event with a JSON payload:
|
||||
//! ```json
|
||||
//! { "ts": "2026-04-04T12:00:00Z", "level": "DEBUG", "module": "skills", "message": "..." }
|
||||
//! ```
|
||||
//! The frontend can filter by module to show logs from specific subsystems
|
||||
//! (skills, screen_recorder, autocomplete, rpc, etc.).
|
||||
|
||||
use chrono::Utc;
|
||||
use parking_lot::Mutex;
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
use tauri::{AppHandle, Emitter};
|
||||
use tracing::field::{Field, Visit};
|
||||
use tracing::span;
|
||||
use tracing_subscriber::layer::Context;
|
||||
use tracing_subscriber::Layer;
|
||||
|
||||
/// A single log entry forwarded to the overlay frontend.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct LogEntry {
|
||||
pub ts: String,
|
||||
pub level: String,
|
||||
pub module: String,
|
||||
pub target: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Ring buffer that keeps the last N log entries so the frontend can fetch
|
||||
/// history on connect without missing early startup logs.
|
||||
pub struct LogBuffer {
|
||||
entries: Mutex<Vec<LogEntry>>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl LogBuffer {
|
||||
pub fn new(capacity: usize) -> Self {
|
||||
Self {
|
||||
entries: Mutex::new(Vec::with_capacity(capacity)),
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push(&self, entry: LogEntry) {
|
||||
let mut entries = self.entries.lock();
|
||||
if entries.len() >= self.capacity {
|
||||
entries.remove(0);
|
||||
}
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> Vec<LogEntry> {
|
||||
self.entries.lock().clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// tracing Layer that captures events and sends them to the Tauri frontend.
|
||||
pub struct TauriLogLayer {
|
||||
app: AppHandle,
|
||||
buffer: Arc<LogBuffer>,
|
||||
}
|
||||
|
||||
impl TauriLogLayer {
|
||||
pub fn new(app: AppHandle, buffer: Arc<LogBuffer>) -> Self {
|
||||
Self { app, buffer }
|
||||
}
|
||||
}
|
||||
|
||||
/// Visitor that extracts the `message` field from tracing events.
|
||||
struct MessageVisitor {
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl MessageVisitor {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
message: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Visit for MessageVisitor {
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
|
||||
if field.name() == "message" {
|
||||
self.message = format!("{:?}", value);
|
||||
} else if self.message.is_empty() {
|
||||
// Fall back to first field if no explicit "message"
|
||||
self.message = format!("{}: {:?}", field.name(), value);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
if field.name() == "message" {
|
||||
self.message = value.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Derive a human-friendly module name from the tracing target.
|
||||
/// e.g. "openhuman::skills::qjs_engine" -> "skills"
|
||||
/// "openhuman::rpc" -> "rpc"
|
||||
/// "core_server::dispatch" -> "core_server"
|
||||
fn module_from_target(target: &str) -> String {
|
||||
let parts: Vec<&str> = target.split("::").collect();
|
||||
// Try to find the second segment under "openhuman::"
|
||||
if parts.len() >= 2 && parts[0] == "openhuman" {
|
||||
return parts[1].to_string();
|
||||
}
|
||||
if parts.len() >= 2 && parts[0] == "openhuman_core" {
|
||||
return parts[1].to_string();
|
||||
}
|
||||
// For other crates, use the first segment
|
||||
parts.first().unwrap_or(&"unknown").to_string()
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for TauriLogLayer
|
||||
where
|
||||
S: tracing::Subscriber + for<'lookup> tracing_subscriber::registry::LookupSpan<'lookup>,
|
||||
{
|
||||
fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
|
||||
let metadata = event.metadata();
|
||||
let level = metadata.level().to_string();
|
||||
let target = metadata.target().to_string();
|
||||
let module = module_from_target(&target);
|
||||
|
||||
let mut visitor = MessageVisitor::new();
|
||||
event.record(&mut visitor);
|
||||
|
||||
let entry = LogEntry {
|
||||
ts: Utc::now().to_rfc3339(),
|
||||
level,
|
||||
module,
|
||||
target,
|
||||
message: visitor.message,
|
||||
};
|
||||
|
||||
// Buffer for late-joining frontends
|
||||
self.buffer.push(entry.clone());
|
||||
|
||||
// Emit to all listening webviews — fire-and-forget
|
||||
let _ = self.app.emit("core:log", &entry);
|
||||
}
|
||||
|
||||
fn on_new_span(&self, _attrs: &span::Attributes<'_>, _id: &span::Id, _ctx: Context<'_, S>) {}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Prevents additional console window on Windows in release, DO NOT REMOVE!!
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||
|
||||
fn main() {
|
||||
overlay_lib::run()
|
||||
}
|
||||
Reference in New Issue
Block a user