mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-30 23:14:37 +00:00
feat(mascot): sleep by default, wake on hover/click (#1493)
This commit is contained in:
@@ -148,6 +148,12 @@ Quick reference for anyone starting with Claude on this project. Updated by the
|
||||
- **Not a Tauri window** — The floating mascot is a native `NSPanel` + `WKWebView` in `app/src-tauri/src/mascot_native_window.rs`. It uses `ignoresMouseEvents=true` (click-through); interaction is detected by polling `NSEvent` via a Foundation timer. macOS-only, uses objc2 bindings.
|
||||
- **`MainThreadOnly` import must stay** — Required by `WKWebView::alloc()` and other AppKit allocators even if not explicitly referenced in user code. Removing it causes compile errors.
|
||||
- **`NSEvent::pressedMouseButtons` not in typed objc2-appkit bindings** — Must be called via `msg_send!(objc2::class!(NSEvent), pressedMouseButtons)` instead of the typed API.
|
||||
- **WKWebView IPC via `evaluateJavaScript`** — The mascot webview is NOT a Tauri runtime; Tauri `invoke`/`emit` do NOT work. Rust-to-mascot communication uses `msg_send!(webview, evaluateJavaScript:completionHandler:)` to dispatch `new CustomEvent(...)` on `window`. React listens with `addEventListener`. This is NOT subject to the CEF JS injection ban (that only applies to `webview_accounts/` third-party origins).
|
||||
- **`MascotCharacter` `sleeping` prop** — Drives the sleep animation (eye close + Zzz). `sleepStartSec` and `sleepFullSec` are hardcoded at 2.5s and 4.0s — they are NOT configurable props. Only toggle `sleeping: boolean`.
|
||||
- **`FACE_PRESETS` is a strict `Record`** — Typed as `Record<Exclude<MascotFace, 'normal'>, FacePreset>` in `Ghosty.tsx`. Adding a new `MascotFace` union variant requires adding a matching entry to `FACE_PRESETS` or it won't compile.
|
||||
- **`_webview` in `spawn_drag_timer`** — The `WKWebView` captured in the drag timer closure was originally unused (prefixed `_`). It can be used for `evaluateJavaScript` calls during the hover polling loop (e.g. to trigger blink/wake events from Rust).
|
||||
- **FrameProvider loops — sleep animation resets** — `FrameProvider` uses `frame % durationInFrames` so animations loop. Default `DURATION_FRAMES = FPS * 6` (6s). Sleep animation completes at 4s, then eyes re-open at 6s when frame resets to 0. Fix: use a much longer `durationInFrames` for sleep face (e.g. `FPS * 600`) so the loop never triggers while sleeping.
|
||||
- **Hover detection needs circular hitbox** — The mascot panel is 79x79 but the character is visually circular. Using the full AABB (`cursor_in_panel`) for hover triggers false positives when cursor is in a panel corner. Use distance-from-center check instead. Also suppress hover events for ~1s after panel shows to let the webview load.
|
||||
|
||||
## PR Checklist CI
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ const DRAG_POLL_SECONDS: f64 = 0.016;
|
||||
/// dropped webview.
|
||||
struct MascotPanel {
|
||||
panel: Retained<NSPanel>,
|
||||
_webview: Retained<WKWebView>,
|
||||
webview: Retained<WKWebView>,
|
||||
drag_timer: Retained<NSTimer>,
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ pub(crate) fn show(app: &AppHandle<AppRuntime>) -> Result<(), String> {
|
||||
MASCOT.with(|cell| {
|
||||
*cell.borrow_mut() = Some(MascotPanel {
|
||||
panel,
|
||||
_webview: webview,
|
||||
webview,
|
||||
drag_timer,
|
||||
});
|
||||
});
|
||||
@@ -275,12 +275,19 @@ unsafe fn build_panel(mtm: MainThreadMarker, frame: NSRect) -> Retained<NSPanel>
|
||||
/// but the visual drag starts on the next timer tick (~16 ms).
|
||||
unsafe fn spawn_drag_timer(
|
||||
panel: Retained<NSPanel>,
|
||||
_webview: Retained<WKWebView>,
|
||||
webview: Retained<WKWebView>,
|
||||
) -> Retained<NSTimer> {
|
||||
let dragging: Rc<Cell<bool>> = Rc::new(Cell::new(false));
|
||||
// Offset from cursor to panel origin at drag start so the panel
|
||||
// doesn't snap its corner to the cursor.
|
||||
let drag_offset: Rc<Cell<(f64, f64)>> = Rc::new(Cell::new((0.0, 0.0)));
|
||||
// Track previous hover state so we only dispatch when it changes.
|
||||
let was_hovering: Rc<Cell<bool>> = Rc::new(Cell::new(false));
|
||||
// Tick counter — suppress hover events for the first ~1s after panel
|
||||
// shows so the webview can load before we start dispatching JS.
|
||||
let tick_count: Rc<Cell<u32>> = Rc::new(Cell::new(0));
|
||||
// ~60 ticks = ~1 second at DRAG_POLL_SECONDS (0.016s).
|
||||
const HOVER_SUPPRESS_TICKS: u32 = 60;
|
||||
|
||||
let block = RcBlock::new(move |_timer: NonNull<NSTimer>| {
|
||||
let cursor = NSEvent::mouseLocation();
|
||||
@@ -314,6 +321,38 @@ unsafe fn spawn_drag_timer(
|
||||
drag_offset.set((cursor.x - frame.origin.x, cursor.y - frame.origin.y));
|
||||
log::debug!("[mascot-native] drag started");
|
||||
}
|
||||
|
||||
// Hover detection: use a circular hitbox (radius = half panel size)
|
||||
// instead of the full AABB so corners don't trigger false positives.
|
||||
// Suppress for the first ~1s so the webview can finish loading.
|
||||
tick_count.set(tick_count.get().saturating_add(1));
|
||||
let center_x = frame.origin.x + frame.size.width / 2.0;
|
||||
let center_y = frame.origin.y + frame.size.height / 2.0;
|
||||
let dx = cursor.x - center_x;
|
||||
let dy = cursor.y - center_y;
|
||||
let radius = frame.size.width / 2.0;
|
||||
let cursor_in_circle = (dx * dx + dy * dy) <= (radius * radius);
|
||||
let hovering_now = cursor_in_circle
|
||||
&& !left_down
|
||||
&& !dragging.get()
|
||||
&& tick_count.get() > HOVER_SUPPRESS_TICKS;
|
||||
if hovering_now != was_hovering.get() {
|
||||
was_hovering.set(hovering_now);
|
||||
let js_str = if hovering_now {
|
||||
"window.dispatchEvent(new CustomEvent('mascot:hover-state',{detail:{hovering:true}}))"
|
||||
} else {
|
||||
"window.dispatchEvent(new CustomEvent('mascot:hover-state',{detail:{hovering:false}}))"
|
||||
};
|
||||
log::debug!("[mascot-native] hover-state hovering={hovering_now}");
|
||||
let js = NSString::from_str(js_str);
|
||||
unsafe {
|
||||
let _: () = msg_send![
|
||||
&*webview,
|
||||
evaluateJavaScript: &*js,
|
||||
completionHandler: std::ptr::null::<objc2::runtime::AnyObject>()
|
||||
];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
unsafe {
|
||||
|
||||
@@ -5,6 +5,7 @@ import { Ghosty, type MascotFace } from './Ghosty';
|
||||
import { VISEMES } from './visemes';
|
||||
|
||||
const FACES: MascotFace[] = [
|
||||
'sleep',
|
||||
'idle',
|
||||
'normal',
|
||||
'listening',
|
||||
@@ -31,7 +32,7 @@ describe('Ghosty', () => {
|
||||
});
|
||||
|
||||
it('omits eyebrows for neutral / acknowledgement states', () => {
|
||||
for (const face of ['idle', 'normal', 'speaking', 'happy'] as MascotFace[]) {
|
||||
for (const face of ['sleep', 'idle', 'normal', 'speaking', 'happy'] as MascotFace[]) {
|
||||
const { container } = render(<Ghosty face={face} />);
|
||||
expect(container.querySelector('g[data-face-brows]')).toBeNull();
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import { visemePath, VISEMES, type VisemeShape } from './visemes';
|
||||
* compatibility with older callers.
|
||||
*/
|
||||
export type MascotFace =
|
||||
| 'sleep'
|
||||
| 'idle'
|
||||
| 'listening'
|
||||
| 'thinking'
|
||||
@@ -59,6 +60,14 @@ interface FacePreset {
|
||||
}
|
||||
|
||||
const FACE_PRESETS: Record<Exclude<MascotFace, 'normal'>, FacePreset> = {
|
||||
sleep: {
|
||||
eyeScaleY: 0.1,
|
||||
eyeScaleX: 1,
|
||||
browTilt: 0,
|
||||
browDy: 2,
|
||||
showBrows: false,
|
||||
blushOpacity: 0.5,
|
||||
},
|
||||
idle: {
|
||||
eyeScaleY: 1,
|
||||
eyeScaleX: 1,
|
||||
@@ -301,6 +310,9 @@ export const Ghosty: React.FC<GhostyProps> = ({
|
||||
*/
|
||||
function restMouthPath(face: MascotFace): string {
|
||||
switch (face) {
|
||||
case 'sleep':
|
||||
// Tiny flat line — sleeping, mouth barely visible.
|
||||
return 'M496,588 Q520,593 544,588 Q520,592 496,588 Z';
|
||||
case 'happy':
|
||||
// Wider grin.
|
||||
return 'M460,565 Q520,635 580,565 Q520,605 460,565 Z';
|
||||
|
||||
@@ -14,6 +14,7 @@ describe('<YellowMascot />', () => {
|
||||
|
||||
it.each([
|
||||
['idle', 'idle'],
|
||||
['sleep', 'sleep'],
|
||||
['speaking', 'speaking'],
|
||||
['thinking', 'thinking'],
|
||||
['confused', 'confused'],
|
||||
@@ -23,6 +24,13 @@ describe('<YellowMascot />', () => {
|
||||
expect(host.getAttribute('data-face')).toBe(expected);
|
||||
});
|
||||
|
||||
it('renders the sleep face with an svg', () => {
|
||||
const { container } = render(<YellowMascot face="sleep" />);
|
||||
const host = container.querySelector('[data-face="sleep"]') as HTMLElement;
|
||||
expect(host).not.toBeNull();
|
||||
expect(container.querySelector('svg')).not.toBeNull();
|
||||
});
|
||||
|
||||
it('forwards a numeric size prop as a pixel width', () => {
|
||||
const { container } = render(<YellowMascot size={48} />);
|
||||
const host = container.querySelector('[data-face]') as HTMLElement;
|
||||
|
||||
@@ -58,6 +58,11 @@ function variantForFace(
|
||||
mascotColor: extras.mascotColor ?? 'yellow',
|
||||
};
|
||||
switch (face) {
|
||||
case 'sleep':
|
||||
return {
|
||||
component: YellowMascotIdle,
|
||||
inputProps: { ...base, sleeping: true, arm: 'none', talking: false, thinking: false },
|
||||
};
|
||||
case 'thinking':
|
||||
case 'confused':
|
||||
return {
|
||||
@@ -117,7 +122,11 @@ export const YellowMascot: FC<YellowMascotProps> = ({
|
||||
<style>{`
|
||||
.mascot-yellow-host svg { width: 100% !important; height: 100% !important; }
|
||||
`}</style>
|
||||
<FrameProvider fps={FPS} width={CANVAS} height={CANVAS} durationInFrames={DURATION_FRAMES}>
|
||||
<FrameProvider
|
||||
fps={FPS}
|
||||
width={CANVAS}
|
||||
height={CANVAS}
|
||||
durationInFrames={face === 'sleep' ? FPS * 600 : DURATION_FRAMES}>
|
||||
<Component {...inputProps} />
|
||||
</FrameProvider>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { act, render } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import MascotWindowApp from './MascotWindowApp';
|
||||
|
||||
describe('MascotWindowApp', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('renders the sleep face by default', () => {
|
||||
const { container } = render(<MascotWindowApp />);
|
||||
const host = container.querySelector('[data-face]') as HTMLElement;
|
||||
expect(host).not.toBeNull();
|
||||
expect(host.getAttribute('data-face')).toBe('sleep');
|
||||
});
|
||||
|
||||
it('switches to idle face on mascot:hover-state with hovering=true', () => {
|
||||
const { container } = render(<MascotWindowApp />);
|
||||
act(() => {
|
||||
window.dispatchEvent(new CustomEvent('mascot:hover-state', { detail: { hovering: true } }));
|
||||
});
|
||||
const host = container.querySelector('[data-face]') as HTMLElement;
|
||||
expect(host.getAttribute('data-face')).toBe('idle');
|
||||
});
|
||||
|
||||
it('returns to sleep face after 2s when hovering=false', () => {
|
||||
const { container } = render(<MascotWindowApp />);
|
||||
|
||||
// First hover in to get to idle.
|
||||
act(() => {
|
||||
window.dispatchEvent(new CustomEvent('mascot:hover-state', { detail: { hovering: true } }));
|
||||
});
|
||||
expect((container.querySelector('[data-face]') as HTMLElement).getAttribute('data-face')).toBe(
|
||||
'idle'
|
||||
);
|
||||
|
||||
// Hover out — should still be idle before the delay elapses.
|
||||
act(() => {
|
||||
window.dispatchEvent(new CustomEvent('mascot:hover-state', { detail: { hovering: false } }));
|
||||
});
|
||||
expect((container.querySelector('[data-face]') as HTMLElement).getAttribute('data-face')).toBe(
|
||||
'idle'
|
||||
);
|
||||
|
||||
// Advance past the 2-second sleep delay.
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(2000);
|
||||
});
|
||||
expect((container.querySelector('[data-face]') as HTMLElement).getAttribute('data-face')).toBe(
|
||||
'sleep'
|
||||
);
|
||||
});
|
||||
|
||||
it('cancels the sleep timeout when hovering again before delay elapses', () => {
|
||||
const { container } = render(<MascotWindowApp />);
|
||||
|
||||
// Hover in → out → back in quickly.
|
||||
act(() => {
|
||||
window.dispatchEvent(new CustomEvent('mascot:hover-state', { detail: { hovering: true } }));
|
||||
});
|
||||
act(() => {
|
||||
window.dispatchEvent(new CustomEvent('mascot:hover-state', { detail: { hovering: false } }));
|
||||
});
|
||||
// Advance only 1s — still within the sleep delay.
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(1000);
|
||||
});
|
||||
// Hover back in — should cancel the pending sleep timer.
|
||||
act(() => {
|
||||
window.dispatchEvent(new CustomEvent('mascot:hover-state', { detail: { hovering: true } }));
|
||||
});
|
||||
// Advance past the original deadline — sleep timer should have been cancelled.
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(2000);
|
||||
});
|
||||
expect((container.querySelector('[data-face]') as HTMLElement).getAttribute('data-face')).toBe(
|
||||
'idle'
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,7 @@
|
||||
import { type MascotFace, YellowMascot } from '../features/human/Mascot';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { YellowMascot } from '../features/human/Mascot';
|
||||
import type { MascotFace } from '../features/human/Mascot/Ghosty';
|
||||
|
||||
/**
|
||||
* Hosted inside a native macOS NSPanel + WKWebView (see
|
||||
@@ -7,18 +10,49 @@ import { type MascotFace, YellowMascot } from '../features/human/Mascot';
|
||||
* - No `@tauri-apps/api/*` calls work here.
|
||||
* - The panel is `ignoresMouseEvents=true` so the cursor passes straight
|
||||
* through. When the Rust host sees the cursor enter the panel frame it
|
||||
* animates the whole NSPanel to the other right-edge corner, so the
|
||||
* mascot bounces out of the way without going off-screen.
|
||||
* dispatches a `mascot:hover-state` CustomEvent with `detail.hovering`.
|
||||
* - Default state is `sleep` (closed eyes). On hover the face switches to
|
||||
* `idle`. After the cursor leaves, a 2-second timer returns to `sleep`.
|
||||
* - Show/hide is driven from the tray menu in the main app.
|
||||
*
|
||||
* [ui-flow] mascot-window: sleep → idle (hover-start) → sleep (hover-end +2s)
|
||||
*/
|
||||
const DEFAULT_FACE: MascotFace = 'idle';
|
||||
|
||||
const SLEEP_DELAY_MS = 2000;
|
||||
|
||||
const MascotWindowApp = () => {
|
||||
const [face, setFace] = useState<MascotFace>('sleep');
|
||||
const timeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const handler = (e: Event) => {
|
||||
const { hovering } = (e as CustomEvent<{ hovering: boolean }>).detail;
|
||||
if (hovering) {
|
||||
if (timeoutRef.current) {
|
||||
clearTimeout(timeoutRef.current);
|
||||
timeoutRef.current = null;
|
||||
}
|
||||
setFace('idle');
|
||||
// [ui-flow] mascot-window: hover-start → face=idle
|
||||
} else {
|
||||
timeoutRef.current = setTimeout(() => {
|
||||
setFace('sleep');
|
||||
timeoutRef.current = null;
|
||||
// [ui-flow] mascot-window: sleep-delay elapsed → face=sleep
|
||||
}, SLEEP_DELAY_MS);
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener('mascot:hover-state', handler);
|
||||
return () => {
|
||||
window.removeEventListener('mascot:hover-state', handler);
|
||||
if (timeoutRef.current) clearTimeout(timeoutRef.current);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{ position: 'fixed', inset: 0, background: 'transparent' }}
|
||||
data-face={DEFAULT_FACE}>
|
||||
<YellowMascot face={DEFAULT_FACE} groundShadowOpacity={0.75} compactArmShading />
|
||||
<div style={{ position: 'fixed', inset: 0, background: 'transparent' }} data-face={face}>
|
||||
<YellowMascot face={face} groundShadowOpacity={0.75} compactArmShading />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user