mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-29 22:23:01 +00:00
270 lines
10 KiB
TypeScript
270 lines
10 KiB
TypeScript
import { useCallback } from 'react';
|
|
|
|
import {
|
|
type ScreenIntelligenceState,
|
|
useScreenIntelligenceState,
|
|
} from '../../features/screen-intelligence/useScreenIntelligenceState';
|
|
import { useT } from '../../lib/i18n/I18nContext';
|
|
|
|
const formatBytes = (bytes: number | null | undefined): string => {
|
|
if (bytes == null) return '-';
|
|
if (bytes < 1024) return `${bytes} B`;
|
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
|
|
};
|
|
|
|
interface ScreenIntelligenceDebugPanelProps {
|
|
state?: Pick<
|
|
ScreenIntelligenceState,
|
|
| 'status'
|
|
| 'captureTestResult'
|
|
| 'isCaptureTestRunning'
|
|
| 'recentVisionSummaries'
|
|
| 'lastError'
|
|
| 'refreshStatus'
|
|
| 'refreshVision'
|
|
| 'runCaptureTest'
|
|
>;
|
|
}
|
|
|
|
const ScreenIntelligenceDebugPanelContent = ({
|
|
state: providedState,
|
|
}: Required<Pick<ScreenIntelligenceDebugPanelProps, 'state'>>) => {
|
|
const { t } = useT();
|
|
const {
|
|
status,
|
|
captureTestResult,
|
|
isCaptureTestRunning,
|
|
recentVisionSummaries,
|
|
lastError,
|
|
refreshStatus,
|
|
refreshVision,
|
|
runCaptureTest,
|
|
} = providedState;
|
|
|
|
const handleCaptureTest = useCallback(() => {
|
|
void runCaptureTest();
|
|
}, [runCaptureTest]);
|
|
|
|
const handleRefreshStatus = useCallback(() => {
|
|
void refreshStatus();
|
|
void refreshVision(5);
|
|
}, [refreshStatus, refreshVision]);
|
|
|
|
const permissions = status?.permissions;
|
|
const session = status?.session;
|
|
|
|
return (
|
|
<div className="space-y-4">
|
|
<div className="flex items-center justify-between">
|
|
<h3 className="text-sm font-semibold text-stone-100">
|
|
{t('intelligence.screenDebug.title')}
|
|
</h3>
|
|
<button
|
|
onClick={handleRefreshStatus}
|
|
className="rounded-lg border border-stone-700 bg-stone-800/60 px-3 py-1 text-xs text-stone-300 dark:text-neutral-600 transition-colors hover:bg-stone-700/60">
|
|
{t('common.refresh')}
|
|
</button>
|
|
</div>
|
|
|
|
{/* Permissions */}
|
|
<div className="rounded-xl border border-stone-700 bg-stone-900/50 p-3">
|
|
<h4 className="mb-2 text-xs font-medium uppercase tracking-wide text-stone-400 dark:text-neutral-500">
|
|
{t('intelligence.screenDebug.permissions')}
|
|
</h4>
|
|
{status?.platform_supported === false ? (
|
|
<p className="text-xs text-stone-500 dark:text-neutral-400">
|
|
{t('intelligence.screenDebug.platformNotSupported')}
|
|
</p>
|
|
) : (
|
|
<div className="grid grid-cols-3 gap-2 text-xs">
|
|
<PermissionDot
|
|
label={t('intelligence.screenDebug.permScreen')}
|
|
value={permissions?.screen_recording}
|
|
/>
|
|
<PermissionDot
|
|
label={t('intelligence.screenDebug.permAccessibility')}
|
|
value={permissions?.accessibility}
|
|
/>
|
|
<PermissionDot
|
|
label={t('intelligence.screenDebug.permInput')}
|
|
value={permissions?.input_monitoring}
|
|
/>
|
|
</div>
|
|
)}
|
|
</div>
|
|
|
|
{/* Session Status */}
|
|
<div className="rounded-xl border border-stone-700 bg-stone-900/50 p-3">
|
|
<h4 className="mb-2 text-xs font-medium uppercase tracking-wide text-stone-400 dark:text-neutral-500">
|
|
{t('intelligence.screenDebug.session')}
|
|
</h4>
|
|
<div className="space-y-1 text-xs text-stone-300 dark:text-neutral-600">
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.active')}</span>
|
|
<span
|
|
className={
|
|
session?.active ? 'text-green-400' : 'text-stone-500 dark:text-neutral-400'
|
|
}>
|
|
{session?.active ? t('common.yes') : t('common.no')}
|
|
</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.frames')}</span>
|
|
<span>{session?.frames_in_memory ?? 0}</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.visionState')}</span>
|
|
<span>{session?.vision_state ?? t('intelligence.screenDebug.idle')}</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.visionQueue')}</span>
|
|
<span>{session?.vision_queue_depth ?? 0}</span>
|
|
</div>
|
|
{session?.last_context && (
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.lastApp')}</span>
|
|
<span className="max-w-[180px] truncate">{session.last_context}</span>
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
{/* Capture Test */}
|
|
<div className="rounded-xl border border-stone-700 bg-stone-900/50 p-3">
|
|
<h4 className="mb-2 text-xs font-medium uppercase tracking-wide text-stone-400 dark:text-neutral-500">
|
|
{t('intelligence.screenDebug.captureTest')}
|
|
</h4>
|
|
<button
|
|
onClick={handleCaptureTest}
|
|
disabled={isCaptureTestRunning}
|
|
className="mb-3 w-full rounded-lg border border-primary-600/40 bg-primary-600/20 px-3 py-2 text-sm font-medium text-primary-300 transition-colors hover:bg-primary-600/30 disabled:opacity-50">
|
|
{isCaptureTestRunning
|
|
? t('intelligence.screenDebug.capturing')
|
|
: t('intelligence.screenDebug.testCapture')}
|
|
</button>
|
|
|
|
{captureTestResult && (
|
|
<div className="space-y-2">
|
|
<div className="space-y-1 text-xs text-stone-300 dark:text-neutral-600">
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.status')}</span>
|
|
<span className={captureTestResult.ok ? 'text-green-400' : 'text-red-400'}>
|
|
{captureTestResult.ok
|
|
? t('intelligence.screenDebug.captureSuccess')
|
|
: t('intelligence.screenDebug.captureFailed')}
|
|
</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.mode')}</span>
|
|
<span>{captureTestResult.capture_mode}</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.time')}</span>
|
|
<span>{captureTestResult.timing_ms}ms</span>
|
|
</div>
|
|
{captureTestResult.bytes_estimate != null && (
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.size')}</span>
|
|
<span>{formatBytes(captureTestResult.bytes_estimate)}</span>
|
|
</div>
|
|
)}
|
|
{captureTestResult.context && (
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.app')}</span>
|
|
<span className="max-w-[180px] truncate">
|
|
{captureTestResult.context.app_name ?? t('intelligence.screenDebug.unknown')}
|
|
</span>
|
|
</div>
|
|
)}
|
|
{captureTestResult.context?.bounds_width != null && (
|
|
<div className="flex justify-between">
|
|
<span>{t('intelligence.screenDebug.bounds')}</span>
|
|
<span>
|
|
{captureTestResult.context.bounds_width}x
|
|
{captureTestResult.context.bounds_height} at (
|
|
{captureTestResult.context.bounds_x},{captureTestResult.context.bounds_y})
|
|
</span>
|
|
</div>
|
|
)}
|
|
</div>
|
|
|
|
{captureTestResult.error && (
|
|
<div className="rounded-lg border border-red-800/50 bg-red-900/20 p-2 text-xs text-red-300">
|
|
{captureTestResult.error}
|
|
</div>
|
|
)}
|
|
|
|
{captureTestResult.image_ref && (
|
|
<div className="overflow-hidden rounded-lg border border-stone-700">
|
|
<img
|
|
src={captureTestResult.image_ref}
|
|
alt={t('intelligence.screenDebug.captureAlt')}
|
|
className="w-full"
|
|
/>
|
|
</div>
|
|
)}
|
|
</div>
|
|
)}
|
|
</div>
|
|
|
|
{/* Recent Vision Summaries */}
|
|
{recentVisionSummaries.length > 0 && (
|
|
<div className="rounded-xl border border-stone-700 bg-stone-900/50 p-3">
|
|
<h4 className="mb-2 text-xs font-medium uppercase tracking-wide text-stone-400 dark:text-neutral-500">
|
|
{t('intelligence.screenDebug.recentVisionSummaries')}
|
|
</h4>
|
|
<div className="space-y-2">
|
|
{recentVisionSummaries.map(summary => (
|
|
<div
|
|
key={summary.id}
|
|
className="rounded-lg border border-stone-700/50 bg-stone-800/30 p-2 text-xs">
|
|
<div className="flex justify-between text-stone-400 dark:text-neutral-500">
|
|
<span>{summary.app_name ?? t('intelligence.screenDebug.unknown')}</span>
|
|
<span>
|
|
{new Date(summary.captured_at_ms).toLocaleTimeString()} ·{' '}
|
|
{(summary.confidence * 100).toFixed(0)}%
|
|
</span>
|
|
</div>
|
|
<div className="mt-1 text-stone-200">{summary.actionable_notes}</div>
|
|
</div>
|
|
))}
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{/* Error Display */}
|
|
{lastError && (
|
|
<div className="rounded-lg border border-red-800/50 bg-red-900/20 p-2 text-xs text-red-300">
|
|
{lastError}
|
|
</div>
|
|
)}
|
|
</div>
|
|
);
|
|
};
|
|
|
|
const OwnedScreenIntelligenceDebugPanel = () => {
|
|
const state = useScreenIntelligenceState({ loadVision: true, visionLimit: 5, pollMs: 2000 });
|
|
return <ScreenIntelligenceDebugPanelContent state={state} />;
|
|
};
|
|
|
|
const ScreenIntelligenceDebugPanel = ({ state }: ScreenIntelligenceDebugPanelProps) => {
|
|
if (state) {
|
|
return <ScreenIntelligenceDebugPanelContent state={state} />;
|
|
}
|
|
return <OwnedScreenIntelligenceDebugPanel />;
|
|
};
|
|
|
|
const PermissionDot = ({ label, value }: { label: string; value?: string }) => {
|
|
const color =
|
|
value === 'granted' ? 'bg-green-500' : value === 'denied' ? 'bg-red-500' : 'bg-stone-600';
|
|
return (
|
|
<div className="flex items-center gap-1.5">
|
|
<div className={`h-2 w-2 rounded-full ${color}`} />
|
|
<span className="text-stone-300 dark:text-neutral-600">{label}</span>
|
|
</div>
|
|
);
|
|
};
|
|
|
|
export default ScreenIntelligenceDebugPanel;
|