From fdd22198bb1c77cd2224f6effcad47730c84155d Mon Sep 17 00:00:00 2001 From: Sathvik Gilakamsetty Date: Thu, 28 May 2026 15:07:17 +0000 Subject: [PATCH] feat(dashboard): per-model health comparison table (#2823) Co-authored-by: sanil-23 --- app/src-tauri/Cargo.lock | 1 + .../settings/panels/DeveloperOptionsPanel.tsx | 16 + .../settings/panels/ModelHealthPanel.tsx | 379 ++++++++++++++++++ .../__tests__/ModelHealthPanel.test.tsx | 235 +++++++++++ app/src/lib/i18n/chunks/ar-5.ts | 24 ++ app/src/lib/i18n/chunks/bn-5.ts | 24 ++ app/src/lib/i18n/chunks/de-5.ts | 24 ++ app/src/lib/i18n/chunks/en-5.ts | 24 ++ app/src/lib/i18n/chunks/es-5.ts | 24 ++ app/src/lib/i18n/chunks/fr-5.ts | 24 ++ app/src/lib/i18n/chunks/hi-5.ts | 24 ++ app/src/lib/i18n/chunks/id-5.ts | 24 ++ app/src/lib/i18n/chunks/it-5.ts | 24 ++ app/src/lib/i18n/chunks/ko-5.ts | 24 ++ app/src/lib/i18n/chunks/pt-5.ts | 24 ++ app/src/lib/i18n/chunks/ru-5.ts | 24 ++ app/src/lib/i18n/chunks/zh-CN-5.ts | 24 ++ app/src/lib/i18n/en.ts | 24 ++ app/src/pages/Settings.tsx | 5 + src/core/all.rs | 5 + src/openhuman/config/schema/dashboard.rs | 87 ++++ src/openhuman/config/schema/mod.rs | 2 + src/openhuman/config/schema/types.rs | 18 + src/openhuman/dashboard/mod.rs | 20 + src/openhuman/dashboard/ops.rs | 132 ++++++ src/openhuman/dashboard/schemas.rs | 177 ++++++++ src/openhuman/dashboard/types.rs | 37 ++ src/openhuman/mod.rs | 1 + 28 files changed, 1451 insertions(+) create mode 100644 app/src/components/settings/panels/ModelHealthPanel.tsx create mode 100644 app/src/components/settings/panels/__tests__/ModelHealthPanel.test.tsx create mode 100644 src/openhuman/config/schema/dashboard.rs create mode 100644 src/openhuman/dashboard/mod.rs create mode 100644 src/openhuman/dashboard/ops.rs create mode 100644 src/openhuman/dashboard/schemas.rs create mode 100644 src/openhuman/dashboard/types.rs diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 7e6f73e10..05750db2c 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -5306,6 +5306,7 @@ dependencies = [ "tracing-appender", "tracing-log", "tracing-subscriber", + "unicode-normalization", "unicode-segmentation", "unicode-width", "url", diff --git a/app/src/components/settings/panels/DeveloperOptionsPanel.tsx b/app/src/components/settings/panels/DeveloperOptionsPanel.tsx index 7ae8fb03d..faf72fd65 100644 --- a/app/src/components/settings/panels/DeveloperOptionsPanel.tsx +++ b/app/src/components/settings/panels/DeveloperOptionsPanel.tsx @@ -189,6 +189,22 @@ const developerItems = [ ), }, + { + id: 'model-health', + titleKey: 'settings.modelHealth.title', + descriptionKey: 'settings.modelHealth.desc', + route: 'model-health', + icon: ( + + + + ), + }, ]; const CoreModeBadge = () => { diff --git a/app/src/components/settings/panels/ModelHealthPanel.tsx b/app/src/components/settings/panels/ModelHealthPanel.tsx new file mode 100644 index 000000000..0890099ae --- /dev/null +++ b/app/src/components/settings/panels/ModelHealthPanel.tsx @@ -0,0 +1,379 @@ +import debug from 'debug'; +import { useEffect, useRef, useState } from 'react'; + +import { useT } from '../../../lib/i18n/I18nContext'; +import { callCoreRpc } from '../../../services/coreRpcClient'; +import SettingsHeader from '../components/SettingsHeader'; +import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; + +const log = debug('openhuman:model-health'); + +interface ModelEntry { + id: string; + provider: string; + cost_per_1m_output: number; + vision: boolean; + quality_score: number | null; + hallucination_rate: number | null; + agents_using: number; + tasks_evaluated: number; +} + +interface HealthConfig { + hallucination_threshold: number; + min_tasks_for_rating: number; + evaluation_window_tasks: number; +} + +interface ModelHealthRpcResponse { + models: ModelEntry[]; + config: HealthConfig; +} + +interface RpcOutcomeEnvelope { + result: T; + logs: string[]; +} + +type RpcModelHealthPayload = ModelHealthRpcResponse | RpcOutcomeEnvelope; + +function unwrapPayload(payload: RpcModelHealthPayload): ModelHealthRpcResponse { + if (payload && typeof payload === 'object' && 'result' in payload && 'logs' in payload) { + return payload.result; + } + return payload as ModelHealthRpcResponse; +} + +type SortCol = + | 'id' + | 'quality_score' + | 'hallucination_rate' + | 'cost_per_1m_output' + | 'agents_using'; +type StatusBadge = 'keep' | 'replace' | 'staging' | 'vision'; + +function getStatus(m: ModelEntry, cfg: HealthConfig): StatusBadge { + if (m.vision) return 'vision'; + if (m.tasks_evaluated < cfg.min_tasks_for_rating) return 'staging'; + if (m.hallucination_rate !== null && m.hallucination_rate > cfg.hallucination_threshold) + return 'replace'; + return 'keep'; +} + +function qualityStars(score: number | null): string { + if (score === null) return '—'; + const full = Math.round(score); + return '★'.repeat(full) + '☆'.repeat(5 - full); +} + +const BADGE_STYLES: Record = { + keep: { bg: 'bg-green-500/20', text: 'text-green-400', label: 'settings.modelHealth.badge.keep' }, + replace: { + bg: 'bg-red-500/20', + text: 'text-red-400', + label: 'settings.modelHealth.badge.replace', + }, + staging: { + bg: 'bg-amber-500/20', + text: 'text-amber-400', + label: 'settings.modelHealth.badge.staging', + }, + vision: { + bg: 'bg-blue-500/20', + text: 'text-blue-400', + label: 'settings.modelHealth.badge.vision', + }, +}; + +const ModelHealthPanel = () => { + const { t } = useT(); + const { navigateBack, breadcrumbs } = useSettingsNavigation(); + const [models, setModels] = useState([]); + const [config, setConfig] = useState({ + hallucination_threshold: 0.1, + min_tasks_for_rating: 10, + evaluation_window_tasks: 50, + }); + const [sortCol, setSortCol] = useState('id'); + const [sortAsc, setSortAsc] = useState(true); + const [filterStatus, setFilterStatus] = useState(''); + const [swapTarget, setSwapTarget] = useState(null); + const [selectedCandidate, setSelectedCandidate] = useState(null); + const [loading, setLoading] = useState(true); + const fetchedRef = useRef(false); + + useEffect(() => { + if (fetchedRef.current) return; + fetchedRef.current = true; + (async () => { + log('[model-health] fetch start'); + try { + const payload = await callCoreRpc({ + method: 'openhuman.dashboard_model_health', + }); + const data = unwrapPayload(payload); + setModels(data.models || []); + if (data.config) setConfig(data.config); + log('[model-health] fetch ok models=%d', data.models?.length ?? 0); + } catch (err) { + log('[model-health] fetch failed: %o', err); + } + setLoading(false); + })(); + }, []); + + const handleSort = (col: SortCol) => { + if (sortCol === col) { + setSortAsc(!sortAsc); + } else { + setSortCol(col); + setSortAsc(true); + } + }; + + const sorted = [...models].sort((a, b) => { + const av = a[sortCol] ?? -1; + const bv = b[sortCol] ?? -1; + if (av < bv) return sortAsc ? -1 : 1; + if (av > bv) return sortAsc ? 1 : -1; + return 0; + }); + + const filtered = sorted.filter(m => { + if (!filterStatus) return true; + return getStatus(m, config) === filterStatus; + }); + + const replaceCandidates = (target: ModelEntry) => + models.filter( + c => + c.id !== target.id && + !c.vision && + (c.hallucination_rate ?? 1) < (target.hallucination_rate ?? 1) && + c.cost_per_1m_output <= target.cost_per_1m_output + ); + + const betterCandidates = (target: ModelEntry) => + models.filter( + c => + c.id !== target.id && + !c.vision && + (c.hallucination_rate ?? 1) < (target.hallucination_rate ?? 1) && + c.cost_per_1m_output > target.cost_per_1m_output + ); + + const sortIcon = (col: SortCol) => (sortCol === col ? (sortAsc ? ' ↑' : ' ↓') : ''); + + return ( +
+ +
+
+ + + {filtered.length} {t('settings.modelHealth.models')} + +
+ + {loading ? ( +

+ {t('settings.modelHealth.loading')} +

+ ) : filtered.length === 0 ? ( +

+ {t('settings.modelHealth.empty')} +

+ ) : ( +
+ + + + + + + + + + + + + {filtered.map(m => { + const status = getStatus(m, config); + const badge = BADGE_STYLES[status]; + const isReplace = status === 'replace'; + const candidates = isReplace + ? [...replaceCandidates(m), ...betterCandidates(m)] + : []; + return ( + + + + + + + + + ); + })} + +
handleSort('id')}> + {t('settings.modelHealth.col.model')} + {sortIcon('id')} + handleSort('quality_score')}> + {t('settings.modelHealth.col.quality')} + {sortIcon('quality_score')} + handleSort('hallucination_rate')}> + {t('settings.modelHealth.col.halluc')} + {sortIcon('hallucination_rate')} + handleSort('cost_per_1m_output')}> + {t('settings.modelHealth.col.cost')} + {sortIcon('cost_per_1m_output')} + handleSort('agents_using')}> + {t('settings.modelHealth.col.agents')} + {sortIcon('agents_using')} + {t('settings.modelHealth.col.status')}
+
+ {m.id} +
+
{m.provider}
+ 		{qualityStars(m.quality_score)} + {m.hallucination_rate !== null ? ( + config.hallucination_threshold + ? 'text-red-400' + : 'text-green-400' + }> + {(m.hallucination_rate * 100).toFixed(1)}% + + ) : ( + '—' + )} + ${m.cost_per_1m_output.toFixed(2)}{m.agents_using} + + {t(badge.label)} + + {isReplace && candidates.length > 0 && ( + + )} +
+
+ )} +
+ + {/* Swap Modal */} + {swapTarget && ( +
{ + setSwapTarget(null); + setSelectedCandidate(null); + }}> +
e.stopPropagation()}> +

{t('settings.modelHealth.modal.title')}

+

+ {swapTarget.id} — {t('settings.modelHealth.modal.hallucRate')}:{' '} + {((swapTarget.hallucination_rate ?? 0) * 100).toFixed(1)}% +

+
+ {[...replaceCandidates(swapTarget), ...betterCandidates(swapTarget)].map(c => { + const isSelected = selectedCandidate?.id === c.id; + return ( + + ); + })} +
+
+ + +
+
+
+ )} +
+ ); +}; + +export default ModelHealthPanel; diff --git a/app/src/components/settings/panels/__tests__/ModelHealthPanel.test.tsx b/app/src/components/settings/panels/__tests__/ModelHealthPanel.test.tsx new file mode 100644 index 000000000..83cfb16c1 --- /dev/null +++ b/app/src/components/settings/panels/__tests__/ModelHealthPanel.test.tsx @@ -0,0 +1,235 @@ +import { fireEvent, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { renderWithProviders } from '../../../../test/test-utils'; +import ModelHealthPanel from '../ModelHealthPanel'; + +vi.mock('../../../../services/coreRpcClient', () => ({ callCoreRpc: vi.fn() })); +vi.mock('../../hooks/useSettingsNavigation', () => ({ + useSettingsNavigation: () => ({ navigateBack: vi.fn(), breadcrumbs: [] }), +})); +vi.mock('../../../../lib/i18n/I18nContext', () => ({ useT: () => ({ t: (k: string) => k }) })); + +const MOCK_RESPONSE = { + models: [ + { + id: 'deepseek-v3', + provider: 'SiliconFlow', + cost_per_1m_output: 0.33, + vision: false, + quality_score: 4, + hallucination_rate: 0.03, + agents_using: 5, + tasks_evaluated: 60, + }, + { + id: 'qwen-2.5-8b', + provider: 'OpenRouter', + cost_per_1m_output: 0.09, + vision: true, + quality_score: 3, + hallucination_rate: 0.04, + agents_using: 1, + tasks_evaluated: 20, + }, + { + id: 'bad-model', + provider: 'Test', + cost_per_1m_output: 1.0, + vision: false, + quality_score: 2, + hallucination_rate: 0.18, + agents_using: 2, + tasks_evaluated: 50, + }, + ], + config: { hallucination_threshold: 0.1, min_tasks_for_rating: 10, evaluation_window_tasks: 50 }, +}; + +async function mockRpc(response: unknown) { + const { callCoreRpc } = await import('../../../../services/coreRpcClient'); + vi.mocked(callCoreRpc).mockResolvedValueOnce(response); +} + +async function mockRpcReject(err: unknown) { + const { callCoreRpc } = await import('../../../../services/coreRpcClient'); + vi.mocked(callCoreRpc).mockRejectedValueOnce(err); +} + +describe('ModelHealthPanel', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('renders panel with table', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('deepseek-v3')).toBeTruthy(); + }); + expect(screen.getByText('qwen-2.5-8b')).toBeTruthy(); + expect(screen.getByText('bad-model')).toBeTruthy(); + }); + + it('shows correct status badges', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('deepseek-v3')).toBeTruthy(); + }); + expect(screen.getAllByText('settings.modelHealth.badge.keep').length).toBeGreaterThan(0); + expect(screen.getAllByText('settings.modelHealth.badge.vision').length).toBeGreaterThan(0); + expect(screen.getAllByText('settings.modelHealth.badge.replace').length).toBeGreaterThan(0); + }); + + it('filters by status', async () => { + await mockRpc(MOCK_RESPONSE); + const { container } = renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('deepseek-v3')).toBeTruthy(); + }); + const select = container.querySelector('select')!; + fireEvent.change(select, { target: { value: 'vision' } }); + await waitFor(() => { + expect(screen.queryByText('deepseek-v3')).toBeNull(); + }); + expect(screen.getByText('qwen-2.5-8b')).toBeTruthy(); + }); + + it('sorts by column', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('deepseek-v3')).toBeTruthy(); + }); + fireEvent.click(screen.getByText('settings.modelHealth.col.cost')); + // After sorting by cost asc, first row should be cheapest model (qwen at $0.09) + const rows = screen.getAllByRole('row'); + expect(rows[1].textContent).toContain('qwen-2.5-8b'); + }); + + it('shows swap button for replace-flagged models', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy(); + }); + }); + + it('opens swap modal on click', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy(); + }); + fireEvent.click(screen.getByText('settings.modelHealth.swap')); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.modal.title')).toBeTruthy(); + }); + }); + + it('shows empty state when no models', async () => { + await mockRpc({ models: [], config: MOCK_RESPONSE.config }); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.empty')).toBeTruthy(); + }); + }); + + it('shows loading then content', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByTestId('model-health-panel')).toBeTruthy(); + }); + }); + + it('unwraps RpcOutcome {result, logs} envelope', async () => { + await mockRpc({ result: MOCK_RESPONSE, logs: ['dashboard.model_health returned 3 models'] }); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('deepseek-v3')).toBeTruthy(); + }); + expect(screen.getByText('qwen-2.5-8b')).toBeTruthy(); + }); + + it('handles rpc failure gracefully', async () => { + await mockRpcReject(new Error('config unavailable')); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.empty')).toBeTruthy(); + }); + }); + + it('toggles sort direction on same column click', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('deepseek-v3')).toBeTruthy(); + }); + const costHeader = screen.getByText('settings.modelHealth.col.cost'); + fireEvent.click(costHeader); // asc + fireEvent.click(costHeader); // desc — toggles direction + const rows = screen.getAllByRole('row'); + expect(rows[1].textContent).toContain('bad-model'); // most expensive first + }); + + it('modal shows candidates and closes on backdrop click', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy(); + }); + fireEvent.click(screen.getByText('settings.modelHealth.swap')); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.modal.title')).toBeTruthy(); + }); + // Verify candidates are shown + expect(screen.getByText('settings.modelHealth.modal.apply')).toBeTruthy(); + expect(screen.getByText('settings.modelHealth.modal.cancel')).toBeTruthy(); + // Close via cancel + fireEvent.click(screen.getByText('settings.modelHealth.modal.cancel')); + await waitFor(() => { + expect(screen.queryByText('settings.modelHealth.modal.title')).toBeNull(); + }); + }); + + it('closes modal on cancel', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy(); + }); + fireEvent.click(screen.getByText('settings.modelHealth.swap')); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.modal.title')).toBeTruthy(); + }); + fireEvent.click(screen.getByText('settings.modelHealth.modal.cancel')); + await waitFor(() => { + expect(screen.queryByText('settings.modelHealth.modal.title')).toBeNull(); + }); + }); + + it('Apply button is disabled until a candidate is selected', async () => { + await mockRpc(MOCK_RESPONSE); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy(); + }); + fireEvent.click(screen.getByText('settings.modelHealth.swap')); + const applyButton = await screen.findByText('settings.modelHealth.modal.apply'); + expect((applyButton as HTMLButtonElement).disabled).toBe(true); + + // Select a candidate then re-check. + const candidates = screen.getAllByRole('radio'); + expect(candidates.length).toBeGreaterThan(0); + fireEvent.click(candidates[0]); + expect((applyButton as HTMLButtonElement).disabled).toBe(false); + + // Clicking Apply closes the modal (UI-side; backend swap is a follow-up). + fireEvent.click(applyButton); + await waitFor(() => { + expect(screen.queryByText('settings.modelHealth.modal.title')).toBeNull(); + }); + }); +}); diff --git a/app/src/lib/i18n/chunks/ar-5.ts b/app/src/lib/i18n/chunks/ar-5.ts index 88d818dd1..89a8d8c16 100644 --- a/app/src/lib/i18n/chunks/ar-5.ts +++ b/app/src/lib/i18n/chunks/ar-5.ts @@ -787,6 +787,30 @@ const ar5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default ar5; diff --git a/app/src/lib/i18n/chunks/bn-5.ts b/app/src/lib/i18n/chunks/bn-5.ts index 2bc1e0d8f..01f88ad64 100644 --- a/app/src/lib/i18n/chunks/bn-5.ts +++ b/app/src/lib/i18n/chunks/bn-5.ts @@ -800,6 +800,30 @@ const bn5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default bn5; diff --git a/app/src/lib/i18n/chunks/de-5.ts b/app/src/lib/i18n/chunks/de-5.ts index 79d8c434a..782dec3e9 100644 --- a/app/src/lib/i18n/chunks/de-5.ts +++ b/app/src/lib/i18n/chunks/de-5.ts @@ -828,6 +828,30 @@ const de5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default de5; diff --git a/app/src/lib/i18n/chunks/en-5.ts b/app/src/lib/i18n/chunks/en-5.ts index 3078c6852..c3f7bbb90 100644 --- a/app/src/lib/i18n/chunks/en-5.ts +++ b/app/src/lib/i18n/chunks/en-5.ts @@ -795,6 +795,30 @@ const en5: TranslationMap = { 'skills.meetingBots.platforms.zoom': 'Zoom', 'skills.meetingBots.soonSuffix': 'soon', 'skills.setup.screenIntel.permissionPathLabel': 'macOS applies privacy to:', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default en5; diff --git a/app/src/lib/i18n/chunks/es-5.ts b/app/src/lib/i18n/chunks/es-5.ts index c239742c6..445357f9c 100644 --- a/app/src/lib/i18n/chunks/es-5.ts +++ b/app/src/lib/i18n/chunks/es-5.ts @@ -814,6 +814,30 @@ const es5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default es5; diff --git a/app/src/lib/i18n/chunks/fr-5.ts b/app/src/lib/i18n/chunks/fr-5.ts index 32c3a086f..4a225ec30 100644 --- a/app/src/lib/i18n/chunks/fr-5.ts +++ b/app/src/lib/i18n/chunks/fr-5.ts @@ -818,6 +818,30 @@ const fr5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default fr5; diff --git a/app/src/lib/i18n/chunks/hi-5.ts b/app/src/lib/i18n/chunks/hi-5.ts index acdb59fcd..88d1d548e 100644 --- a/app/src/lib/i18n/chunks/hi-5.ts +++ b/app/src/lib/i18n/chunks/hi-5.ts @@ -801,6 +801,30 @@ const hi5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default hi5; diff --git a/app/src/lib/i18n/chunks/id-5.ts b/app/src/lib/i18n/chunks/id-5.ts index 525c97dde..9d3214cf1 100644 --- a/app/src/lib/i18n/chunks/id-5.ts +++ b/app/src/lib/i18n/chunks/id-5.ts @@ -801,6 +801,30 @@ const id5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default id5; diff --git a/app/src/lib/i18n/chunks/it-5.ts b/app/src/lib/i18n/chunks/it-5.ts index e1f960ef1..ada387796 100644 --- a/app/src/lib/i18n/chunks/it-5.ts +++ b/app/src/lib/i18n/chunks/it-5.ts @@ -812,6 +812,30 @@ const it5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default it5; diff --git a/app/src/lib/i18n/chunks/ko-5.ts b/app/src/lib/i18n/chunks/ko-5.ts index c6bca4826..54652c7ea 100644 --- a/app/src/lib/i18n/chunks/ko-5.ts +++ b/app/src/lib/i18n/chunks/ko-5.ts @@ -790,6 +790,30 @@ const ko5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default ko5; diff --git a/app/src/lib/i18n/chunks/pt-5.ts b/app/src/lib/i18n/chunks/pt-5.ts index 37509c707..ed78b21b2 100644 --- a/app/src/lib/i18n/chunks/pt-5.ts +++ b/app/src/lib/i18n/chunks/pt-5.ts @@ -811,6 +811,30 @@ const pt5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default pt5; diff --git a/app/src/lib/i18n/chunks/ru-5.ts b/app/src/lib/i18n/chunks/ru-5.ts index 325a370f2..abdb43faf 100644 --- a/app/src/lib/i18n/chunks/ru-5.ts +++ b/app/src/lib/i18n/chunks/ru-5.ts @@ -807,6 +807,30 @@ const ru5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default ru5; diff --git a/app/src/lib/i18n/chunks/zh-CN-5.ts b/app/src/lib/i18n/chunks/zh-CN-5.ts index 3c305758e..e5152906c 100644 --- a/app/src/lib/i18n/chunks/zh-CN-5.ts +++ b/app/src/lib/i18n/chunks/zh-CN-5.ts @@ -761,6 +761,30 @@ const zhCN5: TranslationMap = { 'settings.agentAccess.add': 'Add', 'settings.agentAccess.saving': 'Saving…', 'settings.agentAccess.changesApply': 'Changes apply on your next message.', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', }; export default zhCN5; diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 606109a0f..42732b7d3 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -3372,6 +3372,30 @@ const en: TranslationMap = { 'Smallest memory window. Cheapest, fastest, least continuity between runs.', 'settings.memoryWindow.minimal.label': 'Minimal', 'settings.memoryWindow.title': 'Long-term memory window', + 'settings.modelHealth.title': 'Model Health', + 'settings.modelHealth.desc': + 'Per-model quality, hallucination rate, and cost comparison across active models', + 'settings.modelHealth.allStatuses': 'All statuses', + 'settings.modelHealth.models': 'models', + 'settings.modelHealth.loading': 'Loading model data...', + 'settings.modelHealth.empty': 'No models registered', + 'settings.modelHealth.col.model': 'Model', + 'settings.modelHealth.col.quality': 'Quality', + 'settings.modelHealth.col.halluc': 'Halluc. Rate', + 'settings.modelHealth.col.cost': 'Cost / 1M out', + 'settings.modelHealth.col.agents': 'Agents', + 'settings.modelHealth.col.status': 'Status', + 'settings.modelHealth.badge.keep': 'Keep', + 'settings.modelHealth.badge.replace': 'Replace', + 'settings.modelHealth.badge.staging': 'Staging test', + 'settings.modelHealth.badge.vision': 'Vision only', + 'settings.modelHealth.swap': 'Swap?', + 'settings.modelHealth.modal.title': 'Replace Model?', + 'settings.modelHealth.modal.hallucRate': 'Hallucination rate', + 'settings.modelHealth.modal.cancel': 'Cancel', + 'settings.modelHealth.modal.apply': 'Apply Replacement', + 'settings.modelHealth.tag.cheaper': 'CHEAPER', + 'settings.modelHealth.tag.better': 'BETTER', 'settings.screenIntel.permissions.accessibility': 'Accessibility', 'settings.screenIntel.permissions.grantHint': 'Grant these permissions in System Settings, then restart the core.', diff --git a/app/src/pages/Settings.tsx b/app/src/pages/Settings.tsx index f9c9ff6a0..4f7af1492 100644 --- a/app/src/pages/Settings.tsx +++ b/app/src/pages/Settings.tsx @@ -28,6 +28,7 @@ import McpServerPanel from '../components/settings/panels/McpServerPanel'; import MemoryDataPanel from '../components/settings/panels/MemoryDataPanel'; import MemoryDebugPanel from '../components/settings/panels/MemoryDebugPanel'; import MigrationPanel from '../components/settings/panels/MigrationPanel'; +import ModelHealthPanel from '../components/settings/panels/ModelHealthPanel'; import NotificationsTabbedPanel from '../components/settings/panels/NotificationsTabbedPanel'; import PersonaPanel from '../components/settings/panels/PersonaPanel'; import PrivacyPanel from '../components/settings/panels/PrivacyPanel'; @@ -461,6 +462,10 @@ const Settings = () => { )} /> )} /> )} /> + , { maxWidthClass: 'max-w-4xl' })} + /> )} /> )} /> } /> diff --git a/src/core/all.rs b/src/core/all.rs index c43faa68e..9d5d4c737 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -114,6 +114,7 @@ fn build_registered_controllers() -> Vec { controllers.extend(crate::openhuman::composio::all_composio_registered_controllers()); // Scheduled job management controllers.extend(crate::openhuman::cron::all_cron_registered_controllers()); + controllers.extend(crate::openhuman::dashboard::all_dashboard_registered_controllers()); // MCP client subsystem: Smithery registry browser, local server install/connect, tool dispatch controllers.extend(crate::openhuman::mcp_registry::all_mcp_registry_registered_controllers()); // Webview APIs bridge — proxies connector calls (Gmail, …) through @@ -285,6 +286,7 @@ fn build_declared_controller_schemas() -> Vec { schemas.extend(crate::openhuman::audio_toolkit::all_audio_toolkit_controller_schemas()); schemas.extend(crate::openhuman::composio::all_composio_controller_schemas()); schemas.extend(crate::openhuman::cron::all_cron_controller_schemas()); + schemas.extend(crate::openhuman::dashboard::all_dashboard_controller_schemas()); schemas.extend(crate::openhuman::mcp_registry::all_mcp_registry_controller_schemas()); schemas.extend(crate::openhuman::webview_apis::all_webview_apis_controller_schemas()); schemas.extend(crate::openhuman::agent::all_agent_controller_schemas()); @@ -402,6 +404,9 @@ pub fn namespace_description(namespace: &str) -> Option<&'static str> { "Connectivity diagnostics for the local sidecar, listening port, and backend Socket.IO state.", ), "cron" => Some("Manage scheduled jobs and run history."), + "dashboard" => Some( + "Operator-facing dashboard aggregations: per-model health comparison rows.", + ), "mcp_clients" => Some( "Browse the Smithery.ai MCP registry, install MCP servers locally, manage their stdio connections, and expose their tools to the agent.", ), diff --git a/src/openhuman/config/schema/dashboard.rs b/src/openhuman/config/schema/dashboard.rs new file mode 100644 index 000000000..abdb07d3d --- /dev/null +++ b/src/openhuman/config/schema/dashboard.rs @@ -0,0 +1,87 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct DashboardConfig { + pub event_stream: EventStreamConfig, + #[serde(default)] + pub model_health: ModelHealthConfig, +} + +impl Default for DashboardConfig { + fn default() -> Self { + Self { + event_stream: EventStreamConfig::default(), + model_health: ModelHealthConfig::default(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct EventStreamConfig { + #[serde(default = "default_es_enabled")] + pub enabled: bool, +} + +fn default_es_enabled() -> bool { + true +} + +impl Default for EventStreamConfig { + fn default() -> Self { + Self { enabled: true } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct ModelHealthConfig { + #[serde(default = "default_mh_enabled")] + pub enabled: bool, + #[serde(default = "default_hallucination_threshold")] + pub hallucination_threshold: f64, + #[serde(default = "default_min_tasks")] + pub min_tasks_for_rating: usize, + #[serde(default = "default_eval_window")] + pub evaluation_window_tasks: usize, +} + +fn default_mh_enabled() -> bool { + true +} +fn default_hallucination_threshold() -> f64 { + 0.10 +} +fn default_min_tasks() -> usize { + 10 +} +fn default_eval_window() -> usize { + 50 +} + +impl Default for ModelHealthConfig { + fn default() -> Self { + Self { + enabled: true, + hallucination_threshold: 0.10, + min_tasks_for_rating: 10, + evaluation_window_tasks: 50, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn model_health_defaults_match_spec() { + let mh = ModelHealthConfig::default(); + assert!(mh.enabled); + assert!((mh.hallucination_threshold - 0.10).abs() < f64::EPSILON); + assert_eq!(mh.min_tasks_for_rating, 10); + assert_eq!(mh.evaluation_window_tasks, 50); + } +} diff --git a/src/openhuman/config/schema/mod.rs b/src/openhuman/config/schema/mod.rs index 310cf93f1..dd3449e91 100644 --- a/src/openhuman/config/schema/mod.rs +++ b/src/openhuman/config/schema/mod.rs @@ -90,5 +90,7 @@ pub use voice_providers::{ generate_voice_provider_id, is_voice_slug_reserved, BuiltinVoiceProvider, SttApiStyle, TtsApiStyle, VoiceCapability, VoiceProviderCreds, BUILTIN_VOICE_PROVIDERS, }; +mod dashboard; +pub use dashboard::{DashboardConfig, EventStreamConfig, ModelHealthConfig}; mod types; pub use types::*; diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index e536127ad..3cec445db 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -36,6 +36,16 @@ pub const MODEL_SUMMARIZATION_V1: &str = "summarization-v1"; /// upgrades any persisted `config.toml` that still holds `chat-v1`. pub const DEFAULT_MODEL: &str = MODEL_REASONING_QUICK_V1; +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct ModelRegistryEntry { + pub id: String, + pub provider: String, + #[serde(default)] + pub cost_per_1m_output: f64, + #[serde(default)] + pub vision: bool, +} + /// Top-level configuration (config.toml root). #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct Config { @@ -350,6 +360,12 @@ pub struct Config { /// to the orchestrator regardless of this flag's value. #[serde(default)] pub chat_onboarding_completed: bool, + + #[serde(default)] + pub dashboard: DashboardConfig, + + #[serde(default)] + pub model_registry: Vec, } /// Shared default so `#[serde(default)]` and `Config::default()` stay in sync. @@ -658,6 +674,8 @@ impl Default for Config { meet: MeetConfig::default(), onboarding_completed: false, chat_onboarding_completed: false, + dashboard: DashboardConfig::default(), + model_registry: Vec::new(), } } } diff --git a/src/openhuman/dashboard/mod.rs b/src/openhuman/dashboard/mod.rs new file mode 100644 index 000000000..1d2f723ba --- /dev/null +++ b/src/openhuman/dashboard/mod.rs @@ -0,0 +1,20 @@ +//! Dashboard domain — aggregate views and operator-facing comparisons. +//! +//! Currently exposes the per-model health comparison table used by the +//! desktop Settings → Developer Options → Model Health panel. The table +//! joins the local `model_registry` config with `dashboard.model_health` +//! thresholds and emits per-model rows with the metric fields the panel +//! expects. Telemetry-driven fields (`quality_score`, `hallucination_rate`, +//! `agents_using`, `tasks_evaluated`) are reported as `null`/`0` until a +//! local telemetry pipeline is wired in — see [`ops::model_health`] for +//! the explicit placeholder contract. + +mod ops; +mod schemas; +mod types; + +pub use ops::model_health; +pub use schemas::{ + all_dashboard_controller_schemas, all_dashboard_registered_controllers, dashboard_schemas, +}; +pub use types::{ModelHealthConfigView, ModelHealthEntry, ModelHealthResponse}; diff --git a/src/openhuman/dashboard/ops.rs b/src/openhuman/dashboard/ops.rs new file mode 100644 index 000000000..a7fc41edf --- /dev/null +++ b/src/openhuman/dashboard/ops.rs @@ -0,0 +1,132 @@ +//! Dashboard model-health aggregation. + +use crate::openhuman::config::Config; +use crate::rpc::RpcOutcome; + +use super::types::{ModelHealthConfigView, ModelHealthEntry, ModelHealthResponse}; + +/// Build the model health response by joining `model_registry` with the +/// `dashboard.model_health` thresholds. +/// +/// Telemetry-driven fields (`quality_score`, `hallucination_rate`, +/// `agents_using`, `tasks_evaluated`) are emitted as placeholders today — +/// there is no local telemetry sink wired in yet. The frontend treats +/// `null` quality / hallucination as "no signal", which collapses status +/// badges to `staging` (under `min_tasks_for_rating`) and keeps the table +/// useful for cost and vision comparison. When a telemetry source lands, +/// populate these fields here rather than at the transport layer. +pub fn model_health(config: &Config) -> Result, String> { + let mh_cfg = &config.dashboard.model_health; + if !mh_cfg.enabled { + log::debug!("[dashboard] model_health request rejected — feature disabled"); + return Err("model health disabled".to_string()); + } + + let models: Vec = config + .model_registry + .iter() + .map(|entry| ModelHealthEntry { + id: entry.id.clone(), + provider: entry.provider.clone(), + cost_per_1m_output: entry.cost_per_1m_output, + vision: entry.vision, + // Placeholder metrics — see module-level docs. + quality_score: None, + hallucination_rate: None, + agents_using: 0, + tasks_evaluated: 0, + }) + .collect(); + + let log = format!( + "dashboard.model_health returned {} models (threshold={:.2}, window={})", + models.len(), + mh_cfg.hallucination_threshold, + mh_cfg.evaluation_window_tasks, + ); + + Ok(RpcOutcome::single_log( + ModelHealthResponse { + models, + config: ModelHealthConfigView { + hallucination_threshold: mh_cfg.hallucination_threshold, + min_tasks_for_rating: mh_cfg.min_tasks_for_rating, + evaluation_window_tasks: mh_cfg.evaluation_window_tasks, + }, + }, + log, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg_with_models() -> Config { + let mut cfg = Config::default(); + cfg.model_registry = vec![ + crate::openhuman::config::schema::ModelRegistryEntry { + id: "deepseek-v3.2".to_string(), + provider: "SiliconFlow".to_string(), + cost_per_1m_output: 0.33, + vision: false, + }, + crate::openhuman::config::schema::ModelRegistryEntry { + id: "qwen-2.5-8b".to_string(), + provider: "OpenRouter".to_string(), + cost_per_1m_output: 0.09, + vision: true, + }, + ]; + cfg + } + + #[test] + fn maps_registry_entries_with_placeholder_metrics() { + let cfg = cfg_with_models(); + let outcome = model_health(&cfg).expect("enabled"); + let resp = &outcome.value; + assert_eq!(resp.models.len(), 2); + + let first = &resp.models[0]; + assert_eq!(first.id, "deepseek-v3.2"); + assert_eq!(first.provider, "SiliconFlow"); + assert!((first.cost_per_1m_output - 0.33).abs() < f64::EPSILON); + assert!(!first.vision); + // Placeholder telemetry fields — assert the contract. + assert!(first.quality_score.is_none()); + assert!(first.hallucination_rate.is_none()); + assert_eq!(first.agents_using, 0); + assert_eq!(first.tasks_evaluated, 0); + + let second = &resp.models[1]; + assert_eq!(second.id, "qwen-2.5-8b"); + assert!(second.vision); + } + + #[test] + fn surfaces_config_thresholds() { + let cfg = cfg_with_models(); + let outcome = model_health(&cfg).expect("enabled"); + let resp = &outcome.value; + assert!((resp.config.hallucination_threshold - 0.10).abs() < f64::EPSILON); + assert_eq!(resp.config.min_tasks_for_rating, 10); + assert_eq!(resp.config.evaluation_window_tasks, 50); + } + + #[test] + fn disabled_feature_errors() { + let mut cfg = cfg_with_models(); + cfg.dashboard.model_health.enabled = false; + let err = model_health(&cfg).expect_err("disabled"); + assert!(err.contains("disabled"), "unexpected error: {err}"); + } + + #[test] + fn empty_registry_returns_empty_models() { + let mut cfg = Config::default(); + cfg.model_registry.clear(); + let outcome = model_health(&cfg).expect("enabled"); + assert!(outcome.value.models.is_empty()); + } +} diff --git a/src/openhuman/dashboard/schemas.rs b/src/openhuman/dashboard/schemas.rs new file mode 100644 index 000000000..23eaa45f5 --- /dev/null +++ b/src/openhuman/dashboard/schemas.rs @@ -0,0 +1,177 @@ +//! Controller schemas + handlers for the dashboard domain. + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::rpc::RpcOutcome; + +use super::ops; +use super::types::ModelHealthResponse; + +pub fn all_dashboard_controller_schemas() -> Vec { + vec![dashboard_schemas("dashboard_model_health")] +} + +pub fn all_dashboard_registered_controllers() -> Vec { + vec![RegisteredController { + schema: dashboard_schemas("dashboard_model_health"), + handler: handle_dashboard_model_health, + }] +} + +pub fn dashboard_schemas(function: &str) -> ControllerSchema { + match function { + "dashboard_model_health" => ControllerSchema { + namespace: "dashboard", + function: "model_health", + description: "Per-model health comparison rows for the desktop dashboard panel \ + — joins local model_registry with dashboard.model_health thresholds. \ + Telemetry-driven metric fields are placeholders until a local \ + telemetry sink is wired in.", + inputs: vec![], + outputs: vec![ + FieldSchema { + name: "models", + ty: TypeSchema::Array(Box::new(TypeSchema::Object { + fields: model_health_entry_fields(), + })), + comment: "Per-model rows: id, provider, cost_per_1m_output, vision, \ + quality_score, hallucination_rate, agents_using, tasks_evaluated.", + required: true, + }, + FieldSchema { + name: "config", + ty: TypeSchema::Object { + fields: model_health_config_fields(), + }, + comment: "Threshold view: hallucination_threshold, min_tasks_for_rating, \ + evaluation_window_tasks.", + required: true, + }, + ], + }, + _ => ControllerSchema { + namespace: "dashboard", + function: "unknown", + description: "Unknown dashboard controller.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "error", + ty: TypeSchema::String, + comment: "Lookup error details.", + required: true, + }], + }, + } +} + +fn model_health_entry_fields() -> Vec { + vec![ + FieldSchema { + name: "id", + ty: TypeSchema::String, + comment: "Model identifier as configured in `model_registry`.", + required: true, + }, + FieldSchema { + name: "provider", + ty: TypeSchema::String, + comment: "Provider label, e.g. SiliconFlow, OpenRouter.", + required: true, + }, + FieldSchema { + name: "cost_per_1m_output", + ty: TypeSchema::F64, + comment: "USD cost per 1M output tokens from local config.", + required: true, + }, + FieldSchema { + name: "vision", + ty: TypeSchema::Bool, + comment: "True when the model accepts image inputs.", + required: true, + }, + FieldSchema { + name: "quality_score", + ty: TypeSchema::Option(Box::new(TypeSchema::F64)), + comment: "Per-model quality score (placeholder — null until telemetry lands).", + required: false, + }, + FieldSchema { + name: "hallucination_rate", + ty: TypeSchema::Option(Box::new(TypeSchema::F64)), + comment: "Observed hallucination rate (placeholder — null until telemetry lands).", + required: false, + }, + FieldSchema { + name: "agents_using", + ty: TypeSchema::U64, + comment: "Number of agents bound to this model (placeholder — 0 until wired).", + required: true, + }, + FieldSchema { + name: "tasks_evaluated", + ty: TypeSchema::U64, + comment: "Tasks evaluated against this model (placeholder — 0 until wired).", + required: true, + }, + ] +} + +fn model_health_config_fields() -> Vec { + vec![ + FieldSchema { + name: "hallucination_threshold", + ty: TypeSchema::F64, + comment: "Rate above which a model is flagged `replace`.", + required: true, + }, + FieldSchema { + name: "min_tasks_for_rating", + ty: TypeSchema::U64, + comment: "Minimum tasks evaluated before quality/hallucination are considered.", + required: true, + }, + FieldSchema { + name: "evaluation_window_tasks", + ty: TypeSchema::U64, + comment: "Sliding window size used when computing telemetry metrics.", + required: true, + }, + ] +} + +fn handle_dashboard_model_health(_params: Map) -> ControllerFuture { + Box::pin(async move { + log::debug!("[dashboard] model_health request received"); + let cfg = crate::openhuman::config::rpc::load_config_with_timeout() + .await + .map_err(|err| { + log::warn!("[dashboard] model_health failed to load config: {err}"); + format!("config unavailable: {err}") + })?; + let outcome: RpcOutcome = ops::model_health(&cfg)?; + outcome.into_cli_compatible_json() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schema_namespace_and_function_are_stable() { + let s = dashboard_schemas("dashboard_model_health"); + assert_eq!(s.namespace, "dashboard"); + assert_eq!(s.function, "model_health"); + } + + #[test] + fn controller_lists_match_lengths() { + assert_eq!( + all_dashboard_controller_schemas().len(), + all_dashboard_registered_controllers().len() + ); + } +} diff --git a/src/openhuman/dashboard/types.rs b/src/openhuman/dashboard/types.rs new file mode 100644 index 000000000..3ff420d0a --- /dev/null +++ b/src/openhuman/dashboard/types.rs @@ -0,0 +1,37 @@ +//! Wire types for the dashboard model health view. + +use serde::{Deserialize, Serialize}; + +/// One row in the model health comparison table. +/// +/// Mirrors the JSON shape consumed by the frontend +/// `ModelHealthPanel` — `id`/`provider`/`cost_per_1m_output`/`vision` +/// come from `Config::model_registry`; the four metric fields are +/// emitted as placeholder (`null` / `0`) values until a local telemetry +/// pipeline is wired in. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ModelHealthEntry { + pub id: String, + pub provider: String, + pub cost_per_1m_output: f64, + pub vision: bool, + pub quality_score: Option, + pub hallucination_rate: Option, + pub agents_using: u32, + pub tasks_evaluated: u32, +} + +/// Thresholds the frontend needs to compute the status badge. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ModelHealthConfigView { + pub hallucination_threshold: f64, + pub min_tasks_for_rating: usize, + pub evaluation_window_tasks: usize, +} + +/// `openhuman.dashboard_model_health` RPC response. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ModelHealthResponse { + pub models: Vec, + pub config: ModelHealthConfigView, +} diff --git a/src/openhuman/mod.rs b/src/openhuman/mod.rs index 8a4f8d658..60d9faa1e 100644 --- a/src/openhuman/mod.rs +++ b/src/openhuman/mod.rs @@ -33,6 +33,7 @@ pub mod cost; pub mod credentials; pub mod cron; pub mod cwd_jail; +pub mod dashboard; pub mod desktop_companion; pub mod dev_paths; pub mod devices;