mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-28 21:44:38 +00:00
feat(dashboard): per-model health comparison table (#2823)
Co-authored-by: sanil-23 <sanil@vezures.xyz>
This commit is contained in:
Sathvik Gilakamsetty
co-authored by
sanil-23
sanil-23
parent
b9c2387c93
commit
fdd22198bb
Generated
+1
@@ -5306,6 +5306,7 @@ dependencies = [
|
||||
"tracing-appender",
|
||||
"tracing-log",
|
||||
"tracing-subscriber",
|
||||
"unicode-normalization",
|
||||
"unicode-segmentation",
|
||||
"unicode-width",
|
||||
"url",
|
||||
|
||||
@@ -189,6 +189,22 @@ const developerItems = [
|
||||
</svg>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: 'model-health',
|
||||
titleKey: 'settings.modelHealth.title',
|
||||
descriptionKey: 'settings.modelHealth.desc',
|
||||
route: 'model-health',
|
||||
icon: (
|
||||
<svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={2}
|
||||
d="M9 19v-6a2 2 0 00-2-2H5a2 2 0 00-2 2v6a2 2 0 002 2h2a2 2 0 002-2zm0 0V9a2 2 0 012-2h2a2 2 0 012 2v10m-6 0a2 2 0 002 2h2a2 2 0 002-2m0 0V5a2 2 0 012-2h2a2 2 0 012 2v14a2 2 0 01-2 2h-2a2 2 0 01-2-2z"
|
||||
/>
|
||||
</svg>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
const CoreModeBadge = () => {
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
import debug from 'debug';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { useT } from '../../../lib/i18n/I18nContext';
|
||||
import { callCoreRpc } from '../../../services/coreRpcClient';
|
||||
import SettingsHeader from '../components/SettingsHeader';
|
||||
import { useSettingsNavigation } from '../hooks/useSettingsNavigation';
|
||||
|
||||
const log = debug('openhuman:model-health');
|
||||
|
||||
interface ModelEntry {
|
||||
id: string;
|
||||
provider: string;
|
||||
cost_per_1m_output: number;
|
||||
vision: boolean;
|
||||
quality_score: number | null;
|
||||
hallucination_rate: number | null;
|
||||
agents_using: number;
|
||||
tasks_evaluated: number;
|
||||
}
|
||||
|
||||
interface HealthConfig {
|
||||
hallucination_threshold: number;
|
||||
min_tasks_for_rating: number;
|
||||
evaluation_window_tasks: number;
|
||||
}
|
||||
|
||||
interface ModelHealthRpcResponse {
|
||||
models: ModelEntry[];
|
||||
config: HealthConfig;
|
||||
}
|
||||
|
||||
interface RpcOutcomeEnvelope<T> {
|
||||
result: T;
|
||||
logs: string[];
|
||||
}
|
||||
|
||||
type RpcModelHealthPayload = ModelHealthRpcResponse | RpcOutcomeEnvelope<ModelHealthRpcResponse>;
|
||||
|
||||
function unwrapPayload(payload: RpcModelHealthPayload): ModelHealthRpcResponse {
|
||||
if (payload && typeof payload === 'object' && 'result' in payload && 'logs' in payload) {
|
||||
return payload.result;
|
||||
}
|
||||
return payload as ModelHealthRpcResponse;
|
||||
}
|
||||
|
||||
type SortCol =
|
||||
| 'id'
|
||||
| 'quality_score'
|
||||
| 'hallucination_rate'
|
||||
| 'cost_per_1m_output'
|
||||
| 'agents_using';
|
||||
type StatusBadge = 'keep' | 'replace' | 'staging' | 'vision';
|
||||
|
||||
function getStatus(m: ModelEntry, cfg: HealthConfig): StatusBadge {
|
||||
if (m.vision) return 'vision';
|
||||
if (m.tasks_evaluated < cfg.min_tasks_for_rating) return 'staging';
|
||||
if (m.hallucination_rate !== null && m.hallucination_rate > cfg.hallucination_threshold)
|
||||
return 'replace';
|
||||
return 'keep';
|
||||
}
|
||||
|
||||
function qualityStars(score: number | null): string {
|
||||
if (score === null) return '—';
|
||||
const full = Math.round(score);
|
||||
return '★'.repeat(full) + '☆'.repeat(5 - full);
|
||||
}
|
||||
|
||||
const BADGE_STYLES: Record<StatusBadge, { bg: string; text: string; label: string }> = {
|
||||
keep: { bg: 'bg-green-500/20', text: 'text-green-400', label: 'settings.modelHealth.badge.keep' },
|
||||
replace: {
|
||||
bg: 'bg-red-500/20',
|
||||
text: 'text-red-400',
|
||||
label: 'settings.modelHealth.badge.replace',
|
||||
},
|
||||
staging: {
|
||||
bg: 'bg-amber-500/20',
|
||||
text: 'text-amber-400',
|
||||
label: 'settings.modelHealth.badge.staging',
|
||||
},
|
||||
vision: {
|
||||
bg: 'bg-blue-500/20',
|
||||
text: 'text-blue-400',
|
||||
label: 'settings.modelHealth.badge.vision',
|
||||
},
|
||||
};
|
||||
|
||||
const ModelHealthPanel = () => {
|
||||
const { t } = useT();
|
||||
const { navigateBack, breadcrumbs } = useSettingsNavigation();
|
||||
const [models, setModels] = useState<ModelEntry[]>([]);
|
||||
const [config, setConfig] = useState<HealthConfig>({
|
||||
hallucination_threshold: 0.1,
|
||||
min_tasks_for_rating: 10,
|
||||
evaluation_window_tasks: 50,
|
||||
});
|
||||
const [sortCol, setSortCol] = useState<SortCol>('id');
|
||||
const [sortAsc, setSortAsc] = useState(true);
|
||||
const [filterStatus, setFilterStatus] = useState<string>('');
|
||||
const [swapTarget, setSwapTarget] = useState<ModelEntry | null>(null);
|
||||
const [selectedCandidate, setSelectedCandidate] = useState<ModelEntry | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const fetchedRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (fetchedRef.current) return;
|
||||
fetchedRef.current = true;
|
||||
(async () => {
|
||||
log('[model-health] fetch start');
|
||||
try {
|
||||
const payload = await callCoreRpc<RpcModelHealthPayload>({
|
||||
method: 'openhuman.dashboard_model_health',
|
||||
});
|
||||
const data = unwrapPayload(payload);
|
||||
setModels(data.models || []);
|
||||
if (data.config) setConfig(data.config);
|
||||
log('[model-health] fetch ok models=%d', data.models?.length ?? 0);
|
||||
} catch (err) {
|
||||
log('[model-health] fetch failed: %o', err);
|
||||
}
|
||||
setLoading(false);
|
||||
})();
|
||||
}, []);
|
||||
|
||||
const handleSort = (col: SortCol) => {
|
||||
if (sortCol === col) {
|
||||
setSortAsc(!sortAsc);
|
||||
} else {
|
||||
setSortCol(col);
|
||||
setSortAsc(true);
|
||||
}
|
||||
};
|
||||
|
||||
const sorted = [...models].sort((a, b) => {
|
||||
const av = a[sortCol] ?? -1;
|
||||
const bv = b[sortCol] ?? -1;
|
||||
if (av < bv) return sortAsc ? -1 : 1;
|
||||
if (av > bv) return sortAsc ? 1 : -1;
|
||||
return 0;
|
||||
});
|
||||
|
||||
const filtered = sorted.filter(m => {
|
||||
if (!filterStatus) return true;
|
||||
return getStatus(m, config) === filterStatus;
|
||||
});
|
||||
|
||||
const replaceCandidates = (target: ModelEntry) =>
|
||||
models.filter(
|
||||
c =>
|
||||
c.id !== target.id &&
|
||||
!c.vision &&
|
||||
(c.hallucination_rate ?? 1) < (target.hallucination_rate ?? 1) &&
|
||||
c.cost_per_1m_output <= target.cost_per_1m_output
|
||||
);
|
||||
|
||||
const betterCandidates = (target: ModelEntry) =>
|
||||
models.filter(
|
||||
c =>
|
||||
c.id !== target.id &&
|
||||
!c.vision &&
|
||||
(c.hallucination_rate ?? 1) < (target.hallucination_rate ?? 1) &&
|
||||
c.cost_per_1m_output > target.cost_per_1m_output
|
||||
);
|
||||
|
||||
const sortIcon = (col: SortCol) => (sortCol === col ? (sortAsc ? ' ↑' : ' ↓') : '');
|
||||
|
||||
return (
|
||||
<div data-testid="model-health-panel">
|
||||
<SettingsHeader
|
||||
title={t('settings.modelHealth.title')}
|
||||
showBackButton={true}
|
||||
onBack={navigateBack}
|
||||
breadcrumbs={breadcrumbs}
|
||||
/>
|
||||
<div className="p-4 space-y-4">
|
||||
<div className="flex items-center gap-2 text-xs">
|
||||
<select
|
||||
className="rounded-lg border border-stone-200 dark:border-neutral-800 bg-stone-50 dark:bg-neutral-800/60 px-3 py-1.5 font-medium text-stone-700 dark:text-neutral-200"
|
||||
value={filterStatus}
|
||||
onChange={e => setFilterStatus(e.target.value)}>
|
||||
<option value="">{t('settings.modelHealth.allStatuses')}</option>
|
||||
<option value="keep">{t('settings.modelHealth.badge.keep')}</option>
|
||||
<option value="replace">{t('settings.modelHealth.badge.replace')}</option>
|
||||
<option value="staging">{t('settings.modelHealth.badge.staging')}</option>
|
||||
<option value="vision">{t('settings.modelHealth.badge.vision')}</option>
|
||||
</select>
|
||||
<span className="text-stone-500 dark:text-neutral-400">
|
||||
{filtered.length} {t('settings.modelHealth.models')}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{loading ? (
|
||||
<p className="text-xs text-stone-400 py-4 text-center">
|
||||
{t('settings.modelHealth.loading')}
|
||||
</p>
|
||||
) : filtered.length === 0 ? (
|
||||
<p className="text-xs text-stone-400 py-4 text-center">
|
||||
{t('settings.modelHealth.empty')}
|
||||
</p>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-xs">
|
||||
<thead>
|
||||
<tr className="border-b border-stone-200 dark:border-neutral-800">
|
||||
<th
|
||||
className="text-left py-2 px-2 cursor-pointer"
|
||||
onClick={() => handleSort('id')}>
|
||||
{t('settings.modelHealth.col.model')}
|
||||
{sortIcon('id')}
|
||||
</th>
|
||||
<th
|
||||
className="text-left py-2 px-2 cursor-pointer"
|
||||
onClick={() => handleSort('quality_score')}>
|
||||
{t('settings.modelHealth.col.quality')}
|
||||
{sortIcon('quality_score')}
|
||||
</th>
|
||||
<th
|
||||
className="text-left py-2 px-2 cursor-pointer"
|
||||
onClick={() => handleSort('hallucination_rate')}>
|
||||
{t('settings.modelHealth.col.halluc')}
|
||||
{sortIcon('hallucination_rate')}
|
||||
</th>
|
||||
<th
|
||||
className="text-left py-2 px-2 cursor-pointer"
|
||||
onClick={() => handleSort('cost_per_1m_output')}>
|
||||
{t('settings.modelHealth.col.cost')}
|
||||
{sortIcon('cost_per_1m_output')}
|
||||
</th>
|
||||
<th
|
||||
className="text-left py-2 px-2 cursor-pointer"
|
||||
onClick={() => handleSort('agents_using')}>
|
||||
{t('settings.modelHealth.col.agents')}
|
||||
{sortIcon('agents_using')}
|
||||
</th>
|
||||
<th className="text-left py-2 px-2">{t('settings.modelHealth.col.status')}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filtered.map(m => {
|
||||
const status = getStatus(m, config);
|
||||
const badge = BADGE_STYLES[status];
|
||||
const isReplace = status === 'replace';
|
||||
const candidates = isReplace
|
||||
? [...replaceCandidates(m), ...betterCandidates(m)]
|
||||
: [];
|
||||
return (
|
||||
<tr
|
||||
key={m.id}
|
||||
className={`border-b border-stone-100 dark:border-neutral-800/50 ${isReplace ? 'bg-red-500/5' : ''}`}>
|
||||
<td className="py-2 px-2">
|
||||
<div className="font-semibold text-stone-900 dark:text-neutral-100">
|
||||
{m.id}
|
||||
</div>
|
||||
<div className="text-[10px] text-stone-400">{m.provider}</div>
|
||||
</td>
|
||||
<td className="py-2 px-2 text-amber-400">{qualityStars(m.quality_score)}</td>
|
||||
<td className="py-2 px-2 font-mono">
|
||||
{m.hallucination_rate !== null ? (
|
||||
<span
|
||||
className={
|
||||
m.hallucination_rate > config.hallucination_threshold
|
||||
? 'text-red-400'
|
||||
: 'text-green-400'
|
||||
}>
|
||||
{(m.hallucination_rate * 100).toFixed(1)}%
|
||||
</span>
|
||||
) : (
|
||||
'—'
|
||||
)}
|
||||
</td>
|
||||
<td className="py-2 px-2 font-mono">${m.cost_per_1m_output.toFixed(2)}</td>
|
||||
<td className="py-2 px-2">{m.agents_using}</td>
|
||||
<td className="py-2 px-2">
|
||||
<span
|
||||
className={`rounded-full ${badge.bg} px-2 py-0.5 text-[10px] ${badge.text}`}>
|
||||
{t(badge.label)}
|
||||
</span>
|
||||
{isReplace && candidates.length > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
className="ml-1 text-[10px] text-amber-400 hover:text-amber-300"
|
||||
onClick={() => setSwapTarget(m)}>
|
||||
{t('settings.modelHealth.swap')}
|
||||
</button>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Swap Modal */}
|
||||
{swapTarget && (
|
||||
<div
|
||||
className="fixed inset-0 bg-black/60 z-50 flex items-center justify-center"
|
||||
onClick={() => {
|
||||
setSwapTarget(null);
|
||||
setSelectedCandidate(null);
|
||||
}}>
|
||||
<div
|
||||
className="bg-white dark:bg-neutral-900 border border-stone-200 dark:border-neutral-700 rounded-xl p-5 max-w-sm w-full mx-4"
|
||||
onClick={e => e.stopPropagation()}>
|
||||
<h3 className="text-sm font-bold mb-2">{t('settings.modelHealth.modal.title')}</h3>
|
||||
<p className="text-xs text-stone-500 dark:text-neutral-400 mb-3">
|
||||
{swapTarget.id} — {t('settings.modelHealth.modal.hallucRate')}:{' '}
|
||||
{((swapTarget.hallucination_rate ?? 0) * 100).toFixed(1)}%
|
||||
</p>
|
||||
<div className="space-y-2 mb-4" role="radiogroup">
|
||||
{[...replaceCandidates(swapTarget), ...betterCandidates(swapTarget)].map(c => {
|
||||
const isSelected = selectedCandidate?.id === c.id;
|
||||
return (
|
||||
<button
|
||||
key={c.id}
|
||||
type="button"
|
||||
role="radio"
|
||||
aria-checked={isSelected}
|
||||
onClick={() => setSelectedCandidate(c)}
|
||||
className={`w-full text-left rounded-lg border p-2 flex items-center justify-between cursor-pointer ${isSelected ? 'border-green-500 bg-green-500/15' : 'border-green-500/30 bg-green-500/5'}`}>
|
||||
<span>
|
||||
<span className="block text-xs font-semibold">{c.id}</span>
|
||||
<span className="block text-[10px] text-stone-400">
|
||||
{c.hallucination_rate !== null
|
||||
? (c.hallucination_rate * 100).toFixed(1)
|
||||
: '?'}
|
||||
% · ${c.cost_per_1m_output.toFixed(2)}/1M
|
||||
</span>
|
||||
</span>
|
||||
<span className="text-[9px] font-bold text-green-400">
|
||||
{c.cost_per_1m_output <= swapTarget.cost_per_1m_output
|
||||
? t('settings.modelHealth.tag.cheaper')
|
||||
: t('settings.modelHealth.tag.better')}
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
type="button"
|
||||
className="flex-1 py-2 rounded-lg border border-stone-200 dark:border-neutral-700 text-xs"
|
||||
onClick={() => {
|
||||
setSwapTarget(null);
|
||||
setSelectedCandidate(null);
|
||||
}}>
|
||||
{t('settings.modelHealth.modal.cancel')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!selectedCandidate}
|
||||
className="flex-1 py-2 rounded-lg bg-blue-600 text-white text-xs font-semibold disabled:opacity-40"
|
||||
onClick={() => {
|
||||
if (selectedCandidate && swapTarget) {
|
||||
// Apply is currently UI-only: the backend swap RPC is a
|
||||
// follow-up (no agent → model rewire wiring yet). Log the
|
||||
// operator's intent so it shows up in support logs.
|
||||
log(
|
||||
'[model-health] swap intent recorded from=%s to=%s (no-op backend follow-up pending)',
|
||||
swapTarget.id,
|
||||
selectedCandidate.id
|
||||
);
|
||||
}
|
||||
setSwapTarget(null);
|
||||
setSelectedCandidate(null);
|
||||
}}>
|
||||
{t('settings.modelHealth.modal.apply')}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ModelHealthPanel;
|
||||
@@ -0,0 +1,235 @@
|
||||
import { fireEvent, screen, waitFor } from '@testing-library/react';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { renderWithProviders } from '../../../../test/test-utils';
|
||||
import ModelHealthPanel from '../ModelHealthPanel';
|
||||
|
||||
vi.mock('../../../../services/coreRpcClient', () => ({ callCoreRpc: vi.fn() }));
|
||||
vi.mock('../../hooks/useSettingsNavigation', () => ({
|
||||
useSettingsNavigation: () => ({ navigateBack: vi.fn(), breadcrumbs: [] }),
|
||||
}));
|
||||
vi.mock('../../../../lib/i18n/I18nContext', () => ({ useT: () => ({ t: (k: string) => k }) }));
|
||||
|
||||
const MOCK_RESPONSE = {
|
||||
models: [
|
||||
{
|
||||
id: 'deepseek-v3',
|
||||
provider: 'SiliconFlow',
|
||||
cost_per_1m_output: 0.33,
|
||||
vision: false,
|
||||
quality_score: 4,
|
||||
hallucination_rate: 0.03,
|
||||
agents_using: 5,
|
||||
tasks_evaluated: 60,
|
||||
},
|
||||
{
|
||||
id: 'qwen-2.5-8b',
|
||||
provider: 'OpenRouter',
|
||||
cost_per_1m_output: 0.09,
|
||||
vision: true,
|
||||
quality_score: 3,
|
||||
hallucination_rate: 0.04,
|
||||
agents_using: 1,
|
||||
tasks_evaluated: 20,
|
||||
},
|
||||
{
|
||||
id: 'bad-model',
|
||||
provider: 'Test',
|
||||
cost_per_1m_output: 1.0,
|
||||
vision: false,
|
||||
quality_score: 2,
|
||||
hallucination_rate: 0.18,
|
||||
agents_using: 2,
|
||||
tasks_evaluated: 50,
|
||||
},
|
||||
],
|
||||
config: { hallucination_threshold: 0.1, min_tasks_for_rating: 10, evaluation_window_tasks: 50 },
|
||||
};
|
||||
|
||||
async function mockRpc(response: unknown) {
|
||||
const { callCoreRpc } = await import('../../../../services/coreRpcClient');
|
||||
vi.mocked(callCoreRpc).mockResolvedValueOnce(response);
|
||||
}
|
||||
|
||||
async function mockRpcReject(err: unknown) {
|
||||
const { callCoreRpc } = await import('../../../../services/coreRpcClient');
|
||||
vi.mocked(callCoreRpc).mockRejectedValueOnce(err);
|
||||
}
|
||||
|
||||
describe('ModelHealthPanel', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('renders panel with table', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('deepseek-v3')).toBeTruthy();
|
||||
});
|
||||
expect(screen.getByText('qwen-2.5-8b')).toBeTruthy();
|
||||
expect(screen.getByText('bad-model')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('shows correct status badges', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('deepseek-v3')).toBeTruthy();
|
||||
});
|
||||
expect(screen.getAllByText('settings.modelHealth.badge.keep').length).toBeGreaterThan(0);
|
||||
expect(screen.getAllByText('settings.modelHealth.badge.vision').length).toBeGreaterThan(0);
|
||||
expect(screen.getAllByText('settings.modelHealth.badge.replace').length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('filters by status', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
const { container } = renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('deepseek-v3')).toBeTruthy();
|
||||
});
|
||||
const select = container.querySelector('select')!;
|
||||
fireEvent.change(select, { target: { value: 'vision' } });
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('deepseek-v3')).toBeNull();
|
||||
});
|
||||
expect(screen.getByText('qwen-2.5-8b')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('sorts by column', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('deepseek-v3')).toBeTruthy();
|
||||
});
|
||||
fireEvent.click(screen.getByText('settings.modelHealth.col.cost'));
|
||||
// After sorting by cost asc, first row should be cheapest model (qwen at $0.09)
|
||||
const rows = screen.getAllByRole('row');
|
||||
expect(rows[1].textContent).toContain('qwen-2.5-8b');
|
||||
});
|
||||
|
||||
it('shows swap button for replace-flagged models', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
it('opens swap modal on click', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy();
|
||||
});
|
||||
fireEvent.click(screen.getByText('settings.modelHealth.swap'));
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.modal.title')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
it('shows empty state when no models', async () => {
|
||||
await mockRpc({ models: [], config: MOCK_RESPONSE.config });
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.empty')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
it('shows loading then content', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId('model-health-panel')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
it('unwraps RpcOutcome {result, logs} envelope', async () => {
|
||||
await mockRpc({ result: MOCK_RESPONSE, logs: ['dashboard.model_health returned 3 models'] });
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('deepseek-v3')).toBeTruthy();
|
||||
});
|
||||
expect(screen.getByText('qwen-2.5-8b')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('handles rpc failure gracefully', async () => {
|
||||
await mockRpcReject(new Error('config unavailable'));
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.empty')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
it('toggles sort direction on same column click', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('deepseek-v3')).toBeTruthy();
|
||||
});
|
||||
const costHeader = screen.getByText('settings.modelHealth.col.cost');
|
||||
fireEvent.click(costHeader); // asc
|
||||
fireEvent.click(costHeader); // desc — toggles direction
|
||||
const rows = screen.getAllByRole('row');
|
||||
expect(rows[1].textContent).toContain('bad-model'); // most expensive first
|
||||
});
|
||||
|
||||
it('modal shows candidates and closes on backdrop click', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy();
|
||||
});
|
||||
fireEvent.click(screen.getByText('settings.modelHealth.swap'));
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.modal.title')).toBeTruthy();
|
||||
});
|
||||
// Verify candidates are shown
|
||||
expect(screen.getByText('settings.modelHealth.modal.apply')).toBeTruthy();
|
||||
expect(screen.getByText('settings.modelHealth.modal.cancel')).toBeTruthy();
|
||||
// Close via cancel
|
||||
fireEvent.click(screen.getByText('settings.modelHealth.modal.cancel'));
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('settings.modelHealth.modal.title')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
it('closes modal on cancel', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy();
|
||||
});
|
||||
fireEvent.click(screen.getByText('settings.modelHealth.swap'));
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.modal.title')).toBeTruthy();
|
||||
});
|
||||
fireEvent.click(screen.getByText('settings.modelHealth.modal.cancel'));
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('settings.modelHealth.modal.title')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
it('Apply button is disabled until a candidate is selected', async () => {
|
||||
await mockRpc(MOCK_RESPONSE);
|
||||
renderWithProviders(<ModelHealthPanel />);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('settings.modelHealth.swap')).toBeTruthy();
|
||||
});
|
||||
fireEvent.click(screen.getByText('settings.modelHealth.swap'));
|
||||
const applyButton = await screen.findByText('settings.modelHealth.modal.apply');
|
||||
expect((applyButton as HTMLButtonElement).disabled).toBe(true);
|
||||
|
||||
// Select a candidate then re-check.
|
||||
const candidates = screen.getAllByRole('radio');
|
||||
expect(candidates.length).toBeGreaterThan(0);
|
||||
fireEvent.click(candidates[0]);
|
||||
expect((applyButton as HTMLButtonElement).disabled).toBe(false);
|
||||
|
||||
// Clicking Apply closes the modal (UI-side; backend swap is a follow-up).
|
||||
fireEvent.click(applyButton);
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('settings.modelHealth.modal.title')).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -787,6 +787,30 @@ const ar5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default ar5;
|
||||
|
||||
@@ -800,6 +800,30 @@ const bn5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default bn5;
|
||||
|
||||
@@ -828,6 +828,30 @@ const de5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default de5;
|
||||
|
||||
@@ -795,6 +795,30 @@ const en5: TranslationMap = {
|
||||
'skills.meetingBots.platforms.zoom': 'Zoom',
|
||||
'skills.meetingBots.soonSuffix': 'soon',
|
||||
'skills.setup.screenIntel.permissionPathLabel': 'macOS applies privacy to:',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default en5;
|
||||
|
||||
@@ -814,6 +814,30 @@ const es5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default es5;
|
||||
|
||||
@@ -818,6 +818,30 @@ const fr5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default fr5;
|
||||
|
||||
@@ -801,6 +801,30 @@ const hi5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default hi5;
|
||||
|
||||
@@ -801,6 +801,30 @@ const id5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default id5;
|
||||
|
||||
@@ -812,6 +812,30 @@ const it5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default it5;
|
||||
|
||||
@@ -790,6 +790,30 @@ const ko5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default ko5;
|
||||
|
||||
@@ -811,6 +811,30 @@ const pt5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default pt5;
|
||||
|
||||
@@ -807,6 +807,30 @@ const ru5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default ru5;
|
||||
|
||||
@@ -761,6 +761,30 @@ const zhCN5: TranslationMap = {
|
||||
'settings.agentAccess.add': 'Add',
|
||||
'settings.agentAccess.saving': 'Saving…',
|
||||
'settings.agentAccess.changesApply': 'Changes apply on your next message.',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
};
|
||||
|
||||
export default zhCN5;
|
||||
|
||||
@@ -3372,6 +3372,30 @@ const en: TranslationMap = {
|
||||
'Smallest memory window. Cheapest, fastest, least continuity between runs.',
|
||||
'settings.memoryWindow.minimal.label': 'Minimal',
|
||||
'settings.memoryWindow.title': 'Long-term memory window',
|
||||
'settings.modelHealth.title': 'Model Health',
|
||||
'settings.modelHealth.desc':
|
||||
'Per-model quality, hallucination rate, and cost comparison across active models',
|
||||
'settings.modelHealth.allStatuses': 'All statuses',
|
||||
'settings.modelHealth.models': 'models',
|
||||
'settings.modelHealth.loading': 'Loading model data...',
|
||||
'settings.modelHealth.empty': 'No models registered',
|
||||
'settings.modelHealth.col.model': 'Model',
|
||||
'settings.modelHealth.col.quality': 'Quality',
|
||||
'settings.modelHealth.col.halluc': 'Halluc. Rate',
|
||||
'settings.modelHealth.col.cost': 'Cost / 1M out',
|
||||
'settings.modelHealth.col.agents': 'Agents',
|
||||
'settings.modelHealth.col.status': 'Status',
|
||||
'settings.modelHealth.badge.keep': 'Keep',
|
||||
'settings.modelHealth.badge.replace': 'Replace',
|
||||
'settings.modelHealth.badge.staging': 'Staging test',
|
||||
'settings.modelHealth.badge.vision': 'Vision only',
|
||||
'settings.modelHealth.swap': 'Swap?',
|
||||
'settings.modelHealth.modal.title': 'Replace Model?',
|
||||
'settings.modelHealth.modal.hallucRate': 'Hallucination rate',
|
||||
'settings.modelHealth.modal.cancel': 'Cancel',
|
||||
'settings.modelHealth.modal.apply': 'Apply Replacement',
|
||||
'settings.modelHealth.tag.cheaper': 'CHEAPER',
|
||||
'settings.modelHealth.tag.better': 'BETTER',
|
||||
'settings.screenIntel.permissions.accessibility': 'Accessibility',
|
||||
'settings.screenIntel.permissions.grantHint':
|
||||
'Grant these permissions in System Settings, then restart the core.',
|
||||
|
||||
@@ -28,6 +28,7 @@ import McpServerPanel from '../components/settings/panels/McpServerPanel';
|
||||
import MemoryDataPanel from '../components/settings/panels/MemoryDataPanel';
|
||||
import MemoryDebugPanel from '../components/settings/panels/MemoryDebugPanel';
|
||||
import MigrationPanel from '../components/settings/panels/MigrationPanel';
|
||||
import ModelHealthPanel from '../components/settings/panels/ModelHealthPanel';
|
||||
import NotificationsTabbedPanel from '../components/settings/panels/NotificationsTabbedPanel';
|
||||
import PersonaPanel from '../components/settings/panels/PersonaPanel';
|
||||
import PrivacyPanel from '../components/settings/panels/PrivacyPanel';
|
||||
@@ -461,6 +462,10 @@ const Settings = () => {
|
||||
<Route path="voice-debug" element={wrapSettingsPage(<VoiceDebugPanel />)} />
|
||||
<Route path="local-model-debug" element={wrapSettingsPage(<LocalModelDebugPanel />)} />
|
||||
<Route path="webhooks-debug" element={wrapSettingsPage(<WebhooksDebugPanel />)} />
|
||||
<Route
|
||||
path="model-health"
|
||||
element={wrapSettingsPage(<ModelHealthPanel />, { maxWidthClass: 'max-w-4xl' })}
|
||||
/>
|
||||
<Route path="memory-data" element={wrapSettingsPage(<MemoryDataPanel />)} />
|
||||
<Route path="memory-debug" element={wrapSettingsPage(<MemoryDebugPanel />)} />
|
||||
<Route path="intelligence" element={<Intelligence />} />
|
||||
|
||||
@@ -114,6 +114,7 @@ fn build_registered_controllers() -> Vec<RegisteredController> {
|
||||
controllers.extend(crate::openhuman::composio::all_composio_registered_controllers());
|
||||
// Scheduled job management
|
||||
controllers.extend(crate::openhuman::cron::all_cron_registered_controllers());
|
||||
controllers.extend(crate::openhuman::dashboard::all_dashboard_registered_controllers());
|
||||
// MCP client subsystem: Smithery registry browser, local server install/connect, tool dispatch
|
||||
controllers.extend(crate::openhuman::mcp_registry::all_mcp_registry_registered_controllers());
|
||||
// Webview APIs bridge — proxies connector calls (Gmail, …) through
|
||||
@@ -285,6 +286,7 @@ fn build_declared_controller_schemas() -> Vec<ControllerSchema> {
|
||||
schemas.extend(crate::openhuman::audio_toolkit::all_audio_toolkit_controller_schemas());
|
||||
schemas.extend(crate::openhuman::composio::all_composio_controller_schemas());
|
||||
schemas.extend(crate::openhuman::cron::all_cron_controller_schemas());
|
||||
schemas.extend(crate::openhuman::dashboard::all_dashboard_controller_schemas());
|
||||
schemas.extend(crate::openhuman::mcp_registry::all_mcp_registry_controller_schemas());
|
||||
schemas.extend(crate::openhuman::webview_apis::all_webview_apis_controller_schemas());
|
||||
schemas.extend(crate::openhuman::agent::all_agent_controller_schemas());
|
||||
@@ -402,6 +404,9 @@ pub fn namespace_description(namespace: &str) -> Option<&'static str> {
|
||||
"Connectivity diagnostics for the local sidecar, listening port, and backend Socket.IO state.",
|
||||
),
|
||||
"cron" => Some("Manage scheduled jobs and run history."),
|
||||
"dashboard" => Some(
|
||||
"Operator-facing dashboard aggregations: per-model health comparison rows.",
|
||||
),
|
||||
"mcp_clients" => Some(
|
||||
"Browse the Smithery.ai MCP registry, install MCP servers locally, manage their stdio connections, and expose their tools to the agent.",
|
||||
),
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(default)]
|
||||
pub struct DashboardConfig {
|
||||
pub event_stream: EventStreamConfig,
|
||||
#[serde(default)]
|
||||
pub model_health: ModelHealthConfig,
|
||||
}
|
||||
|
||||
impl Default for DashboardConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
event_stream: EventStreamConfig::default(),
|
||||
model_health: ModelHealthConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(default)]
|
||||
pub struct EventStreamConfig {
|
||||
#[serde(default = "default_es_enabled")]
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
fn default_es_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
impl Default for EventStreamConfig {
|
||||
fn default() -> Self {
|
||||
Self { enabled: true }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(default)]
|
||||
pub struct ModelHealthConfig {
|
||||
#[serde(default = "default_mh_enabled")]
|
||||
pub enabled: bool,
|
||||
#[serde(default = "default_hallucination_threshold")]
|
||||
pub hallucination_threshold: f64,
|
||||
#[serde(default = "default_min_tasks")]
|
||||
pub min_tasks_for_rating: usize,
|
||||
#[serde(default = "default_eval_window")]
|
||||
pub evaluation_window_tasks: usize,
|
||||
}
|
||||
|
||||
fn default_mh_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
fn default_hallucination_threshold() -> f64 {
|
||||
0.10
|
||||
}
|
||||
fn default_min_tasks() -> usize {
|
||||
10
|
||||
}
|
||||
fn default_eval_window() -> usize {
|
||||
50
|
||||
}
|
||||
|
||||
impl Default for ModelHealthConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
hallucination_threshold: 0.10,
|
||||
min_tasks_for_rating: 10,
|
||||
evaluation_window_tasks: 50,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn model_health_defaults_match_spec() {
|
||||
let mh = ModelHealthConfig::default();
|
||||
assert!(mh.enabled);
|
||||
assert!((mh.hallucination_threshold - 0.10).abs() < f64::EPSILON);
|
||||
assert_eq!(mh.min_tasks_for_rating, 10);
|
||||
assert_eq!(mh.evaluation_window_tasks, 50);
|
||||
}
|
||||
}
|
||||
@@ -90,5 +90,7 @@ pub use voice_providers::{
|
||||
generate_voice_provider_id, is_voice_slug_reserved, BuiltinVoiceProvider, SttApiStyle,
|
||||
TtsApiStyle, VoiceCapability, VoiceProviderCreds, BUILTIN_VOICE_PROVIDERS,
|
||||
};
|
||||
mod dashboard;
|
||||
pub use dashboard::{DashboardConfig, EventStreamConfig, ModelHealthConfig};
|
||||
mod types;
|
||||
pub use types::*;
|
||||
|
||||
@@ -36,6 +36,16 @@ pub const MODEL_SUMMARIZATION_V1: &str = "summarization-v1";
|
||||
/// upgrades any persisted `config.toml` that still holds `chat-v1`.
|
||||
pub const DEFAULT_MODEL: &str = MODEL_REASONING_QUICK_V1;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ModelRegistryEntry {
|
||||
pub id: String,
|
||||
pub provider: String,
|
||||
#[serde(default)]
|
||||
pub cost_per_1m_output: f64,
|
||||
#[serde(default)]
|
||||
pub vision: bool,
|
||||
}
|
||||
|
||||
/// Top-level configuration (config.toml root).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Config {
|
||||
@@ -350,6 +360,12 @@ pub struct Config {
|
||||
/// to the orchestrator regardless of this flag's value.
|
||||
#[serde(default)]
|
||||
pub chat_onboarding_completed: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub dashboard: DashboardConfig,
|
||||
|
||||
#[serde(default)]
|
||||
pub model_registry: Vec<ModelRegistryEntry>,
|
||||
}
|
||||
|
||||
/// Shared default so `#[serde(default)]` and `Config::default()` stay in sync.
|
||||
@@ -658,6 +674,8 @@ impl Default for Config {
|
||||
meet: MeetConfig::default(),
|
||||
onboarding_completed: false,
|
||||
chat_onboarding_completed: false,
|
||||
dashboard: DashboardConfig::default(),
|
||||
model_registry: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
//! Dashboard domain — aggregate views and operator-facing comparisons.
|
||||
//!
|
||||
//! Currently exposes the per-model health comparison table used by the
|
||||
//! desktop Settings → Developer Options → Model Health panel. The table
|
||||
//! joins the local `model_registry` config with `dashboard.model_health`
|
||||
//! thresholds and emits per-model rows with the metric fields the panel
|
||||
//! expects. Telemetry-driven fields (`quality_score`, `hallucination_rate`,
|
||||
//! `agents_using`, `tasks_evaluated`) are reported as `null`/`0` until a
|
||||
//! local telemetry pipeline is wired in — see [`ops::model_health`] for
|
||||
//! the explicit placeholder contract.
|
||||
|
||||
mod ops;
|
||||
mod schemas;
|
||||
mod types;
|
||||
|
||||
pub use ops::model_health;
|
||||
pub use schemas::{
|
||||
all_dashboard_controller_schemas, all_dashboard_registered_controllers, dashboard_schemas,
|
||||
};
|
||||
pub use types::{ModelHealthConfigView, ModelHealthEntry, ModelHealthResponse};
|
||||
@@ -0,0 +1,132 @@
|
||||
//! Dashboard model-health aggregation.
|
||||
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::rpc::RpcOutcome;
|
||||
|
||||
use super::types::{ModelHealthConfigView, ModelHealthEntry, ModelHealthResponse};
|
||||
|
||||
/// Build the model health response by joining `model_registry` with the
|
||||
/// `dashboard.model_health` thresholds.
|
||||
///
|
||||
/// Telemetry-driven fields (`quality_score`, `hallucination_rate`,
|
||||
/// `agents_using`, `tasks_evaluated`) are emitted as placeholders today —
|
||||
/// there is no local telemetry sink wired in yet. The frontend treats
|
||||
/// `null` quality / hallucination as "no signal", which collapses status
|
||||
/// badges to `staging` (under `min_tasks_for_rating`) and keeps the table
|
||||
/// useful for cost and vision comparison. When a telemetry source lands,
|
||||
/// populate these fields here rather than at the transport layer.
|
||||
pub fn model_health(config: &Config) -> Result<RpcOutcome<ModelHealthResponse>, String> {
|
||||
let mh_cfg = &config.dashboard.model_health;
|
||||
if !mh_cfg.enabled {
|
||||
log::debug!("[dashboard] model_health request rejected — feature disabled");
|
||||
return Err("model health disabled".to_string());
|
||||
}
|
||||
|
||||
let models: Vec<ModelHealthEntry> = config
|
||||
.model_registry
|
||||
.iter()
|
||||
.map(|entry| ModelHealthEntry {
|
||||
id: entry.id.clone(),
|
||||
provider: entry.provider.clone(),
|
||||
cost_per_1m_output: entry.cost_per_1m_output,
|
||||
vision: entry.vision,
|
||||
// Placeholder metrics — see module-level docs.
|
||||
quality_score: None,
|
||||
hallucination_rate: None,
|
||||
agents_using: 0,
|
||||
tasks_evaluated: 0,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let log = format!(
|
||||
"dashboard.model_health returned {} models (threshold={:.2}, window={})",
|
||||
models.len(),
|
||||
mh_cfg.hallucination_threshold,
|
||||
mh_cfg.evaluation_window_tasks,
|
||||
);
|
||||
|
||||
Ok(RpcOutcome::single_log(
|
||||
ModelHealthResponse {
|
||||
models,
|
||||
config: ModelHealthConfigView {
|
||||
hallucination_threshold: mh_cfg.hallucination_threshold,
|
||||
min_tasks_for_rating: mh_cfg.min_tasks_for_rating,
|
||||
evaluation_window_tasks: mh_cfg.evaluation_window_tasks,
|
||||
},
|
||||
},
|
||||
log,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cfg_with_models() -> Config {
|
||||
let mut cfg = Config::default();
|
||||
cfg.model_registry = vec![
|
||||
crate::openhuman::config::schema::ModelRegistryEntry {
|
||||
id: "deepseek-v3.2".to_string(),
|
||||
provider: "SiliconFlow".to_string(),
|
||||
cost_per_1m_output: 0.33,
|
||||
vision: false,
|
||||
},
|
||||
crate::openhuman::config::schema::ModelRegistryEntry {
|
||||
id: "qwen-2.5-8b".to_string(),
|
||||
provider: "OpenRouter".to_string(),
|
||||
cost_per_1m_output: 0.09,
|
||||
vision: true,
|
||||
},
|
||||
];
|
||||
cfg
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_registry_entries_with_placeholder_metrics() {
|
||||
let cfg = cfg_with_models();
|
||||
let outcome = model_health(&cfg).expect("enabled");
|
||||
let resp = &outcome.value;
|
||||
assert_eq!(resp.models.len(), 2);
|
||||
|
||||
let first = &resp.models[0];
|
||||
assert_eq!(first.id, "deepseek-v3.2");
|
||||
assert_eq!(first.provider, "SiliconFlow");
|
||||
assert!((first.cost_per_1m_output - 0.33).abs() < f64::EPSILON);
|
||||
assert!(!first.vision);
|
||||
// Placeholder telemetry fields — assert the contract.
|
||||
assert!(first.quality_score.is_none());
|
||||
assert!(first.hallucination_rate.is_none());
|
||||
assert_eq!(first.agents_using, 0);
|
||||
assert_eq!(first.tasks_evaluated, 0);
|
||||
|
||||
let second = &resp.models[1];
|
||||
assert_eq!(second.id, "qwen-2.5-8b");
|
||||
assert!(second.vision);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surfaces_config_thresholds() {
|
||||
let cfg = cfg_with_models();
|
||||
let outcome = model_health(&cfg).expect("enabled");
|
||||
let resp = &outcome.value;
|
||||
assert!((resp.config.hallucination_threshold - 0.10).abs() < f64::EPSILON);
|
||||
assert_eq!(resp.config.min_tasks_for_rating, 10);
|
||||
assert_eq!(resp.config.evaluation_window_tasks, 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_feature_errors() {
|
||||
let mut cfg = cfg_with_models();
|
||||
cfg.dashboard.model_health.enabled = false;
|
||||
let err = model_health(&cfg).expect_err("disabled");
|
||||
assert!(err.contains("disabled"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_registry_returns_empty_models() {
|
||||
let mut cfg = Config::default();
|
||||
cfg.model_registry.clear();
|
||||
let outcome = model_health(&cfg).expect("enabled");
|
||||
assert!(outcome.value.models.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
//! Controller schemas + handlers for the dashboard domain.
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::core::all::{ControllerFuture, RegisteredController};
|
||||
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
|
||||
use crate::rpc::RpcOutcome;
|
||||
|
||||
use super::ops;
|
||||
use super::types::ModelHealthResponse;
|
||||
|
||||
pub fn all_dashboard_controller_schemas() -> Vec<ControllerSchema> {
|
||||
vec![dashboard_schemas("dashboard_model_health")]
|
||||
}
|
||||
|
||||
pub fn all_dashboard_registered_controllers() -> Vec<RegisteredController> {
|
||||
vec![RegisteredController {
|
||||
schema: dashboard_schemas("dashboard_model_health"),
|
||||
handler: handle_dashboard_model_health,
|
||||
}]
|
||||
}
|
||||
|
||||
pub fn dashboard_schemas(function: &str) -> ControllerSchema {
|
||||
match function {
|
||||
"dashboard_model_health" => ControllerSchema {
|
||||
namespace: "dashboard",
|
||||
function: "model_health",
|
||||
description: "Per-model health comparison rows for the desktop dashboard panel \
|
||||
— joins local model_registry with dashboard.model_health thresholds. \
|
||||
Telemetry-driven metric fields are placeholders until a local \
|
||||
telemetry sink is wired in.",
|
||||
inputs: vec![],
|
||||
outputs: vec![
|
||||
FieldSchema {
|
||||
name: "models",
|
||||
ty: TypeSchema::Array(Box::new(TypeSchema::Object {
|
||||
fields: model_health_entry_fields(),
|
||||
})),
|
||||
comment: "Per-model rows: id, provider, cost_per_1m_output, vision, \
|
||||
quality_score, hallucination_rate, agents_using, tasks_evaluated.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "config",
|
||||
ty: TypeSchema::Object {
|
||||
fields: model_health_config_fields(),
|
||||
},
|
||||
comment: "Threshold view: hallucination_threshold, min_tasks_for_rating, \
|
||||
evaluation_window_tasks.",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
_ => ControllerSchema {
|
||||
namespace: "dashboard",
|
||||
function: "unknown",
|
||||
description: "Unknown dashboard controller.",
|
||||
inputs: vec![],
|
||||
outputs: vec![FieldSchema {
|
||||
name: "error",
|
||||
ty: TypeSchema::String,
|
||||
comment: "Lookup error details.",
|
||||
required: true,
|
||||
}],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn model_health_entry_fields() -> Vec<FieldSchema> {
|
||||
vec![
|
||||
FieldSchema {
|
||||
name: "id",
|
||||
ty: TypeSchema::String,
|
||||
comment: "Model identifier as configured in `model_registry`.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "provider",
|
||||
ty: TypeSchema::String,
|
||||
comment: "Provider label, e.g. SiliconFlow, OpenRouter.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "cost_per_1m_output",
|
||||
ty: TypeSchema::F64,
|
||||
comment: "USD cost per 1M output tokens from local config.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "vision",
|
||||
ty: TypeSchema::Bool,
|
||||
comment: "True when the model accepts image inputs.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "quality_score",
|
||||
ty: TypeSchema::Option(Box::new(TypeSchema::F64)),
|
||||
comment: "Per-model quality score (placeholder — null until telemetry lands).",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "hallucination_rate",
|
||||
ty: TypeSchema::Option(Box::new(TypeSchema::F64)),
|
||||
comment: "Observed hallucination rate (placeholder — null until telemetry lands).",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "agents_using",
|
||||
ty: TypeSchema::U64,
|
||||
comment: "Number of agents bound to this model (placeholder — 0 until wired).",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "tasks_evaluated",
|
||||
ty: TypeSchema::U64,
|
||||
comment: "Tasks evaluated against this model (placeholder — 0 until wired).",
|
||||
required: true,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn model_health_config_fields() -> Vec<FieldSchema> {
|
||||
vec![
|
||||
FieldSchema {
|
||||
name: "hallucination_threshold",
|
||||
ty: TypeSchema::F64,
|
||||
comment: "Rate above which a model is flagged `replace`.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "min_tasks_for_rating",
|
||||
ty: TypeSchema::U64,
|
||||
comment: "Minimum tasks evaluated before quality/hallucination are considered.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "evaluation_window_tasks",
|
||||
ty: TypeSchema::U64,
|
||||
comment: "Sliding window size used when computing telemetry metrics.",
|
||||
required: true,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn handle_dashboard_model_health(_params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move {
|
||||
log::debug!("[dashboard] model_health request received");
|
||||
let cfg = crate::openhuman::config::rpc::load_config_with_timeout()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
log::warn!("[dashboard] model_health failed to load config: {err}");
|
||||
format!("config unavailable: {err}")
|
||||
})?;
|
||||
let outcome: RpcOutcome<ModelHealthResponse> = ops::model_health(&cfg)?;
|
||||
outcome.into_cli_compatible_json()
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn schema_namespace_and_function_are_stable() {
|
||||
let s = dashboard_schemas("dashboard_model_health");
|
||||
assert_eq!(s.namespace, "dashboard");
|
||||
assert_eq!(s.function, "model_health");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn controller_lists_match_lengths() {
|
||||
assert_eq!(
|
||||
all_dashboard_controller_schemas().len(),
|
||||
all_dashboard_registered_controllers().len()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
//! Wire types for the dashboard model health view.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// One row in the model health comparison table.
|
||||
///
|
||||
/// Mirrors the JSON shape consumed by the frontend
|
||||
/// `ModelHealthPanel` — `id`/`provider`/`cost_per_1m_output`/`vision`
|
||||
/// come from `Config::model_registry`; the four metric fields are
|
||||
/// emitted as placeholder (`null` / `0`) values until a local telemetry
|
||||
/// pipeline is wired in.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct ModelHealthEntry {
|
||||
pub id: String,
|
||||
pub provider: String,
|
||||
pub cost_per_1m_output: f64,
|
||||
pub vision: bool,
|
||||
pub quality_score: Option<f64>,
|
||||
pub hallucination_rate: Option<f64>,
|
||||
pub agents_using: u32,
|
||||
pub tasks_evaluated: u32,
|
||||
}
|
||||
|
||||
/// Thresholds the frontend needs to compute the status badge.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct ModelHealthConfigView {
|
||||
pub hallucination_threshold: f64,
|
||||
pub min_tasks_for_rating: usize,
|
||||
pub evaluation_window_tasks: usize,
|
||||
}
|
||||
|
||||
/// `openhuman.dashboard_model_health` RPC response.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct ModelHealthResponse {
|
||||
pub models: Vec<ModelHealthEntry>,
|
||||
pub config: ModelHealthConfigView,
|
||||
}
|
||||
@@ -33,6 +33,7 @@ pub mod cost;
|
||||
pub mod credentials;
|
||||
pub mod cron;
|
||||
pub mod cwd_jail;
|
||||
pub mod dashboard;
|
||||
pub mod desktop_companion;
|
||||
pub mod dev_paths;
|
||||
pub mod devices;
|
||||
|
||||
Reference in New Issue
Block a user