mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-29 14:02:19 +00:00
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
9b85a64b58
commit
f891fb1508
@@ -81,7 +81,12 @@ export default function ChatComposer({
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept={allowedMimeTypes.join(',')}
|
||||
// No `accept` filter: Chromium 146 / CEF on macOS greys out valid files
|
||||
// at the native open panel regardless of the filter shape (MIME, mixed,
|
||||
// or extension-only). Selection is gated in `onAttachFiles` →
|
||||
// `validateAndReadFile` instead (rejects unsupported types + images on
|
||||
// non-vision models). `allowedMimeTypes` is kept for that JS validation.
|
||||
accept={allowedMimeTypes.length ? allowedMimeTypes.join(',') : undefined}
|
||||
multiple
|
||||
className="hidden"
|
||||
onChange={e => {
|
||||
|
||||
@@ -26,11 +26,14 @@ import {
|
||||
loadLocalProviderSnapshot,
|
||||
type LocalProviderSnapshot,
|
||||
type ModelInfo,
|
||||
type ModelRegistryEntry,
|
||||
modelRegistryVision,
|
||||
OPENAI_CODEX_OAUTH_MISSING_AUTH_URL,
|
||||
OPENAI_CODEX_OAUTH_MISSING_CALLBACK_URL,
|
||||
saveAISettings,
|
||||
setCloudProviderKey,
|
||||
testProviderModel,
|
||||
upsertModelRegistryVision,
|
||||
} from '../../../services/api/aiSettingsApi';
|
||||
import {
|
||||
creditsApi,
|
||||
@@ -144,12 +147,17 @@ const BUILTIN_PROVIDER_META: Record<string, { tone: string; label: string }> = {
|
||||
};
|
||||
|
||||
const WORKLOADS: Workload[] = [
|
||||
{ id: 'chat', group: 'chat', label: 'Chat', description: 'Direct conversational back-and-forth' },
|
||||
{
|
||||
id: 'chat',
|
||||
group: 'chat',
|
||||
label: 'Chat',
|
||||
description: 'Direct conversational back-and-forth — “Quick” mode in Conversations',
|
||||
},
|
||||
{
|
||||
id: 'reasoning',
|
||||
group: 'chat',
|
||||
label: 'Reasoning',
|
||||
description: 'Main chat agent, meeting summarizer',
|
||||
description: 'Main chat agent, meeting summarizer — “Reasoning” mode in Conversations',
|
||||
},
|
||||
{
|
||||
id: 'agentic',
|
||||
@@ -217,7 +225,11 @@ const WORKLOAD_MODEL_HINTS: Record<WorkloadId, string> = {
|
||||
// just derives the `maskedKey` display string from `has_api_key`.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
type AISettings = { cloudProviders: CloudProvider[]; routing: RoutingMap };
|
||||
type AISettings = {
|
||||
cloudProviders: CloudProvider[];
|
||||
routing: RoutingMap;
|
||||
modelRegistry: ModelRegistryEntry[];
|
||||
};
|
||||
|
||||
const EMPTY_ROUTING: RoutingMap = {
|
||||
chat: { kind: 'default' },
|
||||
@@ -230,7 +242,11 @@ const EMPTY_ROUTING: RoutingMap = {
|
||||
subconscious: { kind: 'default' },
|
||||
};
|
||||
|
||||
const EMPTY_SETTINGS: AISettings = { cloudProviders: [], routing: EMPTY_ROUTING };
|
||||
const EMPTY_SETTINGS: AISettings = {
|
||||
cloudProviders: [],
|
||||
routing: EMPTY_ROUTING,
|
||||
modelRegistry: [],
|
||||
};
|
||||
|
||||
function maskKeyLabel(hasKey: boolean): string {
|
||||
return hasKey ? '•••• configured' : 'Not configured';
|
||||
@@ -280,7 +296,7 @@ function toPanelRoutingFromApi(api: ApiAISettings): { panel: AISettings } {
|
||||
learning: liftRef(api.routing.learning),
|
||||
subconscious: liftRef(api.routing.subconscious),
|
||||
};
|
||||
return { panel: { cloudProviders, routing } };
|
||||
return { panel: { cloudProviders, routing, modelRegistry: api.modelRegistry } };
|
||||
}
|
||||
|
||||
function toApiSettings(panel: AISettings): ApiAISettings {
|
||||
@@ -303,6 +319,7 @@ function toApiSettings(panel: AISettings): ApiAISettings {
|
||||
learning: panel.routing.learning,
|
||||
subconscious: panel.routing.subconscious,
|
||||
},
|
||||
modelRegistry: panel.modelRegistry,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1725,8 +1742,11 @@ interface CustomRoutingDialogProps {
|
||||
cloudProviders: CloudProvider[];
|
||||
localModels: OllamaModel[];
|
||||
ollamaRunning: boolean;
|
||||
/** Current per-model vision registry, used to prefill the vision checkbox. */
|
||||
modelRegistry: ModelRegistryEntry[];
|
||||
onClose: () => void;
|
||||
onSubmit: (next: ProviderRef) => void;
|
||||
/** Emits the chosen provider ref plus the user's vision flag for that model. */
|
||||
onSubmit: (next: ProviderRef, vision: boolean) => void;
|
||||
}
|
||||
|
||||
type CustomDialogSource =
|
||||
@@ -1811,6 +1831,7 @@ const CustomRoutingDialog = ({
|
||||
cloudProviders,
|
||||
localModels,
|
||||
ollamaRunning,
|
||||
modelRegistry,
|
||||
onClose,
|
||||
onSubmit,
|
||||
}: CustomRoutingDialogProps) => {
|
||||
@@ -1862,6 +1883,35 @@ const CustomRoutingDialog = ({
|
||||
: null
|
||||
);
|
||||
|
||||
// Registry slug for the selected source — keys the per-model vision flag.
|
||||
// Cloud uses the provider slug; local → `ollama`; claude-code → `claude-code`.
|
||||
const registrySlug =
|
||||
source?.kind === 'cloud'
|
||||
? source.providerSlug
|
||||
: source?.kind === 'local'
|
||||
? 'ollama'
|
||||
: source?.kind === 'claude-code'
|
||||
? 'claude-code'
|
||||
: null;
|
||||
|
||||
// User-set vision flag for this (provider, model). Prefilled from the registry,
|
||||
// re-prefilled whenever the selected provider/model changes.
|
||||
const [vision, setVision] = useState<boolean>(() =>
|
||||
registrySlug && model.trim()
|
||||
? modelRegistryVision(modelRegistry, registrySlug, model.trim())
|
||||
: false
|
||||
);
|
||||
useEffect(() => {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
setVision(
|
||||
registrySlug && model.trim()
|
||||
? modelRegistryVision(modelRegistry, registrySlug, model.trim())
|
||||
: false
|
||||
);
|
||||
// modelRegistry is stable for the dialog's lifetime (prop doesn't change mid-open).
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [registrySlug, model]);
|
||||
|
||||
const selectedCloud =
|
||||
source?.kind === 'cloud' ? customCloud.find(c => c.slug === source.providerSlug) : undefined;
|
||||
|
||||
@@ -1935,16 +1985,19 @@ const CustomRoutingDialog = ({
|
||||
if (!source || !canSave) return;
|
||||
const temp = temperature == null || !Number.isFinite(temperature) ? null : temperature;
|
||||
if (source.kind === 'cloud') {
|
||||
onSubmit({
|
||||
kind: 'cloud',
|
||||
providerSlug: source.providerSlug,
|
||||
model: model.trim(),
|
||||
temperature: temp,
|
||||
});
|
||||
onSubmit(
|
||||
{
|
||||
kind: 'cloud',
|
||||
providerSlug: source.providerSlug,
|
||||
model: model.trim(),
|
||||
temperature: temp,
|
||||
},
|
||||
vision
|
||||
);
|
||||
} else if (source.kind === 'claude-code') {
|
||||
onSubmit({ kind: 'claude-code', model: model.trim(), temperature: temp });
|
||||
onSubmit({ kind: 'claude-code', model: model.trim(), temperature: temp }, vision);
|
||||
} else {
|
||||
onSubmit({ kind: 'local', model: model.trim(), temperature: temp });
|
||||
onSubmit({ kind: 'local', model: model.trim(), temperature: temp }, vision);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2220,6 +2273,26 @@ const CustomRoutingDialog = ({
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Vision capability (optional). Marks a custom/BYOK model as
|
||||
accepting image input so the chat composer offers image
|
||||
attachments for it. Only shown once a concrete model is chosen. */}
|
||||
{registrySlug && model.trim().length > 0 && (
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<label className="inline-flex items-center gap-2 text-xs font-medium text-stone-700 dark:text-neutral-200">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={vision}
|
||||
onChange={e => setVision(e.target.checked)}
|
||||
className="h-3.5 w-3.5 rounded border-stone-300 dark:border-neutral-700 text-primary-500 focus:ring-primary-500"
|
||||
/>
|
||||
{t('settings.ai.modelVision')}
|
||||
</label>
|
||||
<p className="text-[11px] text-stone-400 dark:text-neutral-500">
|
||||
{t('settings.ai.modelVisionDesc')}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(testBusy || testReply || testError || testStartedAt) && (
|
||||
<div
|
||||
role={testError ? 'alert' : 'status'}
|
||||
@@ -2357,6 +2430,7 @@ const GlobalOwnModelSelector = ({
|
||||
cloudProviders,
|
||||
localModels,
|
||||
ollamaRunning,
|
||||
modelRegistry,
|
||||
onApply,
|
||||
}: {
|
||||
current: ProviderRef | null;
|
||||
@@ -2364,7 +2438,8 @@ const GlobalOwnModelSelector = ({
|
||||
cloudProviders: CloudProvider[];
|
||||
localModels: OllamaModel[];
|
||||
ollamaRunning: boolean;
|
||||
onApply: (next: ProviderRef) => Promise<void>;
|
||||
modelRegistry: ModelRegistryEntry[];
|
||||
onApply: (next: ProviderRef, vision: boolean) => Promise<void>;
|
||||
}) => {
|
||||
const { t } = useT();
|
||||
const customCloud = cloudProviders.filter(p => p.slug !== 'openhuman');
|
||||
@@ -2385,6 +2460,29 @@ const GlobalOwnModelSelector = ({
|
||||
const [model, setModel] = useState<string>(
|
||||
current?.kind === 'cloud' || current?.kind === 'local' ? current.model : ''
|
||||
);
|
||||
// Registry slug for the selected source — keys the per-model vision flag.
|
||||
const registrySlug =
|
||||
source?.kind === 'cloud'
|
||||
? source.providerSlug
|
||||
: source?.kind === 'local'
|
||||
? 'ollama'
|
||||
: source?.kind === 'claude-code'
|
||||
? 'claude-code'
|
||||
: null;
|
||||
const [vision, setVision] = useState<boolean>(() =>
|
||||
registrySlug && model.trim()
|
||||
? modelRegistryVision(modelRegistry, registrySlug, model.trim())
|
||||
: false
|
||||
);
|
||||
useEffect(() => {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
setVision(
|
||||
registrySlug && model.trim()
|
||||
? modelRegistryVision(modelRegistry, registrySlug, model.trim())
|
||||
: false
|
||||
);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [registrySlug, model]);
|
||||
const [cloudModels, setCloudModels] = useState<ModelInfo[]>([]);
|
||||
const [cloudModelsLoading, setCloudModelsLoading] = useState(false);
|
||||
const [cloudModelsError, setCloudModelsError] = useState<string | null>(null);
|
||||
@@ -2453,15 +2551,14 @@ const GlobalOwnModelSelector = ({
|
||||
setSaving(true);
|
||||
try {
|
||||
if (nextSource.kind === 'local') {
|
||||
await onApply({ kind: 'local', model: nextModel.trim() });
|
||||
await onApply({ kind: 'local', model: nextModel.trim() }, vision);
|
||||
} else if (nextSource.kind === 'claude-code') {
|
||||
await onApply({ kind: 'claude-code', model: nextModel.trim() });
|
||||
await onApply({ kind: 'claude-code', model: nextModel.trim() }, vision);
|
||||
} else {
|
||||
await onApply({
|
||||
kind: 'cloud',
|
||||
providerSlug: nextSource.providerSlug,
|
||||
model: nextModel.trim(),
|
||||
});
|
||||
await onApply(
|
||||
{ kind: 'cloud', providerSlug: nextSource.providerSlug, model: nextModel.trim() },
|
||||
vision
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
setSaving(false);
|
||||
@@ -2566,6 +2663,23 @@ const GlobalOwnModelSelector = ({
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
{registrySlug && model.trim().length > 0 && (
|
||||
<label className="flex items-start gap-2 text-xs font-medium text-stone-700 dark:text-neutral-200">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={vision}
|
||||
onChange={e => setVision(e.target.checked)}
|
||||
className="mt-0.5 h-3.5 w-3.5 rounded border-stone-300 dark:border-neutral-700 text-primary-500 focus:ring-primary-500"
|
||||
/>
|
||||
<span>
|
||||
{t('settings.ai.modelVision')}
|
||||
<span className="block font-normal text-[11px] text-stone-400 dark:text-neutral-500">
|
||||
{t('settings.ai.modelVisionDesc')}
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
)}
|
||||
|
||||
<div className="rounded-lg bg-stone-50 dark:bg-neutral-800/60 px-3 py-2 text-xs text-stone-500 dark:text-neutral-400">
|
||||
{t('settings.ai.globalModel.appliesToAll')}
|
||||
</div>
|
||||
@@ -3103,8 +3217,23 @@ const AIPanel = ({ embedded = false }: AIPanelProps = {}) => {
|
||||
cloudProviders={draft.cloudProviders}
|
||||
localModels={installed}
|
||||
ollamaRunning={ollama.state === 'running'}
|
||||
onApply={async next => {
|
||||
await persist({ ...draft, routing: routingWithAllWorkloads(next) });
|
||||
modelRegistry={draft.modelRegistry}
|
||||
onApply={async (next, vision) => {
|
||||
const reg =
|
||||
next.kind === 'cloud'
|
||||
? { slug: next.providerSlug, model: next.model }
|
||||
: next.kind === 'local'
|
||||
? { slug: 'ollama', model: next.model }
|
||||
: next.kind === 'claude-code'
|
||||
? { slug: 'claude-code', model: next.model }
|
||||
: null;
|
||||
await persist({
|
||||
...draft,
|
||||
routing: routingWithAllWorkloads(next),
|
||||
modelRegistry: reg
|
||||
? upsertModelRegistryVision(draft.modelRegistry, reg.slug, reg.model, vision)
|
||||
: draft.modelRegistry,
|
||||
});
|
||||
}}
|
||||
/>
|
||||
) : null}
|
||||
@@ -3290,11 +3419,24 @@ const AIPanel = ({ embedded = false }: AIPanelProps = {}) => {
|
||||
cloudProviders={draft.cloudProviders}
|
||||
localModels={installed}
|
||||
ollamaRunning={ollama.state === 'running'}
|
||||
modelRegistry={draft.modelRegistry}
|
||||
onClose={() => setCustomDialogFor(null)}
|
||||
onSubmit={async next => {
|
||||
onSubmit={async (next, vision) => {
|
||||
// (provider slug, model id) the vision flag keys on.
|
||||
const reg =
|
||||
next.kind === 'cloud'
|
||||
? { slug: next.providerSlug, model: next.model }
|
||||
: next.kind === 'local'
|
||||
? { slug: 'ollama', model: next.model }
|
||||
: next.kind === 'claude-code'
|
||||
? { slug: 'claude-code', model: next.model }
|
||||
: null;
|
||||
const nextDraft = {
|
||||
...draft,
|
||||
routing: { ...draft.routing, [customDialogFor]: next },
|
||||
modelRegistry: reg
|
||||
? upsertModelRegistryVision(draft.modelRegistry, reg.slug, reg.model, vision)
|
||||
: draft.modelRegistry,
|
||||
};
|
||||
await persist(nextDraft);
|
||||
setCustomDialogFor(null);
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
setCloudProviderKey,
|
||||
startOpenAiCodexOAuth,
|
||||
testProviderModel,
|
||||
upsertModelRegistryVision,
|
||||
} from '../../../../services/api/aiSettingsApi';
|
||||
import { creditsApi } from '../../../../services/api/creditsApi';
|
||||
import { renderWithProviders } from '../../../../test/test-utils';
|
||||
@@ -45,6 +46,8 @@ vi.mock('../../../../services/api/aiSettingsApi', () => ({
|
||||
saveAISettings: vi.fn(),
|
||||
loadLocalProviderSnapshot: vi.fn(),
|
||||
testProviderModel: vi.fn(),
|
||||
modelRegistryVision: vi.fn(() => false),
|
||||
upsertModelRegistryVision: vi.fn((registry: unknown[]) => registry),
|
||||
setCloudProviderKey: vi.fn().mockResolvedValue(undefined),
|
||||
clearCloudProviderKey: vi.fn().mockResolvedValue(undefined),
|
||||
serializeProviderRef: vi.fn((r: { kind: string; providerSlug?: string; model?: string }) =>
|
||||
@@ -123,6 +126,7 @@ const baseSettings = {
|
||||
learning: { kind: 'openhuman' as const },
|
||||
subconscious: { kind: 'openhuman' as const },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
|
||||
const baseLocalSnapshot = { status: null, diagnostics: null, presets: null, installedModels: [] };
|
||||
@@ -304,6 +308,52 @@ describe('AIPanel', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// ─── per-model vision flag (BYOK) ───────────────────────────────────────────
|
||||
|
||||
it('flags a custom BYOK model as vision-capable via the Own-model selector', async () => {
|
||||
vi.mocked(loadAISettings).mockResolvedValue({
|
||||
...baseSettings,
|
||||
cloudProviders: [
|
||||
...baseSettings.cloudProviders,
|
||||
{
|
||||
id: 'p_custom_openai',
|
||||
slug: 'openai',
|
||||
label: 'OpenAI',
|
||||
endpoint: 'https://api.openai.com/v1',
|
||||
auth_style: 'bearer' as const,
|
||||
has_api_key: true,
|
||||
},
|
||||
],
|
||||
});
|
||||
renderWithProviders(<AIPanel />);
|
||||
await waitFor(() =>
|
||||
expect(screen.getByRole('button', { name: /Use Your Own Models/i })).toBeInTheDocument()
|
||||
);
|
||||
fireEvent.click(screen.getByRole('button', { name: /Use Your Own Models/i }));
|
||||
|
||||
// Enter a model id → the per-model "Supports vision" checkbox appears.
|
||||
const modelInput = await screen.findByPlaceholderText('Enter model id');
|
||||
fireEvent.change(modelInput, { target: { value: 'gpt-4o' } });
|
||||
|
||||
const visionCheckbox = await screen.findByRole('checkbox', { name: /Supports vision/i });
|
||||
expect(visionCheckbox).not.toBeChecked();
|
||||
fireEvent.click(visionCheckbox);
|
||||
expect(visionCheckbox).toBeChecked();
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: /^Save$/ }));
|
||||
|
||||
// The vision flag is threaded through to the registry upsert + persisted.
|
||||
await waitFor(() =>
|
||||
expect(vi.mocked(upsertModelRegistryVision)).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'openai',
|
||||
'gpt-4o',
|
||||
true
|
||||
)
|
||||
);
|
||||
expect(saveAISettings).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ─── auth_style preservation ────────────────────────────────────────────────
|
||||
|
||||
it('preserves auth_style: "anthropic" through save when Anthropic provider is configured', async () => {
|
||||
@@ -333,6 +383,7 @@ describe('AIPanel', () => {
|
||||
learning: { kind: 'openhuman' as const },
|
||||
subconscious: { kind: 'openhuman' as const },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithAnthropic);
|
||||
@@ -557,6 +608,7 @@ describe('AIPanel', () => {
|
||||
learning: { kind: 'openhuman' as const },
|
||||
subconscious: { kind: 'openhuman' as const },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
|
||||
vi.mocked(saveAISettings).mockResolvedValue(undefined);
|
||||
@@ -612,6 +664,7 @@ describe('AIPanel', () => {
|
||||
learning: { kind: 'openhuman' as const },
|
||||
subconscious: { kind: 'openhuman' as const },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOllama);
|
||||
vi.mocked(saveAISettings).mockResolvedValue(undefined);
|
||||
@@ -986,6 +1039,7 @@ describe('AIPanel', () => {
|
||||
has_api_key: false,
|
||||
},
|
||||
],
|
||||
modelRegistry: [],
|
||||
};
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOllama);
|
||||
renderWithProviders(<AIPanel />);
|
||||
@@ -1008,6 +1062,7 @@ describe('AIPanel', () => {
|
||||
has_api_key: false,
|
||||
},
|
||||
],
|
||||
modelRegistry: [],
|
||||
};
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOllama);
|
||||
renderWithProviders(<AIPanel />);
|
||||
@@ -1047,6 +1102,7 @@ describe('AIPanel', () => {
|
||||
...baseSettings.routing,
|
||||
reasoning: { kind: 'cloud' as const, providerSlug: 'openai', model: 'gpt-4o' },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
|
||||
vi.mocked(saveAISettings).mockResolvedValue(undefined);
|
||||
@@ -1103,6 +1159,7 @@ describe('AIPanel', () => {
|
||||
...baseSettings.routing,
|
||||
reasoning: { kind: 'cloud' as const, providerSlug: 'openai', model: 'gpt-4o' },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
|
||||
vi.mocked(listProviderModels).mockResolvedValue([{ id: 'gpt-4o' }, { id: 'gpt-4o-mini' }]);
|
||||
@@ -1146,6 +1203,7 @@ describe('AIPanel', () => {
|
||||
...baseSettings.routing,
|
||||
reasoning: { kind: 'cloud' as const, providerSlug: 'openai', model: 'gpt-4o' },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
|
||||
vi.mocked(listProviderModels).mockResolvedValue([{ id: 'gpt-4o' }]);
|
||||
@@ -1190,6 +1248,7 @@ describe('AIPanel', () => {
|
||||
...baseSettings.routing,
|
||||
reasoning: { kind: 'cloud' as const, providerSlug: 'openai', model: 'gpt-4o' },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
|
||||
vi.mocked(listProviderModels).mockResolvedValue([{ id: 'gpt-4o' }]);
|
||||
|
||||
@@ -85,6 +85,40 @@ describe('validateAndReadFile', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects non-extractable documents (docx/pptx/xlsx) the agent cannot read', async () => {
|
||||
for (const mime of [
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'application/zip',
|
||||
]) {
|
||||
const result = await validateAndReadFile(makeFile('f', mime, 8), 0);
|
||||
expect('error' in result && result.error.code).toBe('unsupported_type');
|
||||
}
|
||||
});
|
||||
|
||||
it('accepts the text-extractable document formats (pdf/txt/markdown)', async () => {
|
||||
for (const mime of ['application/pdf', 'text/plain', 'text/markdown']) {
|
||||
const result = await validateAndReadFile(makeFile('f', mime, 8), 0);
|
||||
expect('attachment' in result && result.attachment.kind === 'file').toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects an image when allowImages is false (non-vision model)', async () => {
|
||||
const result = await validateAndReadFile(makeFile('p.png', 'image/png', 8), 0, 0, false);
|
||||
expect('error' in result && result.error.code).toBe('image_not_supported');
|
||||
});
|
||||
|
||||
it('still accepts a document when allowImages is false', async () => {
|
||||
const result = await validateAndReadFile(makeFile('d.pdf', 'application/pdf', 8), 0, 0, false);
|
||||
expect('attachment' in result && result.attachment.kind === 'file').toBe(true);
|
||||
});
|
||||
|
||||
it('accepts an image when allowImages is true (vision model)', async () => {
|
||||
const result = await validateAndReadFile(makeFile('p.png', 'image/png', 8), 0, 0, true);
|
||||
expect('attachment' in result && result.attachment.kind === 'image').toBe(true);
|
||||
});
|
||||
|
||||
it('rejects files that exceed the file size limit', async () => {
|
||||
const oversizedFile = makeFile(
|
||||
'big.pdf',
|
||||
|
||||
@@ -41,6 +41,17 @@ export const ALLOWED_ATTACHMENT_MIME_TYPES = [
|
||||
...ALLOWED_FILE_MIME_TYPES,
|
||||
] as const;
|
||||
|
||||
// Document formats the backend actually text-extracts (PDF via pdf_extract;
|
||||
// TXT/Markdown via UTF-8). DOCX/PPTX/XLSX/ZIP are intentionally excluded — the
|
||||
// agent would only see a reference stub, not their content. `text/csv` is also
|
||||
// deliberately left out: the backend *can* extract it, but the chat composer is
|
||||
// scoped to PDF/TXT/Markdown by product decision (revisit here if CSV is wanted).
|
||||
// Used by the ingest validator below, not by a native `accept` filter:
|
||||
// Chromium/CEF on macOS greys valid files at the open panel regardless of the
|
||||
// filter shape, so selection is gated in `validateAndReadFile` after the user
|
||||
// picks, not at the dialog.
|
||||
const EXTRACTABLE_FILE_MIME_TYPES = ['application/pdf', 'text/plain', 'text/markdown'] as const;
|
||||
|
||||
export const ATTACHMENT_MAX_IMAGES = 4;
|
||||
export const ATTACHMENT_MAX_FILES = 4;
|
||||
export const ATTACHMENT_MAX_IMAGE_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
|
||||
@@ -63,6 +74,7 @@ export type AttachmentError =
|
||||
| { code: 'unsupported_type'; mimeType: string }
|
||||
| { code: 'too_large'; sizeBytes: number; maxBytes: number }
|
||||
| { code: 'too_many'; kind: AttachmentKind; max: number }
|
||||
| { code: 'image_not_supported' }
|
||||
| { code: 'read_failed'; reason: string };
|
||||
|
||||
export function isAllowedMimeType(mime: string): mime is AllowedImageMimeType {
|
||||
@@ -73,6 +85,29 @@ export function isAllowedAttachmentMimeType(mime: string): mime is AllowedAttach
|
||||
return (ALLOWED_ATTACHMENT_MIME_TYPES as readonly string[]).includes(mime);
|
||||
}
|
||||
|
||||
/**
|
||||
* The exact MIME set the ingest validator accepts — images plus the
|
||||
* text-extractable documents. A strict subset of {@link AllowedAttachmentMimeType}
|
||||
* (which also lists reference-only types like CSV/DOCX/ZIP that we reject).
|
||||
*/
|
||||
export type SupportedAttachmentMimeType =
|
||||
| AllowedImageMimeType
|
||||
| (typeof EXTRACTABLE_FILE_MIME_TYPES)[number];
|
||||
|
||||
/**
|
||||
* Stricter gate than {@link isAllowedAttachmentMimeType}: only the formats the
|
||||
* backend actually reads — images, plus the text-extractable documents (PDF via
|
||||
* pdf_extract; TXT/Markdown via UTF-8). DOCX/PPTX/XLSX/ZIP are excluded so they
|
||||
* can't be attached as content-less reference stubs. Applied on every ingest
|
||||
* path (picker, drag-drop, paste).
|
||||
*/
|
||||
export function isSupportedAttachmentMimeType(mime: string): mime is SupportedAttachmentMimeType {
|
||||
return (
|
||||
(ALLOWED_IMAGE_MIME_TYPES as readonly string[]).includes(mime) ||
|
||||
(EXTRACTABLE_FILE_MIME_TYPES as readonly string[]).includes(mime)
|
||||
);
|
||||
}
|
||||
|
||||
export function attachmentKindForMime(mime: AllowedAttachmentMimeType): AttachmentKind {
|
||||
return isAllowedMimeType(mime) ? 'image' : 'file';
|
||||
}
|
||||
@@ -148,13 +183,20 @@ async function buildAttachmentDataUri(
|
||||
export async function validateAndReadFile(
|
||||
file: File,
|
||||
existingCount: number,
|
||||
existingFileCount = 0
|
||||
existingFileCount = 0,
|
||||
// When `false` (the active chat model isn't vision-capable), image files are
|
||||
// rejected; documents (PDF/Word/etc.) still flow. Defaults `true` so non-chat
|
||||
// callers are unaffected.
|
||||
allowImages = true
|
||||
): Promise<{ attachment: Attachment } | { error: AttachmentError }> {
|
||||
if (!isAllowedAttachmentMimeType(file.type)) {
|
||||
if (!isSupportedAttachmentMimeType(file.type)) {
|
||||
return { error: { code: 'unsupported_type', mimeType: file.type || 'unknown' } };
|
||||
}
|
||||
|
||||
const kind = attachmentKindForMime(file.type);
|
||||
if (!allowImages && kind === 'image') {
|
||||
return { error: { code: 'image_not_supported' } };
|
||||
}
|
||||
const maxCount = kind === 'image' ? ATTACHMENT_MAX_IMAGES : ATTACHMENT_MAX_FILES;
|
||||
const count = kind === 'image' ? existingCount : existingFileCount;
|
||||
if (count >= maxCount) {
|
||||
|
||||
@@ -1822,7 +1822,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooMany': 'الحد الأقصى {max} صور لكل رسالة',
|
||||
'chat.attachment.tooManyFiles': 'الحد الأقصى {max} ملفات لكل رسالة',
|
||||
'chat.attachment.tooLarge': 'حجم الصورة يتجاوز الحد المسموح {max}',
|
||||
'chat.attachment.unsupportedType': 'نوع ملف غير مدعوم. استخدم PNG أو JPEG أو WebP أو GIF أو BMP.',
|
||||
'chat.attachment.unsupportedType':
|
||||
'نوع ملف غير مدعوم. استخدم صورة (PNG أو JPEG أو WebP أو GIF أو BMP) أو ملف PDF أو TXT أو Markdown.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'لا يمكن لهذا النموذج قراءة الصور. أرفق ملف PDF أو TXT أو Markdown بدلاً من ذلك.',
|
||||
'chat.attachment.readFailed': 'تعذر قراءة الملف',
|
||||
'memory.searchAria': 'البحث في الذاكرة',
|
||||
'memory.searchPlaceholder': 'البحث في إدخالات الذاكرة...',
|
||||
@@ -3046,6 +3049,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'تجاوز درجة الحرارة (القيمة)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'أقل = أكثر تحديدا. إتركْ غير مُمْكَنَّلَ لإسْتِعْمال قَبْل المُقدّمِ.',
|
||||
'settings.ai.modelVision': 'يدعم الرؤية (إدخال الصور)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'فعّل هذا إذا كان النموذج يقبل الصور. يتيح لمربع الدردشة إرفاق الصور عند اختيار هذا النموذج.',
|
||||
'settings.ai.testFailed': 'فشل الاختبار',
|
||||
'settings.ai.testingModel': 'نموذج الاختبار...',
|
||||
'settings.ai.modelResponse': 'استجابة النموذج',
|
||||
|
||||
@@ -1865,7 +1865,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': 'প্রতি বার্তায় সর্বোচ্চ {max}টি ফাইল',
|
||||
'chat.attachment.tooLarge': 'ছবি {max} আকারের সীমা অতিক্রম করেছে',
|
||||
'chat.attachment.unsupportedType':
|
||||
'অসমর্থিত ফাইল প্রকার। PNG, JPEG, WebP, GIF, বা BMP ব্যবহার করুন।',
|
||||
'অসমর্থিত ফাইল প্রকার। একটি ছবি (PNG, JPEG, WebP, GIF, BMP) অথবা একটি PDF, TXT, বা Markdown ফাইল ব্যবহার করুন।',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'এই মডেলটি ছবি পড়তে পারে না। পরিবর্তে একটি PDF, TXT, বা Markdown ফাইল সংযুক্ত করুন।',
|
||||
'chat.attachment.readFailed': 'ফাইল পড়া যায়নি',
|
||||
'memory.searchAria': 'মেমোরি খুঁজুন',
|
||||
'memory.searchPlaceholder': 'মেমোরি এন্ট্রি খুঁজুন...',
|
||||
@@ -3112,6 +3114,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'তাপমাত্রা ওভাররাইড (মান)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'নিচে = আরও বুদ্ধিমান. পরিসেবা উপলব্ধকারীর ডিফল্ট ব্যবহারের জন্য সীমা ধার্য না করা হলে, বন্ধ করুন ।',
|
||||
'settings.ai.modelVision': 'ভিশন সমর্থন করে (ছবি ইনপুট)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'এই মডেল ছবি গ্রহণ করলে সক্ষম করুন। এই মডেল নির্বাচিত থাকলে চ্যাটে ছবি সংযুক্ত করতে দেয়।',
|
||||
'settings.ai.testFailed': 'পরীক্ষা ব্যর্থ হয়েছে',
|
||||
'settings.ai.testingModel': 'পরীক্ষার মডেল...',
|
||||
'settings.ai.modelResponse': 'মডেল প্রতিক্রিয়া',
|
||||
|
||||
@@ -1912,7 +1912,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': 'Maximal {max} Dateien pro Nachricht',
|
||||
'chat.attachment.tooLarge': 'Bild überschreitet die Größenbeschränkung von {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Nicht unterstützter Dateityp. Verwenden Sie PNG, JPEG, WebP, GIF oder BMP.',
|
||||
'Nicht unterstützter Dateityp. Verwenden Sie ein Bild (PNG, JPEG, WebP, GIF, BMP) oder eine PDF-, TXT- oder Markdown-Datei.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'Dieses Modell kann keine Bilder lesen. Hängen Sie stattdessen eine PDF-, TXT- oder Markdown-Datei an.',
|
||||
'chat.attachment.readFailed': 'Datei konnte nicht gelesen werden',
|
||||
'memory.searchAria': 'Speicher durchsuchen',
|
||||
'memory.searchPlaceholder': 'Speichereinträge durchsuchen...',
|
||||
@@ -3191,6 +3193,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Temperatur-Override (Wert)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Niedriger = deterministischer. Deaktivieren Sie diese Option, um den Anbieterstandard zu verwenden.',
|
||||
'settings.ai.modelVision': 'Unterstützt Bildeingabe (Vision)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Aktivieren, wenn dieses Modell Bilder akzeptiert. Ermöglicht das Anhängen von Bildern im Chat, wenn dieses Modell ausgewählt ist.',
|
||||
'settings.ai.testFailed': 'Test fehlgeschlagen.',
|
||||
'settings.ai.testingModel': 'Modell wird getestet...',
|
||||
'settings.ai.modelResponse': 'Modellantwort',
|
||||
|
||||
@@ -2238,7 +2238,10 @@ const en: TranslationMap = {
|
||||
'chat.attachment.tooMany': 'Maximum {max} images per message',
|
||||
'chat.attachment.tooManyFiles': 'Maximum {max} files per message',
|
||||
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
|
||||
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Unsupported file type. Use an image (PNG, JPEG, WebP, GIF, BMP) or a PDF, TXT, or Markdown file.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'This model can’t read images. You can attach a pdf, txt, or md file.',
|
||||
'chat.attachment.readFailed': 'Could not read file',
|
||||
|
||||
// Memory (additional)
|
||||
@@ -3633,6 +3636,9 @@ const en: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Temperature override (value)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Lower = more deterministic. Leave unchecked to use the provider default.',
|
||||
'settings.ai.modelVision': 'Supports vision (image input)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Enable if this model accepts images. Lets the chat composer attach images when this model is selected.',
|
||||
'settings.ai.testFailed': 'Test failed',
|
||||
'settings.ai.testingModel': 'Testing model...',
|
||||
'settings.ai.modelResponse': 'Model response',
|
||||
|
||||
@@ -1904,7 +1904,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': 'Máximo {max} archivos por mensaje',
|
||||
'chat.attachment.tooLarge': 'La imagen supera el límite de tamaño de {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Tipo de archivo no compatible. Use PNG, JPEG, WebP, GIF o BMP.',
|
||||
'Tipo de archivo no compatible. Use una imagen (PNG, JPEG, WebP, GIF, BMP) o un archivo PDF, TXT o Markdown.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'Este modelo no puede leer imágenes. Adjunta un archivo PDF, TXT o Markdown en su lugar.',
|
||||
'chat.attachment.readFailed': 'No se pudo leer el archivo',
|
||||
'memory.searchAria': 'Buscar en memoria',
|
||||
'memory.searchPlaceholder': 'Buscar entradas de memoria...',
|
||||
@@ -3173,6 +3175,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Anulación de temperatura (valor)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Más bajo = más determinista. Déjelo sin marcar para usar el valor predeterminado del proveedor.',
|
||||
'settings.ai.modelVision': 'Admite visión (entrada de imágenes)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Actívalo si este modelo acepta imágenes. Permite adjuntar imágenes en el chat cuando este modelo está seleccionado.',
|
||||
'settings.ai.testFailed': 'Prueba fallida',
|
||||
'settings.ai.testingModel': 'Modelo de prueba...',
|
||||
'settings.ai.modelResponse': 'Respuesta modelo',
|
||||
|
||||
@@ -1912,7 +1912,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': 'Maximum {max} fichiers par message',
|
||||
'chat.attachment.tooLarge': "L'image dépasse la taille limite de {max}",
|
||||
'chat.attachment.unsupportedType':
|
||||
'Type de fichier non pris en charge. Utilisez PNG, JPEG, WebP, GIF ou BMP.',
|
||||
'Type de fichier non pris en charge. Utilise une image (PNG, JPEG, WebP, GIF, BMP) ou un fichier PDF, TXT ou Markdown.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'Ce modèle ne peut pas lire les images. Joins plutôt un fichier PDF, TXT ou Markdown.',
|
||||
'chat.attachment.readFailed': 'Impossible de lire le fichier',
|
||||
'memory.searchAria': 'Rechercher dans la mémoire',
|
||||
'memory.searchPlaceholder': 'Rechercher des entrées de mémoire…',
|
||||
@@ -3184,6 +3186,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Override de température (valeur)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Plus bas = plus déterministe. Laissez décoché pour utiliser la valeur par défaut du fournisseur.',
|
||||
'settings.ai.modelVision': 'Prend en charge la vision (entrée d’images)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Activez si ce modèle accepte les images. Permet de joindre des images dans le chat lorsque ce modèle est sélectionné.',
|
||||
'settings.ai.testFailed': 'Test échoué',
|
||||
'settings.ai.testingModel': 'Modèle de test...',
|
||||
'settings.ai.modelResponse': 'Réponse du modèle',
|
||||
|
||||
@@ -1864,7 +1864,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': 'प्रति संदेश अधिकतम {max} फ़ाइलें',
|
||||
'chat.attachment.tooLarge': 'छवि {max} आकार सीमा से अधिक है',
|
||||
'chat.attachment.unsupportedType':
|
||||
'असमर्थित फ़ाइल प्रकार। PNG, JPEG, WebP, GIF, या BMP का उपयोग करें।',
|
||||
'असमर्थित फ़ाइल प्रकार। कोई छवि (PNG, JPEG, WebP, GIF, BMP) या PDF, TXT, या Markdown फ़ाइल का उपयोग करें।',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'यह मॉडल छवियाँ नहीं पढ़ सकता। इसके बजाय कोई PDF, TXT या Markdown फ़ाइल संलग्न करें।',
|
||||
'chat.attachment.readFailed': 'फ़ाइल पढ़ नहीं सकी',
|
||||
'memory.searchAria': 'मेमोरी सर्च करें',
|
||||
'memory.searchPlaceholder': 'मेमोरी एंट्रीज़ सर्च करें...',
|
||||
@@ -3120,6 +3122,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'तापमान ओवरराइड (मान)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'लोअर = अधिक नियतिवादी। प्रदाता डिफ़ॉल्ट का उपयोग करने के लिए अनचेक छोड़ दें।',
|
||||
'settings.ai.modelVision': 'विज़न समर्थित है (छवि इनपुट)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'यदि यह मॉडल छवियाँ स्वीकार करता है तो सक्षम करें। यह मॉडल चुने जाने पर चैट में छवियाँ संलग्न करने देता है।',
|
||||
'settings.ai.testFailed': 'परीक्षण विफल रहा',
|
||||
'settings.ai.testingModel': 'परीक्षण मॉडल...',
|
||||
'settings.ai.modelResponse': 'मॉडल प्रतिक्रिया',
|
||||
|
||||
@@ -1867,7 +1867,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': 'Maksimal {max} file per pesan',
|
||||
'chat.attachment.tooLarge': 'Gambar melebihi batas ukuran {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Jenis file tidak didukung. Gunakan PNG, JPEG, WebP, GIF, atau BMP.',
|
||||
'Jenis file tidak didukung. Gunakan gambar (PNG, JPEG, WebP, GIF, BMP) atau file PDF, TXT, atau Markdown.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'Model ini tidak dapat membaca gambar. Lampirkan file PDF, TXT, atau Markdown sebagai gantinya.',
|
||||
'chat.attachment.readFailed': 'Tidak dapat membaca file',
|
||||
'memory.searchAria': 'Cari memori',
|
||||
'memory.searchPlaceholder': 'Cari entri memori...',
|
||||
@@ -3124,6 +3126,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Penggantian suhu (nilai)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Turunkan = lebih deterministik. Biarkan tak diperiksa untuk memakai penyedia bawaan.',
|
||||
'settings.ai.modelVision': 'Mendukung visi (input gambar)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Aktifkan jika model ini menerima gambar. Memungkinkan penyusun obrolan melampirkan gambar saat model ini dipilih.',
|
||||
'settings.ai.testFailed': 'Pengujian gagal',
|
||||
'settings.ai.testingModel': 'Pengujian model...',
|
||||
'settings.ai.modelResponse': 'Respons model',
|
||||
|
||||
@@ -1895,7 +1895,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooMany': 'Massimo {max} immagini per messaggio',
|
||||
'chat.attachment.tooManyFiles': 'Massimo {max} file per messaggio',
|
||||
'chat.attachment.tooLarge': "L'immagine supera il limite di dimensione di {max}",
|
||||
'chat.attachment.unsupportedType': 'Tipo di file non supportato. Usa PNG, JPEG, WebP, GIF o BMP.',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Tipo di file non supportato. Usa un file immagine (PNG, JPEG, WebP, GIF, BMP) oppure PDF, TXT o Markdown.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'Questo modello non può leggere le immagini. Allega invece un file PDF, TXT o Markdown.',
|
||||
'chat.attachment.readFailed': 'Impossibile leggere il file',
|
||||
'memory.searchAria': 'Cerca memoria',
|
||||
'memory.searchPlaceholder': 'Cerca voci di memoria...',
|
||||
@@ -3166,6 +3169,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Ignora temperatura (valore)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Più basso = più deterministico. Lascia deselezionato per usare il valore predefinito del provider.',
|
||||
'settings.ai.modelVision': 'Supporta la visione (input immagini)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Attiva se questo modello accetta immagini. Consente di allegare immagini nella chat quando è selezionato questo modello.',
|
||||
'settings.ai.testFailed': 'Test non riuscito',
|
||||
'settings.ai.testingModel': 'Test del modello...',
|
||||
'settings.ai.modelResponse': 'Risposta del modello',
|
||||
|
||||
@@ -1844,7 +1844,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': '메시지당 최대 {max}개 파일',
|
||||
'chat.attachment.tooLarge': '이미지가 {max} 크기 제한을 초과합니다',
|
||||
'chat.attachment.unsupportedType':
|
||||
'지원되지 않는 파일 형식입니다. PNG, JPEG, WebP, GIF 또는 BMP를 사용하세요.',
|
||||
'지원되지 않는 파일 형식입니다. 이미지(PNG, JPEG, WebP, GIF, BMP) 또는 PDF, TXT, Markdown 파일을 사용하세요.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'이 모델은 이미지를 읽을 수 없습니다. 대신 PDF, TXT 또는 Markdown 파일을 첨부하세요.',
|
||||
'chat.attachment.readFailed': '파일을 읽을 수 없습니다',
|
||||
'memory.searchAria': '메모리 검색',
|
||||
'memory.searchPlaceholder': '메모리 항목 검색...',
|
||||
@@ -3086,6 +3088,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': '온도 재정의(값)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'낮을수록 더 결정적입니다. 제공업체 기본값을 사용하려면 선택하지 않은 상태로 두세요.',
|
||||
'settings.ai.modelVision': '비전 지원 (이미지 입력)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'이 모델이 이미지를 지원하면 활성화하세요. 이 모델을 선택하면 채팅 작성기에서 이미지를 첨부할 수 있습니다.',
|
||||
'settings.ai.testFailed': '테스트 실패',
|
||||
'settings.ai.testingModel': '모델 테스트 중...',
|
||||
'settings.ai.modelResponse': '모델 응답',
|
||||
|
||||
@@ -1884,7 +1884,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooMany': 'Maksymalnie {max} obrazów na wiadomość',
|
||||
'chat.attachment.tooManyFiles': 'Maksymalnie {max} plików na wiadomość',
|
||||
'chat.attachment.tooLarge': 'Obraz przekracza limit rozmiaru {max}',
|
||||
'chat.attachment.unsupportedType': 'Nieobsługiwany typ pliku. Użyj PNG, JPEG, WebP, GIF lub BMP.',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Nieobsługiwany typ pliku. Użyj obrazu (PNG, JPEG, WebP, GIF, BMP) lub pliku PDF, TXT albo Markdown.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'Ten model nie może odczytywać obrazów. Zamiast tego dołącz plik PDF, TXT lub Markdown.',
|
||||
'chat.attachment.readFailed': 'Nie można odczytać pliku',
|
||||
'memory.searchAria': 'Szukaj w pamięci',
|
||||
'memory.searchPlaceholder': 'Szukaj wpisów pamięci...',
|
||||
@@ -3166,6 +3169,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Nadpisanie temperatury (wartość)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Niższa = bardziej deterministyczna. Pozostaw niezaznaczone, aby użyć domyślnej wartości dostawcy.',
|
||||
'settings.ai.modelVision': 'Obsługuje wizję (wejście obrazu)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Włącz, jeśli ten model akceptuje obrazy. Pozwala dołączać obrazy w czacie, gdy ten model jest wybrany.',
|
||||
'settings.ai.testFailed': 'Test nie powiódł się',
|
||||
'settings.ai.testingModel': 'Testowanie modelu...',
|
||||
'settings.ai.modelResponse': 'Odpowiedź modelu',
|
||||
|
||||
@@ -1903,7 +1903,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': 'Máximo de {max} arquivos por mensagem',
|
||||
'chat.attachment.tooLarge': 'A imagem excede o limite de tamanho de {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Tipo de arquivo não suportado. Use PNG, JPEG, WebP, GIF ou BMP.',
|
||||
'Tipo de arquivo não suportado. Use uma imagem (PNG, JPEG, WebP, GIF, BMP) ou um arquivo PDF, TXT ou Markdown.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'Este modelo não consegue ler imagens. Anexe um arquivo PDF, TXT ou Markdown.',
|
||||
'chat.attachment.readFailed': 'Não foi possível ler o arquivo',
|
||||
'memory.searchAria': 'Pesquisar memória',
|
||||
'memory.searchPlaceholder': 'Pesquisar entradas de memória...',
|
||||
@@ -3169,6 +3171,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Substituição de temperatura (valor)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Mais baixo = mais determinístico. Deixe desmarcado para usar o padrão do provedor.',
|
||||
'settings.ai.modelVision': 'Suporta visão (entrada de imagem)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Ative se este modelo aceitar imagens. Permite anexar imagens no chat quando este modelo está selecionado.',
|
||||
'settings.ai.testFailed': 'Teste falhou',
|
||||
'settings.ai.testingModel': 'Modelo de teste...',
|
||||
'settings.ai.modelResponse': 'Resposta do modelo',
|
||||
|
||||
@@ -1879,7 +1879,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooManyFiles': 'Максимум {max} файлов на сообщение',
|
||||
'chat.attachment.tooLarge': 'Изображение превышает ограничение размера {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Неподдерживаемый тип файла. Используйте PNG, JPEG, WebP, GIF или BMP.',
|
||||
'Неподдерживаемый тип файла. Используйте изображение (PNG, JPEG, WebP, GIF, BMP) или файл PDF, TXT либо Markdown.',
|
||||
'chat.attachment.imageNotSupported':
|
||||
'Эта модель не может читать изображения. Вместо этого прикрепите файл PDF, TXT или Markdown.',
|
||||
'chat.attachment.readFailed': 'Не удалось прочитать файл',
|
||||
'memory.searchAria': 'Поиск в памяти',
|
||||
'memory.searchPlaceholder': 'Поиск записей памяти...',
|
||||
@@ -3140,6 +3142,9 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideValue': 'Переопределение температуры (значение)',
|
||||
'settings.ai.temperatureOverrideDesc':
|
||||
'Ниже = более детерминированно. Оставьте флажок неактивным, чтобы использовать поставщика по умолчанию.',
|
||||
'settings.ai.modelVision': 'Поддерживает зрение (ввод изображений)',
|
||||
'settings.ai.modelVisionDesc':
|
||||
'Включите, если модель принимает изображения. Позволяет прикреплять изображения в чате, когда выбрана эта модель.',
|
||||
'settings.ai.testFailed': 'Тест не пройден.',
|
||||
'settings.ai.testingModel': 'Модель тестирования...',
|
||||
'settings.ai.modelResponse': 'Ответ модели',
|
||||
|
||||
@@ -1760,7 +1760,9 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.tooMany': '每条消息最多 {max} 张图片',
|
||||
'chat.attachment.tooManyFiles': '每条消息最多 {max} 个文件',
|
||||
'chat.attachment.tooLarge': '图片超过 {max} 大小限制',
|
||||
'chat.attachment.unsupportedType': '不支持的文件类型。请使用 PNG、JPEG、WebP、GIF 或 BMP。',
|
||||
'chat.attachment.unsupportedType':
|
||||
'不支持的文件类型。请使用图片(PNG、JPEG、WebP、GIF、BMP)或 PDF、TXT、Markdown 文件。',
|
||||
'chat.attachment.imageNotSupported': '此模型无法读取图像。请改为附加 PDF、TXT 或 Markdown 文件。',
|
||||
'chat.attachment.readFailed': '无法读取文件',
|
||||
'memory.searchAria': '搜索记忆',
|
||||
'memory.searchPlaceholder': '搜索记忆条目...',
|
||||
@@ -2955,6 +2957,8 @@ const messages: TranslationMap = {
|
||||
'settings.ai.temperatureOverrideSlider': '温度超控(滑块)',
|
||||
'settings.ai.temperatureOverrideValue': '温度超控(值)',
|
||||
'settings.ai.temperatureOverrideDesc': '值越低,结果越确定。保持未勾选则使用提供商默认值。',
|
||||
'settings.ai.modelVision': '支持视觉(图像输入)',
|
||||
'settings.ai.modelVisionDesc': '如果此模型接受图像,请启用。选择此模型后可在聊天框中附加图像。',
|
||||
'settings.ai.testFailed': '测试失败',
|
||||
'settings.ai.testingModel': '测试模型...',
|
||||
'settings.ai.modelResponse': '模型响应',
|
||||
|
||||
@@ -17,7 +17,6 @@ import MicComposer from '../features/human/MicComposer';
|
||||
import { useStickToBottom } from '../hooks/useStickToBottom';
|
||||
import { useUsageState } from '../hooks/useUsageState';
|
||||
import {
|
||||
ALLOWED_ATTACHMENT_MIME_TYPES,
|
||||
type Attachment,
|
||||
ATTACHMENT_MAX_FILES,
|
||||
ATTACHMENT_MAX_IMAGES,
|
||||
@@ -270,6 +269,11 @@ const Conversations = ({
|
||||
onCancel: () => {},
|
||||
});
|
||||
const [resolvedModel, setResolvedModel] = useState<string | null>(null);
|
||||
// Whether the resolved model for the active profile accepts image input.
|
||||
// Managed tiers do; custom/BYOK models only when the user flagged them. Gates
|
||||
// the composer's image-attachment affordance (docs flow regardless). Resolved
|
||||
// against the non-attachment hint so the affordance is stable as you attach.
|
||||
const [modelSupportsVision, setModelSupportsVision] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
@@ -280,13 +284,19 @@ const Conversations = ({
|
||||
attachments.length > 0
|
||||
? MULTIMODAL_MODEL_HINT
|
||||
: (profile?.modelOverride ?? CHAT_MODEL_HINT);
|
||||
const res = await callCoreRpc<{ model: string }>({
|
||||
const res = await callCoreRpc<{ model: string; vision?: boolean }>({
|
||||
method: 'openhuman.inference_resolve_model',
|
||||
params: { hint },
|
||||
});
|
||||
if (!cancelled) setResolvedModel(res.model);
|
||||
if (!cancelled) {
|
||||
setResolvedModel(res.model);
|
||||
setModelSupportsVision(res.vision === true);
|
||||
}
|
||||
} catch {
|
||||
if (!cancelled) setResolvedModel(null);
|
||||
if (!cancelled) {
|
||||
setResolvedModel(null);
|
||||
setModelSupportsVision(false);
|
||||
}
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
@@ -680,10 +690,19 @@ const Conversations = ({
|
||||
let acceptedImageCount = attachments.filter(attachment => attachment.kind === 'image').length;
|
||||
let acceptedFileCount = attachments.filter(attachment => attachment.kind === 'file').length;
|
||||
for (const file of Array.from(files)) {
|
||||
const result = await validateAndReadFile(file, acceptedImageCount, acceptedFileCount);
|
||||
const result = await validateAndReadFile(
|
||||
file,
|
||||
acceptedImageCount,
|
||||
acceptedFileCount,
|
||||
modelSupportsVision
|
||||
);
|
||||
if ('error' in result) {
|
||||
const { error } = result;
|
||||
if (error.code === 'too_many') {
|
||||
if (error.code === 'image_not_supported') {
|
||||
setAttachError(
|
||||
chatSendError('attachment_invalid', t('chat.attachment.imageNotSupported'))
|
||||
);
|
||||
} else if (error.code === 'too_many') {
|
||||
const key =
|
||||
error.kind === 'image' ? 'chat.attachment.tooMany' : 'chat.attachment.tooManyFiles';
|
||||
setAttachError(
|
||||
@@ -1724,6 +1743,18 @@ const Conversations = ({
|
||||
? (msg.extraMetadata.attachmentDataUris as string[])
|
||||
: parseMessageImages(msg.content ?? '').dataUris;
|
||||
const hasImages = dataUris.length > 0;
|
||||
// Document attachments carry no image data-URI (only
|
||||
// images do); surface them as filename chips from the
|
||||
// persisted attachmentKinds/attachmentNames metadata.
|
||||
const kinds = Array.isArray(msg.extraMetadata?.attachmentKinds)
|
||||
? (msg.extraMetadata.attachmentKinds as string[])
|
||||
: [];
|
||||
const names = Array.isArray(msg.extraMetadata?.attachmentNames)
|
||||
? (msg.extraMetadata.attachmentNames as string[])
|
||||
: [];
|
||||
const fileNames = kinds
|
||||
.map((k, i) => (k === 'file' ? names[i] : null))
|
||||
.filter((n): n is string => Boolean(n));
|
||||
const showTime = latestVisibleMessage?.id === msg.id;
|
||||
return (
|
||||
<>
|
||||
@@ -1739,6 +1770,35 @@ const Conversations = ({
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{fileNames.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5 justify-end">
|
||||
{fileNames.map((name, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="flex items-center gap-2 rounded-lg border border-stone-200 dark:border-neutral-700 bg-stone-50 dark:bg-neutral-800 px-2.5 py-1.5 text-xs text-stone-700 dark:text-neutral-300 max-w-[220px]">
|
||||
<svg
|
||||
className="w-4 h-4 flex-shrink-0 text-stone-500 dark:text-neutral-400"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24">
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={1.8}
|
||||
d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"
|
||||
/>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={1.8}
|
||||
d="M14 2v6h6"
|
||||
/>
|
||||
</svg>
|
||||
<span className="truncate font-medium">{name}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{(msg.content || showTime) && (
|
||||
<div className="rounded-2xl px-4 py-2.5 bg-primary-500 text-white rounded-br-md break-words overflow-hidden">
|
||||
{msg.content && (
|
||||
@@ -2213,7 +2273,10 @@ const Conversations = ({
|
||||
inlineCompletionSuffix={inlineCompletionSuffix}
|
||||
isComposingTextRef={isComposingTextRef}
|
||||
maxAttachments={ATTACHMENT_MAX_IMAGES + ATTACHMENT_MAX_FILES}
|
||||
allowedMimeTypes={ALLOWED_ATTACHMENT_MIME_TYPES}
|
||||
// Empty → no native `accept` filter (it greys valid files on
|
||||
// macOS/CEF). Type enforcement happens in handleAttachFiles via
|
||||
// validateAndReadFile, which honors modelSupportsVision.
|
||||
allowedMimeTypes={[]}
|
||||
attachmentsEnabled={CHAT_ATTACHMENTS_ENABLED}
|
||||
/>
|
||||
) : (
|
||||
|
||||
@@ -140,6 +140,18 @@ vi.mock('../../features/autocomplete/useAutocompleteSkillStatus', () => ({
|
||||
|
||||
vi.mock('../../utils/openUrl', () => ({ openUrl: vi.fn() }));
|
||||
|
||||
// The composer gates image attachments on the resolved model's vision capability
|
||||
// (inference_resolve_model). These tests exercise image attachments, so resolve
|
||||
// the active model as vision-capable.
|
||||
vi.mock('../../services/coreRpcClient', () => ({
|
||||
callCoreRpc: vi.fn(async ({ method }: { method: string }) =>
|
||||
method === 'openhuman.inference_resolve_model'
|
||||
? { model: 'test-vision-model', vision: true }
|
||||
: {}
|
||||
),
|
||||
CoreRpcError: class CoreRpcError extends Error {},
|
||||
}));
|
||||
|
||||
vi.mock('../../lib/coreState/store', () => ({
|
||||
getCoreStateSnapshot: vi.fn(() => ({
|
||||
isBootstrapping: false,
|
||||
@@ -506,4 +518,53 @@ describe('Conversations — attachment feature', () => {
|
||||
expect(img).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
it('renders a document filename chip in the user bubble from attachmentKinds/Names', async () => {
|
||||
const thread = makeThread({ id: 'file-thread', title: 'File Thread' });
|
||||
const message = {
|
||||
id: 'msg-file-1',
|
||||
content: 'whats in this file',
|
||||
type: 'text' as const,
|
||||
sender: 'user' as const,
|
||||
createdAt: new Date().toISOString(),
|
||||
extraMetadata: {
|
||||
attachmentCount: 1,
|
||||
attachmentKinds: ['file'],
|
||||
attachmentNames: ['report.pdf'],
|
||||
},
|
||||
};
|
||||
|
||||
mockGetThreads.mockResolvedValue({ threads: [thread], count: 1 });
|
||||
mockGetThreadMessages.mockResolvedValue({ messages: [message], count: 1 });
|
||||
|
||||
const store = buildStore({
|
||||
thread: {
|
||||
threads: [thread],
|
||||
selectedThreadId: thread.id,
|
||||
activeThreadId: null,
|
||||
welcomeThreadId: null,
|
||||
messagesByThreadId: { [thread.id]: [message] },
|
||||
messages: [message],
|
||||
isLoadingThreads: false,
|
||||
isLoadingMessages: false,
|
||||
messagesError: null,
|
||||
},
|
||||
socket: socketState('connected'),
|
||||
});
|
||||
|
||||
const { default: Conversations } = await import('../Conversations');
|
||||
|
||||
render(
|
||||
<Provider store={store}>
|
||||
<MemoryRouter>
|
||||
<Conversations />
|
||||
</MemoryRouter>
|
||||
</Provider>
|
||||
);
|
||||
|
||||
// The document attachment surfaces as a filename chip (not an <img>).
|
||||
await waitFor(() => {
|
||||
expect(document.body.textContent).toContain('report.pdf');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
loadAISettings,
|
||||
loadLocalProviderSnapshot,
|
||||
localProvider,
|
||||
modelRegistryVision,
|
||||
OPENAI_CODEX_OAUTH_MISSING_AUTH_URL,
|
||||
OPENAI_CODEX_OAUTH_MISSING_CALLBACK_URL,
|
||||
parseProviderString,
|
||||
@@ -28,6 +29,7 @@ import {
|
||||
setLocalRuntimeEnabled,
|
||||
startOpenAiCodexOAuth,
|
||||
testProviderModel,
|
||||
upsertModelRegistryVision,
|
||||
} from '../aiSettingsApi';
|
||||
|
||||
// ─── Mock declarations (must be hoisted before imports) ───────────────────────
|
||||
@@ -83,6 +85,7 @@ function makeClientConfigResult(overrides: Record<string, unknown> = {}) {
|
||||
api_key_set: false,
|
||||
model_routes: [],
|
||||
cloud_providers: [],
|
||||
model_registry: [],
|
||||
primary_cloud: null,
|
||||
reasoning_provider: null,
|
||||
agentic_provider: null,
|
||||
@@ -587,6 +590,7 @@ describe('saveAISettings', () => {
|
||||
learning: { kind: 'openhuman' },
|
||||
subconscious: { kind: 'openhuman' },
|
||||
},
|
||||
modelRegistry: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -673,8 +677,13 @@ describe('saveAISettings', () => {
|
||||
learning: { kind: 'openhuman' },
|
||||
subconscious: { kind: 'openhuman' },
|
||||
},
|
||||
modelRegistry: [],
|
||||
};
|
||||
const next: AISettings = {
|
||||
cloudProviders: [anthropicProvider],
|
||||
routing: { ...prev.routing },
|
||||
modelRegistry: [],
|
||||
};
|
||||
const next: AISettings = { cloudProviders: [anthropicProvider], routing: { ...prev.routing } };
|
||||
|
||||
await saveAISettings(prev, next);
|
||||
|
||||
@@ -697,6 +706,34 @@ describe('saveAISettings', () => {
|
||||
expect(patch.cloud_providers).toBeDefined();
|
||||
expect(patch.coding_provider).toBe('openai:gpt-4o-mini');
|
||||
});
|
||||
|
||||
it('sends model_registry when the vision flag changes', async () => {
|
||||
const prev = makeSettings({ modelRegistry: [] });
|
||||
const next = makeSettings({
|
||||
modelRegistry: [{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true }],
|
||||
});
|
||||
await saveAISettings(prev, next);
|
||||
const patch = mockOpenhumanUpdateModelSettings.mock.calls[0][0];
|
||||
expect(patch.model_registry).toEqual([
|
||||
{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true },
|
||||
]);
|
||||
});
|
||||
|
||||
it('omits model_registry when unchanged', async () => {
|
||||
const registry = [{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true }];
|
||||
const prev = makeSettings({ modelRegistry: registry });
|
||||
const next = makeSettings({
|
||||
modelRegistry: [...registry],
|
||||
routing: {
|
||||
...makeSettings().routing,
|
||||
coding: { kind: 'cloud', providerSlug: 'openai', model: 'gpt-4o-mini' },
|
||||
},
|
||||
});
|
||||
await saveAISettings(prev, next);
|
||||
const patch = mockOpenhumanUpdateModelSettings.mock.calls[0][0];
|
||||
expect(patch.model_registry).toBeUndefined();
|
||||
expect(patch.coding_provider).toBe('openai:gpt-4o-mini');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── setCloudProviderKey ──────────────────────────────────────────────────────
|
||||
@@ -918,3 +955,32 @@ describe('flushCloudProviders', () => {
|
||||
expect(mockOpenhumanUpdateModelSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('model registry vision helpers', () => {
|
||||
const reg = [
|
||||
{ id: 'gpt-4o', provider: 'openai', cost_per_1m_output: 0, vision: true },
|
||||
{ id: 'text-only', provider: 'openai', cost_per_1m_output: 0, vision: false },
|
||||
];
|
||||
|
||||
it('modelRegistryVision matches by (provider, id)', () => {
|
||||
expect(modelRegistryVision(reg, 'openai', 'gpt-4o')).toBe(true);
|
||||
expect(modelRegistryVision(reg, 'openai', 'text-only')).toBe(false);
|
||||
expect(modelRegistryVision(reg, 'openai', 'unlisted')).toBe(false);
|
||||
expect(modelRegistryVision(reg, 'azure', 'gpt-4o')).toBe(false);
|
||||
});
|
||||
|
||||
it('upsertModelRegistryVision adds, flips, and removes entries', () => {
|
||||
const added = upsertModelRegistryVision([], 'openai', 'my-llava', true);
|
||||
expect(added).toEqual([
|
||||
{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true },
|
||||
]);
|
||||
// vision:false removes the entry (absence ⇒ no vision).
|
||||
const removed = upsertModelRegistryVision(reg, 'openai', 'gpt-4o', false);
|
||||
expect(removed.find(e => e.id === 'gpt-4o')).toBeUndefined();
|
||||
expect(removed.find(e => e.id === 'text-only')).toBeDefined();
|
||||
// Flipping an existing entry on stays idempotent on the key.
|
||||
const flipped = upsertModelRegistryVision(reg, 'openai', 'text-only', true);
|
||||
expect(flipped.filter(e => e.id === 'text-only')).toHaveLength(1);
|
||||
expect(modelRegistryVision(flipped, 'openai', 'text-only')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -26,6 +26,7 @@ import { isTauri } from '../../utils/tauriCommands/common';
|
||||
import {
|
||||
type ClientConfig,
|
||||
type CloudProviderCreds,
|
||||
type ModelRegistryEntry,
|
||||
type ModelSettingsUpdate,
|
||||
openhumanGetClientConfig,
|
||||
openhumanUpdateLocalAiSettings,
|
||||
@@ -132,6 +133,47 @@ const PROVIDER_MODEL_TEST_TIMEOUT_MS = 120_000;
|
||||
export interface AISettings {
|
||||
cloudProviders: CloudProviderView[];
|
||||
routing: Record<WorkloadId, ProviderRef>;
|
||||
/**
|
||||
* Per-model registry carrying each model's user-set `vision` flag, keyed by
|
||||
* `(provider slug, model id)`. Surfaced in the custom-model dialog; gates chat
|
||||
* image attachments for custom/BYOK models.
|
||||
*/
|
||||
modelRegistry: ModelRegistryEntry[];
|
||||
}
|
||||
|
||||
/** Re-export so callers (e.g. the AI panel) can reference the entry type. */
|
||||
export type { ModelRegistryEntry };
|
||||
|
||||
/** Find a model's vision flag in the registry, matching by (provider, id).
|
||||
* Tolerates an undefined registry (older snapshots / transient load state). */
|
||||
export function modelRegistryVision(
|
||||
registry: ModelRegistryEntry[] | undefined,
|
||||
provider: string,
|
||||
id: string
|
||||
): boolean {
|
||||
return (registry ?? []).some(e => e.provider === provider && e.id === id && e.vision);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert a model's vision flag, returning a new array. Matches by
|
||||
* `(provider, id)`; a `vision: false` entry is removed (absence ⇒ no vision).
|
||||
*/
|
||||
export function upsertModelRegistryVision(
|
||||
registry: ModelRegistryEntry[] | undefined,
|
||||
provider: string,
|
||||
id: string,
|
||||
vision: boolean
|
||||
): ModelRegistryEntry[] {
|
||||
const base = registry ?? [];
|
||||
const without = base.filter(e => !(e.provider === provider && e.id === id));
|
||||
if (!vision) {
|
||||
return without;
|
||||
}
|
||||
const existing = base.find(e => e.provider === provider && e.id === id);
|
||||
return [
|
||||
...without,
|
||||
{ id, provider, cost_per_1m_output: existing?.cost_per_1m_output ?? 0, vision: true },
|
||||
];
|
||||
}
|
||||
|
||||
// ─── Read path: load + parse ───────────────────────────────────────────────
|
||||
@@ -264,7 +306,10 @@ export async function loadAISettings(): Promise<AISettings> {
|
||||
);
|
||||
}
|
||||
|
||||
return { cloudProviders, routing };
|
||||
// Per-model registry (vision flags). Defensive default for older snapshots.
|
||||
const modelRegistry: ModelRegistryEntry[] = config.model_registry ?? [];
|
||||
|
||||
return { cloudProviders, routing, modelRegistry };
|
||||
}
|
||||
// ─── Write path: diff + save ───────────────────────────────────────────────
|
||||
|
||||
@@ -310,12 +355,37 @@ export async function saveAISettings(prev: AISettings, next: AISettings): Promis
|
||||
}
|
||||
}
|
||||
|
||||
// Per-model registry (vision flags): any change → send the full list.
|
||||
if (!modelRegistriesEqual(prev.modelRegistry, next.modelRegistry)) {
|
||||
patch.model_registry = next.modelRegistry.map(
|
||||
({ id, provider, cost_per_1m_output, vision }) => ({
|
||||
id,
|
||||
provider,
|
||||
cost_per_1m_output,
|
||||
vision,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
if (Object.keys(patch).length === 0) {
|
||||
return;
|
||||
}
|
||||
await openhumanUpdateModelSettings(patch);
|
||||
}
|
||||
|
||||
/** Order-insensitive structural equality for two model registries. */
|
||||
function modelRegistriesEqual(a: ModelRegistryEntry[], b: ModelRegistryEntry[]): boolean {
|
||||
if (a.length !== b.length) {
|
||||
return false;
|
||||
}
|
||||
const key = (e: ModelRegistryEntry) => `${e.provider} ${e.id}`;
|
||||
const bByKey = new Map(b.map(e => [key(e), e]));
|
||||
return a.every(e => {
|
||||
const m = bByKey.get(key(e));
|
||||
return !!m && m.vision === e.vision && m.cost_per_1m_output === e.cost_per_1m_output;
|
||||
});
|
||||
}
|
||||
|
||||
// ─── API key management (per cloud provider slug) ──────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
@@ -49,6 +49,18 @@ export interface CloudProviderCreds {
|
||||
auth_style: AuthStyle;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-model registry entry. Mirrors the Rust `ModelRegistryEntry`
|
||||
* (`config/schema/types.rs`). Carries the user-set `vision` flag that lets a
|
||||
* custom/BYOK model accept chat image attachments.
|
||||
*/
|
||||
export interface ModelRegistryEntry {
|
||||
id: string;
|
||||
provider: string;
|
||||
cost_per_1m_output: number;
|
||||
vision: boolean;
|
||||
}
|
||||
|
||||
export interface ModelSettingsUpdate {
|
||||
/**
|
||||
* OpenHuman product backend URL. Almost always left untouched; the
|
||||
@@ -77,6 +89,12 @@ export interface ModelSettingsUpdate {
|
||||
* Each entry: { id?, slug, label?, endpoint, auth_style? }
|
||||
*/
|
||||
cloud_providers?: CloudProviderCreds[] | null;
|
||||
/**
|
||||
* When present, REPLACES `config.model_registry` wholesale. Carries each
|
||||
* model's `vision` flag (Settings → Advanced LLM → custom model → "Supports
|
||||
* vision"). Send `[]` to clear; omit to leave untouched.
|
||||
*/
|
||||
model_registry?: ModelRegistryEntry[] | null;
|
||||
/** @deprecated No longer used — slug-based routing replaces primary_cloud. */
|
||||
primary_cloud?: string | null;
|
||||
/** Per-workload provider strings — see Rust `providers::factory` grammar. */
|
||||
@@ -215,6 +233,8 @@ export interface ClientConfig {
|
||||
model_routes: ModelRoute[];
|
||||
/** Configured cloud providers (no API keys — those live in auth-profiles.json). */
|
||||
cloud_providers: CloudProviderCreds[];
|
||||
/** Per-model registry carrying each model's `vision` flag. */
|
||||
model_registry: ModelRegistryEntry[];
|
||||
/** Id of the `cloud_providers` entry resolved by the `"cloud"` sentinel. */
|
||||
primary_cloud: string | null;
|
||||
/** Per-workload provider strings (e.g. `"cloud"`, `"ollama:llama3.1:8b"`, `"openai:gpt-4o"`). */
|
||||
|
||||
@@ -2,7 +2,6 @@ import { expect, type Page, test } from '@playwright/test';
|
||||
|
||||
import {
|
||||
bootAuthenticatedPage,
|
||||
callCoreRpc,
|
||||
dismissWalkthroughIfPresent,
|
||||
waitForAppReady,
|
||||
} from '../helpers/core-rpc';
|
||||
@@ -73,7 +72,7 @@ async function newThread(page: Page): Promise<string> {
|
||||
}
|
||||
|
||||
test.describe('Chat management functional coverage', () => {
|
||||
test('attachment preview, remove, and multimodal send path remain interactive', async ({
|
||||
test('attachment preview, remove, and document send path remain interactive', async ({
|
||||
page,
|
||||
}) => {
|
||||
await resetMock();
|
||||
@@ -89,34 +88,24 @@ test.describe('Chat management functional coverage', () => {
|
||||
|
||||
const fileInput = page.locator('input[type="file"]');
|
||||
await expect(fileInput).toHaveCount(1);
|
||||
await fileInput.setInputFiles({
|
||||
name: 'pixel.png',
|
||||
mimeType: 'image/png',
|
||||
buffer: Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
|
||||
'base64'
|
||||
),
|
||||
});
|
||||
|
||||
await expect(page.getByText('pixel.png')).toBeVisible();
|
||||
await page.getByRole('button', { name: /Remove pixel\.png/ }).click();
|
||||
await expect(page.getByText('pixel.png')).toHaveCount(0);
|
||||
// Documents work on every model; images require a vision-capable model.
|
||||
// Managed OpenHuman tiers are text-only (see `oh_tier_supports_vision`), so
|
||||
// the attachment mechanics here are exercised with a text document. Image
|
||||
// upload is gated to vision-flagged models and validated in unit tests.
|
||||
const txtBuffer = Buffer.from('renderer uploaded text document', 'utf8');
|
||||
const attachTxt = () =>
|
||||
fileInput.setInputFiles({ name: 'notes.txt', mimeType: 'text/plain', buffer: txtBuffer });
|
||||
|
||||
const pngBuffer = Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
|
||||
'base64'
|
||||
);
|
||||
await fileInput.setInputFiles({ name: 'pixel.png', mimeType: 'image/png', buffer: pngBuffer });
|
||||
await expect(page.getByText('pixel.png')).toBeVisible();
|
||||
await attachTxt();
|
||||
await expect(page.getByText('notes.txt')).toBeVisible();
|
||||
await page.getByRole('button', { name: /Remove notes\.txt/ }).click();
|
||||
await expect(page.getByText('notes.txt')).toHaveCount(0);
|
||||
|
||||
await fileInput.setInputFiles({
|
||||
name: 'notes.txt',
|
||||
mimeType: 'text/plain',
|
||||
buffer: Buffer.from('renderer uploaded text document', 'utf8'),
|
||||
});
|
||||
await attachTxt();
|
||||
await expect(page.getByText('notes.txt')).toBeVisible();
|
||||
|
||||
await page.getByPlaceholder('How can I help you today?').fill('Describe this image');
|
||||
await page.getByPlaceholder('How can I help you today?').fill('Summarize this file');
|
||||
await page.getByTestId('send-message-button').click();
|
||||
await expect(page.getByText('Attachment received by the assistant.')).toBeVisible({
|
||||
timeout: 30_000,
|
||||
@@ -132,13 +121,13 @@ test.describe('Chat management functional coverage', () => {
|
||||
request.method === 'POST' &&
|
||||
request.url?.includes('/chat/completions') &&
|
||||
typeof request.body === 'string' &&
|
||||
request.body.includes('Describe this image')
|
||||
request.body.includes('Summarize this file')
|
||||
);
|
||||
return completion?.body ?? '';
|
||||
},
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
.toContain('image_url');
|
||||
.toContain('[FILE-EXTRACTED:');
|
||||
|
||||
const log = await requests();
|
||||
const completionBody =
|
||||
@@ -147,13 +136,21 @@ test.describe('Chat management functional coverage', () => {
|
||||
request.method === 'POST' &&
|
||||
request.url?.includes('/chat/completions') &&
|
||||
typeof request.body === 'string' &&
|
||||
request.body.includes('Describe this image')
|
||||
request.body.includes('Summarize this file')
|
||||
)?.body ?? '';
|
||||
expect(completionBody).toContain('reasoning');
|
||||
expect(completionBody).toContain('[FILE-EXTRACTED:');
|
||||
expect(completionBody).toContain('renderer uploaded text document');
|
||||
});
|
||||
|
||||
// NOTE: image attachments require a vision-capable model. Managed tiers are
|
||||
// text-only by default (`oh_tier_supports_vision`); a model is flagged
|
||||
// vision-capable via `model_registry` (Settings → "Supports vision"). The
|
||||
// `image_url` promotion path is covered by Rust unit tests
|
||||
// (`inference::provider::compatible` MessageContent). An E2E that toggles the
|
||||
// vision flag at runtime is a follow-up (needs a page re-mount so the
|
||||
// composer's resolve picks up the flag — out of scope here).
|
||||
|
||||
test('thread rename and delete remain usable from the conversation UI', async ({ page }) => {
|
||||
await resetMock();
|
||||
await openChat(page, 'pw-chat-rename-delete');
|
||||
|
||||
@@ -248,7 +248,14 @@ pub(crate) async fn run_turn_engine(
|
||||
|
||||
tracing::debug!(iteration, "[agent_loop] sending LLM request");
|
||||
let image_marker_count = multimodal::count_image_markers(history);
|
||||
if image_marker_count > 0 && !provider.supports_vision() {
|
||||
// A model accepts images when its provider advertises vision (managed
|
||||
// backend) OR the user marked it vision-capable in `model_registry`
|
||||
// (custom/BYOK) — surfaced via the `current_model_vision` task-local set
|
||||
// at session build. Unset (CLI/tests) falls back to the provider flag.
|
||||
let has_vision = provider.supports_vision()
|
||||
|| crate::openhuman::agent::harness::model_vision_context::current_model_vision()
|
||||
.unwrap_or(false);
|
||||
if image_marker_count > 0 && !has_vision {
|
||||
let cap_err = ProviderCapabilityError {
|
||||
provider: provider_name.to_string(),
|
||||
capability: "vision".to_string(),
|
||||
|
||||
@@ -30,6 +30,7 @@ mod instructions;
|
||||
pub mod interrupt;
|
||||
pub(crate) mod memory_context;
|
||||
pub(crate) mod memory_context_safety;
|
||||
pub mod model_vision_context;
|
||||
mod parse;
|
||||
pub(crate) mod payload_summarizer;
|
||||
pub mod run_queue;
|
||||
@@ -50,6 +51,7 @@ pub use definition::{
|
||||
};
|
||||
pub use fork_context::{current_parent, with_parent_context, ParentExecutionContext};
|
||||
pub use interrupt::{check_interrupt, InterruptFence, InterruptedError};
|
||||
pub use model_vision_context::{current_model_vision, with_current_model_vision};
|
||||
pub use sandbox_context::{current_sandbox_mode, with_current_sandbox_mode};
|
||||
pub(crate) use spawn_depth_context::{current_spawn_depth, with_spawn_depth, MAX_SPAWN_DEPTH};
|
||||
pub use subagent_runner::{run_subagent, SubagentRunError, SubagentRunOptions};
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Task-local carrier for the **current session/sub-agent's user-configured
|
||||
//! vision capability** so the deep turn engine's image gate can honor a custom
|
||||
//! (BYOK) model's `model_registry.vision` flag without widening
|
||||
//! [`crate::openhuman::agent::harness::engine::run_turn_engine`]'s signature.
|
||||
//!
|
||||
//! Managed-backend models advertise vision via `Provider::supports_vision()`, so
|
||||
//! the gate accepts their image turns already. Custom OpenAI-compatible providers
|
||||
//! report `supports_vision() == false` (a provider endpoint can't know a per-model
|
||||
//! property), so without this the gate would reject image turns for a model the
|
||||
//! user explicitly marked vision-capable. This task-local surfaces that per-model
|
||||
//! flag — computed once at session build (where the full `Config` / `model_registry`
|
||||
//! and the resolved model id coexist) — to the gate.
|
||||
//!
|
||||
//! Mirrors [`super::sandbox_context`]. When unset (CLI / direct invocation / tests
|
||||
//! that never wrapped the call) [`current_model_vision`] returns `None` and the
|
||||
//! gate falls back to the provider capability only — strictly additive.
|
||||
|
||||
tokio::task_local! {
|
||||
/// User-configured vision capability for the currently-executing
|
||||
/// session/sub-agent's model. Scoped per turn by the turn loop + subagent
|
||||
/// runner. `None` when unset.
|
||||
pub static CURRENT_MODEL_VISION: bool;
|
||||
}
|
||||
|
||||
/// Returns the current model's user-configured vision flag, if scope is active.
|
||||
pub fn current_model_vision() -> Option<bool> {
|
||||
CURRENT_MODEL_VISION.try_with(|v| *v).ok()
|
||||
}
|
||||
|
||||
/// Run `future` with `vision` installed as the current model's vision flag.
|
||||
/// Intended call site is around each `run_turn_engine` invocation.
|
||||
pub async fn with_current_model_vision<F, R>(vision: bool, future: F) -> R
|
||||
where
|
||||
F: std::future::Future<Output = R>,
|
||||
{
|
||||
CURRENT_MODEL_VISION.scope(vision, future).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_model_vision_returns_none_outside_scope() {
|
||||
assert_eq!(current_model_vision(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn with_current_model_vision_installs_value() {
|
||||
let observed = with_current_model_vision(true, async { current_model_vision() }).await;
|
||||
assert_eq!(observed, Some(true));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn with_current_model_vision_does_not_leak_across_scopes() {
|
||||
with_current_model_vision(true, async {
|
||||
assert_eq!(current_model_vision(), Some(true));
|
||||
})
|
||||
.await;
|
||||
assert_eq!(current_model_vision(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nested_scope_overrides_outer() {
|
||||
with_current_model_vision(false, async {
|
||||
assert_eq!(current_model_vision(), Some(false));
|
||||
with_current_model_vision(true, async {
|
||||
assert_eq!(current_model_vision(), Some(true));
|
||||
})
|
||||
.await;
|
||||
assert_eq!(current_model_vision(), Some(false));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
@@ -439,6 +439,15 @@ impl Agent {
|
||||
model_name = pinned_model.to_string();
|
||||
}
|
||||
|
||||
// Resolve the user-configured vision flag for the (now-final) model while
|
||||
// the full `Config` / `model_registry` is in scope — the turn engine only
|
||||
// sees `MultimodalConfig`. Stored on the session and surfaced to the image
|
||||
// gate via the `current_model_vision` task-local (covers custom/BYOK models
|
||||
// the provider can't introspect). Computed with `&model_name` since it's
|
||||
// moved into the builder below.
|
||||
let model_vision =
|
||||
crate::openhuman::inference::model_context::model_supports_vision(&model_name, config);
|
||||
|
||||
// Dispatcher selection is deferred until after the tool list is
|
||||
// finalised (orchestrator tools are appended below). We capture
|
||||
// the choice string now so the provider borrow doesn't conflict
|
||||
@@ -1047,6 +1056,7 @@ impl Agent {
|
||||
.config(config.agent.clone())
|
||||
.context_config(config.context.clone())
|
||||
.model_name(model_name)
|
||||
.model_vision(model_vision)
|
||||
.temperature(effective_temperature)
|
||||
.workspace_dir(config.workspace_dir.clone())
|
||||
.action_dir(config.action_dir.clone())
|
||||
|
||||
@@ -29,6 +29,7 @@ impl AgentBuilder {
|
||||
config: None,
|
||||
context_config: None,
|
||||
model_name: None,
|
||||
model_vision: None,
|
||||
temperature: None,
|
||||
workspace_dir: None,
|
||||
action_dir: None,
|
||||
@@ -142,6 +143,14 @@ impl AgentBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the user-configured vision capability for the resolved model.
|
||||
/// Surfaced to the turn engine's image gate via the `current_model_vision`
|
||||
/// task-local. Defaults to `false` when unset.
|
||||
pub fn model_vision(mut self, model_vision: bool) -> Self {
|
||||
self.model_vision = Some(model_vision);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the temperature for chat requests.
|
||||
pub fn temperature(mut self, temperature: f64) -> Self {
|
||||
self.temperature = Some(temperature);
|
||||
@@ -514,6 +523,7 @@ impl AgentBuilder {
|
||||
.unwrap_or_else(|| Box::new(DefaultMemoryLoader::default())),
|
||||
config,
|
||||
model_name,
|
||||
model_vision: self.model_vision.unwrap_or(false),
|
||||
temperature: self.temperature.unwrap_or(0.7),
|
||||
workspace_dir,
|
||||
action_dir,
|
||||
|
||||
@@ -323,6 +323,10 @@ impl Agent {
|
||||
// model prompt, not in the Rust-side classifier. Sub-agents pick
|
||||
// their own tier via `ModelSpec::Hint(...)` in their definition.
|
||||
let effective_model = self.model_name.clone();
|
||||
// Capture before `self` is borrowed by the turn observer below, so it can
|
||||
// be installed as the `current_model_vision` task-local around the engine
|
||||
// call (read by the image gate for custom/BYOK vision models).
|
||||
let model_vision = self.model_vision;
|
||||
log::info!(
|
||||
"[agent_loop] model pinned model={} (per-turn classification disabled for KV cache stability)",
|
||||
effective_model
|
||||
@@ -436,25 +440,28 @@ impl Agent {
|
||||
// overflow happens during the parent's poll on the way in
|
||||
// — verified against the `chat-harness-subagent` Playwright
|
||||
// lane crash on PR #3151.
|
||||
let outcome = Box::pin(super::super::super::engine::run_turn_engine(
|
||||
provider.as_ref(),
|
||||
&mut buf,
|
||||
&mut tool_source,
|
||||
&progress,
|
||||
&mut observer,
|
||||
&checkpoint,
|
||||
&parser,
|
||||
&provider_name,
|
||||
&effective_model,
|
||||
temperature,
|
||||
true, // silent — the channel/UI renders via progress + the return value
|
||||
&multimodal,
|
||||
&multimodal_files,
|
||||
max_iterations,
|
||||
None, // the web bridge streams via on_progress deltas, not on_delta
|
||||
&[],
|
||||
turn_run_queue,
|
||||
))
|
||||
let outcome = super::super::super::model_vision_context::with_current_model_vision(
|
||||
model_vision,
|
||||
Box::pin(super::super::super::engine::run_turn_engine(
|
||||
provider.as_ref(),
|
||||
&mut buf,
|
||||
&mut tool_source,
|
||||
&progress,
|
||||
&mut observer,
|
||||
&checkpoint,
|
||||
&parser,
|
||||
&provider_name,
|
||||
&effective_model,
|
||||
temperature,
|
||||
true, // silent — the channel/UI renders via progress + the return value
|
||||
&multimodal,
|
||||
&multimodal_files,
|
||||
max_iterations,
|
||||
None, // the web bridge streams via on_progress deltas, not on_delta
|
||||
&[],
|
||||
turn_run_queue,
|
||||
)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Pull the observer's accounting out, then drop it to release the
|
||||
|
||||
@@ -53,6 +53,11 @@ pub struct Agent {
|
||||
pub(super) memory_loader: Box<dyn MemoryLoader>,
|
||||
pub(super) config: crate::openhuman::config::AgentConfig,
|
||||
pub(super) model_name: String,
|
||||
/// User-configured vision capability for [`Self::model_name`], evaluated at
|
||||
/// session build from `model_vision_enabled(&model, config)`. Surfaced to the
|
||||
/// turn engine's image gate via the `current_model_vision` task-local so a
|
||||
/// custom/BYOK model the user flagged can forward images. Defaults to `false`.
|
||||
pub(super) model_vision: bool,
|
||||
pub(super) temperature: f64,
|
||||
pub(super) workspace_dir: std::path::PathBuf,
|
||||
pub(super) action_dir: std::path::PathBuf,
|
||||
@@ -272,6 +277,8 @@ pub struct AgentBuilder {
|
||||
/// [`crate::openhuman::config::ContextConfig::default`].
|
||||
pub(super) context_config: Option<crate::openhuman::config::ContextConfig>,
|
||||
pub(super) model_name: Option<String>,
|
||||
/// User vision flag for the resolved model; `None` → `false` in `build()`.
|
||||
pub(super) model_vision: Option<bool>,
|
||||
pub(super) temperature: Option<f64>,
|
||||
pub(super) workspace_dir: Option<std::path::PathBuf>,
|
||||
pub(super) action_dir: Option<std::path::PathBuf>,
|
||||
|
||||
@@ -48,6 +48,10 @@ pub(super) async fn run_inner_loop(
|
||||
allowed_names: HashSet<String>,
|
||||
lazy_resolver: Option<LazyToolkitResolver>,
|
||||
model: &str,
|
||||
// User-configured vision flag for `model` (computed at the call site), set as
|
||||
// the `current_model_vision` task-local around the engine call so a flagged
|
||||
// custom/BYOK sub-agent model can forward images.
|
||||
model_vision: bool,
|
||||
temperature: f64,
|
||||
max_iterations: usize,
|
||||
task_id: &str,
|
||||
@@ -176,25 +180,28 @@ pub(super) async fn run_inner_loop(
|
||||
// `nested_subagent_dispatch_runs_on_a_constrained_worker_stack`;
|
||||
// the deep end-to-end catcher is the `chat-harness-subagent`
|
||||
// Playwright spec.
|
||||
let outcome = Box::pin(super::super::super::engine::run_turn_engine(
|
||||
provider,
|
||||
history,
|
||||
&mut tool_source,
|
||||
&progress,
|
||||
&mut observer,
|
||||
&checkpoint,
|
||||
&parser,
|
||||
"subagent",
|
||||
model,
|
||||
temperature,
|
||||
true, // silent — sub-agents never echo to stdout
|
||||
&crate::openhuman::config::MultimodalConfig::default(),
|
||||
&crate::openhuman::config::MultimodalFileConfig::default(),
|
||||
max_iterations,
|
||||
None, // sub-agents don't stream a draft
|
||||
&["ask_user_clarification"],
|
||||
None, // sub-agents don't support run-queue steering
|
||||
))
|
||||
let outcome = super::super::super::model_vision_context::with_current_model_vision(
|
||||
model_vision,
|
||||
Box::pin(super::super::super::engine::run_turn_engine(
|
||||
provider,
|
||||
history,
|
||||
&mut tool_source,
|
||||
&progress,
|
||||
&mut observer,
|
||||
&checkpoint,
|
||||
&parser,
|
||||
"subagent",
|
||||
model,
|
||||
temperature,
|
||||
true, // silent — sub-agents never echo to stdout
|
||||
&crate::openhuman::config::MultimodalConfig::default(),
|
||||
&crate::openhuman::config::MultimodalFileConfig::default(),
|
||||
max_iterations,
|
||||
None, // sub-agents don't stream a draft
|
||||
&["ask_user_clarification"],
|
||||
None, // sub-agents don't support run-queue steering
|
||||
)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok((
|
||||
|
||||
@@ -645,6 +645,20 @@ async fn run_typed_mode(
|
||||
};
|
||||
|
||||
// ── Run the inner tool-call loop ───────────────────────────────────
|
||||
// Resolve the sub-agent model's user-configured vision flag; defaults to
|
||||
// `false` when config can't be loaded. Combined with the provider capability
|
||||
// at the gate, this lets a flagged custom/BYOK sub-agent model forward images.
|
||||
let model_vision = crate::openhuman::config::Config::load_or_init()
|
||||
.await
|
||||
.ok()
|
||||
.map(|cfg| crate::openhuman::inference::model_context::model_supports_vision(&model, &cfg))
|
||||
.unwrap_or(false);
|
||||
tracing::debug!(
|
||||
target: "subagent_runner",
|
||||
model = %model,
|
||||
model_vision,
|
||||
"[subagent_runner] resolved sub-agent model vision capability"
|
||||
);
|
||||
let (output, iterations, _agg_usage, early_exit_tool) = Box::pin(run_inner_loop(
|
||||
subagent_provider.as_ref(),
|
||||
&mut history,
|
||||
@@ -654,6 +668,7 @@ async fn run_typed_mode(
|
||||
allowed_names,
|
||||
lazy_resolver,
|
||||
&model,
|
||||
model_vision,
|
||||
temperature,
|
||||
definition.effective_max_iterations(),
|
||||
task_id,
|
||||
|
||||
@@ -11,6 +11,14 @@ pub(super) fn autonomy_signature(config: &Config) -> String {
|
||||
serde_json::to_string(&config.autonomy).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Signature of `config.model_registry` for the session-cache fingerprint.
|
||||
/// Captures every per-model `vision` flag so toggling one in Settings forces a
|
||||
/// rebuild (picking up the new build-time `model_vision`). Mirrors
|
||||
/// [`autonomy_signature`].
|
||||
pub(super) fn model_registry_signature(config: &Config) -> String {
|
||||
serde_json::to_string(&config.model_registry).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(super) fn pick_target_agent_id(_config: &Config, profile: &AgentProfile) -> String {
|
||||
if profile.id == DEFAULT_PROFILE_ID {
|
||||
"orchestrator".to_string()
|
||||
@@ -186,5 +194,6 @@ pub(super) fn build_session_fingerprint(
|
||||
),
|
||||
target_agent_id,
|
||||
autonomy_signature: autonomy_signature(config),
|
||||
model_registry_signature: model_registry_signature(config),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,12 @@ pub(crate) struct SessionCacheFingerprint {
|
||||
pub(super) target_agent_id: String,
|
||||
pub(super) provider_binding: String,
|
||||
pub(super) autonomy_signature: String,
|
||||
/// Signature of `config.model_registry`. The cached `Agent` stores a
|
||||
/// build-time `model_vision` bool; toggling a model's "Supports vision" flag
|
||||
/// keeps the same model id (so neither `model_override` nor `provider_binding`
|
||||
/// change) — without this the stale session would be reused. Mirrors
|
||||
/// [`Self::autonomy_signature`].
|
||||
pub(super) model_registry_signature: String,
|
||||
}
|
||||
|
||||
pub(super) struct SessionEntry {
|
||||
|
||||
@@ -1208,6 +1208,7 @@ fn fp(
|
||||
target_agent_id: target.to_string(),
|
||||
provider_binding: provider_binding.to_string(),
|
||||
autonomy_signature: "sig-default".to_string(),
|
||||
model_registry_signature: "registry-default".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1225,6 +1226,20 @@ fn fingerprint_autonomy_change_is_cache_miss() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_model_registry_change_is_cache_miss() {
|
||||
// Toggling a model's "Supports vision" flag keeps the same model id, so it
|
||||
// changes neither model_override nor provider_binding. Without the registry
|
||||
// signature the stale Agent (old build-time model_vision) would be reused.
|
||||
let base = fp(None, None, "orchestrator", "openai:my-llava");
|
||||
let mut changed = fp(None, None, "orchestrator", "openai:my-llava");
|
||||
changed.model_registry_signature = "registry-after-vision-toggle".to_string();
|
||||
assert_ne!(
|
||||
base, changed,
|
||||
"a model_registry change (vision toggle) must produce a cache miss → rebuild"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_identical_inputs_are_cache_hit() {
|
||||
let a = fp(None, None, "orchestrator", "anthropic:claude-sonnet-4-6");
|
||||
|
||||
@@ -254,6 +254,18 @@ pub fn client_config_json(config: &Config) -> serde_json::Value {
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let model_registry: Vec<serde_json::Value> = config
|
||||
.model_registry
|
||||
.iter()
|
||||
.map(|m| {
|
||||
serde_json::json!({
|
||||
"id": m.id,
|
||||
"provider": m.provider,
|
||||
"cost_per_1m_output": m.cost_per_1m_output,
|
||||
"vision": m.vision,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
serde_json::json!({
|
||||
"api_url": config.api_url,
|
||||
@@ -263,6 +275,7 @@ pub fn client_config_json(config: &Config) -> serde_json::Value {
|
||||
"api_key_set": api_key_set,
|
||||
"model_routes": model_routes,
|
||||
"cloud_providers": cloud_providers,
|
||||
"model_registry": model_registry,
|
||||
"primary_cloud": config.primary_cloud,
|
||||
"chat_provider": config.chat_provider,
|
||||
"reasoning_provider": config.reasoning_provider,
|
||||
|
||||
@@ -26,6 +26,10 @@ pub struct ModelSettingsPatch {
|
||||
/// Pass `Some(vec![])` to clear all third-party cloud providers.
|
||||
pub cloud_providers:
|
||||
Option<Vec<crate::openhuman::config::schema::cloud_providers::CloudProviderCreds>>,
|
||||
/// When `Some`, REPLACES the entire `config.model_registry` array. Carries
|
||||
/// each model's user-set `vision` flag (Settings → Advanced LLM → custom
|
||||
/// model → "Supports vision"). Pass `Some(vec![])` to clear; `None` keeps it.
|
||||
pub model_registry: Option<Vec<crate::openhuman::config::schema::ModelRegistryEntry>>,
|
||||
/// Id of the `cloud_providers` entry used when a workload routes to
|
||||
/// `"cloud"`. Empty string clears (factory falls back to OpenHuman).
|
||||
pub primary_cloud: Option<String>,
|
||||
@@ -139,6 +143,24 @@ pub async fn apply_model_settings(
|
||||
if let Some(routes) = update.model_routes {
|
||||
config.model_routes = routes;
|
||||
}
|
||||
if let Some(registry) = update.model_registry {
|
||||
// Full replacement — the UI sends the canonical per-model registry
|
||||
// (carrying each model's `vision` flag). Empty vec clears it.
|
||||
log::debug!(
|
||||
"[config] apply_model_settings: replacing model_registry ({} entries)",
|
||||
registry.len()
|
||||
);
|
||||
// Normalize ids: `model_vision_enabled` matches the resolved model id
|
||||
// exactly, so stray surrounding whitespace would silently disable vision
|
||||
// for an otherwise valid model.
|
||||
config.model_registry = registry
|
||||
.into_iter()
|
||||
.map(|mut entry| {
|
||||
entry.id = entry.id.trim().to_string();
|
||||
entry
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
if let Some(providers) = update.cloud_providers {
|
||||
use crate::openhuman::config::schema::cloud_providers::is_slug_reserved;
|
||||
let preserved: Vec<_> = config
|
||||
|
||||
@@ -588,6 +588,77 @@ async fn apply_model_settings_replaces_model_routes_when_some_and_keeps_when_non
|
||||
assert!(cfg.model_routes.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn apply_model_settings_replaces_model_registry_when_some_and_keeps_when_none() {
|
||||
// Per-model vision registry follows Some=replace / None=keep / empty=clear —
|
||||
// this persists the "Supports vision" flag set in Settings → Advanced LLM.
|
||||
use crate::openhuman::config::schema::ModelRegistryEntry;
|
||||
let tmp = tempdir().unwrap();
|
||||
let mut cfg = tmp_config(&tmp);
|
||||
|
||||
let set = ModelSettingsPatch {
|
||||
model_registry: Some(vec![ModelRegistryEntry {
|
||||
id: "my-llava".into(),
|
||||
provider: "openai".into(),
|
||||
cost_per_1m_output: 0.0,
|
||||
vision: true,
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
let _ = apply_model_settings(&mut cfg, set).await.expect("set");
|
||||
assert_eq!(cfg.model_registry.len(), 1);
|
||||
assert!(cfg
|
||||
.model_registry
|
||||
.iter()
|
||||
.any(|e| e.id == "my-llava" && e.vision));
|
||||
|
||||
// None — leave registry alone.
|
||||
let _ = apply_model_settings(
|
||||
&mut cfg,
|
||||
ModelSettingsPatch {
|
||||
api_url: Some("https://x.test/v1".into()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("touch");
|
||||
assert_eq!(cfg.model_registry.len(), 1);
|
||||
|
||||
// Empty vec — clear.
|
||||
let _ = apply_model_settings(
|
||||
&mut cfg,
|
||||
ModelSettingsPatch {
|
||||
model_registry: Some(vec![]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("clear");
|
||||
assert!(cfg.model_registry.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn apply_model_settings_trims_model_registry_ids() {
|
||||
// `model_vision_enabled` matches the resolved id exactly, so persisted ids
|
||||
// must be trimmed or stray whitespace would silently disable vision.
|
||||
use crate::openhuman::config::schema::ModelRegistryEntry;
|
||||
let tmp = tempdir().unwrap();
|
||||
let mut cfg = tmp_config(&tmp);
|
||||
|
||||
let set = ModelSettingsPatch {
|
||||
model_registry: Some(vec![ModelRegistryEntry {
|
||||
id: " spaced-model ".into(),
|
||||
provider: "openai".into(),
|
||||
cost_per_1m_output: 0.0,
|
||||
vision: true,
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
let _ = apply_model_settings(&mut cfg, set).await.expect("set");
|
||||
assert_eq!(cfg.model_registry.len(), 1);
|
||||
assert_eq!(cfg.model_registry[0].id, "spaced-model");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn apply_model_settings_empty_strings_clear_optional_fields() {
|
||||
let tmp = tempdir().unwrap();
|
||||
|
||||
@@ -360,6 +360,10 @@ fn handle_update_model_settings(params: Map<String, Value>) -> ControllerFuture
|
||||
.collect::<Result<Vec<_>, String>>()
|
||||
})
|
||||
.transpose()?,
|
||||
// The config-domain RPC doesn't carry a model-registry payload — the
|
||||
// per-model vision registry is updated via the inference-domain
|
||||
// `inference_update_model_settings` path.
|
||||
model_registry: None,
|
||||
primary_cloud: update.primary_cloud,
|
||||
chat_provider: update.chat_provider,
|
||||
reasoning_provider: update.reasoning_provider,
|
||||
|
||||
@@ -137,6 +137,46 @@ pub fn context_window_for_model_with_local_fallback(
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether the model resolved for a chat hint/agent/profile accepts image input
|
||||
/// according to the **user-configured** vision flag in `config.model_registry`.
|
||||
///
|
||||
/// This is the per-model override that lets a user mark a **custom / BYOK** model
|
||||
/// as vision-capable (Settings → Advanced LLM → custom model → "Supports
|
||||
/// vision"). Managed-backend models already advertise vision via
|
||||
/// [`crate::openhuman::inference::provider::Provider::supports_vision`]; this flag
|
||||
/// covers OpenAI-compatible providers the backend can't introspect per-model.
|
||||
/// Returns `false` for models the user has not flagged.
|
||||
pub fn model_vision_enabled(model: &str, config: &crate::openhuman::config::Config) -> bool {
|
||||
let normalized = model.trim();
|
||||
if normalized.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let enabled = config
|
||||
.model_registry
|
||||
.iter()
|
||||
.any(|entry| entry.id == normalized && entry.vision);
|
||||
tracing::debug!(
|
||||
model = normalized,
|
||||
vision_enabled = enabled,
|
||||
"[model_context] resolved user-configured vision flag"
|
||||
);
|
||||
enabled
|
||||
}
|
||||
|
||||
/// Whether a resolved model accepts image input. The single predicate shared by
|
||||
/// the chat UI resolve and the server-side session/sub-agent gates.
|
||||
///
|
||||
/// - **Managed OpenHuman tiers** consult the hardcoded per-tier map
|
||||
/// ([`crate::openhuman::inference::provider::factory::oh_tier_supports_vision`]) —
|
||||
/// the remote backend does not advertise per-tier capability, so the core owns
|
||||
/// it. Currently every managed tier is `false`.
|
||||
/// - **Custom/BYOK models** consult the user-set `model_registry.vision` flag
|
||||
/// ([`model_vision_enabled`]).
|
||||
pub fn model_supports_vision(model: &str, config: &crate::openhuman::config::Config) -> bool {
|
||||
crate::openhuman::inference::provider::factory::oh_tier_supports_vision(model)
|
||||
|| model_vision_enabled(model, config)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -210,6 +250,50 @@ mod tests {
|
||||
assert_eq!(context_window_for_model(" "), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_vision_enabled_reads_registry_only() {
|
||||
use crate::openhuman::config::schema::ModelRegistryEntry;
|
||||
let mut config = crate::openhuman::config::Config::default();
|
||||
config.model_registry = vec![
|
||||
ModelRegistryEntry {
|
||||
id: "my-llava".into(),
|
||||
provider: "openai".into(),
|
||||
cost_per_1m_output: 0.0,
|
||||
vision: true,
|
||||
},
|
||||
ModelRegistryEntry {
|
||||
id: "text-only".into(),
|
||||
provider: "openai".into(),
|
||||
cost_per_1m_output: 0.0,
|
||||
vision: false,
|
||||
},
|
||||
];
|
||||
assert!(model_vision_enabled("my-llava", &config));
|
||||
assert!(!model_vision_enabled("text-only", &config));
|
||||
assert!(!model_vision_enabled("unlisted", &config));
|
||||
assert!(!model_vision_enabled(" ", &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_supports_vision_combines_tier_map_and_registry() {
|
||||
use crate::openhuman::config::schema::ModelRegistryEntry;
|
||||
let mut config = crate::openhuman::config::Config::default();
|
||||
config.model_registry = vec![ModelRegistryEntry {
|
||||
id: "my-llava".into(),
|
||||
provider: "openai".into(),
|
||||
cost_per_1m_output: 0.0,
|
||||
vision: true,
|
||||
}];
|
||||
// Managed tiers are non-vision (the per-tier map is all `false` today).
|
||||
assert!(!model_supports_vision("chat-v1", &config));
|
||||
assert!(!model_supports_vision("reasoning-v1", &config));
|
||||
assert!(!model_supports_vision("hint:chat", &config));
|
||||
// BYOK model flagged in the registry is vision-capable.
|
||||
assert!(model_supports_vision("my-llava", &config));
|
||||
// Unlisted custom model is not.
|
||||
assert!(!model_supports_vision("gpt-5", &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn o1_o3_segment_match_does_not_overmatch() {
|
||||
// Real OpenAI o1/o3 model ids must still resolve.
|
||||
|
||||
@@ -172,6 +172,34 @@ pub(crate) fn is_known_openhuman_tier(model: &str) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// Per-tier vision (image-input) capability for the managed OpenHuman backend.
|
||||
///
|
||||
/// The remote managed backend (`api.tinyhumans.ai`) does not advertise per-tier
|
||||
/// capabilities, so the core maintains this map itself. Accepts both the tier
|
||||
/// constants and their `hint:*` forms (callers may pass either pre- or
|
||||
/// post-resolution).
|
||||
///
|
||||
/// Every managed tier currently returns `false` — flip an individual arm to
|
||||
/// `true` once that tier is confirmed multimodal on the backend. This is the
|
||||
/// **only** place to change managed-model vision; BYOK/custom models are handled
|
||||
/// separately by the user-set `model_registry.vision` flag
|
||||
/// ([`crate::openhuman::inference::model_context::model_vision_enabled`]).
|
||||
pub(crate) fn oh_tier_supports_vision(model: &str) -> bool {
|
||||
use crate::openhuman::config::{
|
||||
MODEL_AGENTIC_V1, MODEL_CHAT_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1,
|
||||
MODEL_REASONING_V1, MODEL_SUMMARIZATION_V1,
|
||||
};
|
||||
match model {
|
||||
MODEL_CHAT_V1 | "hint:chat" => false,
|
||||
MODEL_REASONING_V1 | "hint:reasoning" => false,
|
||||
MODEL_REASONING_QUICK_V1 => false,
|
||||
MODEL_AGENTIC_V1 | "hint:agentic" => false,
|
||||
MODEL_CODING_V1 | "hint:coding" => false,
|
||||
MODEL_SUMMARIZATION_V1 | "hint:summarization" => false,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the configured provider string for a named workload role.
|
||||
///
|
||||
/// Empty / `"cloud"` resolves through BYOK fallback first for the three
|
||||
|
||||
@@ -867,6 +867,40 @@ fn invalid_models_fail() {
|
||||
assert!(!is_known_openhuman_tier("hint:"));
|
||||
}
|
||||
|
||||
// ── oh_tier_supports_vision ──────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn no_managed_tier_is_vision_capable_yet() {
|
||||
// Every managed tier (and its hint form) is non-vision until confirmed
|
||||
// multimodal on the backend. Flip the corresponding arm in
|
||||
// `oh_tier_supports_vision` to enable one.
|
||||
for model in [
|
||||
"reasoning-v1",
|
||||
"chat-v1",
|
||||
"agentic-v1",
|
||||
"coding-v1",
|
||||
"reasoning-quick-v1",
|
||||
"summarization-v1",
|
||||
"hint:reasoning",
|
||||
"hint:chat",
|
||||
"hint:agentic",
|
||||
"hint:coding",
|
||||
"hint:summarization",
|
||||
] {
|
||||
assert!(
|
||||
!oh_tier_supports_vision(model),
|
||||
"expected managed tier '{model}' to be non-vision"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_models_are_not_vision_capable() {
|
||||
assert!(!oh_tier_supports_vision("gpt-5"));
|
||||
assert!(!oh_tier_supports_vision("claude-opus-4-7"));
|
||||
assert!(!oh_tier_supports_vision(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_openhuman_backend_forwards_unknown_hint_verbatim() {
|
||||
// Unrecognised hint:* strings (e.g. hint:reaction for lightweight models)
|
||||
|
||||
@@ -80,6 +80,8 @@ struct InferenceUpdateModelSettingsParams {
|
||||
default_temperature: Option<f64>,
|
||||
model_routes: Option<Vec<InferenceModelRouteUpdate>>,
|
||||
cloud_providers: Option<Vec<InferenceCloudProviderUpdate>>,
|
||||
#[serde(default)]
|
||||
model_registry: Option<Vec<crate::openhuman::config::schema::ModelRegistryEntry>>,
|
||||
primary_cloud: Option<String>,
|
||||
chat_provider: Option<String>,
|
||||
reasoning_provider: Option<String>,
|
||||
@@ -255,7 +257,13 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
||||
function: "resolve_model",
|
||||
description: "Resolve a model hint or tier name to the concrete model the provider router would use.",
|
||||
inputs: vec![required_string("hint", "Model hint (e.g. hint:reasoning) or tier name (e.g. reasoning-v1).")],
|
||||
outputs: vec![json_output("model", "Resolved concrete model id.")],
|
||||
outputs: vec![
|
||||
json_output("model", "Resolved concrete model id."),
|
||||
json_output(
|
||||
"vision",
|
||||
"Whether the resolved model accepts image input (vision-capable).",
|
||||
),
|
||||
],
|
||||
},
|
||||
"status" => ControllerSchema {
|
||||
namespace: "inference",
|
||||
@@ -283,6 +291,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
||||
optional_f64("default_temperature", "Optional default temperature override."),
|
||||
optional_json("model_routes", "Optional full replacement for legacy model routes."),
|
||||
optional_json("cloud_providers", "Optional full replacement for configured cloud providers."),
|
||||
optional_json("model_registry", "Optional full replacement for the per-model registry (carries each model's `vision` flag)."),
|
||||
optional_string("primary_cloud", "Optional primary cloud provider id."),
|
||||
optional_string("chat_provider", "Optional chat workload provider string."),
|
||||
optional_string("reasoning_provider", "Optional reasoning workload provider string."),
|
||||
@@ -557,8 +566,14 @@ fn handle_inference_resolve_model(params: Map<String, Value>) -> ControllerFutur
|
||||
let resolved = crate::openhuman::inference::provider::factory::resolve_model_for_hint(
|
||||
&p.hint, &config,
|
||||
);
|
||||
// Whether the resolved model accepts image input — drives the chat UI's
|
||||
// image-attachment affordance. Managed OpenHuman tiers consult the
|
||||
// core-owned per-tier map (currently all `false`); custom/BYOK models are
|
||||
// covered by the user's per-model `model_registry.vision` flag.
|
||||
let vision =
|
||||
crate::openhuman::inference::model_context::model_supports_vision(&resolved, &config);
|
||||
to_json(RpcOutcome::new(
|
||||
serde_json::json!({ "model": resolved }),
|
||||
serde_json::json!({ "model": resolved, "vision": vision }),
|
||||
vec![],
|
||||
))
|
||||
})
|
||||
@@ -676,6 +691,7 @@ fn handle_inference_update_model_settings(params: Map<String, Value>) -> Control
|
||||
.collect::<Result<Vec<_>, String>>()
|
||||
})
|
||||
.transpose()?,
|
||||
model_registry: update.model_registry,
|
||||
primary_cloud: update.primary_cloud,
|
||||
chat_provider: update.chat_provider,
|
||||
reasoning_provider: update.reasoning_provider,
|
||||
|
||||
Reference in New Issue
Block a user