feat(chat): per-model vision flag for custom/BYOK image attachments (#3282) (#3420)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
sanil-23
2026-06-08 20:01:03 +05:30
committed by GitHub
co-authored by Claude Opus 4.8
parent 9b85a64b58
commit f891fb1508
45 changed files with 1194 additions and 120 deletions
+6 -1
View File
@@ -81,7 +81,12 @@ export default function ChatComposer({
<input
ref={fileInputRef}
type="file"
accept={allowedMimeTypes.join(',')}
// No `accept` filter: Chromium 146 / CEF on macOS greys out valid files
// at the native open panel regardless of the filter shape (MIME, mixed,
// or extension-only). Selection is gated in `onAttachFiles` →
// `validateAndReadFile` instead (rejects unsupported types + images on
// non-vision models). `allowedMimeTypes` is kept for that JS validation.
accept={allowedMimeTypes.length ? allowedMimeTypes.join(',') : undefined}
multiple
className="hidden"
onChange={e => {
+167 -25
View File
@@ -26,11 +26,14 @@ import {
loadLocalProviderSnapshot,
type LocalProviderSnapshot,
type ModelInfo,
type ModelRegistryEntry,
modelRegistryVision,
OPENAI_CODEX_OAUTH_MISSING_AUTH_URL,
OPENAI_CODEX_OAUTH_MISSING_CALLBACK_URL,
saveAISettings,
setCloudProviderKey,
testProviderModel,
upsertModelRegistryVision,
} from '../../../services/api/aiSettingsApi';
import {
creditsApi,
@@ -144,12 +147,17 @@ const BUILTIN_PROVIDER_META: Record<string, { tone: string; label: string }> = {
};
const WORKLOADS: Workload[] = [
{ id: 'chat', group: 'chat', label: 'Chat', description: 'Direct conversational back-and-forth' },
{
id: 'chat',
group: 'chat',
label: 'Chat',
description: 'Direct conversational back-and-forth — “Quick” mode in Conversations',
},
{
id: 'reasoning',
group: 'chat',
label: 'Reasoning',
description: 'Main chat agent, meeting summarizer',
description: 'Main chat agent, meeting summarizer — “Reasoning” mode in Conversations',
},
{
id: 'agentic',
@@ -217,7 +225,11 @@ const WORKLOAD_MODEL_HINTS: Record<WorkloadId, string> = {
// just derives the `maskedKey` display string from `has_api_key`.
// ─────────────────────────────────────────────────────────────────────────────
type AISettings = { cloudProviders: CloudProvider[]; routing: RoutingMap };
type AISettings = {
cloudProviders: CloudProvider[];
routing: RoutingMap;
modelRegistry: ModelRegistryEntry[];
};
const EMPTY_ROUTING: RoutingMap = {
chat: { kind: 'default' },
@@ -230,7 +242,11 @@ const EMPTY_ROUTING: RoutingMap = {
subconscious: { kind: 'default' },
};
const EMPTY_SETTINGS: AISettings = { cloudProviders: [], routing: EMPTY_ROUTING };
const EMPTY_SETTINGS: AISettings = {
cloudProviders: [],
routing: EMPTY_ROUTING,
modelRegistry: [],
};
function maskKeyLabel(hasKey: boolean): string {
return hasKey ? '•••• configured' : 'Not configured';
@@ -280,7 +296,7 @@ function toPanelRoutingFromApi(api: ApiAISettings): { panel: AISettings } {
learning: liftRef(api.routing.learning),
subconscious: liftRef(api.routing.subconscious),
};
return { panel: { cloudProviders, routing } };
return { panel: { cloudProviders, routing, modelRegistry: api.modelRegistry } };
}
function toApiSettings(panel: AISettings): ApiAISettings {
@@ -303,6 +319,7 @@ function toApiSettings(panel: AISettings): ApiAISettings {
learning: panel.routing.learning,
subconscious: panel.routing.subconscious,
},
modelRegistry: panel.modelRegistry,
};
}
@@ -1725,8 +1742,11 @@ interface CustomRoutingDialogProps {
cloudProviders: CloudProvider[];
localModels: OllamaModel[];
ollamaRunning: boolean;
/** Current per-model vision registry, used to prefill the vision checkbox. */
modelRegistry: ModelRegistryEntry[];
onClose: () => void;
onSubmit: (next: ProviderRef) => void;
/** Emits the chosen provider ref plus the user's vision flag for that model. */
onSubmit: (next: ProviderRef, vision: boolean) => void;
}
type CustomDialogSource =
@@ -1811,6 +1831,7 @@ const CustomRoutingDialog = ({
cloudProviders,
localModels,
ollamaRunning,
modelRegistry,
onClose,
onSubmit,
}: CustomRoutingDialogProps) => {
@@ -1862,6 +1883,35 @@ const CustomRoutingDialog = ({
: null
);
// Registry slug for the selected source — keys the per-model vision flag.
// Cloud uses the provider slug; local → `ollama`; claude-code → `claude-code`.
const registrySlug =
source?.kind === 'cloud'
? source.providerSlug
: source?.kind === 'local'
? 'ollama'
: source?.kind === 'claude-code'
? 'claude-code'
: null;
// User-set vision flag for this (provider, model). Prefilled from the registry,
// re-prefilled whenever the selected provider/model changes.
const [vision, setVision] = useState<boolean>(() =>
registrySlug && model.trim()
? modelRegistryVision(modelRegistry, registrySlug, model.trim())
: false
);
useEffect(() => {
// eslint-disable-next-line react-hooks/set-state-in-effect
setVision(
registrySlug && model.trim()
? modelRegistryVision(modelRegistry, registrySlug, model.trim())
: false
);
// modelRegistry is stable for the dialog's lifetime (prop doesn't change mid-open).
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [registrySlug, model]);
const selectedCloud =
source?.kind === 'cloud' ? customCloud.find(c => c.slug === source.providerSlug) : undefined;
@@ -1935,16 +1985,19 @@ const CustomRoutingDialog = ({
if (!source || !canSave) return;
const temp = temperature == null || !Number.isFinite(temperature) ? null : temperature;
if (source.kind === 'cloud') {
onSubmit({
kind: 'cloud',
providerSlug: source.providerSlug,
model: model.trim(),
temperature: temp,
});
onSubmit(
{
kind: 'cloud',
providerSlug: source.providerSlug,
model: model.trim(),
temperature: temp,
},
vision
);
} else if (source.kind === 'claude-code') {
onSubmit({ kind: 'claude-code', model: model.trim(), temperature: temp });
onSubmit({ kind: 'claude-code', model: model.trim(), temperature: temp }, vision);
} else {
onSubmit({ kind: 'local', model: model.trim(), temperature: temp });
onSubmit({ kind: 'local', model: model.trim(), temperature: temp }, vision);
}
};
@@ -2220,6 +2273,26 @@ const CustomRoutingDialog = ({
</p>
</div>
{/* Vision capability (optional). Marks a custom/BYOK model as
accepting image input so the chat composer offers image
attachments for it. Only shown once a concrete model is chosen. */}
{registrySlug && model.trim().length > 0 && (
<div className="flex flex-col gap-1.5">
<label className="inline-flex items-center gap-2 text-xs font-medium text-stone-700 dark:text-neutral-200">
<input
type="checkbox"
checked={vision}
onChange={e => setVision(e.target.checked)}
className="h-3.5 w-3.5 rounded border-stone-300 dark:border-neutral-700 text-primary-500 focus:ring-primary-500"
/>
{t('settings.ai.modelVision')}
</label>
<p className="text-[11px] text-stone-400 dark:text-neutral-500">
{t('settings.ai.modelVisionDesc')}
</p>
</div>
)}
{(testBusy || testReply || testError || testStartedAt) && (
<div
role={testError ? 'alert' : 'status'}
@@ -2357,6 +2430,7 @@ const GlobalOwnModelSelector = ({
cloudProviders,
localModels,
ollamaRunning,
modelRegistry,
onApply,
}: {
current: ProviderRef | null;
@@ -2364,7 +2438,8 @@ const GlobalOwnModelSelector = ({
cloudProviders: CloudProvider[];
localModels: OllamaModel[];
ollamaRunning: boolean;
onApply: (next: ProviderRef) => Promise<void>;
modelRegistry: ModelRegistryEntry[];
onApply: (next: ProviderRef, vision: boolean) => Promise<void>;
}) => {
const { t } = useT();
const customCloud = cloudProviders.filter(p => p.slug !== 'openhuman');
@@ -2385,6 +2460,29 @@ const GlobalOwnModelSelector = ({
const [model, setModel] = useState<string>(
current?.kind === 'cloud' || current?.kind === 'local' ? current.model : ''
);
// Registry slug for the selected source — keys the per-model vision flag.
const registrySlug =
source?.kind === 'cloud'
? source.providerSlug
: source?.kind === 'local'
? 'ollama'
: source?.kind === 'claude-code'
? 'claude-code'
: null;
const [vision, setVision] = useState<boolean>(() =>
registrySlug && model.trim()
? modelRegistryVision(modelRegistry, registrySlug, model.trim())
: false
);
useEffect(() => {
// eslint-disable-next-line react-hooks/set-state-in-effect
setVision(
registrySlug && model.trim()
? modelRegistryVision(modelRegistry, registrySlug, model.trim())
: false
);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [registrySlug, model]);
const [cloudModels, setCloudModels] = useState<ModelInfo[]>([]);
const [cloudModelsLoading, setCloudModelsLoading] = useState(false);
const [cloudModelsError, setCloudModelsError] = useState<string | null>(null);
@@ -2453,15 +2551,14 @@ const GlobalOwnModelSelector = ({
setSaving(true);
try {
if (nextSource.kind === 'local') {
await onApply({ kind: 'local', model: nextModel.trim() });
await onApply({ kind: 'local', model: nextModel.trim() }, vision);
} else if (nextSource.kind === 'claude-code') {
await onApply({ kind: 'claude-code', model: nextModel.trim() });
await onApply({ kind: 'claude-code', model: nextModel.trim() }, vision);
} else {
await onApply({
kind: 'cloud',
providerSlug: nextSource.providerSlug,
model: nextModel.trim(),
});
await onApply(
{ kind: 'cloud', providerSlug: nextSource.providerSlug, model: nextModel.trim() },
vision
);
}
} finally {
setSaving(false);
@@ -2566,6 +2663,23 @@ const GlobalOwnModelSelector = ({
) : null}
</div>
</div>
{registrySlug && model.trim().length > 0 && (
<label className="flex items-start gap-2 text-xs font-medium text-stone-700 dark:text-neutral-200">
<input
type="checkbox"
checked={vision}
onChange={e => setVision(e.target.checked)}
className="mt-0.5 h-3.5 w-3.5 rounded border-stone-300 dark:border-neutral-700 text-primary-500 focus:ring-primary-500"
/>
<span>
{t('settings.ai.modelVision')}
<span className="block font-normal text-[11px] text-stone-400 dark:text-neutral-500">
{t('settings.ai.modelVisionDesc')}
</span>
</span>
</label>
)}
<div className="rounded-lg bg-stone-50 dark:bg-neutral-800/60 px-3 py-2 text-xs text-stone-500 dark:text-neutral-400">
{t('settings.ai.globalModel.appliesToAll')}
</div>
@@ -3103,8 +3217,23 @@ const AIPanel = ({ embedded = false }: AIPanelProps = {}) => {
cloudProviders={draft.cloudProviders}
localModels={installed}
ollamaRunning={ollama.state === 'running'}
onApply={async next => {
await persist({ ...draft, routing: routingWithAllWorkloads(next) });
modelRegistry={draft.modelRegistry}
onApply={async (next, vision) => {
const reg =
next.kind === 'cloud'
? { slug: next.providerSlug, model: next.model }
: next.kind === 'local'
? { slug: 'ollama', model: next.model }
: next.kind === 'claude-code'
? { slug: 'claude-code', model: next.model }
: null;
await persist({
...draft,
routing: routingWithAllWorkloads(next),
modelRegistry: reg
? upsertModelRegistryVision(draft.modelRegistry, reg.slug, reg.model, vision)
: draft.modelRegistry,
});
}}
/>
) : null}
@@ -3290,11 +3419,24 @@ const AIPanel = ({ embedded = false }: AIPanelProps = {}) => {
cloudProviders={draft.cloudProviders}
localModels={installed}
ollamaRunning={ollama.state === 'running'}
modelRegistry={draft.modelRegistry}
onClose={() => setCustomDialogFor(null)}
onSubmit={async next => {
onSubmit={async (next, vision) => {
// (provider slug, model id) the vision flag keys on.
const reg =
next.kind === 'cloud'
? { slug: next.providerSlug, model: next.model }
: next.kind === 'local'
? { slug: 'ollama', model: next.model }
: next.kind === 'claude-code'
? { slug: 'claude-code', model: next.model }
: null;
const nextDraft = {
...draft,
routing: { ...draft.routing, [customDialogFor]: next },
modelRegistry: reg
? upsertModelRegistryVision(draft.modelRegistry, reg.slug, reg.model, vision)
: draft.modelRegistry,
};
await persist(nextDraft);
setCustomDialogFor(null);
@@ -15,6 +15,7 @@ import {
setCloudProviderKey,
startOpenAiCodexOAuth,
testProviderModel,
upsertModelRegistryVision,
} from '../../../../services/api/aiSettingsApi';
import { creditsApi } from '../../../../services/api/creditsApi';
import { renderWithProviders } from '../../../../test/test-utils';
@@ -45,6 +46,8 @@ vi.mock('../../../../services/api/aiSettingsApi', () => ({
saveAISettings: vi.fn(),
loadLocalProviderSnapshot: vi.fn(),
testProviderModel: vi.fn(),
modelRegistryVision: vi.fn(() => false),
upsertModelRegistryVision: vi.fn((registry: unknown[]) => registry),
setCloudProviderKey: vi.fn().mockResolvedValue(undefined),
clearCloudProviderKey: vi.fn().mockResolvedValue(undefined),
serializeProviderRef: vi.fn((r: { kind: string; providerSlug?: string; model?: string }) =>
@@ -123,6 +126,7 @@ const baseSettings = {
learning: { kind: 'openhuman' as const },
subconscious: { kind: 'openhuman' as const },
},
modelRegistry: [],
};
const baseLocalSnapshot = { status: null, diagnostics: null, presets: null, installedModels: [] };
@@ -304,6 +308,52 @@ describe('AIPanel', () => {
}
});
// ─── per-model vision flag (BYOK) ───────────────────────────────────────────
it('flags a custom BYOK model as vision-capable via the Own-model selector', async () => {
vi.mocked(loadAISettings).mockResolvedValue({
...baseSettings,
cloudProviders: [
...baseSettings.cloudProviders,
{
id: 'p_custom_openai',
slug: 'openai',
label: 'OpenAI',
endpoint: 'https://api.openai.com/v1',
auth_style: 'bearer' as const,
has_api_key: true,
},
],
});
renderWithProviders(<AIPanel />);
await waitFor(() =>
expect(screen.getByRole('button', { name: /Use Your Own Models/i })).toBeInTheDocument()
);
fireEvent.click(screen.getByRole('button', { name: /Use Your Own Models/i }));
// Enter a model id → the per-model "Supports vision" checkbox appears.
const modelInput = await screen.findByPlaceholderText('Enter model id');
fireEvent.change(modelInput, { target: { value: 'gpt-4o' } });
const visionCheckbox = await screen.findByRole('checkbox', { name: /Supports vision/i });
expect(visionCheckbox).not.toBeChecked();
fireEvent.click(visionCheckbox);
expect(visionCheckbox).toBeChecked();
fireEvent.click(screen.getByRole('button', { name: /^Save$/ }));
// The vision flag is threaded through to the registry upsert + persisted.
await waitFor(() =>
expect(vi.mocked(upsertModelRegistryVision)).toHaveBeenCalledWith(
expect.anything(),
'openai',
'gpt-4o',
true
)
);
expect(saveAISettings).toHaveBeenCalled();
});
// ─── auth_style preservation ────────────────────────────────────────────────
it('preserves auth_style: "anthropic" through save when Anthropic provider is configured', async () => {
@@ -333,6 +383,7 @@ describe('AIPanel', () => {
learning: { kind: 'openhuman' as const },
subconscious: { kind: 'openhuman' as const },
},
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithAnthropic);
@@ -557,6 +608,7 @@ describe('AIPanel', () => {
learning: { kind: 'openhuman' as const },
subconscious: { kind: 'openhuman' as const },
},
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
vi.mocked(saveAISettings).mockResolvedValue(undefined);
@@ -612,6 +664,7 @@ describe('AIPanel', () => {
learning: { kind: 'openhuman' as const },
subconscious: { kind: 'openhuman' as const },
},
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOllama);
vi.mocked(saveAISettings).mockResolvedValue(undefined);
@@ -986,6 +1039,7 @@ describe('AIPanel', () => {
has_api_key: false,
},
],
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOllama);
renderWithProviders(<AIPanel />);
@@ -1008,6 +1062,7 @@ describe('AIPanel', () => {
has_api_key: false,
},
],
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOllama);
renderWithProviders(<AIPanel />);
@@ -1047,6 +1102,7 @@ describe('AIPanel', () => {
...baseSettings.routing,
reasoning: { kind: 'cloud' as const, providerSlug: 'openai', model: 'gpt-4o' },
},
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
vi.mocked(saveAISettings).mockResolvedValue(undefined);
@@ -1103,6 +1159,7 @@ describe('AIPanel', () => {
...baseSettings.routing,
reasoning: { kind: 'cloud' as const, providerSlug: 'openai', model: 'gpt-4o' },
},
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
vi.mocked(listProviderModels).mockResolvedValue([{ id: 'gpt-4o' }, { id: 'gpt-4o-mini' }]);
@@ -1146,6 +1203,7 @@ describe('AIPanel', () => {
...baseSettings.routing,
reasoning: { kind: 'cloud' as const, providerSlug: 'openai', model: 'gpt-4o' },
},
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
vi.mocked(listProviderModels).mockResolvedValue([{ id: 'gpt-4o' }]);
@@ -1190,6 +1248,7 @@ describe('AIPanel', () => {
...baseSettings.routing,
reasoning: { kind: 'cloud' as const, providerSlug: 'openai', model: 'gpt-4o' },
},
modelRegistry: [],
};
vi.mocked(loadAISettings).mockResolvedValue(settingsWithOpenAI);
vi.mocked(listProviderModels).mockResolvedValue([{ id: 'gpt-4o' }]);
+34
View File
@@ -85,6 +85,40 @@ describe('validateAndReadFile', () => {
}
});
it('rejects non-extractable documents (docx/pptx/xlsx) the agent cannot read', async () => {
for (const mime of [
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/zip',
]) {
const result = await validateAndReadFile(makeFile('f', mime, 8), 0);
expect('error' in result && result.error.code).toBe('unsupported_type');
}
});
it('accepts the text-extractable document formats (pdf/txt/markdown)', async () => {
for (const mime of ['application/pdf', 'text/plain', 'text/markdown']) {
const result = await validateAndReadFile(makeFile('f', mime, 8), 0);
expect('attachment' in result && result.attachment.kind === 'file').toBe(true);
}
});
it('rejects an image when allowImages is false (non-vision model)', async () => {
const result = await validateAndReadFile(makeFile('p.png', 'image/png', 8), 0, 0, false);
expect('error' in result && result.error.code).toBe('image_not_supported');
});
it('still accepts a document when allowImages is false', async () => {
const result = await validateAndReadFile(makeFile('d.pdf', 'application/pdf', 8), 0, 0, false);
expect('attachment' in result && result.attachment.kind === 'file').toBe(true);
});
it('accepts an image when allowImages is true (vision model)', async () => {
const result = await validateAndReadFile(makeFile('p.png', 'image/png', 8), 0, 0, true);
expect('attachment' in result && result.attachment.kind === 'image').toBe(true);
});
it('rejects files that exceed the file size limit', async () => {
const oversizedFile = makeFile(
'big.pdf',
+44 -2
View File
@@ -41,6 +41,17 @@ export const ALLOWED_ATTACHMENT_MIME_TYPES = [
...ALLOWED_FILE_MIME_TYPES,
] as const;
// Document formats the backend actually text-extracts (PDF via pdf_extract;
// TXT/Markdown via UTF-8). DOCX/PPTX/XLSX/ZIP are intentionally excluded — the
// agent would only see a reference stub, not their content. `text/csv` is also
// deliberately left out: the backend *can* extract it, but the chat composer is
// scoped to PDF/TXT/Markdown by product decision (revisit here if CSV is wanted).
// Used by the ingest validator below, not by a native `accept` filter:
// Chromium/CEF on macOS greys valid files at the open panel regardless of the
// filter shape, so selection is gated in `validateAndReadFile` after the user
// picks, not at the dialog.
const EXTRACTABLE_FILE_MIME_TYPES = ['application/pdf', 'text/plain', 'text/markdown'] as const;
export const ATTACHMENT_MAX_IMAGES = 4;
export const ATTACHMENT_MAX_FILES = 4;
export const ATTACHMENT_MAX_IMAGE_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
@@ -63,6 +74,7 @@ export type AttachmentError =
| { code: 'unsupported_type'; mimeType: string }
| { code: 'too_large'; sizeBytes: number; maxBytes: number }
| { code: 'too_many'; kind: AttachmentKind; max: number }
| { code: 'image_not_supported' }
| { code: 'read_failed'; reason: string };
export function isAllowedMimeType(mime: string): mime is AllowedImageMimeType {
@@ -73,6 +85,29 @@ export function isAllowedAttachmentMimeType(mime: string): mime is AllowedAttach
return (ALLOWED_ATTACHMENT_MIME_TYPES as readonly string[]).includes(mime);
}
/**
* The exact MIME set the ingest validator accepts — images plus the
* text-extractable documents. A strict subset of {@link AllowedAttachmentMimeType}
* (which also lists reference-only types like CSV/DOCX/ZIP that we reject).
*/
export type SupportedAttachmentMimeType =
| AllowedImageMimeType
| (typeof EXTRACTABLE_FILE_MIME_TYPES)[number];
/**
* Stricter gate than {@link isAllowedAttachmentMimeType}: only the formats the
* backend actually reads — images, plus the text-extractable documents (PDF via
* pdf_extract; TXT/Markdown via UTF-8). DOCX/PPTX/XLSX/ZIP are excluded so they
* can't be attached as content-less reference stubs. Applied on every ingest
* path (picker, drag-drop, paste).
*/
export function isSupportedAttachmentMimeType(mime: string): mime is SupportedAttachmentMimeType {
return (
(ALLOWED_IMAGE_MIME_TYPES as readonly string[]).includes(mime) ||
(EXTRACTABLE_FILE_MIME_TYPES as readonly string[]).includes(mime)
);
}
export function attachmentKindForMime(mime: AllowedAttachmentMimeType): AttachmentKind {
return isAllowedMimeType(mime) ? 'image' : 'file';
}
@@ -148,13 +183,20 @@ async function buildAttachmentDataUri(
export async function validateAndReadFile(
file: File,
existingCount: number,
existingFileCount = 0
existingFileCount = 0,
// When `false` (the active chat model isn't vision-capable), image files are
// rejected; documents (PDF/Word/etc.) still flow. Defaults `true` so non-chat
// callers are unaffected.
allowImages = true
): Promise<{ attachment: Attachment } | { error: AttachmentError }> {
if (!isAllowedAttachmentMimeType(file.type)) {
if (!isSupportedAttachmentMimeType(file.type)) {
return { error: { code: 'unsupported_type', mimeType: file.type || 'unknown' } };
}
const kind = attachmentKindForMime(file.type);
if (!allowImages && kind === 'image') {
return { error: { code: 'image_not_supported' } };
}
const maxCount = kind === 'image' ? ATTACHMENT_MAX_IMAGES : ATTACHMENT_MAX_FILES;
const count = kind === 'image' ? existingCount : existingFileCount;
if (count >= maxCount) {
+7 -1
View File
@@ -1822,7 +1822,10 @@ const messages: TranslationMap = {
'chat.attachment.tooMany': 'الحد الأقصى {max} صور لكل رسالة',
'chat.attachment.tooManyFiles': 'الحد الأقصى {max} ملفات لكل رسالة',
'chat.attachment.tooLarge': 'حجم الصورة يتجاوز الحد المسموح {max}',
'chat.attachment.unsupportedType': 'نوع ملف غير مدعوم. استخدم PNG أو JPEG أو WebP أو GIF أو BMP.',
'chat.attachment.unsupportedType':
'نوع ملف غير مدعوم. استخدم صورة (PNG أو JPEG أو WebP أو GIF أو BMP) أو ملف PDF أو TXT أو Markdown.',
'chat.attachment.imageNotSupported':
'لا يمكن لهذا النموذج قراءة الصور. أرفق ملف PDF أو TXT أو Markdown بدلاً من ذلك.',
'chat.attachment.readFailed': 'تعذر قراءة الملف',
'memory.searchAria': 'البحث في الذاكرة',
'memory.searchPlaceholder': 'البحث في إدخالات الذاكرة...',
@@ -3046,6 +3049,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'تجاوز درجة الحرارة (القيمة)',
'settings.ai.temperatureOverrideDesc':
'أقل = أكثر تحديدا. إتركْ غير مُمْكَنَّلَ لإسْتِعْمال قَبْل المُقدّمِ.',
'settings.ai.modelVision': 'يدعم الرؤية (إدخال الصور)',
'settings.ai.modelVisionDesc':
'فعّل هذا إذا كان النموذج يقبل الصور. يتيح لمربع الدردشة إرفاق الصور عند اختيار هذا النموذج.',
'settings.ai.testFailed': 'فشل الاختبار',
'settings.ai.testingModel': 'نموذج الاختبار...',
'settings.ai.modelResponse': 'استجابة النموذج',
+6 -1
View File
@@ -1865,7 +1865,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': 'প্রতি বার্তায় সর্বোচ্চ {max}টি ফাইল',
'chat.attachment.tooLarge': 'ছবি {max} আকারের সীমা অতিক্রম করেছে',
'chat.attachment.unsupportedType':
'অসমর্থিত ফাইল প্রকার। PNG, JPEG, WebP, GIF, বা BMP ব্যবহার করুন।',
'অসমর্থিত ফাইল প্রকার। একটি ছবি (PNG, JPEG, WebP, GIF, BMP) অথবা একটি PDF, TXT, বা Markdown ফাইল ব্যবহার করুন।',
'chat.attachment.imageNotSupported':
'এই মডেলটি ছবি পড়তে পারে না। পরিবর্তে একটি PDF, TXT, বা Markdown ফাইল সংযুক্ত করুন।',
'chat.attachment.readFailed': 'ফাইল পড়া যায়নি',
'memory.searchAria': 'মেমোরি খুঁজুন',
'memory.searchPlaceholder': 'মেমোরি এন্ট্রি খুঁজুন...',
@@ -3112,6 +3114,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'তাপমাত্রা ওভাররাইড (মান)',
'settings.ai.temperatureOverrideDesc':
'নিচে = আরও বুদ্ধিমান. পরিসেবা উপলব্ধকারীর ডিফল্ট ব্যবহারের জন্য সীমা ধার্য না করা হলে, বন্ধ করুন ।',
'settings.ai.modelVision': 'ভিশন সমর্থন করে (ছবি ইনপুট)',
'settings.ai.modelVisionDesc':
'এই মডেল ছবি গ্রহণ করলে সক্ষম করুন। এই মডেল নির্বাচিত থাকলে চ্যাটে ছবি সংযুক্ত করতে দেয়।',
'settings.ai.testFailed': 'পরীক্ষা ব্যর্থ হয়েছে',
'settings.ai.testingModel': 'পরীক্ষার মডেল...',
'settings.ai.modelResponse': 'মডেল প্রতিক্রিয়া',
+6 -1
View File
@@ -1912,7 +1912,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': 'Maximal {max} Dateien pro Nachricht',
'chat.attachment.tooLarge': 'Bild überschreitet die Größenbeschränkung von {max}',
'chat.attachment.unsupportedType':
'Nicht unterstützter Dateityp. Verwenden Sie PNG, JPEG, WebP, GIF oder BMP.',
'Nicht unterstützter Dateityp. Verwenden Sie ein Bild (PNG, JPEG, WebP, GIF, BMP) oder eine PDF-, TXT- oder Markdown-Datei.',
'chat.attachment.imageNotSupported':
'Dieses Modell kann keine Bilder lesen. Hängen Sie stattdessen eine PDF-, TXT- oder Markdown-Datei an.',
'chat.attachment.readFailed': 'Datei konnte nicht gelesen werden',
'memory.searchAria': 'Speicher durchsuchen',
'memory.searchPlaceholder': 'Speichereinträge durchsuchen...',
@@ -3191,6 +3193,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Temperatur-Override (Wert)',
'settings.ai.temperatureOverrideDesc':
'Niedriger = deterministischer. Deaktivieren Sie diese Option, um den Anbieterstandard zu verwenden.',
'settings.ai.modelVision': 'Unterstützt Bildeingabe (Vision)',
'settings.ai.modelVisionDesc':
'Aktivieren, wenn dieses Modell Bilder akzeptiert. Ermöglicht das Anhängen von Bildern im Chat, wenn dieses Modell ausgewählt ist.',
'settings.ai.testFailed': 'Test fehlgeschlagen.',
'settings.ai.testingModel': 'Modell wird getestet...',
'settings.ai.modelResponse': 'Modellantwort',
+7 -1
View File
@@ -2238,7 +2238,10 @@ const en: TranslationMap = {
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooManyFiles': 'Maximum {max} files per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.unsupportedType':
'Unsupported file type. Use an image (PNG, JPEG, WebP, GIF, BMP) or a PDF, TXT, or Markdown file.',
'chat.attachment.imageNotSupported':
'This model cant read images. You can attach a pdf, txt, or md file.',
'chat.attachment.readFailed': 'Could not read file',
// Memory (additional)
@@ -3633,6 +3636,9 @@ const en: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Temperature override (value)',
'settings.ai.temperatureOverrideDesc':
'Lower = more deterministic. Leave unchecked to use the provider default.',
'settings.ai.modelVision': 'Supports vision (image input)',
'settings.ai.modelVisionDesc':
'Enable if this model accepts images. Lets the chat composer attach images when this model is selected.',
'settings.ai.testFailed': 'Test failed',
'settings.ai.testingModel': 'Testing model...',
'settings.ai.modelResponse': 'Model response',
+6 -1
View File
@@ -1904,7 +1904,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': 'Máximo {max} archivos por mensaje',
'chat.attachment.tooLarge': 'La imagen supera el límite de tamaño de {max}',
'chat.attachment.unsupportedType':
'Tipo de archivo no compatible. Use PNG, JPEG, WebP, GIF o BMP.',
'Tipo de archivo no compatible. Use una imagen (PNG, JPEG, WebP, GIF, BMP) o un archivo PDF, TXT o Markdown.',
'chat.attachment.imageNotSupported':
'Este modelo no puede leer imágenes. Adjunta un archivo PDF, TXT o Markdown en su lugar.',
'chat.attachment.readFailed': 'No se pudo leer el archivo',
'memory.searchAria': 'Buscar en memoria',
'memory.searchPlaceholder': 'Buscar entradas de memoria...',
@@ -3173,6 +3175,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Anulación de temperatura (valor)',
'settings.ai.temperatureOverrideDesc':
'Más bajo = más determinista. Déjelo sin marcar para usar el valor predeterminado del proveedor.',
'settings.ai.modelVision': 'Admite visión (entrada de imágenes)',
'settings.ai.modelVisionDesc':
'Actívalo si este modelo acepta imágenes. Permite adjuntar imágenes en el chat cuando este modelo está seleccionado.',
'settings.ai.testFailed': 'Prueba fallida',
'settings.ai.testingModel': 'Modelo de prueba...',
'settings.ai.modelResponse': 'Respuesta modelo',
+6 -1
View File
@@ -1912,7 +1912,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': 'Maximum {max} fichiers par message',
'chat.attachment.tooLarge': "L'image dépasse la taille limite de {max}",
'chat.attachment.unsupportedType':
'Type de fichier non pris en charge. Utilisez PNG, JPEG, WebP, GIF ou BMP.',
'Type de fichier non pris en charge. Utilise une image (PNG, JPEG, WebP, GIF, BMP) ou un fichier PDF, TXT ou Markdown.',
'chat.attachment.imageNotSupported':
'Ce modèle ne peut pas lire les images. Joins plutôt un fichier PDF, TXT ou Markdown.',
'chat.attachment.readFailed': 'Impossible de lire le fichier',
'memory.searchAria': 'Rechercher dans la mémoire',
'memory.searchPlaceholder': 'Rechercher des entrées de mémoire…',
@@ -3184,6 +3186,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Override de température (valeur)',
'settings.ai.temperatureOverrideDesc':
'Plus bas = plus déterministe. Laissez décoché pour utiliser la valeur par défaut du fournisseur.',
'settings.ai.modelVision': 'Prend en charge la vision (entrée dimages)',
'settings.ai.modelVisionDesc':
'Activez si ce modèle accepte les images. Permet de joindre des images dans le chat lorsque ce modèle est sélectionné.',
'settings.ai.testFailed': 'Test échoué',
'settings.ai.testingModel': 'Modèle de test...',
'settings.ai.modelResponse': 'Réponse du modèle',
+6 -1
View File
@@ -1864,7 +1864,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': 'प्रति संदेश अधिकतम {max} फ़ाइलें',
'chat.attachment.tooLarge': 'छवि {max} आकार सीमा से अधिक है',
'chat.attachment.unsupportedType':
'असमर्थित फ़ाइल प्रकार। PNG, JPEG, WebP, GIF, या BMP का उपयोग करें।',
'असमर्थित फ़ाइल प्रकार। कोई छवि (PNG, JPEG, WebP, GIF, BMP) या PDF, TXT, या Markdown फ़ाइल का उपयोग करें।',
'chat.attachment.imageNotSupported':
'यह मॉडल छवियाँ नहीं पढ़ सकता। इसके बजाय कोई PDF, TXT या Markdown फ़ाइल संलग्न करें।',
'chat.attachment.readFailed': 'फ़ाइल पढ़ नहीं सकी',
'memory.searchAria': 'मेमोरी सर्च करें',
'memory.searchPlaceholder': 'मेमोरी एंट्रीज़ सर्च करें...',
@@ -3120,6 +3122,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'तापमान ओवरराइड (मान)',
'settings.ai.temperatureOverrideDesc':
'लोअर = अधिक नियतिवादी। प्रदाता डिफ़ॉल्ट का उपयोग करने के लिए अनचेक छोड़ दें।',
'settings.ai.modelVision': 'विज़न समर्थित है (छवि इनपुट)',
'settings.ai.modelVisionDesc':
'यदि यह मॉडल छवियाँ स्वीकार करता है तो सक्षम करें। यह मॉडल चुने जाने पर चैट में छवियाँ संलग्न करने देता है।',
'settings.ai.testFailed': 'परीक्षण विफल रहा',
'settings.ai.testingModel': 'परीक्षण मॉडल...',
'settings.ai.modelResponse': 'मॉडल प्रतिक्रिया',
+6 -1
View File
@@ -1867,7 +1867,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': 'Maksimal {max} file per pesan',
'chat.attachment.tooLarge': 'Gambar melebihi batas ukuran {max}',
'chat.attachment.unsupportedType':
'Jenis file tidak didukung. Gunakan PNG, JPEG, WebP, GIF, atau BMP.',
'Jenis file tidak didukung. Gunakan gambar (PNG, JPEG, WebP, GIF, BMP) atau file PDF, TXT, atau Markdown.',
'chat.attachment.imageNotSupported':
'Model ini tidak dapat membaca gambar. Lampirkan file PDF, TXT, atau Markdown sebagai gantinya.',
'chat.attachment.readFailed': 'Tidak dapat membaca file',
'memory.searchAria': 'Cari memori',
'memory.searchPlaceholder': 'Cari entri memori...',
@@ -3124,6 +3126,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Penggantian suhu (nilai)',
'settings.ai.temperatureOverrideDesc':
'Turunkan = lebih deterministik. Biarkan tak diperiksa untuk memakai penyedia bawaan.',
'settings.ai.modelVision': 'Mendukung visi (input gambar)',
'settings.ai.modelVisionDesc':
'Aktifkan jika model ini menerima gambar. Memungkinkan penyusun obrolan melampirkan gambar saat model ini dipilih.',
'settings.ai.testFailed': 'Pengujian gagal',
'settings.ai.testingModel': 'Pengujian model...',
'settings.ai.modelResponse': 'Respons model',
+7 -1
View File
@@ -1895,7 +1895,10 @@ const messages: TranslationMap = {
'chat.attachment.tooMany': 'Massimo {max} immagini per messaggio',
'chat.attachment.tooManyFiles': 'Massimo {max} file per messaggio',
'chat.attachment.tooLarge': "L'immagine supera il limite di dimensione di {max}",
'chat.attachment.unsupportedType': 'Tipo di file non supportato. Usa PNG, JPEG, WebP, GIF o BMP.',
'chat.attachment.unsupportedType':
'Tipo di file non supportato. Usa un file immagine (PNG, JPEG, WebP, GIF, BMP) oppure PDF, TXT o Markdown.',
'chat.attachment.imageNotSupported':
'Questo modello non può leggere le immagini. Allega invece un file PDF, TXT o Markdown.',
'chat.attachment.readFailed': 'Impossibile leggere il file',
'memory.searchAria': 'Cerca memoria',
'memory.searchPlaceholder': 'Cerca voci di memoria...',
@@ -3166,6 +3169,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Ignora temperatura (valore)',
'settings.ai.temperatureOverrideDesc':
'Più basso = più deterministico. Lascia deselezionato per usare il valore predefinito del provider.',
'settings.ai.modelVision': 'Supporta la visione (input immagini)',
'settings.ai.modelVisionDesc':
'Attiva se questo modello accetta immagini. Consente di allegare immagini nella chat quando è selezionato questo modello.',
'settings.ai.testFailed': 'Test non riuscito',
'settings.ai.testingModel': 'Test del modello...',
'settings.ai.modelResponse': 'Risposta del modello',
+6 -1
View File
@@ -1844,7 +1844,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': '메시지당 최대 {max}개 파일',
'chat.attachment.tooLarge': '이미지가 {max} 크기 제한을 초과합니다',
'chat.attachment.unsupportedType':
'지원되지 않는 파일 형식입니다. PNG, JPEG, WebP, GIF 또는 BMP를 사용하세요.',
'지원되지 않는 파일 형식입니다. 이미지(PNG, JPEG, WebP, GIF, BMP) 또는 PDF, TXT, Markdown 파일을 사용하세요.',
'chat.attachment.imageNotSupported':
'이 모델은 이미지를 읽을 수 없습니다. 대신 PDF, TXT 또는 Markdown 파일을 첨부하세요.',
'chat.attachment.readFailed': '파일을 읽을 수 없습니다',
'memory.searchAria': '메모리 검색',
'memory.searchPlaceholder': '메모리 항목 검색...',
@@ -3086,6 +3088,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': '온도 재정의(값)',
'settings.ai.temperatureOverrideDesc':
'낮을수록 더 결정적입니다. 제공업체 기본값을 사용하려면 선택하지 않은 상태로 두세요.',
'settings.ai.modelVision': '비전 지원 (이미지 입력)',
'settings.ai.modelVisionDesc':
'이 모델이 이미지를 지원하면 활성화하세요. 이 모델을 선택하면 채팅 작성기에서 이미지를 첨부할 수 있습니다.',
'settings.ai.testFailed': '테스트 실패',
'settings.ai.testingModel': '모델 테스트 중...',
'settings.ai.modelResponse': '모델 응답',
+7 -1
View File
@@ -1884,7 +1884,10 @@ const messages: TranslationMap = {
'chat.attachment.tooMany': 'Maksymalnie {max} obrazów na wiadomość',
'chat.attachment.tooManyFiles': 'Maksymalnie {max} plików na wiadomość',
'chat.attachment.tooLarge': 'Obraz przekracza limit rozmiaru {max}',
'chat.attachment.unsupportedType': 'Nieobsługiwany typ pliku. Użyj PNG, JPEG, WebP, GIF lub BMP.',
'chat.attachment.unsupportedType':
'Nieobsługiwany typ pliku. Użyj obrazu (PNG, JPEG, WebP, GIF, BMP) lub pliku PDF, TXT albo Markdown.',
'chat.attachment.imageNotSupported':
'Ten model nie może odczytywać obrazów. Zamiast tego dołącz plik PDF, TXT lub Markdown.',
'chat.attachment.readFailed': 'Nie można odczytać pliku',
'memory.searchAria': 'Szukaj w pamięci',
'memory.searchPlaceholder': 'Szukaj wpisów pamięci...',
@@ -3166,6 +3169,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Nadpisanie temperatury (wartość)',
'settings.ai.temperatureOverrideDesc':
'Niższa = bardziej deterministyczna. Pozostaw niezaznaczone, aby użyć domyślnej wartości dostawcy.',
'settings.ai.modelVision': 'Obsługuje wizję (wejście obrazu)',
'settings.ai.modelVisionDesc':
'Włącz, jeśli ten model akceptuje obrazy. Pozwala dołączać obrazy w czacie, gdy ten model jest wybrany.',
'settings.ai.testFailed': 'Test nie powiódł się',
'settings.ai.testingModel': 'Testowanie modelu...',
'settings.ai.modelResponse': 'Odpowiedź modelu',
+6 -1
View File
@@ -1903,7 +1903,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': 'Máximo de {max} arquivos por mensagem',
'chat.attachment.tooLarge': 'A imagem excede o limite de tamanho de {max}',
'chat.attachment.unsupportedType':
'Tipo de arquivo não suportado. Use PNG, JPEG, WebP, GIF ou BMP.',
'Tipo de arquivo não suportado. Use uma imagem (PNG, JPEG, WebP, GIF, BMP) ou um arquivo PDF, TXT ou Markdown.',
'chat.attachment.imageNotSupported':
'Este modelo não consegue ler imagens. Anexe um arquivo PDF, TXT ou Markdown.',
'chat.attachment.readFailed': 'Não foi possível ler o arquivo',
'memory.searchAria': 'Pesquisar memória',
'memory.searchPlaceholder': 'Pesquisar entradas de memória...',
@@ -3169,6 +3171,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Substituição de temperatura (valor)',
'settings.ai.temperatureOverrideDesc':
'Mais baixo = mais determinístico. Deixe desmarcado para usar o padrão do provedor.',
'settings.ai.modelVision': 'Suporta visão (entrada de imagem)',
'settings.ai.modelVisionDesc':
'Ative se este modelo aceitar imagens. Permite anexar imagens no chat quando este modelo está selecionado.',
'settings.ai.testFailed': 'Teste falhou',
'settings.ai.testingModel': 'Modelo de teste...',
'settings.ai.modelResponse': 'Resposta do modelo',
+6 -1
View File
@@ -1879,7 +1879,9 @@ const messages: TranslationMap = {
'chat.attachment.tooManyFiles': 'Максимум {max} файлов на сообщение',
'chat.attachment.tooLarge': 'Изображение превышает ограничение размера {max}',
'chat.attachment.unsupportedType':
'Неподдерживаемый тип файла. Используйте PNG, JPEG, WebP, GIF или BMP.',
'Неподдерживаемый тип файла. Используйте изображение (PNG, JPEG, WebP, GIF, BMP) или файл PDF, TXT либо Markdown.',
'chat.attachment.imageNotSupported':
'Эта модель не может читать изображения. Вместо этого прикрепите файл PDF, TXT или Markdown.',
'chat.attachment.readFailed': 'Не удалось прочитать файл',
'memory.searchAria': 'Поиск в памяти',
'memory.searchPlaceholder': 'Поиск записей памяти...',
@@ -3140,6 +3142,9 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideValue': 'Переопределение температуры (значение)',
'settings.ai.temperatureOverrideDesc':
'Ниже = более детерминированно. Оставьте флажок неактивным, чтобы использовать поставщика по умолчанию.',
'settings.ai.modelVision': 'Поддерживает зрение (ввод изображений)',
'settings.ai.modelVisionDesc':
'Включите, если модель принимает изображения. Позволяет прикреплять изображения в чате, когда выбрана эта модель.',
'settings.ai.testFailed': 'Тест не пройден.',
'settings.ai.testingModel': 'Модель тестирования...',
'settings.ai.modelResponse': 'Ответ модели',
+5 -1
View File
@@ -1760,7 +1760,9 @@ const messages: TranslationMap = {
'chat.attachment.tooMany': '每条消息最多 {max} 张图片',
'chat.attachment.tooManyFiles': '每条消息最多 {max} 个文件',
'chat.attachment.tooLarge': '图片超过 {max} 大小限制',
'chat.attachment.unsupportedType': '不支持的文件类型。请使用 PNG、JPEG、WebP、GIF 或 BMP。',
'chat.attachment.unsupportedType':
'不支持的文件类型。请使用图片(PNG、JPEG、WebP、GIF、BMP)或 PDF、TXT、Markdown 文件。',
'chat.attachment.imageNotSupported': '此模型无法读取图像。请改为附加 PDF、TXT 或 Markdown 文件。',
'chat.attachment.readFailed': '无法读取文件',
'memory.searchAria': '搜索记忆',
'memory.searchPlaceholder': '搜索记忆条目...',
@@ -2955,6 +2957,8 @@ const messages: TranslationMap = {
'settings.ai.temperatureOverrideSlider': '温度超控(滑块)',
'settings.ai.temperatureOverrideValue': '温度超控(值)',
'settings.ai.temperatureOverrideDesc': '值越低,结果越确定。保持未勾选则使用提供商默认值。',
'settings.ai.modelVision': '支持视觉(图像输入)',
'settings.ai.modelVisionDesc': '如果此模型接受图像,请启用。选择此模型后可在聊天框中附加图像。',
'settings.ai.testFailed': '测试失败',
'settings.ai.testingModel': '测试模型...',
'settings.ai.modelResponse': '模型响应',
+70 -7
View File
@@ -17,7 +17,6 @@ import MicComposer from '../features/human/MicComposer';
import { useStickToBottom } from '../hooks/useStickToBottom';
import { useUsageState } from '../hooks/useUsageState';
import {
ALLOWED_ATTACHMENT_MIME_TYPES,
type Attachment,
ATTACHMENT_MAX_FILES,
ATTACHMENT_MAX_IMAGES,
@@ -270,6 +269,11 @@ const Conversations = ({
onCancel: () => {},
});
const [resolvedModel, setResolvedModel] = useState<string | null>(null);
// Whether the resolved model for the active profile accepts image input.
// Managed tiers do; custom/BYOK models only when the user flagged them. Gates
// the composer's image-attachment affordance (docs flow regardless). Resolved
// against the non-attachment hint so the affordance is stable as you attach.
const [modelSupportsVision, setModelSupportsVision] = useState(false);
useEffect(() => {
let cancelled = false;
@@ -280,13 +284,19 @@ const Conversations = ({
attachments.length > 0
? MULTIMODAL_MODEL_HINT
: (profile?.modelOverride ?? CHAT_MODEL_HINT);
const res = await callCoreRpc<{ model: string }>({
const res = await callCoreRpc<{ model: string; vision?: boolean }>({
method: 'openhuman.inference_resolve_model',
params: { hint },
});
if (!cancelled) setResolvedModel(res.model);
if (!cancelled) {
setResolvedModel(res.model);
setModelSupportsVision(res.vision === true);
}
} catch {
if (!cancelled) setResolvedModel(null);
if (!cancelled) {
setResolvedModel(null);
setModelSupportsVision(false);
}
}
})();
return () => {
@@ -680,10 +690,19 @@ const Conversations = ({
let acceptedImageCount = attachments.filter(attachment => attachment.kind === 'image').length;
let acceptedFileCount = attachments.filter(attachment => attachment.kind === 'file').length;
for (const file of Array.from(files)) {
const result = await validateAndReadFile(file, acceptedImageCount, acceptedFileCount);
const result = await validateAndReadFile(
file,
acceptedImageCount,
acceptedFileCount,
modelSupportsVision
);
if ('error' in result) {
const { error } = result;
if (error.code === 'too_many') {
if (error.code === 'image_not_supported') {
setAttachError(
chatSendError('attachment_invalid', t('chat.attachment.imageNotSupported'))
);
} else if (error.code === 'too_many') {
const key =
error.kind === 'image' ? 'chat.attachment.tooMany' : 'chat.attachment.tooManyFiles';
setAttachError(
@@ -1724,6 +1743,18 @@ const Conversations = ({
? (msg.extraMetadata.attachmentDataUris as string[])
: parseMessageImages(msg.content ?? '').dataUris;
const hasImages = dataUris.length > 0;
// Document attachments carry no image data-URI (only
// images do); surface them as filename chips from the
// persisted attachmentKinds/attachmentNames metadata.
const kinds = Array.isArray(msg.extraMetadata?.attachmentKinds)
? (msg.extraMetadata.attachmentKinds as string[])
: [];
const names = Array.isArray(msg.extraMetadata?.attachmentNames)
? (msg.extraMetadata.attachmentNames as string[])
: [];
const fileNames = kinds
.map((k, i) => (k === 'file' ? names[i] : null))
.filter((n): n is string => Boolean(n));
const showTime = latestVisibleMessage?.id === msg.id;
return (
<>
@@ -1739,6 +1770,35 @@ const Conversations = ({
))}
</div>
)}
{fileNames.length > 0 && (
<div className="flex flex-wrap gap-1.5 justify-end">
{fileNames.map((name, i) => (
<div
key={i}
className="flex items-center gap-2 rounded-lg border border-stone-200 dark:border-neutral-700 bg-stone-50 dark:bg-neutral-800 px-2.5 py-1.5 text-xs text-stone-700 dark:text-neutral-300 max-w-[220px]">
<svg
className="w-4 h-4 flex-shrink-0 text-stone-500 dark:text-neutral-400"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24">
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={1.8}
d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"
/>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={1.8}
d="M14 2v6h6"
/>
</svg>
<span className="truncate font-medium">{name}</span>
</div>
))}
</div>
)}
{(msg.content || showTime) && (
<div className="rounded-2xl px-4 py-2.5 bg-primary-500 text-white rounded-br-md break-words overflow-hidden">
{msg.content && (
@@ -2213,7 +2273,10 @@ const Conversations = ({
inlineCompletionSuffix={inlineCompletionSuffix}
isComposingTextRef={isComposingTextRef}
maxAttachments={ATTACHMENT_MAX_IMAGES + ATTACHMENT_MAX_FILES}
allowedMimeTypes={ALLOWED_ATTACHMENT_MIME_TYPES}
// Empty → no native `accept` filter (it greys valid files on
// macOS/CEF). Type enforcement happens in handleAttachFiles via
// validateAndReadFile, which honors modelSupportsVision.
allowedMimeTypes={[]}
attachmentsEnabled={CHAT_ATTACHMENTS_ENABLED}
/>
) : (
@@ -140,6 +140,18 @@ vi.mock('../../features/autocomplete/useAutocompleteSkillStatus', () => ({
vi.mock('../../utils/openUrl', () => ({ openUrl: vi.fn() }));
// The composer gates image attachments on the resolved model's vision capability
// (inference_resolve_model). These tests exercise image attachments, so resolve
// the active model as vision-capable.
vi.mock('../../services/coreRpcClient', () => ({
callCoreRpc: vi.fn(async ({ method }: { method: string }) =>
method === 'openhuman.inference_resolve_model'
? { model: 'test-vision-model', vision: true }
: {}
),
CoreRpcError: class CoreRpcError extends Error {},
}));
vi.mock('../../lib/coreState/store', () => ({
getCoreStateSnapshot: vi.fn(() => ({
isBootstrapping: false,
@@ -506,4 +518,53 @@ describe('Conversations — attachment feature', () => {
expect(img).not.toBeNull();
});
});
it('renders a document filename chip in the user bubble from attachmentKinds/Names', async () => {
const thread = makeThread({ id: 'file-thread', title: 'File Thread' });
const message = {
id: 'msg-file-1',
content: 'whats in this file',
type: 'text' as const,
sender: 'user' as const,
createdAt: new Date().toISOString(),
extraMetadata: {
attachmentCount: 1,
attachmentKinds: ['file'],
attachmentNames: ['report.pdf'],
},
};
mockGetThreads.mockResolvedValue({ threads: [thread], count: 1 });
mockGetThreadMessages.mockResolvedValue({ messages: [message], count: 1 });
const store = buildStore({
thread: {
threads: [thread],
selectedThreadId: thread.id,
activeThreadId: null,
welcomeThreadId: null,
messagesByThreadId: { [thread.id]: [message] },
messages: [message],
isLoadingThreads: false,
isLoadingMessages: false,
messagesError: null,
},
socket: socketState('connected'),
});
const { default: Conversations } = await import('../Conversations');
render(
<Provider store={store}>
<MemoryRouter>
<Conversations />
</MemoryRouter>
</Provider>
);
// The document attachment surfaces as a filename chip (not an <img>).
await waitFor(() => {
expect(document.body.textContent).toContain('report.pdf');
});
});
});
@@ -18,6 +18,7 @@ import {
loadAISettings,
loadLocalProviderSnapshot,
localProvider,
modelRegistryVision,
OPENAI_CODEX_OAUTH_MISSING_AUTH_URL,
OPENAI_CODEX_OAUTH_MISSING_CALLBACK_URL,
parseProviderString,
@@ -28,6 +29,7 @@ import {
setLocalRuntimeEnabled,
startOpenAiCodexOAuth,
testProviderModel,
upsertModelRegistryVision,
} from '../aiSettingsApi';
// ─── Mock declarations (must be hoisted before imports) ───────────────────────
@@ -83,6 +85,7 @@ function makeClientConfigResult(overrides: Record<string, unknown> = {}) {
api_key_set: false,
model_routes: [],
cloud_providers: [],
model_registry: [],
primary_cloud: null,
reasoning_provider: null,
agentic_provider: null,
@@ -587,6 +590,7 @@ describe('saveAISettings', () => {
learning: { kind: 'openhuman' },
subconscious: { kind: 'openhuman' },
},
modelRegistry: [],
...overrides,
};
}
@@ -673,8 +677,13 @@ describe('saveAISettings', () => {
learning: { kind: 'openhuman' },
subconscious: { kind: 'openhuman' },
},
modelRegistry: [],
};
const next: AISettings = {
cloudProviders: [anthropicProvider],
routing: { ...prev.routing },
modelRegistry: [],
};
const next: AISettings = { cloudProviders: [anthropicProvider], routing: { ...prev.routing } };
await saveAISettings(prev, next);
@@ -697,6 +706,34 @@ describe('saveAISettings', () => {
expect(patch.cloud_providers).toBeDefined();
expect(patch.coding_provider).toBe('openai:gpt-4o-mini');
});
it('sends model_registry when the vision flag changes', async () => {
const prev = makeSettings({ modelRegistry: [] });
const next = makeSettings({
modelRegistry: [{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true }],
});
await saveAISettings(prev, next);
const patch = mockOpenhumanUpdateModelSettings.mock.calls[0][0];
expect(patch.model_registry).toEqual([
{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true },
]);
});
it('omits model_registry when unchanged', async () => {
const registry = [{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true }];
const prev = makeSettings({ modelRegistry: registry });
const next = makeSettings({
modelRegistry: [...registry],
routing: {
...makeSettings().routing,
coding: { kind: 'cloud', providerSlug: 'openai', model: 'gpt-4o-mini' },
},
});
await saveAISettings(prev, next);
const patch = mockOpenhumanUpdateModelSettings.mock.calls[0][0];
expect(patch.model_registry).toBeUndefined();
expect(patch.coding_provider).toBe('openai:gpt-4o-mini');
});
});
// ─── setCloudProviderKey ──────────────────────────────────────────────────────
@@ -918,3 +955,32 @@ describe('flushCloudProviders', () => {
expect(mockOpenhumanUpdateModelSettings).not.toHaveBeenCalled();
});
});
describe('model registry vision helpers', () => {
const reg = [
{ id: 'gpt-4o', provider: 'openai', cost_per_1m_output: 0, vision: true },
{ id: 'text-only', provider: 'openai', cost_per_1m_output: 0, vision: false },
];
it('modelRegistryVision matches by (provider, id)', () => {
expect(modelRegistryVision(reg, 'openai', 'gpt-4o')).toBe(true);
expect(modelRegistryVision(reg, 'openai', 'text-only')).toBe(false);
expect(modelRegistryVision(reg, 'openai', 'unlisted')).toBe(false);
expect(modelRegistryVision(reg, 'azure', 'gpt-4o')).toBe(false);
});
it('upsertModelRegistryVision adds, flips, and removes entries', () => {
const added = upsertModelRegistryVision([], 'openai', 'my-llava', true);
expect(added).toEqual([
{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true },
]);
// vision:false removes the entry (absence ⇒ no vision).
const removed = upsertModelRegistryVision(reg, 'openai', 'gpt-4o', false);
expect(removed.find(e => e.id === 'gpt-4o')).toBeUndefined();
expect(removed.find(e => e.id === 'text-only')).toBeDefined();
// Flipping an existing entry on stays idempotent on the key.
const flipped = upsertModelRegistryVision(reg, 'openai', 'text-only', true);
expect(flipped.filter(e => e.id === 'text-only')).toHaveLength(1);
expect(modelRegistryVision(flipped, 'openai', 'text-only')).toBe(true);
});
});
+71 -1
View File
@@ -26,6 +26,7 @@ import { isTauri } from '../../utils/tauriCommands/common';
import {
type ClientConfig,
type CloudProviderCreds,
type ModelRegistryEntry,
type ModelSettingsUpdate,
openhumanGetClientConfig,
openhumanUpdateLocalAiSettings,
@@ -132,6 +133,47 @@ const PROVIDER_MODEL_TEST_TIMEOUT_MS = 120_000;
export interface AISettings {
cloudProviders: CloudProviderView[];
routing: Record<WorkloadId, ProviderRef>;
/**
* Per-model registry carrying each model's user-set `vision` flag, keyed by
* `(provider slug, model id)`. Surfaced in the custom-model dialog; gates chat
* image attachments for custom/BYOK models.
*/
modelRegistry: ModelRegistryEntry[];
}
/** Re-export so callers (e.g. the AI panel) can reference the entry type. */
export type { ModelRegistryEntry };
/** Find a model's vision flag in the registry, matching by (provider, id).
* Tolerates an undefined registry (older snapshots / transient load state). */
export function modelRegistryVision(
registry: ModelRegistryEntry[] | undefined,
provider: string,
id: string
): boolean {
return (registry ?? []).some(e => e.provider === provider && e.id === id && e.vision);
}
/**
* Upsert a model's vision flag, returning a new array. Matches by
* `(provider, id)`; a `vision: false` entry is removed (absence ⇒ no vision).
*/
export function upsertModelRegistryVision(
registry: ModelRegistryEntry[] | undefined,
provider: string,
id: string,
vision: boolean
): ModelRegistryEntry[] {
const base = registry ?? [];
const without = base.filter(e => !(e.provider === provider && e.id === id));
if (!vision) {
return without;
}
const existing = base.find(e => e.provider === provider && e.id === id);
return [
...without,
{ id, provider, cost_per_1m_output: existing?.cost_per_1m_output ?? 0, vision: true },
];
}
// ─── Read path: load + parse ───────────────────────────────────────────────
@@ -264,7 +306,10 @@ export async function loadAISettings(): Promise<AISettings> {
);
}
return { cloudProviders, routing };
// Per-model registry (vision flags). Defensive default for older snapshots.
const modelRegistry: ModelRegistryEntry[] = config.model_registry ?? [];
return { cloudProviders, routing, modelRegistry };
}
// ─── Write path: diff + save ───────────────────────────────────────────────
@@ -310,12 +355,37 @@ export async function saveAISettings(prev: AISettings, next: AISettings): Promis
}
}
// Per-model registry (vision flags): any change → send the full list.
if (!modelRegistriesEqual(prev.modelRegistry, next.modelRegistry)) {
patch.model_registry = next.modelRegistry.map(
({ id, provider, cost_per_1m_output, vision }) => ({
id,
provider,
cost_per_1m_output,
vision,
})
);
}
if (Object.keys(patch).length === 0) {
return;
}
await openhumanUpdateModelSettings(patch);
}
/** Order-insensitive structural equality for two model registries. */
function modelRegistriesEqual(a: ModelRegistryEntry[], b: ModelRegistryEntry[]): boolean {
if (a.length !== b.length) {
return false;
}
const key = (e: ModelRegistryEntry) => `${e.provider} ${e.id}`;
const bByKey = new Map(b.map(e => [key(e), e]));
return a.every(e => {
const m = bByKey.get(key(e));
return !!m && m.vision === e.vision && m.cost_per_1m_output === e.cost_per_1m_output;
});
}
// ─── API key management (per cloud provider slug) ──────────────────────────
/**
+20
View File
@@ -49,6 +49,18 @@ export interface CloudProviderCreds {
auth_style: AuthStyle;
}
/**
* Per-model registry entry. Mirrors the Rust `ModelRegistryEntry`
* (`config/schema/types.rs`). Carries the user-set `vision` flag that lets a
* custom/BYOK model accept chat image attachments.
*/
export interface ModelRegistryEntry {
id: string;
provider: string;
cost_per_1m_output: number;
vision: boolean;
}
export interface ModelSettingsUpdate {
/**
* OpenHuman product backend URL. Almost always left untouched; the
@@ -77,6 +89,12 @@ export interface ModelSettingsUpdate {
* Each entry: { id?, slug, label?, endpoint, auth_style? }
*/
cloud_providers?: CloudProviderCreds[] | null;
/**
* When present, REPLACES `config.model_registry` wholesale. Carries each
* model's `vision` flag (Settings → Advanced LLM → custom model → "Supports
* vision"). Send `[]` to clear; omit to leave untouched.
*/
model_registry?: ModelRegistryEntry[] | null;
/** @deprecated No longer used — slug-based routing replaces primary_cloud. */
primary_cloud?: string | null;
/** Per-workload provider strings — see Rust `providers::factory` grammar. */
@@ -215,6 +233,8 @@ export interface ClientConfig {
model_routes: ModelRoute[];
/** Configured cloud providers (no API keys — those live in auth-profiles.json). */
cloud_providers: CloudProviderCreds[];
/** Per-model registry carrying each model's `vision` flag. */
model_registry: ModelRegistryEntry[];
/** Id of the `cloud_providers` entry resolved by the `"cloud"` sentinel. */
primary_cloud: string | null;
/** Per-workload provider strings (e.g. `"cloud"`, `"ollama:llama3.1:8b"`, `"openai:gpt-4o"`). */
@@ -2,7 +2,6 @@ import { expect, type Page, test } from '@playwright/test';
import {
bootAuthenticatedPage,
callCoreRpc,
dismissWalkthroughIfPresent,
waitForAppReady,
} from '../helpers/core-rpc';
@@ -73,7 +72,7 @@ async function newThread(page: Page): Promise<string> {
}
test.describe('Chat management functional coverage', () => {
test('attachment preview, remove, and multimodal send path remain interactive', async ({
test('attachment preview, remove, and document send path remain interactive', async ({
page,
}) => {
await resetMock();
@@ -89,34 +88,24 @@ test.describe('Chat management functional coverage', () => {
const fileInput = page.locator('input[type="file"]');
await expect(fileInput).toHaveCount(1);
await fileInput.setInputFiles({
name: 'pixel.png',
mimeType: 'image/png',
buffer: Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
'base64'
),
});
await expect(page.getByText('pixel.png')).toBeVisible();
await page.getByRole('button', { name: /Remove pixel\.png/ }).click();
await expect(page.getByText('pixel.png')).toHaveCount(0);
// Documents work on every model; images require a vision-capable model.
// Managed OpenHuman tiers are text-only (see `oh_tier_supports_vision`), so
// the attachment mechanics here are exercised with a text document. Image
// upload is gated to vision-flagged models and validated in unit tests.
const txtBuffer = Buffer.from('renderer uploaded text document', 'utf8');
const attachTxt = () =>
fileInput.setInputFiles({ name: 'notes.txt', mimeType: 'text/plain', buffer: txtBuffer });
const pngBuffer = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
'base64'
);
await fileInput.setInputFiles({ name: 'pixel.png', mimeType: 'image/png', buffer: pngBuffer });
await expect(page.getByText('pixel.png')).toBeVisible();
await attachTxt();
await expect(page.getByText('notes.txt')).toBeVisible();
await page.getByRole('button', { name: /Remove notes\.txt/ }).click();
await expect(page.getByText('notes.txt')).toHaveCount(0);
await fileInput.setInputFiles({
name: 'notes.txt',
mimeType: 'text/plain',
buffer: Buffer.from('renderer uploaded text document', 'utf8'),
});
await attachTxt();
await expect(page.getByText('notes.txt')).toBeVisible();
await page.getByPlaceholder('How can I help you today?').fill('Describe this image');
await page.getByPlaceholder('How can I help you today?').fill('Summarize this file');
await page.getByTestId('send-message-button').click();
await expect(page.getByText('Attachment received by the assistant.')).toBeVisible({
timeout: 30_000,
@@ -132,13 +121,13 @@ test.describe('Chat management functional coverage', () => {
request.method === 'POST' &&
request.url?.includes('/chat/completions') &&
typeof request.body === 'string' &&
request.body.includes('Describe this image')
request.body.includes('Summarize this file')
);
return completion?.body ?? '';
},
{ timeout: 10_000 }
)
.toContain('image_url');
.toContain('[FILE-EXTRACTED:');
const log = await requests();
const completionBody =
@@ -147,13 +136,21 @@ test.describe('Chat management functional coverage', () => {
request.method === 'POST' &&
request.url?.includes('/chat/completions') &&
typeof request.body === 'string' &&
request.body.includes('Describe this image')
request.body.includes('Summarize this file')
)?.body ?? '';
expect(completionBody).toContain('reasoning');
expect(completionBody).toContain('[FILE-EXTRACTED:');
expect(completionBody).toContain('renderer uploaded text document');
});
// NOTE: image attachments require a vision-capable model. Managed tiers are
// text-only by default (`oh_tier_supports_vision`); a model is flagged
// vision-capable via `model_registry` (Settings → "Supports vision"). The
// `image_url` promotion path is covered by Rust unit tests
// (`inference::provider::compatible` MessageContent). An E2E that toggles the
// vision flag at runtime is a follow-up (needs a page re-mount so the
// composer's resolve picks up the flag — out of scope here).
test('thread rename and delete remain usable from the conversation UI', async ({ page }) => {
await resetMock();
await openChat(page, 'pw-chat-rename-delete');
+8 -1
View File
@@ -248,7 +248,14 @@ pub(crate) async fn run_turn_engine(
tracing::debug!(iteration, "[agent_loop] sending LLM request");
let image_marker_count = multimodal::count_image_markers(history);
if image_marker_count > 0 && !provider.supports_vision() {
// A model accepts images when its provider advertises vision (managed
// backend) OR the user marked it vision-capable in `model_registry`
// (custom/BYOK) — surfaced via the `current_model_vision` task-local set
// at session build. Unset (CLI/tests) falls back to the provider flag.
let has_vision = provider.supports_vision()
|| crate::openhuman::agent::harness::model_vision_context::current_model_vision()
.unwrap_or(false);
if image_marker_count > 0 && !has_vision {
let cap_err = ProviderCapabilityError {
provider: provider_name.to_string(),
capability: "vision".to_string(),
+2
View File
@@ -30,6 +30,7 @@ mod instructions;
pub mod interrupt;
pub(crate) mod memory_context;
pub(crate) mod memory_context_safety;
pub mod model_vision_context;
mod parse;
pub(crate) mod payload_summarizer;
pub mod run_queue;
@@ -50,6 +51,7 @@ pub use definition::{
};
pub use fork_context::{current_parent, with_parent_context, ParentExecutionContext};
pub use interrupt::{check_interrupt, InterruptFence, InterruptedError};
pub use model_vision_context::{current_model_vision, with_current_model_vision};
pub use sandbox_context::{current_sandbox_mode, with_current_sandbox_mode};
pub(crate) use spawn_depth_context::{current_spawn_depth, with_spawn_depth, MAX_SPAWN_DEPTH};
pub use subagent_runner::{run_subagent, SubagentRunError, SubagentRunOptions};
@@ -0,0 +1,75 @@
//! Task-local carrier for the **current session/sub-agent's user-configured
//! vision capability** so the deep turn engine's image gate can honor a custom
//! (BYOK) model's `model_registry.vision` flag without widening
//! [`crate::openhuman::agent::harness::engine::run_turn_engine`]'s signature.
//!
//! Managed-backend models advertise vision via `Provider::supports_vision()`, so
//! the gate accepts their image turns already. Custom OpenAI-compatible providers
//! report `supports_vision() == false` (a provider endpoint can't know a per-model
//! property), so without this the gate would reject image turns for a model the
//! user explicitly marked vision-capable. This task-local surfaces that per-model
//! flag — computed once at session build (where the full `Config` / `model_registry`
//! and the resolved model id coexist) — to the gate.
//!
//! Mirrors [`super::sandbox_context`]. When unset (CLI / direct invocation / tests
//! that never wrapped the call) [`current_model_vision`] returns `None` and the
//! gate falls back to the provider capability only — strictly additive.
tokio::task_local! {
/// User-configured vision capability for the currently-executing
/// session/sub-agent's model. Scoped per turn by the turn loop + subagent
/// runner. `None` when unset.
pub static CURRENT_MODEL_VISION: bool;
}
/// Returns the current model's user-configured vision flag, if scope is active.
pub fn current_model_vision() -> Option<bool> {
CURRENT_MODEL_VISION.try_with(|v| *v).ok()
}
/// Run `future` with `vision` installed as the current model's vision flag.
/// Intended call site is around each `run_turn_engine` invocation.
pub async fn with_current_model_vision<F, R>(vision: bool, future: F) -> R
where
F: std::future::Future<Output = R>,
{
CURRENT_MODEL_VISION.scope(vision, future).await
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn current_model_vision_returns_none_outside_scope() {
assert_eq!(current_model_vision(), None);
}
#[tokio::test]
async fn with_current_model_vision_installs_value() {
let observed = with_current_model_vision(true, async { current_model_vision() }).await;
assert_eq!(observed, Some(true));
}
#[tokio::test]
async fn with_current_model_vision_does_not_leak_across_scopes() {
with_current_model_vision(true, async {
assert_eq!(current_model_vision(), Some(true));
})
.await;
assert_eq!(current_model_vision(), None);
}
#[tokio::test]
async fn nested_scope_overrides_outer() {
with_current_model_vision(false, async {
assert_eq!(current_model_vision(), Some(false));
with_current_model_vision(true, async {
assert_eq!(current_model_vision(), Some(true));
})
.await;
assert_eq!(current_model_vision(), Some(false));
})
.await;
}
}
@@ -439,6 +439,15 @@ impl Agent {
model_name = pinned_model.to_string();
}
// Resolve the user-configured vision flag for the (now-final) model while
// the full `Config` / `model_registry` is in scope — the turn engine only
// sees `MultimodalConfig`. Stored on the session and surfaced to the image
// gate via the `current_model_vision` task-local (covers custom/BYOK models
// the provider can't introspect). Computed with `&model_name` since it's
// moved into the builder below.
let model_vision =
crate::openhuman::inference::model_context::model_supports_vision(&model_name, config);
// Dispatcher selection is deferred until after the tool list is
// finalised (orchestrator tools are appended below). We capture
// the choice string now so the provider borrow doesn't conflict
@@ -1047,6 +1056,7 @@ impl Agent {
.config(config.agent.clone())
.context_config(config.context.clone())
.model_name(model_name)
.model_vision(model_vision)
.temperature(effective_temperature)
.workspace_dir(config.workspace_dir.clone())
.action_dir(config.action_dir.clone())
@@ -29,6 +29,7 @@ impl AgentBuilder {
config: None,
context_config: None,
model_name: None,
model_vision: None,
temperature: None,
workspace_dir: None,
action_dir: None,
@@ -142,6 +143,14 @@ impl AgentBuilder {
self
}
/// Sets the user-configured vision capability for the resolved model.
/// Surfaced to the turn engine's image gate via the `current_model_vision`
/// task-local. Defaults to `false` when unset.
pub fn model_vision(mut self, model_vision: bool) -> Self {
self.model_vision = Some(model_vision);
self
}
/// Sets the temperature for chat requests.
pub fn temperature(mut self, temperature: f64) -> Self {
self.temperature = Some(temperature);
@@ -514,6 +523,7 @@ impl AgentBuilder {
.unwrap_or_else(|| Box::new(DefaultMemoryLoader::default())),
config,
model_name,
model_vision: self.model_vision.unwrap_or(false),
temperature: self.temperature.unwrap_or(0.7),
workspace_dir,
action_dir,
@@ -323,6 +323,10 @@ impl Agent {
// model prompt, not in the Rust-side classifier. Sub-agents pick
// their own tier via `ModelSpec::Hint(...)` in their definition.
let effective_model = self.model_name.clone();
// Capture before `self` is borrowed by the turn observer below, so it can
// be installed as the `current_model_vision` task-local around the engine
// call (read by the image gate for custom/BYOK vision models).
let model_vision = self.model_vision;
log::info!(
"[agent_loop] model pinned model={} (per-turn classification disabled for KV cache stability)",
effective_model
@@ -436,25 +440,28 @@ impl Agent {
// overflow happens during the parent's poll on the way in
// — verified against the `chat-harness-subagent` Playwright
// lane crash on PR #3151.
let outcome = Box::pin(super::super::super::engine::run_turn_engine(
provider.as_ref(),
&mut buf,
&mut tool_source,
&progress,
&mut observer,
&checkpoint,
&parser,
&provider_name,
&effective_model,
temperature,
true, // silent — the channel/UI renders via progress + the return value
&multimodal,
&multimodal_files,
max_iterations,
None, // the web bridge streams via on_progress deltas, not on_delta
&[],
turn_run_queue,
))
let outcome = super::super::super::model_vision_context::with_current_model_vision(
model_vision,
Box::pin(super::super::super::engine::run_turn_engine(
provider.as_ref(),
&mut buf,
&mut tool_source,
&progress,
&mut observer,
&checkpoint,
&parser,
&provider_name,
&effective_model,
temperature,
true, // silent — the channel/UI renders via progress + the return value
&multimodal,
&multimodal_files,
max_iterations,
None, // the web bridge streams via on_progress deltas, not on_delta
&[],
turn_run_queue,
)),
)
.await?;
// Pull the observer's accounting out, then drop it to release the
@@ -53,6 +53,11 @@ pub struct Agent {
pub(super) memory_loader: Box<dyn MemoryLoader>,
pub(super) config: crate::openhuman::config::AgentConfig,
pub(super) model_name: String,
/// User-configured vision capability for [`Self::model_name`], evaluated at
/// session build from `model_vision_enabled(&model, config)`. Surfaced to the
/// turn engine's image gate via the `current_model_vision` task-local so a
/// custom/BYOK model the user flagged can forward images. Defaults to `false`.
pub(super) model_vision: bool,
pub(super) temperature: f64,
pub(super) workspace_dir: std::path::PathBuf,
pub(super) action_dir: std::path::PathBuf,
@@ -272,6 +277,8 @@ pub struct AgentBuilder {
/// [`crate::openhuman::config::ContextConfig::default`].
pub(super) context_config: Option<crate::openhuman::config::ContextConfig>,
pub(super) model_name: Option<String>,
/// User vision flag for the resolved model; `None` → `false` in `build()`.
pub(super) model_vision: Option<bool>,
pub(super) temperature: Option<f64>,
pub(super) workspace_dir: Option<std::path::PathBuf>,
pub(super) action_dir: Option<std::path::PathBuf>,
@@ -48,6 +48,10 @@ pub(super) async fn run_inner_loop(
allowed_names: HashSet<String>,
lazy_resolver: Option<LazyToolkitResolver>,
model: &str,
// User-configured vision flag for `model` (computed at the call site), set as
// the `current_model_vision` task-local around the engine call so a flagged
// custom/BYOK sub-agent model can forward images.
model_vision: bool,
temperature: f64,
max_iterations: usize,
task_id: &str,
@@ -176,25 +180,28 @@ pub(super) async fn run_inner_loop(
// `nested_subagent_dispatch_runs_on_a_constrained_worker_stack`;
// the deep end-to-end catcher is the `chat-harness-subagent`
// Playwright spec.
let outcome = Box::pin(super::super::super::engine::run_turn_engine(
provider,
history,
&mut tool_source,
&progress,
&mut observer,
&checkpoint,
&parser,
"subagent",
model,
temperature,
true, // silent — sub-agents never echo to stdout
&crate::openhuman::config::MultimodalConfig::default(),
&crate::openhuman::config::MultimodalFileConfig::default(),
max_iterations,
None, // sub-agents don't stream a draft
&["ask_user_clarification"],
None, // sub-agents don't support run-queue steering
))
let outcome = super::super::super::model_vision_context::with_current_model_vision(
model_vision,
Box::pin(super::super::super::engine::run_turn_engine(
provider,
history,
&mut tool_source,
&progress,
&mut observer,
&checkpoint,
&parser,
"subagent",
model,
temperature,
true, // silent — sub-agents never echo to stdout
&crate::openhuman::config::MultimodalConfig::default(),
&crate::openhuman::config::MultimodalFileConfig::default(),
max_iterations,
None, // sub-agents don't stream a draft
&["ask_user_clarification"],
None, // sub-agents don't support run-queue steering
)),
)
.await?;
Ok((
@@ -645,6 +645,20 @@ async fn run_typed_mode(
};
// ── Run the inner tool-call loop ───────────────────────────────────
// Resolve the sub-agent model's user-configured vision flag; defaults to
// `false` when config can't be loaded. Combined with the provider capability
// at the gate, this lets a flagged custom/BYOK sub-agent model forward images.
let model_vision = crate::openhuman::config::Config::load_or_init()
.await
.ok()
.map(|cfg| crate::openhuman::inference::model_context::model_supports_vision(&model, &cfg))
.unwrap_or(false);
tracing::debug!(
target: "subagent_runner",
model = %model,
model_vision,
"[subagent_runner] resolved sub-agent model vision capability"
);
let (output, iterations, _agg_usage, early_exit_tool) = Box::pin(run_inner_loop(
subagent_provider.as_ref(),
&mut history,
@@ -654,6 +668,7 @@ async fn run_typed_mode(
allowed_names,
lazy_resolver,
&model,
model_vision,
temperature,
definition.effective_max_iterations(),
task_id,
@@ -11,6 +11,14 @@ pub(super) fn autonomy_signature(config: &Config) -> String {
serde_json::to_string(&config.autonomy).unwrap_or_default()
}
/// Signature of `config.model_registry` for the session-cache fingerprint.
/// Captures every per-model `vision` flag so toggling one in Settings forces a
/// rebuild (picking up the new build-time `model_vision`). Mirrors
/// [`autonomy_signature`].
pub(super) fn model_registry_signature(config: &Config) -> String {
serde_json::to_string(&config.model_registry).unwrap_or_default()
}
pub(super) fn pick_target_agent_id(_config: &Config, profile: &AgentProfile) -> String {
if profile.id == DEFAULT_PROFILE_ID {
"orchestrator".to_string()
@@ -186,5 +194,6 @@ pub(super) fn build_session_fingerprint(
),
target_agent_id,
autonomy_signature: autonomy_signature(config),
model_registry_signature: model_registry_signature(config),
}
}
@@ -19,6 +19,12 @@ pub(crate) struct SessionCacheFingerprint {
pub(super) target_agent_id: String,
pub(super) provider_binding: String,
pub(super) autonomy_signature: String,
/// Signature of `config.model_registry`. The cached `Agent` stores a
/// build-time `model_vision` bool; toggling a model's "Supports vision" flag
/// keeps the same model id (so neither `model_override` nor `provider_binding`
/// change) — without this the stale session would be reused. Mirrors
/// [`Self::autonomy_signature`].
pub(super) model_registry_signature: String,
}
pub(super) struct SessionEntry {
@@ -1208,6 +1208,7 @@ fn fp(
target_agent_id: target.to_string(),
provider_binding: provider_binding.to_string(),
autonomy_signature: "sig-default".to_string(),
model_registry_signature: "registry-default".to_string(),
}
}
@@ -1225,6 +1226,20 @@ fn fingerprint_autonomy_change_is_cache_miss() {
);
}
#[test]
fn fingerprint_model_registry_change_is_cache_miss() {
// Toggling a model's "Supports vision" flag keeps the same model id, so it
// changes neither model_override nor provider_binding. Without the registry
// signature the stale Agent (old build-time model_vision) would be reused.
let base = fp(None, None, "orchestrator", "openai:my-llava");
let mut changed = fp(None, None, "orchestrator", "openai:my-llava");
changed.model_registry_signature = "registry-after-vision-toggle".to_string();
assert_ne!(
base, changed,
"a model_registry change (vision toggle) must produce a cache miss → rebuild"
);
}
#[test]
fn fingerprint_identical_inputs_are_cache_hit() {
let a = fp(None, None, "orchestrator", "anthropic:claude-sonnet-4-6");
+13
View File
@@ -254,6 +254,18 @@ pub fn client_config_json(config: &Config) -> serde_json::Value {
})
})
.collect();
let model_registry: Vec<serde_json::Value> = config
.model_registry
.iter()
.map(|m| {
serde_json::json!({
"id": m.id,
"provider": m.provider,
"cost_per_1m_output": m.cost_per_1m_output,
"vision": m.vision,
})
})
.collect();
serde_json::json!({
"api_url": config.api_url,
@@ -263,6 +275,7 @@ pub fn client_config_json(config: &Config) -> serde_json::Value {
"api_key_set": api_key_set,
"model_routes": model_routes,
"cloud_providers": cloud_providers,
"model_registry": model_registry,
"primary_cloud": config.primary_cloud,
"chat_provider": config.chat_provider,
"reasoning_provider": config.reasoning_provider,
+22
View File
@@ -26,6 +26,10 @@ pub struct ModelSettingsPatch {
/// Pass `Some(vec![])` to clear all third-party cloud providers.
pub cloud_providers:
Option<Vec<crate::openhuman::config::schema::cloud_providers::CloudProviderCreds>>,
/// When `Some`, REPLACES the entire `config.model_registry` array. Carries
/// each model's user-set `vision` flag (Settings → Advanced LLM → custom
/// model → "Supports vision"). Pass `Some(vec![])` to clear; `None` keeps it.
pub model_registry: Option<Vec<crate::openhuman::config::schema::ModelRegistryEntry>>,
/// Id of the `cloud_providers` entry used when a workload routes to
/// `"cloud"`. Empty string clears (factory falls back to OpenHuman).
pub primary_cloud: Option<String>,
@@ -139,6 +143,24 @@ pub async fn apply_model_settings(
if let Some(routes) = update.model_routes {
config.model_routes = routes;
}
if let Some(registry) = update.model_registry {
// Full replacement — the UI sends the canonical per-model registry
// (carrying each model's `vision` flag). Empty vec clears it.
log::debug!(
"[config] apply_model_settings: replacing model_registry ({} entries)",
registry.len()
);
// Normalize ids: `model_vision_enabled` matches the resolved model id
// exactly, so stray surrounding whitespace would silently disable vision
// for an otherwise valid model.
config.model_registry = registry
.into_iter()
.map(|mut entry| {
entry.id = entry.id.trim().to_string();
entry
})
.collect();
}
if let Some(providers) = update.cloud_providers {
use crate::openhuman::config::schema::cloud_providers::is_slug_reserved;
let preserved: Vec<_> = config
+71
View File
@@ -588,6 +588,77 @@ async fn apply_model_settings_replaces_model_routes_when_some_and_keeps_when_non
assert!(cfg.model_routes.is_empty());
}
#[tokio::test]
async fn apply_model_settings_replaces_model_registry_when_some_and_keeps_when_none() {
// Per-model vision registry follows Some=replace / None=keep / empty=clear —
// this persists the "Supports vision" flag set in Settings → Advanced LLM.
use crate::openhuman::config::schema::ModelRegistryEntry;
let tmp = tempdir().unwrap();
let mut cfg = tmp_config(&tmp);
let set = ModelSettingsPatch {
model_registry: Some(vec![ModelRegistryEntry {
id: "my-llava".into(),
provider: "openai".into(),
cost_per_1m_output: 0.0,
vision: true,
}]),
..Default::default()
};
let _ = apply_model_settings(&mut cfg, set).await.expect("set");
assert_eq!(cfg.model_registry.len(), 1);
assert!(cfg
.model_registry
.iter()
.any(|e| e.id == "my-llava" && e.vision));
// None — leave registry alone.
let _ = apply_model_settings(
&mut cfg,
ModelSettingsPatch {
api_url: Some("https://x.test/v1".into()),
..Default::default()
},
)
.await
.expect("touch");
assert_eq!(cfg.model_registry.len(), 1);
// Empty vec — clear.
let _ = apply_model_settings(
&mut cfg,
ModelSettingsPatch {
model_registry: Some(vec![]),
..Default::default()
},
)
.await
.expect("clear");
assert!(cfg.model_registry.is_empty());
}
#[tokio::test]
async fn apply_model_settings_trims_model_registry_ids() {
// `model_vision_enabled` matches the resolved id exactly, so persisted ids
// must be trimmed or stray whitespace would silently disable vision.
use crate::openhuman::config::schema::ModelRegistryEntry;
let tmp = tempdir().unwrap();
let mut cfg = tmp_config(&tmp);
let set = ModelSettingsPatch {
model_registry: Some(vec![ModelRegistryEntry {
id: " spaced-model ".into(),
provider: "openai".into(),
cost_per_1m_output: 0.0,
vision: true,
}]),
..Default::default()
};
let _ = apply_model_settings(&mut cfg, set).await.expect("set");
assert_eq!(cfg.model_registry.len(), 1);
assert_eq!(cfg.model_registry[0].id, "spaced-model");
}
#[tokio::test]
async fn apply_model_settings_empty_strings_clear_optional_fields() {
let tmp = tempdir().unwrap();
@@ -360,6 +360,10 @@ fn handle_update_model_settings(params: Map<String, Value>) -> ControllerFuture
.collect::<Result<Vec<_>, String>>()
})
.transpose()?,
// The config-domain RPC doesn't carry a model-registry payload — the
// per-model vision registry is updated via the inference-domain
// `inference_update_model_settings` path.
model_registry: None,
primary_cloud: update.primary_cloud,
chat_provider: update.chat_provider,
reasoning_provider: update.reasoning_provider,
+84
View File
@@ -137,6 +137,46 @@ pub fn context_window_for_model_with_local_fallback(
None
}
/// Whether the model resolved for a chat hint/agent/profile accepts image input
/// according to the **user-configured** vision flag in `config.model_registry`.
///
/// This is the per-model override that lets a user mark a **custom / BYOK** model
/// as vision-capable (Settings → Advanced LLM → custom model → "Supports
/// vision"). Managed-backend models already advertise vision via
/// [`crate::openhuman::inference::provider::Provider::supports_vision`]; this flag
/// covers OpenAI-compatible providers the backend can't introspect per-model.
/// Returns `false` for models the user has not flagged.
pub fn model_vision_enabled(model: &str, config: &crate::openhuman::config::Config) -> bool {
let normalized = model.trim();
if normalized.is_empty() {
return false;
}
let enabled = config
.model_registry
.iter()
.any(|entry| entry.id == normalized && entry.vision);
tracing::debug!(
model = normalized,
vision_enabled = enabled,
"[model_context] resolved user-configured vision flag"
);
enabled
}
/// Whether a resolved model accepts image input. The single predicate shared by
/// the chat UI resolve and the server-side session/sub-agent gates.
///
/// - **Managed OpenHuman tiers** consult the hardcoded per-tier map
/// ([`crate::openhuman::inference::provider::factory::oh_tier_supports_vision`]) —
/// the remote backend does not advertise per-tier capability, so the core owns
/// it. Currently every managed tier is `false`.
/// - **Custom/BYOK models** consult the user-set `model_registry.vision` flag
/// ([`model_vision_enabled`]).
pub fn model_supports_vision(model: &str, config: &crate::openhuman::config::Config) -> bool {
crate::openhuman::inference::provider::factory::oh_tier_supports_vision(model)
|| model_vision_enabled(model, config)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -210,6 +250,50 @@ mod tests {
assert_eq!(context_window_for_model(" "), None);
}
#[test]
fn model_vision_enabled_reads_registry_only() {
use crate::openhuman::config::schema::ModelRegistryEntry;
let mut config = crate::openhuman::config::Config::default();
config.model_registry = vec![
ModelRegistryEntry {
id: "my-llava".into(),
provider: "openai".into(),
cost_per_1m_output: 0.0,
vision: true,
},
ModelRegistryEntry {
id: "text-only".into(),
provider: "openai".into(),
cost_per_1m_output: 0.0,
vision: false,
},
];
assert!(model_vision_enabled("my-llava", &config));
assert!(!model_vision_enabled("text-only", &config));
assert!(!model_vision_enabled("unlisted", &config));
assert!(!model_vision_enabled(" ", &config));
}
#[test]
fn model_supports_vision_combines_tier_map_and_registry() {
use crate::openhuman::config::schema::ModelRegistryEntry;
let mut config = crate::openhuman::config::Config::default();
config.model_registry = vec![ModelRegistryEntry {
id: "my-llava".into(),
provider: "openai".into(),
cost_per_1m_output: 0.0,
vision: true,
}];
// Managed tiers are non-vision (the per-tier map is all `false` today).
assert!(!model_supports_vision("chat-v1", &config));
assert!(!model_supports_vision("reasoning-v1", &config));
assert!(!model_supports_vision("hint:chat", &config));
// BYOK model flagged in the registry is vision-capable.
assert!(model_supports_vision("my-llava", &config));
// Unlisted custom model is not.
assert!(!model_supports_vision("gpt-5", &config));
}
#[test]
fn o1_o3_segment_match_does_not_overmatch() {
// Real OpenAI o1/o3 model ids must still resolve.
@@ -172,6 +172,34 @@ pub(crate) fn is_known_openhuman_tier(model: &str) -> bool {
)
}
/// Per-tier vision (image-input) capability for the managed OpenHuman backend.
///
/// The remote managed backend (`api.tinyhumans.ai`) does not advertise per-tier
/// capabilities, so the core maintains this map itself. Accepts both the tier
/// constants and their `hint:*` forms (callers may pass either pre- or
/// post-resolution).
///
/// Every managed tier currently returns `false` — flip an individual arm to
/// `true` once that tier is confirmed multimodal on the backend. This is the
/// **only** place to change managed-model vision; BYOK/custom models are handled
/// separately by the user-set `model_registry.vision` flag
/// ([`crate::openhuman::inference::model_context::model_vision_enabled`]).
pub(crate) fn oh_tier_supports_vision(model: &str) -> bool {
use crate::openhuman::config::{
MODEL_AGENTIC_V1, MODEL_CHAT_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1,
MODEL_REASONING_V1, MODEL_SUMMARIZATION_V1,
};
match model {
MODEL_CHAT_V1 | "hint:chat" => false,
MODEL_REASONING_V1 | "hint:reasoning" => false,
MODEL_REASONING_QUICK_V1 => false,
MODEL_AGENTIC_V1 | "hint:agentic" => false,
MODEL_CODING_V1 | "hint:coding" => false,
MODEL_SUMMARIZATION_V1 | "hint:summarization" => false,
_ => false,
}
}
/// Return the configured provider string for a named workload role.
///
/// Empty / `"cloud"` resolves through BYOK fallback first for the three
@@ -867,6 +867,40 @@ fn invalid_models_fail() {
assert!(!is_known_openhuman_tier("hint:"));
}
// ── oh_tier_supports_vision ──────────────────────────────────────────────────────
#[test]
fn no_managed_tier_is_vision_capable_yet() {
// Every managed tier (and its hint form) is non-vision until confirmed
// multimodal on the backend. Flip the corresponding arm in
// `oh_tier_supports_vision` to enable one.
for model in [
"reasoning-v1",
"chat-v1",
"agentic-v1",
"coding-v1",
"reasoning-quick-v1",
"summarization-v1",
"hint:reasoning",
"hint:chat",
"hint:agentic",
"hint:coding",
"hint:summarization",
] {
assert!(
!oh_tier_supports_vision(model),
"expected managed tier '{model}' to be non-vision"
);
}
}
#[test]
fn unknown_models_are_not_vision_capable() {
assert!(!oh_tier_supports_vision("gpt-5"));
assert!(!oh_tier_supports_vision("claude-opus-4-7"));
assert!(!oh_tier_supports_vision(""));
}
#[test]
fn make_openhuman_backend_forwards_unknown_hint_verbatim() {
// Unrecognised hint:* strings (e.g. hint:reaction for lightweight models)
+18 -2
View File
@@ -80,6 +80,8 @@ struct InferenceUpdateModelSettingsParams {
default_temperature: Option<f64>,
model_routes: Option<Vec<InferenceModelRouteUpdate>>,
cloud_providers: Option<Vec<InferenceCloudProviderUpdate>>,
#[serde(default)]
model_registry: Option<Vec<crate::openhuman::config::schema::ModelRegistryEntry>>,
primary_cloud: Option<String>,
chat_provider: Option<String>,
reasoning_provider: Option<String>,
@@ -255,7 +257,13 @@ pub fn schemas(function: &str) -> ControllerSchema {
function: "resolve_model",
description: "Resolve a model hint or tier name to the concrete model the provider router would use.",
inputs: vec![required_string("hint", "Model hint (e.g. hint:reasoning) or tier name (e.g. reasoning-v1).")],
outputs: vec![json_output("model", "Resolved concrete model id.")],
outputs: vec![
json_output("model", "Resolved concrete model id."),
json_output(
"vision",
"Whether the resolved model accepts image input (vision-capable).",
),
],
},
"status" => ControllerSchema {
namespace: "inference",
@@ -283,6 +291,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
optional_f64("default_temperature", "Optional default temperature override."),
optional_json("model_routes", "Optional full replacement for legacy model routes."),
optional_json("cloud_providers", "Optional full replacement for configured cloud providers."),
optional_json("model_registry", "Optional full replacement for the per-model registry (carries each model's `vision` flag)."),
optional_string("primary_cloud", "Optional primary cloud provider id."),
optional_string("chat_provider", "Optional chat workload provider string."),
optional_string("reasoning_provider", "Optional reasoning workload provider string."),
@@ -557,8 +566,14 @@ fn handle_inference_resolve_model(params: Map<String, Value>) -> ControllerFutur
let resolved = crate::openhuman::inference::provider::factory::resolve_model_for_hint(
&p.hint, &config,
);
// Whether the resolved model accepts image input — drives the chat UI's
// image-attachment affordance. Managed OpenHuman tiers consult the
// core-owned per-tier map (currently all `false`); custom/BYOK models are
// covered by the user's per-model `model_registry.vision` flag.
let vision =
crate::openhuman::inference::model_context::model_supports_vision(&resolved, &config);
to_json(RpcOutcome::new(
serde_json::json!({ "model": resolved }),
serde_json::json!({ "model": resolved, "vision": vision }),
vec![],
))
})
@@ -676,6 +691,7 @@ fn handle_inference_update_model_settings(params: Map<String, Value>) -> Control
.collect::<Result<Vec<_>, String>>()
})
.transpose()?,
model_registry: update.model_registry,
primary_cloud: update.primary_cloud,
chat_provider: update.chat_provider,
reasoning_provider: update.reasoning_provider,