+
+
+ )}
+
{(testBusy || testReply || testError || testStartedAt) && (
+ {t('settings.ai.modelVisionDesc')} +
+ Promise;
+ modelRegistry: ModelRegistryEntry[];
+ onApply: (next: ProviderRef, vision: boolean) => Promise;
}) => {
const { t } = useT();
const customCloud = cloudProviders.filter(p => p.slug !== 'openhuman');
@@ -2385,6 +2460,29 @@ const GlobalOwnModelSelector = ({
const [model, setModel] = useState(
current?.kind === 'cloud' || current?.kind === 'local' ? current.model : ''
);
+ // Registry slug for the selected source — keys the per-model vision flag.
+ const registrySlug =
+ source?.kind === 'cloud'
+ ? source.providerSlug
+ : source?.kind === 'local'
+ ? 'ollama'
+ : source?.kind === 'claude-code'
+ ? 'claude-code'
+ : null;
+ const [vision, setVision] = useState(() =>
+ registrySlug && model.trim()
+ ? modelRegistryVision(modelRegistry, registrySlug, model.trim())
+ : false
+ );
+ useEffect(() => {
+ // eslint-disable-next-line react-hooks/set-state-in-effect
+ setVision(
+ registrySlug && model.trim()
+ ? modelRegistryVision(modelRegistry, registrySlug, model.trim())
+ : false
+ );
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [registrySlug, model]);
const [cloudModels, setCloudModels] = useState([]);
const [cloudModelsLoading, setCloudModelsLoading] = useState(false);
const [cloudModelsError, setCloudModelsError] = useState(null);
@@ -2453,15 +2551,14 @@ const GlobalOwnModelSelector = ({
setSaving(true);
try {
if (nextSource.kind === 'local') {
- await onApply({ kind: 'local', model: nextModel.trim() });
+ await onApply({ kind: 'local', model: nextModel.trim() }, vision);
} else if (nextSource.kind === 'claude-code') {
- await onApply({ kind: 'claude-code', model: nextModel.trim() });
+ await onApply({ kind: 'claude-code', model: nextModel.trim() }, vision);
} else {
- await onApply({
- kind: 'cloud',
- providerSlug: nextSource.providerSlug,
- model: nextModel.trim(),
- });
+ await onApply(
+ { kind: 'cloud', providerSlug: nextSource.providerSlug, model: nextModel.trim() },
+ vision
+ );
}
} finally {
setSaving(false);
@@ -2566,6 +2663,23 @@ const GlobalOwnModelSelector = ({
) : null}
+ {registrySlug && model.trim().length > 0 && (
+
+ )}
+
{t('settings.ai.globalModel.appliesToAll')}
@@ -3103,8 +3217,23 @@ const AIPanel = ({ embedded = false }: AIPanelProps = {}) => {
cloudProviders={draft.cloudProviders}
localModels={installed}
ollamaRunning={ollama.state === 'running'}
- onApply={async next => {
- await persist({ ...draft, routing: routingWithAllWorkloads(next) });
+ modelRegistry={draft.modelRegistry}
+ onApply={async (next, vision) => {
+ const reg =
+ next.kind === 'cloud'
+ ? { slug: next.providerSlug, model: next.model }
+ : next.kind === 'local'
+ ? { slug: 'ollama', model: next.model }
+ : next.kind === 'claude-code'
+ ? { slug: 'claude-code', model: next.model }
+ : null;
+ await persist({
+ ...draft,
+ routing: routingWithAllWorkloads(next),
+ modelRegistry: reg
+ ? upsertModelRegistryVision(draft.modelRegistry, reg.slug, reg.model, vision)
+ : draft.modelRegistry,
+ });
}}
/>
) : null}
@@ -3290,11 +3419,24 @@ const AIPanel = ({ embedded = false }: AIPanelProps = {}) => {
cloudProviders={draft.cloudProviders}
localModels={installed}
ollamaRunning={ollama.state === 'running'}
+ modelRegistry={draft.modelRegistry}
onClose={() => setCustomDialogFor(null)}
- onSubmit={async next => {
+ onSubmit={async (next, vision) => {
+ // (provider slug, model id) the vision flag keys on.
+ const reg =
+ next.kind === 'cloud'
+ ? { slug: next.providerSlug, model: next.model }
+ : next.kind === 'local'
+ ? { slug: 'ollama', model: next.model }
+ : next.kind === 'claude-code'
+ ? { slug: 'claude-code', model: next.model }
+ : null;
const nextDraft = {
...draft,
routing: { ...draft.routing, [customDialogFor]: next },
+ modelRegistry: reg
+ ? upsertModelRegistryVision(draft.modelRegistry, reg.slug, reg.model, vision)
+ : draft.modelRegistry,
};
await persist(nextDraft);
setCustomDialogFor(null);
diff --git a/app/src/components/settings/panels/__tests__/AIPanel.test.tsx b/app/src/components/settings/panels/__tests__/AIPanel.test.tsx
index 484fd48c8..40cc7741d 100644
--- a/app/src/components/settings/panels/__tests__/AIPanel.test.tsx
+++ b/app/src/components/settings/panels/__tests__/AIPanel.test.tsx
@@ -15,6 +15,7 @@ import {
setCloudProviderKey,
startOpenAiCodexOAuth,
testProviderModel,
+ upsertModelRegistryVision,
} from '../../../../services/api/aiSettingsApi';
import { creditsApi } from '../../../../services/api/creditsApi';
import { renderWithProviders } from '../../../../test/test-utils';
@@ -45,6 +46,8 @@ vi.mock('../../../../services/api/aiSettingsApi', () => ({
saveAISettings: vi.fn(),
loadLocalProviderSnapshot: vi.fn(),
testProviderModel: vi.fn(),
+ modelRegistryVision: vi.fn(() => false),
+ upsertModelRegistryVision: vi.fn((registry: unknown[]) => registry),
setCloudProviderKey: vi.fn().mockResolvedValue(undefined),
clearCloudProviderKey: vi.fn().mockResolvedValue(undefined),
serializeProviderRef: vi.fn((r: { kind: string; providerSlug?: string; model?: string }) =>
@@ -123,6 +126,7 @@ const baseSettings = {
learning: { kind: 'openhuman' as const },
subconscious: { kind: 'openhuman' as const },
},
+ modelRegistry: [],
};
const baseLocalSnapshot = { status: null, diagnostics: null, presets: null, installedModels: [] };
@@ -304,6 +308,52 @@ describe('AIPanel', () => {
}
});
+ // ─── per-model vision flag (BYOK) ───────────────────────────────────────────
+
+ it('flags a custom BYOK model as vision-capable via the Own-model selector', async () => {
+ vi.mocked(loadAISettings).mockResolvedValue({
+ ...baseSettings,
+ cloudProviders: [
+ ...baseSettings.cloudProviders,
+ {
+ id: 'p_custom_openai',
+ slug: 'openai',
+ label: 'OpenAI',
+ endpoint: 'https://api.openai.com/v1',
+ auth_style: 'bearer' as const,
+ has_api_key: true,
+ },
+ ],
+ });
+ renderWithProviders(
+ {fileNames.map((name, i) => (
+
+ )}
{(msg.content || showTime) && (
+
+ {name}
+
+ ))}
+
{msg.content && (
@@ -2213,7 +2273,10 @@ const Conversations = ({
inlineCompletionSuffix={inlineCompletionSuffix}
isComposingTextRef={isComposingTextRef}
maxAttachments={ATTACHMENT_MAX_IMAGES + ATTACHMENT_MAX_FILES}
- allowedMimeTypes={ALLOWED_ATTACHMENT_MIME_TYPES}
+ // Empty → no native `accept` filter (it greys valid files on
+ // macOS/CEF). Type enforcement happens in handleAttachFiles via
+ // validateAndReadFile, which honors modelSupportsVision.
+ allowedMimeTypes={[]}
attachmentsEnabled={CHAT_ATTACHMENTS_ENABLED}
/>
) : (
diff --git a/app/src/pages/__tests__/Conversations.attachments.test.tsx b/app/src/pages/__tests__/Conversations.attachments.test.tsx
index 38923907d..9052b16b4 100644
--- a/app/src/pages/__tests__/Conversations.attachments.test.tsx
+++ b/app/src/pages/__tests__/Conversations.attachments.test.tsx
@@ -140,6 +140,18 @@ vi.mock('../../features/autocomplete/useAutocompleteSkillStatus', () => ({
vi.mock('../../utils/openUrl', () => ({ openUrl: vi.fn() }));
+// The composer gates image attachments on the resolved model's vision capability
+// (inference_resolve_model). These tests exercise image attachments, so resolve
+// the active model as vision-capable.
+vi.mock('../../services/coreRpcClient', () => ({
+ callCoreRpc: vi.fn(async ({ method }: { method: string }) =>
+ method === 'openhuman.inference_resolve_model'
+ ? { model: 'test-vision-model', vision: true }
+ : {}
+ ),
+ CoreRpcError: class CoreRpcError extends Error {},
+}));
+
vi.mock('../../lib/coreState/store', () => ({
getCoreStateSnapshot: vi.fn(() => ({
isBootstrapping: false,
@@ -506,4 +518,53 @@ describe('Conversations — attachment feature', () => {
expect(img).not.toBeNull();
});
});
+
+ it('renders a document filename chip in the user bubble from attachmentKinds/Names', async () => {
+ const thread = makeThread({ id: 'file-thread', title: 'File Thread' });
+ const message = {
+ id: 'msg-file-1',
+ content: 'whats in this file',
+ type: 'text' as const,
+ sender: 'user' as const,
+ createdAt: new Date().toISOString(),
+ extraMetadata: {
+ attachmentCount: 1,
+ attachmentKinds: ['file'],
+ attachmentNames: ['report.pdf'],
+ },
+ };
+
+ mockGetThreads.mockResolvedValue({ threads: [thread], count: 1 });
+ mockGetThreadMessages.mockResolvedValue({ messages: [message], count: 1 });
+
+ const store = buildStore({
+ thread: {
+ threads: [thread],
+ selectedThreadId: thread.id,
+ activeThreadId: null,
+ welcomeThreadId: null,
+ messagesByThreadId: { [thread.id]: [message] },
+ messages: [message],
+ isLoadingThreads: false,
+ isLoadingMessages: false,
+ messagesError: null,
+ },
+ socket: socketState('connected'),
+ });
+
+ const { default: Conversations } = await import('../Conversations');
+
+ render(
+
+
+
+
+ );
+
+ // The document attachment surfaces as a filename chip (not an ![]()
).
+ await waitFor(() => {
+ expect(document.body.textContent).toContain('report.pdf');
+ });
+ });
});
diff --git a/app/src/services/api/__tests__/aiSettingsApi.test.ts b/app/src/services/api/__tests__/aiSettingsApi.test.ts
index 9dbc8413c..7fed06f68 100644
--- a/app/src/services/api/__tests__/aiSettingsApi.test.ts
+++ b/app/src/services/api/__tests__/aiSettingsApi.test.ts
@@ -18,6 +18,7 @@ import {
loadAISettings,
loadLocalProviderSnapshot,
localProvider,
+ modelRegistryVision,
OPENAI_CODEX_OAUTH_MISSING_AUTH_URL,
OPENAI_CODEX_OAUTH_MISSING_CALLBACK_URL,
parseProviderString,
@@ -28,6 +29,7 @@ import {
setLocalRuntimeEnabled,
startOpenAiCodexOAuth,
testProviderModel,
+ upsertModelRegistryVision,
} from '../aiSettingsApi';
// ─── Mock declarations (must be hoisted before imports) ───────────────────────
@@ -83,6 +85,7 @@ function makeClientConfigResult(overrides: Record = {}) {
api_key_set: false,
model_routes: [],
cloud_providers: [],
+ model_registry: [],
primary_cloud: null,
reasoning_provider: null,
agentic_provider: null,
@@ -587,6 +590,7 @@ describe('saveAISettings', () => {
learning: { kind: 'openhuman' },
subconscious: { kind: 'openhuman' },
},
+ modelRegistry: [],
...overrides,
};
}
@@ -673,8 +677,13 @@ describe('saveAISettings', () => {
learning: { kind: 'openhuman' },
subconscious: { kind: 'openhuman' },
},
+ modelRegistry: [],
+ };
+ const next: AISettings = {
+ cloudProviders: [anthropicProvider],
+ routing: { ...prev.routing },
+ modelRegistry: [],
};
- const next: AISettings = { cloudProviders: [anthropicProvider], routing: { ...prev.routing } };
await saveAISettings(prev, next);
@@ -697,6 +706,34 @@ describe('saveAISettings', () => {
expect(patch.cloud_providers).toBeDefined();
expect(patch.coding_provider).toBe('openai:gpt-4o-mini');
});
+
+ it('sends model_registry when the vision flag changes', async () => {
+ const prev = makeSettings({ modelRegistry: [] });
+ const next = makeSettings({
+ modelRegistry: [{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true }],
+ });
+ await saveAISettings(prev, next);
+ const patch = mockOpenhumanUpdateModelSettings.mock.calls[0][0];
+ expect(patch.model_registry).toEqual([
+ { id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true },
+ ]);
+ });
+
+ it('omits model_registry when unchanged', async () => {
+ const registry = [{ id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true }];
+ const prev = makeSettings({ modelRegistry: registry });
+ const next = makeSettings({
+ modelRegistry: [...registry],
+ routing: {
+ ...makeSettings().routing,
+ coding: { kind: 'cloud', providerSlug: 'openai', model: 'gpt-4o-mini' },
+ },
+ });
+ await saveAISettings(prev, next);
+ const patch = mockOpenhumanUpdateModelSettings.mock.calls[0][0];
+ expect(patch.model_registry).toBeUndefined();
+ expect(patch.coding_provider).toBe('openai:gpt-4o-mini');
+ });
});
// ─── setCloudProviderKey ──────────────────────────────────────────────────────
@@ -918,3 +955,32 @@ describe('flushCloudProviders', () => {
expect(mockOpenhumanUpdateModelSettings).not.toHaveBeenCalled();
});
});
+
+describe('model registry vision helpers', () => {
+ const reg = [
+ { id: 'gpt-4o', provider: 'openai', cost_per_1m_output: 0, vision: true },
+ { id: 'text-only', provider: 'openai', cost_per_1m_output: 0, vision: false },
+ ];
+
+ it('modelRegistryVision matches by (provider, id)', () => {
+ expect(modelRegistryVision(reg, 'openai', 'gpt-4o')).toBe(true);
+ expect(modelRegistryVision(reg, 'openai', 'text-only')).toBe(false);
+ expect(modelRegistryVision(reg, 'openai', 'unlisted')).toBe(false);
+ expect(modelRegistryVision(reg, 'azure', 'gpt-4o')).toBe(false);
+ });
+
+ it('upsertModelRegistryVision adds, flips, and removes entries', () => {
+ const added = upsertModelRegistryVision([], 'openai', 'my-llava', true);
+ expect(added).toEqual([
+ { id: 'my-llava', provider: 'openai', cost_per_1m_output: 0, vision: true },
+ ]);
+ // vision:false removes the entry (absence ⇒ no vision).
+ const removed = upsertModelRegistryVision(reg, 'openai', 'gpt-4o', false);
+ expect(removed.find(e => e.id === 'gpt-4o')).toBeUndefined();
+ expect(removed.find(e => e.id === 'text-only')).toBeDefined();
+ // Flipping an existing entry on stays idempotent on the key.
+ const flipped = upsertModelRegistryVision(reg, 'openai', 'text-only', true);
+ expect(flipped.filter(e => e.id === 'text-only')).toHaveLength(1);
+ expect(modelRegistryVision(flipped, 'openai', 'text-only')).toBe(true);
+ });
+});
diff --git a/app/src/services/api/aiSettingsApi.ts b/app/src/services/api/aiSettingsApi.ts
index 89c89eba6..ed2ba173d 100644
--- a/app/src/services/api/aiSettingsApi.ts
+++ b/app/src/services/api/aiSettingsApi.ts
@@ -26,6 +26,7 @@ import { isTauri } from '../../utils/tauriCommands/common';
import {
type ClientConfig,
type CloudProviderCreds,
+ type ModelRegistryEntry,
type ModelSettingsUpdate,
openhumanGetClientConfig,
openhumanUpdateLocalAiSettings,
@@ -132,6 +133,47 @@ const PROVIDER_MODEL_TEST_TIMEOUT_MS = 120_000;
export interface AISettings {
cloudProviders: CloudProviderView[];
routing: Record;
+ /**
+ * Per-model registry carrying each model's user-set `vision` flag, keyed by
+ * `(provider slug, model id)`. Surfaced in the custom-model dialog; gates chat
+ * image attachments for custom/BYOK models.
+ */
+ modelRegistry: ModelRegistryEntry[];
+}
+
+/** Re-export so callers (e.g. the AI panel) can reference the entry type. */
+export type { ModelRegistryEntry };
+
+/** Find a model's vision flag in the registry, matching by (provider, id).
+ * Tolerates an undefined registry (older snapshots / transient load state). */
+export function modelRegistryVision(
+ registry: ModelRegistryEntry[] | undefined,
+ provider: string,
+ id: string
+): boolean {
+ return (registry ?? []).some(e => e.provider === provider && e.id === id && e.vision);
+}
+
+/**
+ * Upsert a model's vision flag, returning a new array. Matches by
+ * `(provider, id)`; a `vision: false` entry is removed (absence ⇒ no vision).
+ */
+export function upsertModelRegistryVision(
+ registry: ModelRegistryEntry[] | undefined,
+ provider: string,
+ id: string,
+ vision: boolean
+): ModelRegistryEntry[] {
+ const base = registry ?? [];
+ const without = base.filter(e => !(e.provider === provider && e.id === id));
+ if (!vision) {
+ return without;
+ }
+ const existing = base.find(e => e.provider === provider && e.id === id);
+ return [
+ ...without,
+ { id, provider, cost_per_1m_output: existing?.cost_per_1m_output ?? 0, vision: true },
+ ];
}
// ─── Read path: load + parse ───────────────────────────────────────────────
@@ -264,7 +306,10 @@ export async function loadAISettings(): Promise {
);
}
- return { cloudProviders, routing };
+ // Per-model registry (vision flags). Defensive default for older snapshots.
+ const modelRegistry: ModelRegistryEntry[] = config.model_registry ?? [];
+
+ return { cloudProviders, routing, modelRegistry };
}
// ─── Write path: diff + save ───────────────────────────────────────────────
@@ -310,12 +355,37 @@ export async function saveAISettings(prev: AISettings, next: AISettings): Promis
}
}
+ // Per-model registry (vision flags): any change → send the full list.
+ if (!modelRegistriesEqual(prev.modelRegistry, next.modelRegistry)) {
+ patch.model_registry = next.modelRegistry.map(
+ ({ id, provider, cost_per_1m_output, vision }) => ({
+ id,
+ provider,
+ cost_per_1m_output,
+ vision,
+ })
+ );
+ }
+
if (Object.keys(patch).length === 0) {
return;
}
await openhumanUpdateModelSettings(patch);
}
+/** Order-insensitive structural equality for two model registries. */
+function modelRegistriesEqual(a: ModelRegistryEntry[], b: ModelRegistryEntry[]): boolean {
+ if (a.length !== b.length) {
+ return false;
+ }
+ const key = (e: ModelRegistryEntry) => `${e.provider} ${e.id}`;
+ const bByKey = new Map(b.map(e => [key(e), e]));
+ return a.every(e => {
+ const m = bByKey.get(key(e));
+ return !!m && m.vision === e.vision && m.cost_per_1m_output === e.cost_per_1m_output;
+ });
+}
+
// ─── API key management (per cloud provider slug) ──────────────────────────
/**
diff --git a/app/src/utils/tauriCommands/config.ts b/app/src/utils/tauriCommands/config.ts
index db1cc3e77..976f2b12b 100644
--- a/app/src/utils/tauriCommands/config.ts
+++ b/app/src/utils/tauriCommands/config.ts
@@ -49,6 +49,18 @@ export interface CloudProviderCreds {
auth_style: AuthStyle;
}
+/**
+ * Per-model registry entry. Mirrors the Rust `ModelRegistryEntry`
+ * (`config/schema/types.rs`). Carries the user-set `vision` flag that lets a
+ * custom/BYOK model accept chat image attachments.
+ */
+export interface ModelRegistryEntry {
+ id: string;
+ provider: string;
+ cost_per_1m_output: number;
+ vision: boolean;
+}
+
export interface ModelSettingsUpdate {
/**
* OpenHuman product backend URL. Almost always left untouched; the
@@ -77,6 +89,12 @@ export interface ModelSettingsUpdate {
* Each entry: { id?, slug, label?, endpoint, auth_style? }
*/
cloud_providers?: CloudProviderCreds[] | null;
+ /**
+ * When present, REPLACES `config.model_registry` wholesale. Carries each
+ * model's `vision` flag (Settings → Advanced LLM → custom model → "Supports
+ * vision"). Send `[]` to clear; omit to leave untouched.
+ */
+ model_registry?: ModelRegistryEntry[] | null;
/** @deprecated No longer used — slug-based routing replaces primary_cloud. */
primary_cloud?: string | null;
/** Per-workload provider strings — see Rust `providers::factory` grammar. */
@@ -215,6 +233,8 @@ export interface ClientConfig {
model_routes: ModelRoute[];
/** Configured cloud providers (no API keys — those live in auth-profiles.json). */
cloud_providers: CloudProviderCreds[];
+ /** Per-model registry carrying each model's `vision` flag. */
+ model_registry: ModelRegistryEntry[];
/** Id of the `cloud_providers` entry resolved by the `"cloud"` sentinel. */
primary_cloud: string | null;
/** Per-workload provider strings (e.g. `"cloud"`, `"ollama:llama3.1:8b"`, `"openai:gpt-4o"`). */
diff --git a/app/test/playwright/specs/chat-management-functional.spec.ts b/app/test/playwright/specs/chat-management-functional.spec.ts
index 62f902270..c4c5a6a62 100644
--- a/app/test/playwright/specs/chat-management-functional.spec.ts
+++ b/app/test/playwright/specs/chat-management-functional.spec.ts
@@ -2,7 +2,6 @@ import { expect, type Page, test } from '@playwright/test';
import {
bootAuthenticatedPage,
- callCoreRpc,
dismissWalkthroughIfPresent,
waitForAppReady,
} from '../helpers/core-rpc';
@@ -73,7 +72,7 @@ async function newThread(page: Page): Promise {
}
test.describe('Chat management functional coverage', () => {
- test('attachment preview, remove, and multimodal send path remain interactive', async ({
+ test('attachment preview, remove, and document send path remain interactive', async ({
page,
}) => {
await resetMock();
@@ -89,34 +88,24 @@ test.describe('Chat management functional coverage', () => {
const fileInput = page.locator('input[type="file"]');
await expect(fileInput).toHaveCount(1);
- await fileInput.setInputFiles({
- name: 'pixel.png',
- mimeType: 'image/png',
- buffer: Buffer.from(
- 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
- 'base64'
- ),
- });
- await expect(page.getByText('pixel.png')).toBeVisible();
- await page.getByRole('button', { name: /Remove pixel\.png/ }).click();
- await expect(page.getByText('pixel.png')).toHaveCount(0);
+ // Documents work on every model; images require a vision-capable model.
+ // Managed OpenHuman tiers are text-only (see `oh_tier_supports_vision`), so
+ // the attachment mechanics here are exercised with a text document. Image
+ // upload is gated to vision-flagged models and validated in unit tests.
+ const txtBuffer = Buffer.from('renderer uploaded text document', 'utf8');
+ const attachTxt = () =>
+ fileInput.setInputFiles({ name: 'notes.txt', mimeType: 'text/plain', buffer: txtBuffer });
- const pngBuffer = Buffer.from(
- 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
- 'base64'
- );
- await fileInput.setInputFiles({ name: 'pixel.png', mimeType: 'image/png', buffer: pngBuffer });
- await expect(page.getByText('pixel.png')).toBeVisible();
+ await attachTxt();
+ await expect(page.getByText('notes.txt')).toBeVisible();
+ await page.getByRole('button', { name: /Remove notes\.txt/ }).click();
+ await expect(page.getByText('notes.txt')).toHaveCount(0);
- await fileInput.setInputFiles({
- name: 'notes.txt',
- mimeType: 'text/plain',
- buffer: Buffer.from('renderer uploaded text document', 'utf8'),
- });
+ await attachTxt();
await expect(page.getByText('notes.txt')).toBeVisible();
- await page.getByPlaceholder('How can I help you today?').fill('Describe this image');
+ await page.getByPlaceholder('How can I help you today?').fill('Summarize this file');
await page.getByTestId('send-message-button').click();
await expect(page.getByText('Attachment received by the assistant.')).toBeVisible({
timeout: 30_000,
@@ -132,13 +121,13 @@ test.describe('Chat management functional coverage', () => {
request.method === 'POST' &&
request.url?.includes('/chat/completions') &&
typeof request.body === 'string' &&
- request.body.includes('Describe this image')
+ request.body.includes('Summarize this file')
);
return completion?.body ?? '';
},
{ timeout: 10_000 }
)
- .toContain('image_url');
+ .toContain('[FILE-EXTRACTED:');
const log = await requests();
const completionBody =
@@ -147,13 +136,21 @@ test.describe('Chat management functional coverage', () => {
request.method === 'POST' &&
request.url?.includes('/chat/completions') &&
typeof request.body === 'string' &&
- request.body.includes('Describe this image')
+ request.body.includes('Summarize this file')
)?.body ?? '';
expect(completionBody).toContain('reasoning');
expect(completionBody).toContain('[FILE-EXTRACTED:');
expect(completionBody).toContain('renderer uploaded text document');
});
+ // NOTE: image attachments require a vision-capable model. Managed tiers are
+ // text-only by default (`oh_tier_supports_vision`); a model is flagged
+ // vision-capable via `model_registry` (Settings → "Supports vision"). The
+ // `image_url` promotion path is covered by Rust unit tests
+ // (`inference::provider::compatible` MessageContent). An E2E that toggles the
+ // vision flag at runtime is a follow-up (needs a page re-mount so the
+ // composer's resolve picks up the flag — out of scope here).
+
test('thread rename and delete remain usable from the conversation UI', async ({ page }) => {
await resetMock();
await openChat(page, 'pw-chat-rename-delete');
diff --git a/src/openhuman/agent/harness/engine/core.rs b/src/openhuman/agent/harness/engine/core.rs
index f130ce3ee..d0f12efe1 100644
--- a/src/openhuman/agent/harness/engine/core.rs
+++ b/src/openhuman/agent/harness/engine/core.rs
@@ -248,7 +248,14 @@ pub(crate) async fn run_turn_engine(
tracing::debug!(iteration, "[agent_loop] sending LLM request");
let image_marker_count = multimodal::count_image_markers(history);
- if image_marker_count > 0 && !provider.supports_vision() {
+ // A model accepts images when its provider advertises vision (managed
+ // backend) OR the user marked it vision-capable in `model_registry`
+ // (custom/BYOK) — surfaced via the `current_model_vision` task-local set
+ // at session build. Unset (CLI/tests) falls back to the provider flag.
+ let has_vision = provider.supports_vision()
+ || crate::openhuman::agent::harness::model_vision_context::current_model_vision()
+ .unwrap_or(false);
+ if image_marker_count > 0 && !has_vision {
let cap_err = ProviderCapabilityError {
provider: provider_name.to_string(),
capability: "vision".to_string(),
diff --git a/src/openhuman/agent/harness/mod.rs b/src/openhuman/agent/harness/mod.rs
index d134842da..379eb189c 100644
--- a/src/openhuman/agent/harness/mod.rs
+++ b/src/openhuman/agent/harness/mod.rs
@@ -30,6 +30,7 @@ mod instructions;
pub mod interrupt;
pub(crate) mod memory_context;
pub(crate) mod memory_context_safety;
+pub mod model_vision_context;
mod parse;
pub(crate) mod payload_summarizer;
pub mod run_queue;
@@ -50,6 +51,7 @@ pub use definition::{
};
pub use fork_context::{current_parent, with_parent_context, ParentExecutionContext};
pub use interrupt::{check_interrupt, InterruptFence, InterruptedError};
+pub use model_vision_context::{current_model_vision, with_current_model_vision};
pub use sandbox_context::{current_sandbox_mode, with_current_sandbox_mode};
pub(crate) use spawn_depth_context::{current_spawn_depth, with_spawn_depth, MAX_SPAWN_DEPTH};
pub use subagent_runner::{run_subagent, SubagentRunError, SubagentRunOptions};
diff --git a/src/openhuman/agent/harness/model_vision_context.rs b/src/openhuman/agent/harness/model_vision_context.rs
new file mode 100644
index 000000000..402ecd426
--- /dev/null
+++ b/src/openhuman/agent/harness/model_vision_context.rs
@@ -0,0 +1,75 @@
+//! Task-local carrier for the **current session/sub-agent's user-configured
+//! vision capability** so the deep turn engine's image gate can honor a custom
+//! (BYOK) model's `model_registry.vision` flag without widening
+//! [`crate::openhuman::agent::harness::engine::run_turn_engine`]'s signature.
+//!
+//! Managed-backend models advertise vision via `Provider::supports_vision()`, so
+//! the gate accepts their image turns already. Custom OpenAI-compatible providers
+//! report `supports_vision() == false` (a provider endpoint can't know a per-model
+//! property), so without this the gate would reject image turns for a model the
+//! user explicitly marked vision-capable. This task-local surfaces that per-model
+//! flag — computed once at session build (where the full `Config` / `model_registry`
+//! and the resolved model id coexist) — to the gate.
+//!
+//! Mirrors [`super::sandbox_context`]. When unset (CLI / direct invocation / tests
+//! that never wrapped the call) [`current_model_vision`] returns `None` and the
+//! gate falls back to the provider capability only — strictly additive.
+
+tokio::task_local! {
+ /// User-configured vision capability for the currently-executing
+ /// session/sub-agent's model. Scoped per turn by the turn loop + subagent
+ /// runner. `None` when unset.
+ pub static CURRENT_MODEL_VISION: bool;
+}
+
+/// Returns the current model's user-configured vision flag, if scope is active.
+pub fn current_model_vision() -> Option {
+ CURRENT_MODEL_VISION.try_with(|v| *v).ok()
+}
+
+/// Run `future` with `vision` installed as the current model's vision flag.
+/// Intended call site is around each `run_turn_engine` invocation.
+pub async fn with_current_model_vision(vision: bool, future: F) -> R
+where
+ F: std::future::Future