admin/openhuman
1
0
Fork 0
mirror of https://github.com/tinyhumansai/openhuman.git synced 2026-07-27 21:08:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(chat): enable reasoning multimodal attachments (#3458)

Browse Source
This commit is contained in:
Steven Enamakel
2026-06-07 01:11:07 -04:00
committed by GitHub
parent ece264ff8f
commit 7133daec05
38 changed files with 870 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/components/chat/AttachmentPreview.tsx
+27 -6
View File
@@ -22,15 +22,36 @@ export default function AttachmentPreview({
<div
key={attachment.id}
className="relative flex items-center gap-2 rounded-lg border border-stone-200 dark:border-neutral-700 bg-stone-50 dark:bg-neutral-800 px-2 py-1.5 text-xs text-stone-700 dark:text-neutral-300 max-w-[180px]">
<img
src={attachment.dataUri}
alt={attachment.file.name}
className="w-8 h-8 rounded object-cover flex-shrink-0"
/>
{attachment.kind === 'image' ? (
<img
src={attachment.previewUri ?? attachment.dataUri}
alt={attachment.file.name}
className="w-8 h-8 rounded object-cover flex-shrink-0"
/>
) : (
<div
aria-hidden
className="w-8 h-8 rounded border border-stone-200 dark:border-neutral-700 bg-white dark:bg-neutral-900 flex items-center justify-center flex-shrink-0 text-stone-500 dark:text-neutral-400">
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={1.8}
d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"
/>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={1.8}
d="M14 2v6h6M8 13h8M8 17h5"
/>
</svg>
</div>
)}
<div className="flex flex-col min-w-0">
<span className="truncate font-medium leading-tight">{attachment.file.name}</span>
<span className="text-stone-400 dark:text-neutral-500 leading-tight">
{formatFileSize(attachment.file.size)}
{formatFileSize(attachment.payloadSizeBytes)}
</span>
</div>
<button
app/src/components/chat/ChatComposer.tsx
+2 -4
View File
@@ -28,10 +28,8 @@ export interface ChatComposerProps {
maxAttachments: number;
allowedMimeTypes: readonly string[];
/**
* Whether chat image attachments are available. When `false`, the attach
* button, hidden file input, and preview strip are not rendered, so the
* (currently backend-unsupported) attachment flow can't be triggered.
* Gated by `CHAT_ATTACHMENTS_ENABLED` at the call site (issue #3205).
* Whether chat multimodal attachments are available. When `false`, the
* attach button, hidden file input, and preview strip are not rendered.
*/
attachmentsEnabled: boolean;
}
app/src/components/chat/__tests__/AttachmentPreview.test.tsx
+19
View File
@@ -8,9 +8,13 @@ function makeAttachment(overrides: Partial<Attachment> = {}): Attachment {
const blob = new Blob([new Uint8Array(512)], { type: 'image/png' });
return {
id: 'att-1',
kind: 'image',
file: new File([blob], 'test.png', { type: 'image/png' }),
dataUri: 'data:image/png;base64,abc',
mimeType: 'image/png',
originalSizeBytes: 512,
payloadSizeBytes: 512,
compressed: false,
...overrides,
};
}
@@ -35,6 +39,21 @@ describe('AttachmentPreview', () => {
expect(img.src).toBe('data:image/png;base64,xyz');
});
it('renders a document icon for non-image files', () => {
const file = new File([new Uint8Array(128)], 'doc.pdf', { type: 'application/pdf' });
const att = makeAttachment({
kind: 'file',
file,
dataUri: 'data:application/pdf;base64,abc',
mimeType: 'application/pdf',
originalSizeBytes: 128,
payloadSizeBytes: 128,
});
render(<AttachmentPreview attachments={[att]} onRemove={vi.fn()} />);
expect(screen.getByText('doc.pdf')).toBeInTheDocument();
expect(screen.queryByAltText('doc.pdf')).not.toBeInTheDocument();
});
it('calls onRemove with the attachment id when × is clicked', () => {
const onRemove = vi.fn();
const att = makeAttachment({ id: 'att-42' });
app/src/components/chat/__tests__/ChatComposer.test.tsx
+4
View File
@@ -12,9 +12,13 @@ function makeAttachment(overrides: Partial<Attachment> = {}): Attachment {
const blob = new Blob([new Uint8Array(256)], { type: 'image/png' });
return {
id: 'att-1',
kind: 'image',
file: new File([blob], 'photo.png', { type: 'image/png' }),
dataUri: 'data:image/png;base64,abc',
mimeType: 'image/png',
originalSizeBytes: 256,
payloadSizeBytes: 256,
compressed: false,
...overrides,
};
}
app/src/lib/attachments.test.ts
+52 -1
View File
@@ -1,7 +1,9 @@
import { describe, expect, it } from 'vitest';
import {
ALLOWED_ATTACHMENT_MIME_TYPES,
type Attachment,
ATTACHMENT_MAX_FILE_SIZE_BYTES,
ATTACHMENT_MAX_IMAGES,
ATTACHMENT_MAX_SIZE_BYTES,
buildMessageWithAttachments,
@@ -20,9 +22,13 @@ function makeFile(name: string, type: string, size = 1024): File {
function makeAttachment(overrides: Partial<Attachment> = {}): Attachment {
return {
id: 'test-id',
kind: 'image',
file: makeFile('test.png', 'image/png'),
dataUri: 'data:image/png;base64,abc',
mimeType: 'image/png',
originalSizeBytes: 1024,
payloadSizeBytes: 1024,
compressed: false,
...overrides,
};
}
@@ -44,6 +50,14 @@ describe('isAllowedMimeType', () => {
});
});
describe('allowed attachment MIME types', () => {
it('includes images and supported document files', () => {
expect(ALLOWED_ATTACHMENT_MIME_TYPES).toContain('image/png');
expect(ALLOWED_ATTACHMENT_MIME_TYPES).toContain('application/pdf');
expect(ALLOWED_ATTACHMENT_MIME_TYPES).toContain('text/plain');
});
});
describe('fileToDataUri', () => {
it('reads a file as a data URI', async () => {
const file = makeFile('photo.png', 'image/png', 4);
@@ -63,7 +77,7 @@ describe('validateAndReadFile', () => {
});
it('rejects unsupported MIME types', async () => {
const file = makeFile('doc.pdf', 'application/pdf');
const file = makeFile('vector.svg', 'image/svg+xml');
const result = await validateAndReadFile(file, 0);
expect('error' in result).toBe(true);
if ('error' in result) {
@@ -71,6 +85,19 @@ describe('validateAndReadFile', () => {
}
});
it('rejects files that exceed the file size limit', async () => {
const oversizedFile = makeFile(
'big.pdf',
'application/pdf',
ATTACHMENT_MAX_FILE_SIZE_BYTES + 1
);
const result = await validateAndReadFile(oversizedFile, 0);
expect('error' in result).toBe(true);
if ('error' in result) {
expect(result.error.code).toBe('too_large');
}
});
it('rejects files that exceed the size limit', async () => {
const oversizedFile = makeFile('big.png', 'image/png', ATTACHMENT_MAX_SIZE_BYTES + 1);
const result = await validateAndReadFile(oversizedFile, 0);
@@ -86,8 +113,21 @@ describe('validateAndReadFile', () => {
expect('attachment' in result).toBe(true);
if ('attachment' in result) {
expect(result.attachment.mimeType).toBe('image/png');
expect(result.attachment.kind).toBe('image');
expect(result.attachment.dataUri).toMatch(/^data:image\/png;base64,/);
expect(result.attachment.file).toBe(file);
expect(result.attachment.compressed).toBe(false);
}
});
it('returns an attachment for a supported file', async () => {
const file = makeFile('doc.pdf', 'application/pdf', 512);
const result = await validateAndReadFile(file, 0);
expect('attachment' in result).toBe(true);
if ('attachment' in result) {
expect(result.attachment.mimeType).toBe('application/pdf');
expect(result.attachment.kind).toBe('file');
expect(result.attachment.dataUri).toMatch(/^data:application\/pdf;base64,/);
}
});
@@ -138,6 +178,17 @@ describe('buildMessageWithAttachments', () => {
expect(result).toBe('look [IMAGE:data:image/png;base64,a] [IMAGE:data:image/jpeg;base64,b]');
});
it('emits FILE markers for non-image attachments', () => {
const a = makeAttachment({
kind: 'file',
file: makeFile('doc.pdf', 'application/pdf'),
dataUri: 'data:application/pdf;base64,abc',
mimeType: 'application/pdf',
});
const result = buildMessageWithAttachments('read this', [a]);
expect(result).toBe('read this [FILE:data:application/pdf;base64,abc]');
});
it('trims leading/trailing whitespace from text', () => {
const a = makeAttachment({ dataUri: 'data:image/png;base64,x' });
const result = buildMessageWithAttachments(' hi ', [a]);
app/src/lib/attachments.ts
+137 -26
View File
@@ -1,14 +1,14 @@
/**
* Utilities for multimodal chat attachments.
*
* Images are embedded in the message text as `[IMAGE:<data-uri>]` markers,
* which the Rust agent harness (`agent/multimodal.rs`) parses and forwards to
* the inference provider. The backend supports up to 4 images per message at
* 8 MB each by default (governed by `MultimodalConfig`).
* Images are embedded as `[IMAGE:<data-uri>]` markers. Other supported files
* are embedded as `[FILE:<data-uri>]` markers. The Rust agent harness
* (`agent/multimodal.rs`) parses, validates, and expands both shapes before
* the provider call.
*/
import debugFactory from 'debug';
export const ATTACHMENT_MAX_IMAGES = 4;
export const ATTACHMENT_MAX_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
const debug = debugFactory('chat:attachments');
export const ALLOWED_IMAGE_MIME_TYPES = [
'image/png',
@@ -20,58 +20,167 @@ export const ALLOWED_IMAGE_MIME_TYPES = [
export type AllowedImageMimeType = (typeof ALLOWED_IMAGE_MIME_TYPES)[number];
export const ALLOWED_FILE_MIME_TYPES = [
'application/pdf',
'text/plain',
'text/csv',
'text/markdown',
'application/zip',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/octet-stream',
] as const;
export type AllowedFileMimeType = (typeof ALLOWED_FILE_MIME_TYPES)[number];
export type AllowedAttachmentMimeType = AllowedImageMimeType | AllowedFileMimeType;
export type AttachmentKind = 'image' | 'file';
export const ALLOWED_ATTACHMENT_MIME_TYPES = [
...ALLOWED_IMAGE_MIME_TYPES,
...ALLOWED_FILE_MIME_TYPES,
] as const;
export const ATTACHMENT_MAX_IMAGES = 4;
export const ATTACHMENT_MAX_FILES = 4;
export const ATTACHMENT_MAX_IMAGE_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
export const ATTACHMENT_MAX_FILE_SIZE_BYTES = 16 * 1024 * 1024; // 16 MB
export const ATTACHMENT_MAX_SIZE_BYTES = ATTACHMENT_MAX_IMAGE_SIZE_BYTES;
export interface Attachment {
id: string;
kind: AttachmentKind;
file: File;
dataUri: string;
mimeType: AllowedImageMimeType;
previewUri?: string;
mimeType: AllowedAttachmentMimeType;
originalSizeBytes: number;
payloadSizeBytes: number;
compressed: boolean;
}
export type AttachmentError =
| { code: 'unsupported_type'; mimeType: string }
| { code: 'too_large'; sizeBytes: number; maxBytes: number }
| { code: 'too_many'; max: number }
| { code: 'too_many'; kind: AttachmentKind; max: number }
| { code: 'read_failed'; reason: string };
export function isAllowedMimeType(mime: string): mime is AllowedImageMimeType {
return (ALLOWED_IMAGE_MIME_TYPES as readonly string[]).includes(mime);
}
export function fileToDataUri(file: File): Promise<string> {
export function isAllowedAttachmentMimeType(mime: string): mime is AllowedAttachmentMimeType {
return (ALLOWED_ATTACHMENT_MIME_TYPES as readonly string[]).includes(mime);
}
export function attachmentKindForMime(mime: AllowedAttachmentMimeType): AttachmentKind {
return isAllowedMimeType(mime) ? 'image' : 'file';
}
export function fileToDataUri(file: Blob): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
const name = file instanceof File ? file.name : 'blob';
reader.onload = () => resolve(reader.result as string);
reader.onerror = () => reject(new Error(`Failed to read file: ${file.name}`));
reader.onerror = () => reject(new Error(`Failed to read file: ${name}`));
reader.readAsDataURL(file);
});
}
export async function validateAndReadFile(
async function blobToDataUri(blob: Blob, mimeType: string): Promise<string> {
const namedBlob = new Blob([blob], { type: mimeType });
return fileToDataUri(namedBlob);
}
async function gzipBlob(file: File): Promise<Blob | null> {
if (!('CompressionStream' in globalThis)) return null;
try {
const compressionStream = new CompressionStream('gzip');
const compressed = file.stream().pipeThrough(compressionStream);
return await new Response(compressed).blob();
} catch (error) {
debug('[chat:attachments] gzip_failed name=%s error=%o', file.name, error);
return null;
}
}
function encodeDataUriParam(value: string): string {
return encodeURIComponent(value).replace(/'/g, '%27');
}
async function buildAttachmentDataUri(
file: File,
existingCount: number
): Promise<{ attachment: Attachment } | { error: AttachmentError }> {
if (existingCount >= ATTACHMENT_MAX_IMAGES) {
return { error: { code: 'too_many', max: ATTACHMENT_MAX_IMAGES } };
mimeType: AllowedAttachmentMimeType
): Promise<{ dataUri: string; payloadSizeBytes: number; compressed: boolean }> {
debug(
'[chat:attachments] compression:start name=%s mime=%s size=%d',
file.name,
mimeType,
file.size
);
const compressed = await gzipBlob(file);
if (compressed && compressed.size < file.size) {
const dataUri = await blobToDataUri(
compressed,
`application/gzip;original_mime=${encodeDataUriParam(mimeType)};name=${encodeDataUriParam(file.name)}`
);
debug(
'[chat:attachments] compression:ok name=%s original=%d compressed=%d',
file.name,
file.size,
compressed.size
);
return { dataUri, payloadSizeBytes: compressed.size, compressed: true };
}
if (!isAllowedMimeType(file.type)) {
const dataUri = await fileToDataUri(file);
debug(
'[chat:attachments] compression:skipped name=%s original=%d compressed=%s',
file.name,
file.size,
compressed?.size ?? 'unavailable'
);
return { dataUri, payloadSizeBytes: file.size, compressed: false };
}
export async function validateAndReadFile(
file: File,
existingCount: number,
existingFileCount = 0
): Promise<{ attachment: Attachment } | { error: AttachmentError }> {
if (!isAllowedAttachmentMimeType(file.type)) {
return { error: { code: 'unsupported_type', mimeType: file.type || 'unknown' } };
}
if (file.size > ATTACHMENT_MAX_SIZE_BYTES) {
return {
error: { code: 'too_large', sizeBytes: file.size, maxBytes: ATTACHMENT_MAX_SIZE_BYTES },
};
const kind = attachmentKindForMime(file.type);
const maxCount = kind === 'image' ? ATTACHMENT_MAX_IMAGES : ATTACHMENT_MAX_FILES;
const count = kind === 'image' ? existingCount : existingFileCount;
if (count >= maxCount) {
return { error: { code: 'too_many', kind, max: maxCount } };
}
const maxBytes =
kind === 'image' ? ATTACHMENT_MAX_IMAGE_SIZE_BYTES : ATTACHMENT_MAX_FILE_SIZE_BYTES;
if (file.size > maxBytes) {
return { error: { code: 'too_large', sizeBytes: file.size, maxBytes } };
}
try {
const dataUri = await fileToDataUri(file);
const { dataUri, payloadSizeBytes, compressed } = await buildAttachmentDataUri(file, file.type);
const previewUri = kind === 'image' ? await fileToDataUri(file) : undefined;
return {
attachment: {
id: globalThis.crypto.randomUUID(),
kind,
file,
dataUri,
mimeType: file.type as AllowedImageMimeType,
previewUri,
mimeType: file.type,
originalSizeBytes: file.size,
payloadSizeBytes,
compressed,
},
};
} catch (err) {
@@ -83,13 +192,15 @@ export async function validateAndReadFile(
/**
* Compose the final message string by appending `[IMAGE:<data-uri>]` markers
* for each attachment after the user's text. The Rust backend parses these
* markers in `parse_image_markers` and strips them from the visible message
* before forwarding clean text + image payload to the inference provider.
* for image attachments and `[FILE:<data-uri>]` markers for other supported
* files after the user's text. The Rust agent harness parses and strips these
* markers before forwarding clean text and attachment payloads to the provider.
*/
export function buildMessageWithAttachments(text: string, attachments: Attachment[]): string {
if (attachments.length === 0) return text;
const markers = attachments.map(a => `[IMAGE:${a.dataUri}]`).join(' ');
const markers = attachments
.map(a => (a.kind === 'image' ? `[IMAGE:${a.dataUri}]` : `[FILE:${a.dataUri}]`))
.join(' ');
return text.trim() ? `${text.trim()} ${markers}` : markers;
}
app/src/lib/i18n/ar.ts
+1
View File
@@ -1795,6 +1795,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'إرفاق صورة',
'chat.attachment.remove': 'إزالة {name}',
'chat.attachment.tooMany': 'الحد الأقصى {max} صور لكل رسالة',
'chat.attachment.tooManyFiles': 'الحد الأقصى {max} ملفات لكل رسالة',
'chat.attachment.tooLarge': 'حجم الصورة يتجاوز الحد المسموح {max}',
'chat.attachment.unsupportedType': 'نوع ملف غير مدعوم. استخدم PNG أو JPEG أو WebP أو GIF أو BMP.',
'chat.attachment.readFailed': 'تعذر قراءة الملف',
app/src/lib/i18n/bn.ts
+1
View File
@@ -1836,6 +1836,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'ছবি সংযুক্ত করুন',
'chat.attachment.remove': '{name} সরান',
'chat.attachment.tooMany': 'প্রতি বার্তায় সর্বোচ্চ {max}টি ছবি',
'chat.attachment.tooManyFiles': 'প্রতি বার্তায় সর্বোচ্চ {max}টি ফাইল',
'chat.attachment.tooLarge': 'ছবি {max} আকারের সীমা অতিক্রম করেছে',
'chat.attachment.unsupportedType':
'অসমর্থিত ফাইল প্রকার। PNG, JPEG, WebP, GIF, বা BMP ব্যবহার করুন।',
app/src/lib/i18n/de.ts
+1
View File
@@ -1883,6 +1883,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'Bild anhängen',
'chat.attachment.remove': '{name} entfernen',
'chat.attachment.tooMany': 'Maximal {max} Bilder pro Nachricht',
'chat.attachment.tooManyFiles': 'Maximal {max} Dateien pro Nachricht',
'chat.attachment.tooLarge': 'Bild überschreitet die Größenbeschränkung von {max}',
'chat.attachment.unsupportedType':
'Nicht unterstützter Dateityp. Verwenden Sie PNG, JPEG, WebP, GIF oder BMP.',
app/src/lib/i18n/en.ts
+1
View File
@@ -2210,6 +2210,7 @@ const en: TranslationMap = {
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooManyFiles': 'Maximum {max} files per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
app/src/lib/i18n/es.ts
+1
View File
@@ -1875,6 +1875,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'Adjuntar imagen',
'chat.attachment.remove': 'Eliminar {name}',
'chat.attachment.tooMany': 'Máximo {max} imágenes por mensaje',
'chat.attachment.tooManyFiles': 'Máximo {max} archivos por mensaje',
'chat.attachment.tooLarge': 'La imagen supera el límite de tamaño de {max}',
'chat.attachment.unsupportedType':
'Tipo de archivo no compatible. Use PNG, JPEG, WebP, GIF o BMP.',
app/src/lib/i18n/fr.ts
+1
View File
@@ -1883,6 +1883,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'Joindre une image',
'chat.attachment.remove': 'Supprimer {name}',
'chat.attachment.tooMany': 'Maximum {max} images par message',
'chat.attachment.tooManyFiles': 'Maximum {max} fichiers par message',
'chat.attachment.tooLarge': "L'image dépasse la taille limite de {max}",
'chat.attachment.unsupportedType':
'Type de fichier non pris en charge. Utilisez PNG, JPEG, WebP, GIF ou BMP.',
app/src/lib/i18n/hi.ts
+1
View File
@@ -1835,6 +1835,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'छवि संलग्न करें',
'chat.attachment.remove': '{name} हटाएं',
'chat.attachment.tooMany': 'प्रति संदेश अधिकतम {max} छवियां',
'chat.attachment.tooManyFiles': 'प्रति संदेश अधिकतम {max} फ़ाइलें',
'chat.attachment.tooLarge': 'छवि {max} आकार सीमा से अधिक है',
'chat.attachment.unsupportedType':
'असमर्थित फ़ाइल प्रकार। PNG, JPEG, WebP, GIF, या BMP का उपयोग करें।',
app/src/lib/i18n/id.ts
+1
View File
@@ -1838,6 +1838,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'Lampirkan gambar',
'chat.attachment.remove': 'Hapus {name}',
'chat.attachment.tooMany': 'Maksimal {max} gambar per pesan',
'chat.attachment.tooManyFiles': 'Maksimal {max} file per pesan',
'chat.attachment.tooLarge': 'Gambar melebihi batas ukuran {max}',
'chat.attachment.unsupportedType':
'Jenis file tidak didukung. Gunakan PNG, JPEG, WebP, GIF, atau BMP.',
app/src/lib/i18n/it.ts
+1
View File
@@ -1867,6 +1867,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'Allega immagine',
'chat.attachment.remove': 'Rimuovi {name}',
'chat.attachment.tooMany': 'Massimo {max} immagini per messaggio',
'chat.attachment.tooManyFiles': 'Massimo {max} file per messaggio',
'chat.attachment.tooLarge': "L'immagine supera il limite di dimensione di {max}",
'chat.attachment.unsupportedType': 'Tipo di file non supportato. Usa PNG, JPEG, WebP, GIF o BMP.',
'chat.attachment.readFailed': 'Impossibile leggere il file',
app/src/lib/i18n/ko.ts
+1
View File
@@ -1815,6 +1815,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': '이미지 첨부',
'chat.attachment.remove': '{name} 제거',
'chat.attachment.tooMany': '메시지당 최대 {max}개 이미지',
'chat.attachment.tooManyFiles': '메시지당 최대 {max}개 파일',
'chat.attachment.tooLarge': '이미지가 {max} 크기 제한을 초과합니다',
'chat.attachment.unsupportedType':
'지원되지 않는 파일 형식입니다. PNG, JPEG, WebP, GIF 또는 BMP를 사용하세요.',
app/src/lib/i18n/pl.ts
+1
View File
@@ -1856,6 +1856,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'Dołącz obraz',
'chat.attachment.remove': 'Usuń {name}',
'chat.attachment.tooMany': 'Maksymalnie {max} obrazów na wiadomość',
'chat.attachment.tooManyFiles': 'Maksymalnie {max} plików na wiadomość',
'chat.attachment.tooLarge': 'Obraz przekracza limit rozmiaru {max}',
'chat.attachment.unsupportedType': 'Nieobsługiwany typ pliku. Użyj PNG, JPEG, WebP, GIF lub BMP.',
'chat.attachment.readFailed': 'Nie można odczytać pliku',
app/src/lib/i18n/pt.ts
+1
View File
@@ -1874,6 +1874,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'Anexar imagem',
'chat.attachment.remove': 'Remover {name}',
'chat.attachment.tooMany': 'Máximo de {max} imagens por mensagem',
'chat.attachment.tooManyFiles': 'Máximo de {max} arquivos por mensagem',
'chat.attachment.tooLarge': 'A imagem excede o limite de tamanho de {max}',
'chat.attachment.unsupportedType':
'Tipo de arquivo não suportado. Use PNG, JPEG, WebP, GIF ou BMP.',
app/src/lib/i18n/ru.ts
+1
View File
@@ -1850,6 +1850,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': 'Прикрепить изображение',
'chat.attachment.remove': 'Удалить {name}',
'chat.attachment.tooMany': 'Максимум {max} изображений на сообщение',
'chat.attachment.tooManyFiles': 'Максимум {max} файлов на сообщение',
'chat.attachment.tooLarge': 'Изображение превышает ограничение размера {max}',
'chat.attachment.unsupportedType':
'Неподдерживаемый тип файла. Используйте PNG, JPEG, WebP, GIF или BMP.',
app/src/lib/i18n/zh-CN.ts
+1
View File
@@ -1733,6 +1733,7 @@ const messages: TranslationMap = {
'chat.attachment.attach': '添加图片',
'chat.attachment.remove': '移除 {name}',
'chat.attachment.tooMany': '每条消息最多 {max} 张图片',
'chat.attachment.tooManyFiles': '每条消息最多 {max} 个文件',
'chat.attachment.tooLarge': '图片超过 {max} 大小限制',
'chat.attachment.unsupportedType': '不支持的文件类型。请使用 PNG、JPEG、WebP、GIF 或 BMP。',
'chat.attachment.readFailed': '无法读取文件',
app/src/pages/Conversations.tsx
+38 -19
View File
@@ -17,10 +17,10 @@ import MicComposer from '../features/human/MicComposer';
import { useStickToBottom } from '../hooks/useStickToBottom';
import { useUsageState } from '../hooks/useUsageState';
import {
ALLOWED_IMAGE_MIME_TYPES,
ALLOWED_ATTACHMENT_MIME_TYPES,
type Attachment,
ATTACHMENT_MAX_FILES,
ATTACHMENT_MAX_IMAGES,
ATTACHMENT_MAX_SIZE_BYTES,
buildMessageWithAttachments,
parseMessageImages,
validateAndReadFile,
@@ -107,9 +107,8 @@ import {
TASKS_TAB_VALUE,
} from './conversations/utils/threadFilter';
// Chat uses the reasoning model; `agentic-v1` is reserved for sub-agents
// that execute tool calls, not the primary user-facing conversation.
const CHAT_MODEL_ID = 'reasoning-v1';
const CHAT_MODEL_HINT = 'hint:chat';
const MULTIMODAL_MODEL_HINT = 'hint:reasoning';
/** Maximum trailing characters rendered in the live-streaming assistant
* preview bubble. The full response is revealed via `addInferenceResponse`
* on `chat_done` — this is purely a ticker-tape affordance to signal
@@ -277,7 +276,10 @@ const Conversations = ({
(async () => {
try {
const profile = agentProfiles.find(p => p.id === selectedAgentProfileId);
const hint = profile?.modelOverride ?? 'hint:chat';
const hint =
attachments.length > 0
? MULTIMODAL_MODEL_HINT
: (profile?.modelOverride ?? CHAT_MODEL_HINT);
const res = await callCoreRpc<{ model: string }>({
method: 'openhuman.inference_resolve_model',
params: { hint },
@@ -290,7 +292,7 @@ const Conversations = ({
return () => {
cancelled = true;
};
}, [agentProfiles, selectedAgentProfileId]);
}, [agentProfiles, attachments.length, selectedAgentProfileId]);
const textInputRef = useRef<HTMLTextAreaElement>(null);
const isComposingTextRef = useRef(false);
@@ -675,20 +677,20 @@ const Conversations = ({
const handleAttachFiles = async (files: FileList | null) => {
if (!files) return;
let acceptedCount = attachments.length;
let acceptedImageCount = attachments.filter(attachment => attachment.kind === 'image').length;
let acceptedFileCount = attachments.filter(attachment => attachment.kind === 'file').length;
for (const file of Array.from(files)) {
const result = await validateAndReadFile(file, acceptedCount);
const result = await validateAndReadFile(file, acceptedImageCount, acceptedFileCount);
if ('error' in result) {
const { error } = result;
if (error.code === 'too_many') {
const key =
error.kind === 'image' ? 'chat.attachment.tooMany' : 'chat.attachment.tooManyFiles';
setAttachError(
chatSendError(
'attachment_invalid',
t('chat.attachment.tooMany').replace('{max}', String(ATTACHMENT_MAX_IMAGES))
)
chatSendError('attachment_invalid', t(key).replace('{max}', String(error.max)))
);
} else if (error.code === 'too_large') {
const maxMb = (ATTACHMENT_MAX_SIZE_BYTES / (1024 * 1024)).toFixed(0);
const maxMb = (error.maxBytes / (1024 * 1024)).toFixed(0);
setAttachError(
chatSendError(
'attachment_invalid',
@@ -702,7 +704,15 @@ const Conversations = ({
}
return;
}
acceptedCount++;
if (result.attachment.kind === 'image') {
acceptedImageCount++;
} else {
acceptedFileCount++;
}
if (selectedAgentProfileId !== 'reasoning') {
debug('attachment accepted; switching chat profile to reasoning for multimodal send');
void handleSelectAgentProfile('reasoning');
}
setAttachments(prev => [...prev, result.attachment]);
}
};
@@ -755,6 +765,11 @@ const Conversations = ({
pendingSendRef.current = sendingThreadId;
setPendingSendingThreadId(sendingThreadId);
const pendingAttachments = attachments.slice();
const modelOverride =
pendingAttachments.length > 0
? MULTIMODAL_MODEL_HINT
: (agentProfiles.find(p => p.id === selectedAgentProfileId)?.modelOverride ??
CHAT_MODEL_HINT);
const messageText = buildMessageWithAttachments(trimmed, pendingAttachments);
const userMessage: ThreadMessage = {
id: `msg_${globalThis.crypto.randomUUID()}`,
@@ -765,7 +780,11 @@ const Conversations = ({
? {
attachmentCount: pendingAttachments.length,
attachmentNames: pendingAttachments.map(a => a.file.name),
attachmentDataUris: pendingAttachments.map(a => a.dataUri),
attachmentKinds: pendingAttachments.map(a => a.kind),
attachmentDataUris: pendingAttachments
.filter(a => a.kind === 'image')
.map(a => a.previewUri ?? a.dataUri),
attachmentCompressed: pendingAttachments.map(a => a.compressed),
}
: {},
sender: 'user',
@@ -818,7 +837,7 @@ const Conversations = ({
await chatSend({
threadId: sendingThreadId,
message: messageText,
model: CHAT_MODEL_ID,
model: modelOverride,
profileId: selectedAgentProfileId,
locale: uiLocale,
});
@@ -2193,8 +2212,8 @@ const Conversations = ({
handleInputKeyDown={handleInputKeyDown}
inlineCompletionSuffix={inlineCompletionSuffix}
isComposingTextRef={isComposingTextRef}
maxAttachments={ATTACHMENT_MAX_IMAGES}
allowedMimeTypes={ALLOWED_IMAGE_MIME_TYPES}
maxAttachments={ATTACHMENT_MAX_IMAGES + ATTACHMENT_MAX_FILES}
allowedMimeTypes={ALLOWED_ATTACHMENT_MIME_TYPES}
attachmentsEnabled={CHAT_ATTACHMENTS_ENABLED}
/>
) : (
app/src/pages/__tests__/Conversations.attachments.test.tsx
+136 -37
View File
@@ -17,23 +17,46 @@ import type { Thread } from '../../types/thread';
// ── Hoisted mock state ──────────────────────────────────────────────────────
const { mockGetThreads, mockGetThreadMessages, mockUseUsageState } = vi.hoisted(() => ({
mockGetThreads: vi.fn().mockResolvedValue({ threads: [], count: 0 }),
mockGetThreadMessages: vi.fn().mockResolvedValue({ messages: [], count: 0 }),
mockUseUsageState: vi.fn(() => ({
teamUsage: null,
currentPlan: null,
currentTier: 'FREE' as const,
isFreeTier: true,
usagePct: 0,
isNearLimit: false,
isAtLimit: false,
isBudgetExhausted: false,
shouldShowBudgetCompletedMessage: false,
isLoading: false,
refresh: vi.fn(),
})),
}));
const { mockGetThreads, mockGetThreadMessages, mockSelectAgentProfile, mockUseUsageState } =
vi.hoisted(() => ({
mockGetThreads: vi.fn().mockResolvedValue({ threads: [], count: 0 }),
mockGetThreadMessages: vi.fn().mockResolvedValue({ messages: [], count: 0 }),
mockSelectAgentProfile: vi.fn().mockImplementation((profileId: string) =>
Promise.resolve({
activeProfileId: profileId,
profiles: [
{
id: 'default',
name: 'Default',
description: 'Default',
agentId: 'orchestrator',
builtIn: true,
},
{
id: 'reasoning',
name: 'Reasoning',
description: 'Reasoning',
agentId: 'orchestrator',
modelOverride: 'hint:reasoning',
builtIn: true,
},
],
})
),
mockUseUsageState: vi.fn(() => ({
teamUsage: null,
currentPlan: null,
currentTier: 'FREE' as const,
isFreeTier: true,
usagePct: 0,
isNearLimit: false,
isAtLimit: false,
isBudgetExhausted: false,
shouldShowBudgetCompletedMessage: false,
isLoading: false,
refresh: vi.fn(),
})),
}));
// ── Module mocks ────────────────────────────────────────────────────────────
@@ -69,21 +92,27 @@ vi.mock('../../services/api/threadApi', () => ({
vi.mock('../../services/api/agentProfilesApi', () => ({
agentProfilesApi: {
list: vi
.fn()
.mockResolvedValue({
activeProfileId: 'default',
profiles: [
{
id: 'default',
name: 'Default',
description: 'Default',
agentId: 'orchestrator',
builtIn: true,
},
],
}),
select: vi.fn().mockResolvedValue({ activeProfileId: 'default', profiles: [] }),
list: vi.fn().mockResolvedValue({
activeProfileId: 'default',
profiles: [
{
id: 'default',
name: 'Default',
description: 'Default',
agentId: 'orchestrator',
builtIn: true,
},
{
id: 'reasoning',
name: 'Reasoning',
description: 'Reasoning',
agentId: 'orchestrator',
modelOverride: 'hint:reasoning',
builtIn: true,
},
],
}),
select: mockSelectAgentProfile,
upsert: vi.fn().mockResolvedValue({ activeProfileId: 'default', profiles: [] }),
delete: vi.fn().mockResolvedValue({ activeProfileId: 'default', profiles: [] }),
},
@@ -91,9 +120,6 @@ vi.mock('../../services/api/agentProfilesApi', () => ({
vi.mock('../../hooks/useUsageState', () => ({ useUsageState: mockUseUsageState }));
// Attachments are gated off by default (CHAT_ATTACHMENTS_ENABLED, #3205); force
// the flag on so these tests still exercise the underlying attachment pipeline
// (validation, preview, attachment-only send) that ships behind the flag.
vi.mock('../../utils/config', async importActual => ({
...(await importActual<typeof import('../../utils/config')>()),
CHAT_ATTACHMENTS_ENABLED: true,
@@ -264,7 +290,22 @@ describe('Conversations — attachment feature', () => {
});
});
it('shows unsupported type error for non-image files', async () => {
it('shows unsupported type error for unsupported files', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const file = makeFile('vector.svg', 'image/svg+xml', 512);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(screen.getByText(/Unsupported file type/i)).toBeInTheDocument();
});
});
it('shows attachment chip after selecting a supported document file', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
@@ -275,7 +316,7 @@ describe('Conversations — attachment feature', () => {
});
await waitFor(() => {
expect(screen.getByText(/Unsupported file type/i)).toBeInTheDocument();
expect(screen.getByText('doc.pdf')).toBeInTheDocument();
});
});
@@ -333,6 +374,25 @@ describe('Conversations — attachment feature', () => {
});
});
it('switches to reasoning when a supported attachment is selected', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const file = makeFile('auto-reasoning.png', 'image/png', 512);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(mockSelectAgentProfile).toHaveBeenCalledWith('reasoning');
expect(screen.getByRole('radio', { name: 'Reasoning' })).toHaveAttribute(
'aria-checked',
'true'
);
});
});
it('clears attachments and calls chatSend after sending with attachment + text', async () => {
const { chatSend } = await import('../../services/chatService');
const { textarea } = await renderWithSelectedThread();
@@ -358,10 +418,49 @@ describe('Conversations — attachment feature', () => {
await waitFor(() => {
expect(chatSend).toHaveBeenCalled();
expect(chatSend).toHaveBeenCalledWith(
expect.objectContaining({
model: 'hint:reasoning',
message: expect.stringContaining('[IMAGE:data:image/png;base64,'),
})
);
expect(screen.queryByText('send.png')).not.toBeInTheDocument();
});
});
it('sends supported document files as FILE markers through the reasoning model', async () => {
const { chatSend } = await import('../../services/chatService');
const { textarea } = await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const file = makeFile('doc.pdf', 'application/pdf', 512);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(screen.getByText('doc.pdf')).toBeInTheDocument();
});
await act(async () => {
fireEvent.change(textarea, { target: { value: 'read this' } });
});
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: 'Send message' }));
});
await waitFor(() => {
expect(chatSend).toHaveBeenCalledWith(
expect.objectContaining({
model: 'hint:reasoning',
message: expect.stringContaining('[FILE:data:application/pdf;base64,'),
})
);
});
});
it('renders image thumbnails in user message bubble from extraMetadata', async () => {
const thread = makeThread({ id: 'img-thread', title: 'Img Thread' });
const dataUri = 'data:image/png;base64,abc123';
app/src/pages/__tests__/Conversations.render.test.tsx
+5 -5
View File
@@ -865,7 +865,7 @@ describe('Conversations — smoke render (#1123 welcome-lock removal)', () => {
expect(chatSend).toHaveBeenCalledWith({
threadId: thread.id,
message: 'hello cloud',
model: 'reasoning-v1',
model: 'hint:chat',
profileId: 'default',
locale: 'en',
});
@@ -889,7 +889,7 @@ describe('Conversations — smoke render (#1123 welcome-lock removal)', () => {
expect(chatSend).toHaveBeenCalledWith({
threadId: thread.id,
message: 'play highway to hell',
model: 'reasoning-v1',
model: 'hint:chat',
profileId: 'default',
locale: 'en',
});
@@ -941,7 +941,7 @@ describe('Conversations — smoke render (#1123 welcome-lock removal)', () => {
expect(chatSend).toHaveBeenCalledWith({
threadId: thread.id,
message: 'slow backend',
model: 'reasoning-v1',
model: 'hint:chat',
profileId: 'default',
locale: 'en',
});
@@ -1309,7 +1309,7 @@ describe('Conversations — smoke render (#1123 welcome-lock removal)', () => {
expect(chatSend).toHaveBeenCalledWith({
threadId: thread.id,
message: 'enter send',
model: 'reasoning-v1',
model: 'hint:chat',
profileId: 'default',
locale: 'en',
});
@@ -1384,7 +1384,7 @@ describe('Conversations — smoke render (#1123 welcome-lock removal)', () => {
expect(chatSend).toHaveBeenCalledWith({
threadId: thread.id,
message: '안녕',
model: 'reasoning-v1',
model: 'hint:chat',
profileId: 'default',
locale: 'en',
});
app/src/test/setup.ts
+1 -1
View File
@@ -180,7 +180,7 @@ vi.mock('../utils/config', () => ({
E2E_DEFAULT_CORE_MODE: '',
E2E_RESTART_APP_AS_RELOAD: false,
DEV_FORCE_ONBOARDING: false,
CHAT_ATTACHMENTS_ENABLED: false,
CHAT_ATTACHMENTS_ENABLED: true,
SKILLS_GITHUB_REPO: 'test/skills',
GA_MEASUREMENT_ID: undefined,
OPENPANEL_API_URL: 'https://panel.tinyhumans.ai/api',
app/src/utils/config.ts
+4 -11
View File
@@ -91,18 +91,11 @@ export const CONSUMER_FIRST_SESSION_ENABLED =
import.meta.env.VITE_CONSUMER_FIRST_SESSION === 'true';
/**
* Chat image attachments (the composer's attach button + file picker).
*
* **Default off.** The end-to-end attachment path is not usable yet: image
* payloads are inlined as base64 into the chat message and rejected by the
* managed backend (`413 Payload Too Large`), so attaching anything surfaces a
* generic "Something went wrong" error (issue #3205). Until images are sent
* out-of-band (uploaded + referenced by URL) and the backend accepts them, we
* hide the entry point rather than ship a button that always fails.
* Re-enable locally to work on the feature: `VITE_CHAT_ATTACHMENTS=true` in
* `app/.env.local`.
* Chat multimodal attachments (image + supported file markers). Enabled by
* default now that core parses `[IMAGE:…]` / `[FILE:…]` payloads and the
* renderer runs a lossless compression pass before sending.
*/
export const CHAT_ATTACHMENTS_ENABLED = import.meta.env.VITE_CHAT_ATTACHMENTS === 'true';
export const CHAT_ATTACHMENTS_ENABLED = import.meta.env.VITE_CHAT_ATTACHMENTS !== 'false';
export const SKILLS_GITHUB_REPO =
import.meta.env.VITE_SKILLS_GITHUB_REPO || 'tinyhumansai/openhuman-skills';
app/test/playwright/specs/chat-harness-subagent.spec.ts
+18 -15
View File
@@ -11,8 +11,9 @@ const USER_ID = 'pw-chat-subagent';
const PROMPT = 'Research the answer to life and tell me a marker phrase.';
const CANARY_FINAL = 'subagent-canary-final-7afe2';
const RESEARCHER_REPLY = 'The researcher answer is 42.';
const FORCED_RESPONSES = [
const KEYWORD_RESPONSES = [
{
keyword: PROMPT,
content: '',
toolCalls: [
{
@@ -22,8 +23,8 @@ const FORCED_RESPONSES = [
},
],
},
{ content: RESEARCHER_REPLY },
{ content: `Done. The result is: ${CANARY_FINAL}` },
{ keyword: 'Tell me a marker phrase', content: RESEARCHER_REPLY },
{ keyword: RESEARCHER_REPLY, content: `Done. The result is: ${CANARY_FINAL}` },
];
interface MockRequest {
@@ -53,6 +54,13 @@ async function requests(): Promise<MockRequest[]> {
return Array.isArray(payload.data) ? payload.data : [];
}
async function completionRequestCount(): Promise<number> {
const log = await requests();
return log.filter(
entry => entry.method === 'POST' && entry.url.includes('/openai/v1/chat/completions')
).length;
}
async function openChat(page: Page): Promise<void> {
await bootAuthenticatedPage(page, USER_ID, '/chat');
await page.goto('/#/chat');
@@ -134,15 +142,19 @@ async function sendMessage(page: Page, prompt: string): Promise<void> {
test.describe('Chat Harness - Subagent', () => {
test('delegates to a subagent and persists the final orchestrator text', async ({ page }) => {
test.setTimeout(150_000);
await resetMock();
await setMockBehavior('llmForcedResponses', JSON.stringify(FORCED_RESPONSES));
await setMockBehavior('llmForcedResponses', '');
await setMockBehavior('llmKeywordRules', JSON.stringify(KEYWORD_RESPONSES));
await setMockBehavior('llmStreamChunkDelayMs', '10');
await openChat(page);
const threadId = await createNewThread(page);
await sendMessage(page, PROMPT);
await expect(page.getByText(CANARY_FINAL)).toBeVisible({ timeout: 75_000 });
await expect.poll(completionRequestCount, { timeout: 90_000 }).toBeGreaterThanOrEqual(3);
await expect(page.getByText(CANARY_FINAL)).toBeVisible({ timeout: 30_000 });
const runtime = await page.evaluate(currentThreadId => {
const store = (
@@ -172,15 +184,6 @@ test.describe('Chat Harness - Subagent', () => {
runtime.ids.some(id => id.includes(':subagent:'))
).toBe(true);
await expect
.poll(async () => {
const log = await requests();
return log.filter(
entry => entry.method === 'POST' && entry.url.includes('/openai/v1/chat/completions')
).length;
})
.toBeGreaterThanOrEqual(2);
await expect(page.getByText(CANARY_FINAL)).toBeVisible();
await expect(page.getByText(CANARY_FINAL)).toBeVisible({ timeout: 15_000 });
});
});
app/test/playwright/specs/chat-management-functional.spec.ts
+64 -8
View File
@@ -25,6 +25,26 @@ async function resetMock(): Promise<void> {
});
}
interface MockRequest {
method?: string;
url?: string;
body?: string;
}
async function requests(): Promise<MockRequest[]> {
const response = await fetch(`${MOCK_BASE}/__admin/requests`);
const payload = (await response.json()) as unknown;
if (Array.isArray(payload)) return payload as MockRequest[];
if (
payload &&
typeof payload === 'object' &&
Array.isArray((payload as { data?: unknown }).data)
) {
return (payload as { data: MockRequest[] }).data;
}
return [];
}
async function selectedThreadId(page: Page): Promise<string | null> {
return page.evaluate(() => {
const store = (
@@ -52,7 +72,7 @@ async function newThread(page: Page): Promise<string> {
}
test.describe('Chat management functional coverage', () => {
test('attachment preview, remove, and attachment send path remain interactive', async ({
test('attachment preview, remove, and multimodal send path remain interactive', async ({
page,
}) => {
await resetMock();
@@ -78,20 +98,56 @@ test.describe('Chat management functional coverage', () => {
await page.getByRole('button', { name: /Remove pixel\.png/ }).click();
await expect(page.getByText('pixel.png')).toHaveCount(0);
const pngBuffer = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
'base64'
);
await fileInput.setInputFiles({ name: 'pixel.png', mimeType: 'image/png', buffer: pngBuffer });
await expect(page.getByText('pixel.png')).toBeVisible();
await fileInput.setInputFiles({
name: 'pixel.png',
mimeType: 'image/png',
buffer: Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
'base64'
),
name: 'notes.txt',
mimeType: 'text/plain',
buffer: Buffer.from('renderer uploaded text document', 'utf8'),
});
await expect(page.getByText('notes.txt')).toBeVisible();
await page.getByPlaceholder('How can I help you today?').fill('Describe this image');
await page.getByTestId('send-message-button').click();
await expect(page.getByText("This model can't process images.")).toBeVisible({
await expect(page.getByText('Attachment received by the assistant.')).toBeVisible({
timeout: 30_000,
});
await expect(page.getByPlaceholder('How can I help you today?')).toBeEnabled();
await expect
.poll(
async () => {
const log = await requests();
const completion = log.find(
request =>
request.method === 'POST' &&
request.url?.includes('/chat/completions') &&
typeof request.body === 'string' &&
request.body.includes('Describe this image')
);
return completion?.body ?? '';
},
{ timeout: 10_000 }
)
.toContain('image_url');
const log = await requests();
const completionBody =
log.find(
request =>
request.method === 'POST' &&
request.url?.includes('/chat/completions') &&
typeof request.body === 'string' &&
request.body.includes('Describe this image')
)?.body ?? '';
expect(completionBody).toContain('reasoning');
expect(completionBody).toContain('[FILE-EXTRACTED:');
expect(completionBody).toContain('renderer uploaded text document');
});
test('thread rename and delete remain usable from the conversation UI', async ({ page }) => {
app/test/playwright/specs/chat-multi-tool-round.spec.ts
+1 -1
View File
@@ -166,7 +166,7 @@ test.describe('Chat Multi Tool Round', () => {
const threadId = await createNewThread(page);
await sendMessage(page, PROMPT);
await expect(page.getByText(CANARY_FINAL)).toBeVisible({ timeout: 50_000 });
await expect(page.getByText(CANARY_FINAL).first()).toBeVisible({ timeout: 50_000 });
await expect
.poll(
app/test/playwright/specs/chat-tool-call-flow.spec.ts
+1 -1
View File
@@ -158,7 +158,7 @@ test.describe('Chat Tool Call Flow', () => {
const threadId = await createNewThread(page);
await sendMessage(page, PROMPT);
await expect(page.getByText(CANARY_FINAL)).toBeVisible({ timeout: 40_000 });
await expect(page.getByText(CANARY_FINAL).first()).toBeVisible({ timeout: 40_000 });
const names = await expect
.poll(async () => toolTimelineNames(page, threadId), { timeout: 20_000 })
src/openhuman/agent/multimodal.rs
+171 -37
View File
@@ -3,8 +3,10 @@ use crate::openhuman::config::{
};
use crate::openhuman::inference::provider::ChatMessage;
use base64::{engine::general_purpose::STANDARD, Engine as _};
use flate2::read::GzDecoder;
use reqwest::Client;
use sha2::{Digest, Sha256};
use std::io::Read;
use std::path::Path;
use std::time::Duration;
@@ -19,10 +21,9 @@ const ALLOWED_IMAGE_MIME_TYPES: &[&str] = &[
/// File-attachment marker prefix. Counterpart to [`IMAGE_MARKER_PREFIX`].
/// Resolution rules mirror images: local paths, optional http(s) URLs
/// gated by [`MultimodalFileConfig::allow_remote_fetch`]. `data:` URIs
/// are intentionally rejected — the file pipeline does not inline
/// base64 the way images do; users wanting inline content should paste
/// it as text.
/// gated by [`MultimodalFileConfig::allow_remote_fetch`], and renderer-owned
/// `data:` URIs. Inline `application/gzip` data URIs are decompressed before
/// validation when they carry an `original_mime=...` parameter.
const FILE_MARKER_PREFIX: &str = "[FILE:";
/// Hard upper bound on how long [`pdf_extract::extract_text_from_mem`]
@@ -488,45 +489,126 @@ async fn normalize_image_reference(
}
fn normalize_data_uri(source: &str, max_bytes: usize) -> anyhow::Result<String> {
let parsed = parse_data_uri(source).map_err(|reason| MultimodalError::InvalidMarker {
input: source.to_string(),
reason,
})?;
let (mime, decoded) = if parsed.mime == "application/gzip" {
let original_mime = data_uri_param(&parsed.params, "original_mime").ok_or_else(|| {
MultimodalError::InvalidMarker {
input: source.to_string(),
reason: "compressed image data URI missing original_mime parameter".to_string(),
}
})?;
(
original_mime.to_ascii_lowercase(),
gunzip_data_uri(source, &parsed.bytes, max_bytes)?,
)
} else {
(parsed.mime, parsed.bytes)
};
validate_mime(source, &mime)?;
validate_size(source, decoded.len(), max_bytes)?;
Ok(format!("data:{mime};base64,{}", STANDARD.encode(decoded)))
}
struct ParsedDataUri {
mime: String,
params: Vec<(String, String)>,
bytes: Vec<u8>,
}
fn parse_data_uri(source: &str) -> Result<ParsedDataUri, String> {
let Some(comma_idx) = source.find(',') else {
return Err(MultimodalError::InvalidMarker {
input: source.to_string(),
reason: "expected data URI payload".to_string(),
}
.into());
return Err("expected data URI payload".to_string());
};
let header = &source[..comma_idx];
let payload = source[comma_idx + 1..].trim();
if !header.contains(";base64") {
return Err("only base64 data URIs are supported".to_string());
}
let mut parts = header.trim_start_matches("data:").split(';');
let mime = parts.next().unwrap_or_default().trim().to_ascii_lowercase();
let params = parts
.filter_map(|part| {
if part.eq_ignore_ascii_case("base64") {
return None;
}
let (key, value) = part.split_once('=')?;
Some((
key.trim().to_ascii_lowercase(),
percent_decode(value.trim()).unwrap_or_else(|| value.trim().to_string()),
))
})
.collect::<Vec<_>>();
let bytes = STANDARD
.decode(payload)
.map_err(|error| format!("invalid base64 payload: {error}"))?;
Ok(ParsedDataUri {
mime,
params,
bytes,
})
}
fn data_uri_param(params: &[(String, String)], key: &str) -> Option<String> {
params
.iter()
.find(|(candidate, _)| candidate.eq_ignore_ascii_case(key))
.map(|(_, value)| value.clone())
}
fn percent_decode(value: &str) -> Option<String> {
let bytes = value.as_bytes();
let mut out = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%' {
if i + 2 >= bytes.len() {
return None;
}
let hex = std::str::from_utf8(&bytes[i + 1..i + 3]).ok()?;
let byte = u8::from_str_radix(hex, 16).ok()?;
out.push(byte);
i += 3;
} else {
out.push(bytes[i]);
i += 1;
}
}
String::from_utf8(out).ok()
}
fn gunzip_data_uri(
source: &str,
bytes: &[u8],
max_decompressed_bytes: usize,
) -> anyhow::Result<Vec<u8>> {
let limit = max_decompressed_bytes.saturating_add(1) as u64;
let mut decoder = GzDecoder::new(bytes).take(limit);
let mut out = Vec::new();
decoder
.read_to_end(&mut out)
.map_err(|error| MultimodalError::InvalidMarker {
input: source.to_string(),
reason: format!("invalid gzip payload: {error}"),
})?;
if out.len() > max_decompressed_bytes {
return Err(MultimodalError::InvalidMarker {
input: source.to_string(),
reason: "only base64 data URIs are supported".to_string(),
reason: format!("decompressed payload exceeds {max_decompressed_bytes} bytes"),
}
.into());
}
let mime = header
.trim_start_matches("data:")
.split(';')
.next()
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
validate_mime(source, &mime)?;
let decoded = STANDARD
.decode(payload)
.map_err(|error| MultimodalError::InvalidMarker {
input: source.to_string(),
reason: format!("invalid base64 payload: {error}"),
})?;
validate_size(source, decoded.len(), max_bytes)?;
Ok(format!("data:{mime};base64,{}", STANDARD.encode(decoded)))
Ok(out)
}
async fn normalize_remote_image(
@@ -727,12 +809,16 @@ async fn normalize_file_reference(
remote_client: &Client,
) -> anyhow::Result<FilePayload> {
if source.starts_with("data:") {
return Err(MultimodalError::InvalidFileMarker {
input: source.to_string(),
reason: "data: URIs are not supported for [FILE:…] markers — paste content as text"
.to_string(),
}
.into());
let (bytes, name, mime) = normalize_file_data_uri(source, max_bytes)?;
return file_payload_from_bytes(
source,
bytes,
name,
mime,
config,
max_extracted_text_chars,
)
.await;
}
let (bytes, path_hint, name, header_content_type) =
@@ -758,6 +844,54 @@ async fn normalize_file_reference(
supported: config.allowed_mime_types.join(", "),
})?;
file_payload_from_bytes(source, bytes, name, mime, config, max_extracted_text_chars).await
}
fn normalize_file_data_uri(
source: &str,
max_bytes: usize,
) -> anyhow::Result<(Vec<u8>, String, String)> {
let parsed = parse_data_uri(source).map_err(|reason| MultimodalError::InvalidFileMarker {
input: source.to_string(),
reason,
})?;
let name = data_uri_param(&parsed.params, "name").unwrap_or_else(|| "attachment".to_string());
let (mime, bytes) = if parsed.mime == "application/gzip" {
let original_mime = data_uri_param(&parsed.params, "original_mime").ok_or_else(|| {
MultimodalError::InvalidFileMarker {
input: source.to_string(),
reason: "compressed file data URI missing original_mime parameter".to_string(),
}
})?;
(
original_mime.to_ascii_lowercase(),
gunzip_data_uri(source, &parsed.bytes, max_bytes)?,
)
} else {
(parsed.mime, parsed.bytes)
};
if bytes.len() > max_bytes {
return Err(MultimodalError::FileTooLarge {
input: source.to_string(),
size_bytes: bytes.len(),
max_bytes,
}
.into());
}
Ok((bytes, name, mime))
}
async fn file_payload_from_bytes(
source: &str,
bytes: Vec<u8>,
name: String,
mime: String,
config: &MultimodalFileConfig,
max_extracted_text_chars: usize,
) -> anyhow::Result<FilePayload> {
if !config.is_mime_allowed(&mime) {
return Err(MultimodalError::UnsupportedFileMime {
input: source.to_string(),
src/openhuman/agent/multimodal_tests.rs
+116 -5
View File
@@ -443,20 +443,131 @@ async fn prepare_messages_rejects_remote_file_when_disabled() {
}
#[tokio::test]
async fn prepare_messages_rejects_data_uri_file_marker() {
async fn prepare_messages_extracts_data_uri_file_marker() {
let messages = vec![ChatMessage::user(
"[FILE:data:text/plain;base64,SGVsbG8=]".to_string(),
"[FILE:data:text/plain;name=note.txt;base64,SGVsbG8=]".to_string(),
)];
let err = prepare_messages_for_provider(
let prepared = prepare_messages_for_provider(
&messages,
&MultimodalConfig::default(),
&MultimodalFileConfig::default(),
)
.await
.expect_err("data: URIs are not supported for FILE markers");
.expect("data: URI files should be supported for renderer uploads");
assert!(err.to_string().contains("data: URIs are not supported"));
assert!(prepared.contains_files);
let body = &prepared.messages[0].content;
assert!(body.contains("[FILE-EXTRACTED:"));
assert!(body.contains("name=\"note.txt\""));
assert!(body.contains("Hello"));
}
#[tokio::test]
async fn prepare_messages_decompresses_gzipped_data_uri_file_marker() {
use base64::Engine as _;
use flate2::{write::GzEncoder, Compression};
use std::io::Write;
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(b"Hello compressed").unwrap();
let gz = encoder.finish().unwrap();
let encoded = base64::engine::general_purpose::STANDARD.encode(gz);
let messages = vec![ChatMessage::user(format!(
"[FILE:data:application/gzip;original_mime=text%2Fplain;name=note.txt;base64,{encoded}]"
))];
let prepared = prepare_messages_for_provider(
&messages,
&MultimodalConfig::default(),
&MultimodalFileConfig::default(),
)
.await
.expect("compressed data URI file should decompress");
assert!(prepared.contains_files);
let body = &prepared.messages[0].content;
assert!(body.contains("Hello compressed"));
}
#[tokio::test]
async fn prepare_messages_decompresses_gzipped_data_uri_image_marker() {
use base64::Engine as _;
use flate2::{write::GzEncoder, Compression};
use std::io::Write;
let png = [0x89, b'P', b'N', b'G', b'\r', b'\n', 0x1a, b'\n'];
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&png).unwrap();
let gz = encoder.finish().unwrap();
let encoded = base64::engine::general_purpose::STANDARD.encode(gz);
let messages = vec![ChatMessage::user(format!(
"[IMAGE:data:application/gzip;original_mime=image%2Fpng;name=shot.png;base64,{encoded}]"
))];
let prepared = prepare_messages_for_provider(
&messages,
&MultimodalConfig::default(),
&MultimodalFileConfig::default(),
)
.await
.expect("compressed data URI image should decompress");
assert!(prepared.contains_images);
let body = &prepared.messages[0].content;
assert!(body.contains("[IMAGE:data:image/png;base64,"));
}
#[tokio::test]
async fn prepare_messages_bounds_gzipped_data_uri_decompression() {
use base64::Engine as _;
use flate2::{write::GzEncoder, Compression};
use std::io::Write;
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&vec![0u8; 1024 * 1024 + 1]).unwrap();
let encoded = base64::engine::general_purpose::STANDARD.encode(encoder.finish().unwrap());
let messages = vec![ChatMessage::user(format!(
"[IMAGE:data:application/gzip;original_mime=image%2Fpng;base64,{encoded}]"
))];
let image_config = MultimodalConfig {
max_images: 4,
max_image_size_mb: 1,
allow_remote_fetch: false,
};
let error =
prepare_messages_for_provider(&messages, &image_config, &MultimodalFileConfig::default())
.await
.expect_err("compressed payload must be capped during decompression");
assert!(error
.to_string()
.contains("decompressed payload exceeds 1048576 bytes"));
}
#[tokio::test]
async fn prepare_messages_rejects_gzip_without_original_mime() {
use base64::Engine as _;
use flate2::{write::GzEncoder, Compression};
use std::io::Write;
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(b"Hello compressed").unwrap();
let encoded = base64::engine::general_purpose::STANDARD.encode(encoder.finish().unwrap());
let messages = vec![ChatMessage::user(format!(
"[FILE:data:application/gzip;name=note.txt;base64,{encoded}]"
))];
let error = prepare_messages_for_provider(
&messages,
&MultimodalConfig::default(),
&MultimodalFileConfig::default(),
)
.await
.expect_err("gzip without original_mime must fail");
assert!(error.to_string().contains("original_mime"));
}
#[tokio::test]
src/openhuman/inference/provider/compatible.rs
+14
View File
@@ -87,6 +87,11 @@ pub struct OpenAiCompatibleProvider {
/// Ollama, whose OpenAI-compat endpoint returns HTTP 400 on `tools`
/// for many models.
native_tool_calling: bool,
/// Value reported by `capabilities().vision`. Defaults to `true` because
/// OpenAI-compatible cloud endpoints accept the `image_url` message
/// content shape. Local compatibility shims opt out unless they can prove
/// the configured model supports vision.
vision: bool,
/// Ollama-specific `options.num_ctx` override. When set, every request
/// to this provider includes `"options": {"num_ctx": <value>}` in the
/// body so Ollama allocates the requested KV-cache size.
@@ -200,6 +205,7 @@ impl OpenAiCompatibleProvider {
temperature_unsupported_models: Vec::new(),
temperature_override: None,
native_tool_calling: true,
vision: true,
ollama_num_ctx: None,
local_provider_kind: None,
}
@@ -214,6 +220,14 @@ impl OpenAiCompatibleProvider {
self
}
/// Toggle whether this provider advertises OpenAI-compatible vision input.
/// Cloud providers default to enabled; local OpenAI-compatible shims use
/// this to stay fail-closed for text-only local models.
pub fn with_vision(mut self, enabled: bool) -> Self {
self.vision = enabled;
self
}
/// Set the list of model glob patterns for which temperature must be
/// omitted from request bodies.
pub fn with_temperature_unsupported_models(mut self, patterns: Vec<String>) -> Self {
src/openhuman/inference/provider/compatible_provider_impl.rs
+1 -4
View File
@@ -20,10 +20,7 @@ impl Provider for OpenAiCompatibleProvider {
fn capabilities(&self) -> crate::openhuman::inference::provider::traits::ProviderCapabilities {
crate::openhuman::inference::provider::traits::ProviderCapabilities {
native_tool_calling: self.native_tool_calling,
// Kept `false` for now — vision is a per-*model* property the provider
// can't know here; the Responses-API path is still text-only. Stays off
// until it can be driven per-model (e.g. from `model_registry.vision`).
vision: false,
vision: self.vision,
}
}
src/openhuman/inference/provider/compatible_tests.rs
+9
View File
@@ -1578,6 +1578,7 @@ fn capabilities_reports_native_tool_calling() {
let p = make_provider("test", "https://example.com", None);
let caps = <OpenAiCompatibleProvider as Provider>::capabilities(&p);
assert!(caps.native_tool_calling);
assert!(caps.vision);
}
// Sub-issue 3 of #3098: Ollama's OpenAI-compat endpoint silently rejects the
@@ -1613,6 +1614,14 @@ fn with_native_tool_calling_is_idempotent() {
assert!(!caps.native_tool_calling);
}
#[test]
fn with_vision_false_disables_capability() {
let p = make_provider("test", "https://example.com", None).with_vision(false);
let caps = <OpenAiCompatibleProvider as Provider>::capabilities(&p);
assert!(!caps.vision);
assert!(!p.supports_vision());
}
/// `supports_native_tools()` is the gate the agent harness reads
/// (`traits.rs:415`) when deciding whether to send tools natively or
/// inject them into the prompt. It MUST agree with
src/openhuman/inference/provider/factory.rs
+4
View File
@@ -1016,6 +1016,7 @@ fn make_ollama_provider(
.with_temperature_unsupported_models(config.temperature_unsupported_models.clone())
.with_temperature_override(temperature_override)
.with_native_tool_calling(false)
.with_vision(false)
.with_ollama_num_ctx(num_ctx)
.with_local_provider_kind(LocalProviderKind::Ollama);
Ok((Box::new(provider), model.to_string()))
@@ -1056,6 +1057,7 @@ fn make_lm_studio_provider(
.with_temperature_unsupported_models(config.temperature_unsupported_models.clone())
.with_temperature_override(temperature_override)
.with_native_tool_calling(false)
.with_vision(false)
.with_local_provider_kind(LocalProviderKind::LmStudio);
Ok((Box::new(provider), model.to_string()))
}
@@ -1092,6 +1094,7 @@ fn make_mlx_provider(
.with_temperature_unsupported_models(config.temperature_unsupported_models.clone())
.with_temperature_override(temperature_override)
.with_native_tool_calling(false)
.with_vision(false)
.with_local_provider_kind(LocalProviderKind::Mlx);
Ok((Box::new(provider), model.to_string()))
}
@@ -1139,6 +1142,7 @@ fn make_local_openai_provider(
.with_temperature_unsupported_models(config.temperature_unsupported_models.clone())
.with_temperature_override(temperature_override)
.with_native_tool_calling(false)
.with_vision(false)
.with_local_provider_kind(LocalProviderKind::LocalOpenai);
Ok((Box::new(provider), model.to_string()))
}
src/openhuman/inference/provider/factory_tests.rs
+26 -1
View File
@@ -227,6 +227,10 @@ fn ollama_provider_opts_out_of_native_tool_calling() {
!caps.native_tool_calling,
"ollama provider must report native_tool_calling=false so the agent harness emits prompt-guided tool specs instead of an OpenAI-style `tools` array"
);
assert!(
!caps.vision,
"local Ollama-compatible providers stay fail-closed for vision until the configured model proves image support"
);
}
#[test]
@@ -241,10 +245,15 @@ fn lmstudio_provider_defaults_to_prompt_guided_tools() {
let (provider, _model) =
create_chat_provider_from_string("chat", "lmstudio:google/gemma-4-e4b", &config)
.expect("lmstudio:<model> must build");
let caps = provider.capabilities();
assert!(
!provider.capabilities().native_tool_calling,
!caps.native_tool_calling,
"lmstudio provider must default to native_tool_calling=false (conservative local dispatch)"
);
assert!(
!caps.vision,
"local LM Studio-compatible providers stay fail-closed for vision until the configured model proves image support"
);
}
// Note: a BYOK-cloud regression test (e.g. `openai:gpt-4o` keeps
@@ -485,6 +494,10 @@ async fn cloud_provider_without_stored_key_fails_with_actionable_error() {
let config = config_with_providers_in_tempdir(&tmp, vec![openai_entry("p_oai", "openai")]);
let (provider, model) = create_chat_provider_from_string("reasoning", "openai:gpt-4o", &config)
.expect("provider should build without eagerly requiring credentials");
assert!(
provider.capabilities().vision,
"cloud OpenAI-compatible providers must advertise vision so reasoning attachment turns reach the provider"
);
let err = provider
.chat_with_system(None, "hello", &model, 0.0)
@@ -878,6 +891,18 @@ fn make_openhuman_backend_translates_summarization_hint() {
assert_eq!(model, crate::openhuman::config::MODEL_SUMMARIZATION_V1);
}
#[test]
fn make_openhuman_backend_reports_vision_capability() {
let config = Config::default();
let (provider, _) = make_openhuman_backend(&config).expect("factory should succeed");
let caps = provider.capabilities();
assert!(caps.native_tool_calling);
assert!(
caps.vision,
"OpenHuman backend must report vision so attachment-driven reasoning turns clear the harness gate"
);
}
#[test]
fn make_openhuman_backend_falls_back_for_invalid_model() {
// An invalid default_model must not be forwarded to the backend.
src/openhuman/inference/provider/openhuman_backend.rs
+5 -6
View File
@@ -130,12 +130,11 @@ impl Provider for OpenHumanBackendProvider {
fn capabilities(&self) -> ProviderCapabilities {
ProviderCapabilities {
native_tool_calling: true,
// Kept `false` for now: the hosted backend's default chat model is
// text-only, so claiming vision would only let image turns through
// the gate to come back empty. The image_url wire format + budgeting
// hygiene ship here (#3205), but the capability stays off until the
// backend routes image turns to a vision-capable model per-model.
vision: false,
// The hosted backend accepts OpenAI-compatible multimodal chat
// payloads and routes attachment turns through the reasoning tier.
// Keep this in sync with the frontend's attachment-driven
// `hint:reasoning` switch so image markers clear the harness gate.
vision: true,
}
}
tests/inference_agent_raw_coverage_e2e.rs
+1 -1
View File
@@ -2068,7 +2068,7 @@ async fn inference_openhuman_backend_provider_covers_authless_and_streaming_edge
},
);
assert!(provider.supports_native_tools());
assert!(!provider.supports_vision());
assert!(provider.supports_vision());
assert!(!provider.supports_streaming());
let missing_session = provider