feat(chat): add image attachment support to composer (#2676)

Co-authored-by: Srinivas Vaddi <38348871+vaddisrinivas@users.noreply.github.com>
Co-authored-by: Steven Enamakel <enamakel@tinyhumans.ai>
Co-authored-by: Ghost Scripter <ghostscripter@zerolend.xyz>
Co-authored-by: sanil-23 <sanil@vezures.xyz>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Darshan Poudel
2026-05-28 23:57:24 +05:30
committed by GitHub
co-authored by Srinivas Vaddi Steven Enamakel Ghost Scripter sanil-23 Claude
parent 092affa22f
commit 32524ebd4c
22 changed files with 1485 additions and 37 deletions
+2 -1
View File
@@ -15,7 +15,8 @@ export type ChatSendErrorCode =
| 'safety_timeout'
| 'usage_limit_reached'
| 'prompt_blocked'
| 'prompt_review';
| 'prompt_review'
| 'attachment_invalid';
export interface ChatSendError {
code: ChatSendErrorCode;
@@ -0,0 +1,55 @@
import { type Attachment, formatFileSize } from '../../lib/attachments';
import { useT } from '../../lib/i18n/I18nContext';
interface AttachmentPreviewProps {
attachments: Attachment[];
onRemove: (id: string) => void;
disabled?: boolean;
}
export default function AttachmentPreview({
attachments,
onRemove,
disabled,
}: AttachmentPreviewProps) {
const { t } = useT();
if (attachments.length === 0) return null;
return (
<div className="flex flex-wrap gap-2 px-1 pb-1">
{attachments.map(attachment => (
<div
key={attachment.id}
className="relative flex items-center gap-2 rounded-lg border border-stone-200 dark:border-neutral-700 bg-stone-50 dark:bg-neutral-800 px-2 py-1.5 text-xs text-stone-700 dark:text-neutral-300 max-w-[180px]">
<img
src={attachment.dataUri}
alt={attachment.file.name}
className="w-8 h-8 rounded object-cover flex-shrink-0"
/>
<div className="flex flex-col min-w-0">
<span className="truncate font-medium leading-tight">{attachment.file.name}</span>
<span className="text-stone-400 dark:text-neutral-500 leading-tight">
{formatFileSize(attachment.file.size)}
</span>
</div>
<button
type="button"
aria-label={t('chat.attachment.remove').replace('{name}', attachment.file.name)}
onClick={() => onRemove(attachment.id)}
disabled={disabled}
className="absolute -top-1.5 -right-1.5 w-4 h-4 flex items-center justify-center rounded-full bg-stone-400 dark:bg-neutral-600 text-white hover:bg-stone-600 dark:hover:bg-neutral-400 transition-colors disabled:opacity-40 disabled:cursor-not-allowed flex-shrink-0">
<svg className="w-2.5 h-2.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={3}
d="M6 18L18 6M6 6l12 12"
/>
</svg>
</button>
</div>
))}
</div>
);
}
@@ -0,0 +1,63 @@
import { fireEvent, render, screen } from '@testing-library/react';
import { describe, expect, it, vi } from 'vitest';
import type { Attachment } from '../../../lib/attachments';
import AttachmentPreview from '../AttachmentPreview';
function makeAttachment(overrides: Partial<Attachment> = {}): Attachment {
const blob = new Blob([new Uint8Array(512)], { type: 'image/png' });
return {
id: 'att-1',
file: new File([blob], 'test.png', { type: 'image/png' }),
dataUri: 'data:image/png;base64,abc',
mimeType: 'image/png',
...overrides,
};
}
describe('AttachmentPreview', () => {
it('renders nothing when attachments list is empty', () => {
const { container } = render(<AttachmentPreview attachments={[]} onRemove={vi.fn()} />);
expect(container.firstChild).toBeNull();
});
it('renders a chip with filename and file size for each attachment', () => {
const att = makeAttachment();
render(<AttachmentPreview attachments={[att]} onRemove={vi.fn()} />);
expect(screen.getByText('test.png')).toBeInTheDocument();
expect(screen.getByText('512 B')).toBeInTheDocument();
});
it('renders a thumbnail image with the dataUri as src', () => {
const att = makeAttachment({ dataUri: 'data:image/png;base64,xyz' });
render(<AttachmentPreview attachments={[att]} onRemove={vi.fn()} />);
const img = screen.getByAltText('test.png') as HTMLImageElement;
expect(img.src).toBe('data:image/png;base64,xyz');
});
it('calls onRemove with the attachment id when × is clicked', () => {
const onRemove = vi.fn();
const att = makeAttachment({ id: 'att-42' });
render(<AttachmentPreview attachments={[att]} onRemove={onRemove} />);
fireEvent.click(screen.getByRole('button', { name: /remove test\.png/i }));
expect(onRemove).toHaveBeenCalledWith('att-42');
});
it('disables the remove button when disabled prop is true', () => {
const att = makeAttachment();
render(<AttachmentPreview attachments={[att]} onRemove={vi.fn()} disabled />);
expect(screen.getByRole('button', { name: /remove test\.png/i })).toBeDisabled();
});
it('renders multiple chips', () => {
const a1 = makeAttachment({ id: '1', file: new File([], 'a.png', { type: 'image/png' }) });
const a2 = makeAttachment({
id: '2',
file: new File([], 'b.jpg', { type: 'image/jpeg' }),
mimeType: 'image/jpeg',
});
render(<AttachmentPreview attachments={[a1, a2]} onRemove={vi.fn()} />);
expect(screen.getByText('a.png')).toBeInTheDocument();
expect(screen.getByText('b.jpg')).toBeInTheDocument();
});
});
+188
View File
@@ -0,0 +1,188 @@
import { describe, expect, it } from 'vitest';
import {
type Attachment,
ATTACHMENT_MAX_IMAGES,
ATTACHMENT_MAX_SIZE_BYTES,
buildMessageWithAttachments,
fileToDataUri,
formatFileSize,
isAllowedMimeType,
parseMessageImages,
validateAndReadFile,
} from './attachments';
function makeFile(name: string, type: string, size = 1024): File {
const blob = new Blob([new Uint8Array(size)], { type });
return new File([blob], name, { type });
}
function makeAttachment(overrides: Partial<Attachment> = {}): Attachment {
return {
id: 'test-id',
file: makeFile('test.png', 'image/png'),
dataUri: 'data:image/png;base64,abc',
mimeType: 'image/png',
...overrides,
};
}
describe('isAllowedMimeType', () => {
it('allows supported image types', () => {
expect(isAllowedMimeType('image/png')).toBe(true);
expect(isAllowedMimeType('image/jpeg')).toBe(true);
expect(isAllowedMimeType('image/webp')).toBe(true);
expect(isAllowedMimeType('image/gif')).toBe(true);
expect(isAllowedMimeType('image/bmp')).toBe(true);
});
it('rejects unsupported types', () => {
expect(isAllowedMimeType('application/pdf')).toBe(false);
expect(isAllowedMimeType('text/plain')).toBe(false);
expect(isAllowedMimeType('image/svg+xml')).toBe(false);
expect(isAllowedMimeType('')).toBe(false);
});
});
describe('fileToDataUri', () => {
it('reads a file as a data URI', async () => {
const file = makeFile('photo.png', 'image/png', 4);
const uri = await fileToDataUri(file);
expect(uri).toMatch(/^data:image\/png;base64,/);
});
});
describe('validateAndReadFile', () => {
it('rejects when at max image count', async () => {
const file = makeFile('x.png', 'image/png');
const result = await validateAndReadFile(file, ATTACHMENT_MAX_IMAGES);
expect('error' in result).toBe(true);
if ('error' in result) {
expect(result.error.code).toBe('too_many');
}
});
it('rejects unsupported MIME types', async () => {
const file = makeFile('doc.pdf', 'application/pdf');
const result = await validateAndReadFile(file, 0);
expect('error' in result).toBe(true);
if ('error' in result) {
expect(result.error.code).toBe('unsupported_type');
}
});
it('rejects files that exceed the size limit', async () => {
const oversizedFile = makeFile('big.png', 'image/png', ATTACHMENT_MAX_SIZE_BYTES + 1);
const result = await validateAndReadFile(oversizedFile, 0);
expect('error' in result).toBe(true);
if ('error' in result) {
expect(result.error.code).toBe('too_large');
}
});
it('returns an attachment for a valid image', async () => {
const file = makeFile('ok.png', 'image/png', 512);
const result = await validateAndReadFile(file, 0);
expect('attachment' in result).toBe(true);
if ('attachment' in result) {
expect(result.attachment.mimeType).toBe('image/png');
expect(result.attachment.dataUri).toMatch(/^data:image\/png;base64,/);
expect(result.attachment.file).toBe(file);
}
});
it('returns read_failed when FileReader errors', async () => {
const file = makeFile('bad.png', 'image/png', 1);
const origFileReader = globalThis.FileReader;
class FailingReader {
onload: (() => void) | null = null;
onerror: ((e: unknown) => void) | null = null;
readAsDataURL() {
setTimeout(() => this.onerror?.(new Error('read error')), 0);
}
}
globalThis.FileReader = FailingReader as unknown as typeof FileReader;
try {
const result = await validateAndReadFile(file, 0);
expect('error' in result).toBe(true);
if ('error' in result) {
expect(result.error.code).toBe('read_failed');
}
} finally {
globalThis.FileReader = origFileReader;
}
});
});
describe('buildMessageWithAttachments', () => {
it('returns text unchanged when no attachments', () => {
expect(buildMessageWithAttachments('hello', [])).toBe('hello');
});
it('appends IMAGE markers after the text', () => {
const a = makeAttachment({ dataUri: 'data:image/png;base64,abc' });
const result = buildMessageWithAttachments('describe this', [a]);
expect(result).toBe('describe this [IMAGE:data:image/png;base64,abc]');
});
it('emits only markers when text is empty', () => {
const a = makeAttachment({ dataUri: 'data:image/png;base64,abc' });
const result = buildMessageWithAttachments('', [a]);
expect(result).toBe('[IMAGE:data:image/png;base64,abc]');
});
it('handles multiple attachments', () => {
const a1 = makeAttachment({ id: '1', dataUri: 'data:image/png;base64,a' });
const a2 = makeAttachment({ id: '2', dataUri: 'data:image/jpeg;base64,b' });
const result = buildMessageWithAttachments('look', [a1, a2]);
expect(result).toBe('look [IMAGE:data:image/png;base64,a] [IMAGE:data:image/jpeg;base64,b]');
});
it('trims leading/trailing whitespace from text', () => {
const a = makeAttachment({ dataUri: 'data:image/png;base64,x' });
const result = buildMessageWithAttachments(' hi ', [a]);
expect(result).toBe('hi [IMAGE:data:image/png;base64,x]');
});
});
describe('parseMessageImages', () => {
it('returns empty dataUris and original text when no markers', () => {
const result = parseMessageImages('hello world');
expect(result.text).toBe('hello world');
expect(result.dataUris).toEqual([]);
});
it('extracts a single image marker and strips it from text', () => {
const result = parseMessageImages('describe this [IMAGE:data:image/png;base64,abc]');
expect(result.text).toBe('describe this');
expect(result.dataUris).toEqual(['data:image/png;base64,abc']);
});
it('extracts multiple image markers', () => {
const result = parseMessageImages(
'[IMAGE:data:image/png;base64,a] [IMAGE:data:image/jpeg;base64,b]'
);
expect(result.text).toBe('');
expect(result.dataUris).toEqual(['data:image/png;base64,a', 'data:image/jpeg;base64,b']);
});
it('returns empty text when message is marker-only', () => {
const result = parseMessageImages('[IMAGE:data:image/png;base64,xyz]');
expect(result.text).toBe('');
expect(result.dataUris).toHaveLength(1);
});
});
describe('formatFileSize', () => {
it('formats bytes', () => {
expect(formatFileSize(500)).toBe('500 B');
});
it('formats kilobytes', () => {
expect(formatFileSize(1536)).toBe('1.5 KB');
});
it('formats megabytes', () => {
expect(formatFileSize(2 * 1024 * 1024)).toBe('2.0 MB');
});
});
+115
View File
@@ -0,0 +1,115 @@
/**
* Utilities for multimodal chat attachments.
*
* Images are embedded in the message text as `[IMAGE:<data-uri>]` markers,
* which the Rust agent harness (`agent/multimodal.rs`) parses and forwards to
* the inference provider. The backend supports up to 4 images per message at
* 8 MB each by default (governed by `MultimodalConfig`).
*/
export const ATTACHMENT_MAX_IMAGES = 4;
export const ATTACHMENT_MAX_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
export const ALLOWED_IMAGE_MIME_TYPES = [
'image/png',
'image/jpeg',
'image/webp',
'image/gif',
'image/bmp',
] as const;
export type AllowedImageMimeType = (typeof ALLOWED_IMAGE_MIME_TYPES)[number];
export interface Attachment {
id: string;
file: File;
dataUri: string;
mimeType: AllowedImageMimeType;
}
export type AttachmentError =
| { code: 'unsupported_type'; mimeType: string }
| { code: 'too_large'; sizeBytes: number; maxBytes: number }
| { code: 'too_many'; max: number }
| { code: 'read_failed'; reason: string };
export function isAllowedMimeType(mime: string): mime is AllowedImageMimeType {
return (ALLOWED_IMAGE_MIME_TYPES as readonly string[]).includes(mime);
}
export function fileToDataUri(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
reader.onerror = () => reject(new Error(`Failed to read file: ${file.name}`));
reader.readAsDataURL(file);
});
}
export async function validateAndReadFile(
file: File,
existingCount: number
): Promise<{ attachment: Attachment } | { error: AttachmentError }> {
if (existingCount >= ATTACHMENT_MAX_IMAGES) {
return { error: { code: 'too_many', max: ATTACHMENT_MAX_IMAGES } };
}
if (!isAllowedMimeType(file.type)) {
return { error: { code: 'unsupported_type', mimeType: file.type || 'unknown' } };
}
if (file.size > ATTACHMENT_MAX_SIZE_BYTES) {
return {
error: { code: 'too_large', sizeBytes: file.size, maxBytes: ATTACHMENT_MAX_SIZE_BYTES },
};
}
try {
const dataUri = await fileToDataUri(file);
return {
attachment: {
id: globalThis.crypto.randomUUID(),
file,
dataUri,
mimeType: file.type as AllowedImageMimeType,
},
};
} catch (err) {
return {
error: { code: 'read_failed', reason: err instanceof Error ? err.message : String(err) },
};
}
}
/**
* Compose the final message string by appending `[IMAGE:<data-uri>]` markers
* for each attachment after the user's text. The Rust backend parses these
* markers in `parse_image_markers` and strips them from the visible message
* before forwarding clean text + image payload to the inference provider.
*/
export function buildMessageWithAttachments(text: string, attachments: Attachment[]): string {
if (attachments.length === 0) return text;
const markers = attachments.map(a => `[IMAGE:${a.dataUri}]`).join(' ');
return text.trim() ? `${text.trim()} ${markers}` : markers;
}
/**
* Parse `[IMAGE:<data-uri>]` markers out of a stored message string.
* Returns the clean text (markers removed) and the list of data URIs found.
*/
export function parseMessageImages(content: string): { text: string; dataUris: string[] } {
const dataUris: string[] = [];
const text = content
.replace(/\[IMAGE:([^\]]+)\]/g, (_match, uri: string) => {
dataUris.push(uri);
return '';
})
.trim();
return { text, dataUris };
}
export function formatFileSize(bytes: number): string {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
+6
View File
@@ -303,6 +303,12 @@ const ar2: TranslationMap = {
'chat.turn': 'دورة',
'chat.turns': 'دورات',
'chat.openWorkerThread': 'فتح محادثة العامل',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'البحث في الذاكرة',
'memory.searchPlaceholder': 'البحث في إدخالات الذاكرة...',
'memory.sourceFilter.all': 'جميع المصادر',
+6
View File
@@ -313,6 +313,12 @@ const bn2: TranslationMap = {
'chat.turn': 'টার্ন',
'chat.turns': 'টার্ন',
'chat.openWorkerThread': 'ওয়ার্কার থ্রেড খুলুন',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'মেমোরি খুঁজুন',
'memory.searchPlaceholder': 'মেমোরি এন্ট্রি খুঁজুন...',
'memory.sourceFilter.all': 'সব উৎস',
+6
View File
@@ -319,6 +319,12 @@ const de2: TranslationMap = {
'chat.turn': 'drehen',
'chat.turns': 'dreht sich',
'chat.openWorkerThread': 'Arbeitsthread öffnen',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'Speicher durchsuchen',
'memory.searchPlaceholder': 'Speichereinträge durchsuchen...',
'memory.sourceFilter.all': 'Alle Quellen',
+6
View File
@@ -309,6 +309,12 @@ const en2: TranslationMap = {
'chat.turn': 'turn',
'chat.turns': 'turns',
'chat.openWorkerThread': 'Open worker thread',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'Search memory',
'memory.searchPlaceholder': 'Search memory entries...',
'memory.sourceFilter.all': 'All sources',
+6
View File
@@ -316,6 +316,12 @@ const es2: TranslationMap = {
'chat.turn': 'turno',
'chat.turns': 'turnos',
'chat.openWorkerThread': 'Abrir hilo de worker',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'Buscar en memoria',
'memory.searchPlaceholder': 'Buscar entradas de memoria...',
'memory.sourceFilter.all': 'Todas las fuentes',
+6
View File
@@ -318,6 +318,12 @@ const fr2: TranslationMap = {
'chat.turn': 'tour',
'chat.turns': 'tours',
'chat.openWorkerThread': 'Ouvrir le fil worker',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'Rechercher dans la mémoire',
'memory.searchPlaceholder': 'Rechercher des entrées de mémoire…',
'memory.sourceFilter.all': 'Toutes les sources',
+6
View File
@@ -310,6 +310,12 @@ const hi2: TranslationMap = {
'chat.turn': 'टर्न',
'chat.turns': 'टर्न्स',
'chat.openWorkerThread': 'वर्कर थ्रेड खोलें',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'मेमोरी सर्च करें',
'memory.searchPlaceholder': 'मेमोरी एंट्रीज़ सर्च करें...',
'memory.sourceFilter.all': 'सभी सोर्स',
+6
View File
@@ -310,6 +310,12 @@ const id2: TranslationMap = {
'chat.turn': 'giliran',
'chat.turns': 'giliran',
'chat.openWorkerThread': 'Buka thread worker',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'Cari memori',
'memory.searchPlaceholder': 'Cari entri memori...',
'memory.sourceFilter.all': 'Semua sumber',
+6
View File
@@ -313,6 +313,12 @@ const it2: TranslationMap = {
'chat.turn': 'turno',
'chat.turns': 'turni',
'chat.openWorkerThread': 'Apri thread worker',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'Cerca memoria',
'memory.searchPlaceholder': 'Cerca voci di memoria...',
'memory.sourceFilter.all': 'Tutte le origini',
+6
View File
@@ -294,6 +294,12 @@ const ko2: TranslationMap = {
'chat.turn': '턴',
'chat.turns': '턴',
'chat.openWorkerThread': '워커 스레드 열기',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': '메모리 검색',
'memory.searchPlaceholder': '메모리 항목 검색...',
'memory.sourceFilter.all': '모든 소스',
+6
View File
@@ -316,6 +316,12 @@ const pt2: TranslationMap = {
'chat.turn': 'turno',
'chat.turns': 'turnos',
'chat.openWorkerThread': 'Abrir thread de worker',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'Pesquisar memória',
'memory.searchPlaceholder': 'Pesquisar entradas de memória...',
'memory.sourceFilter.all': 'Todas as fontes',
+6
View File
@@ -312,6 +312,12 @@ const ru2: TranslationMap = {
'chat.turn': 'ход',
'chat.turns': 'ходов',
'chat.openWorkerThread': 'Открыть чат воркера',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': 'Поиск в памяти',
'memory.searchPlaceholder': 'Поиск записей памяти...',
'memory.sourceFilter.all': 'Все источники',
+6
View File
@@ -293,6 +293,12 @@ const zhCN2: TranslationMap = {
'chat.turn': '轮',
'chat.turns': '轮',
'chat.openWorkerThread': '打开工作线程',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
'memory.searchAria': '搜索记忆',
'memory.searchPlaceholder': '搜索记忆条目...',
'memory.sourceFilter.all': '所有来源',
+6
View File
@@ -1605,6 +1605,12 @@ const en: TranslationMap = {
'chat.turn': 'turn',
'chat.turns': 'turns',
'chat.openWorkerThread': 'Open worker thread',
'chat.attachment.attach': 'Attach image',
'chat.attachment.remove': 'Remove {name}',
'chat.attachment.tooMany': 'Maximum {max} images per message',
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
'chat.attachment.unsupportedType': 'Unsupported file type. Use PNG, JPEG, WebP, GIF, or BMP.',
'chat.attachment.readFailed': 'Could not read file',
// Memory (additional)
'memory.searchAria': 'Search memory',
+183 -36
View File
@@ -6,6 +6,7 @@ import { useLocation, useNavigate } from 'react-router-dom';
import { type ChatSendError, chatSendError } from '../chat/chatSendError';
import { checkPromptInjection, promptGuardMessage } from '../chat/promptInjectionGuard';
import ApprovalRequestCard from '../components/chat/ApprovalRequestCard';
import AttachmentPreview from '../components/chat/AttachmentPreview';
import TokenUsagePill from '../components/chat/TokenUsagePill';
import { ConfirmationModal } from '../components/intelligence/ConfirmationModal';
import PillTabBar from '../components/PillTabBar';
@@ -14,6 +15,15 @@ import { dismissBanner, shouldShowBanner } from '../components/upsell/upsellDism
import MicComposer from '../features/human/MicComposer';
import { useStickToBottom } from '../hooks/useStickToBottom';
import { useUsageState } from '../hooks/useUsageState';
import {
ALLOWED_IMAGE_MIME_TYPES,
type Attachment,
ATTACHMENT_MAX_IMAGES,
ATTACHMENT_MAX_SIZE_BYTES,
buildMessageWithAttachments,
parseMessageImages,
validateAndReadFile,
} from '../lib/attachments';
import { useT } from '../lib/i18n/I18nContext';
import { trackEvent } from '../services/analytics';
import { threadApi } from '../services/api/threadApi';
@@ -186,6 +196,8 @@ const Conversations = ({
const [showSidebar, setShowSidebar] = useState(false);
const [inputValue, setInputValue] = useState('');
const [attachments, setAttachments] = useState<Attachment[]>([]);
const fileInputRef = useRef<HTMLInputElement>(null);
const [copiedMessageId, setCopiedMessageId] = useState<string | null>(null);
const [inputMode, setInputMode] = useState<InputMode>('text');
const [replyMode, setReplyMode] = useState<ReplyMode>('text');
@@ -196,6 +208,7 @@ const Conversations = ({
const [selectedLabel, setSelectedLabel] = useState<string>('all');
const [inlineSuggestionValue, setInlineSuggestionValue] = useState('');
const [sendError, setSendError] = useState<ChatSendError | null>(null);
const [attachError, setAttachError] = useState<ChatSendError | null>(null);
const [sendAdvisory, setSendAdvisory] = useState<string | null>(null);
const [pendingSendingThreadId, setPendingSendingThreadId] = useState<string | null>(null);
const [profileDraftOpen, setProfileDraftOpen] = useState(false);
@@ -604,6 +617,40 @@ const Conversations = ({
return true;
};
const handleAttachFiles = async (files: FileList | null) => {
if (!files) return;
let acceptedCount = attachments.length;
for (const file of Array.from(files)) {
const result = await validateAndReadFile(file, acceptedCount);
if ('error' in result) {
const { error } = result;
if (error.code === 'too_many') {
setAttachError(
chatSendError(
'attachment_invalid',
t('chat.attachment.tooMany').replace('{max}', String(ATTACHMENT_MAX_IMAGES))
)
);
} else if (error.code === 'too_large') {
const maxMb = (ATTACHMENT_MAX_SIZE_BYTES / (1024 * 1024)).toFixed(0);
setAttachError(
chatSendError(
'attachment_invalid',
t('chat.attachment.tooLarge').replace('{max}', `${maxMb} MB`)
)
);
} else if (error.code === 'unsupported_type') {
setAttachError(chatSendError('attachment_invalid', t('chat.attachment.unsupportedType')));
} else {
setAttachError(chatSendError('attachment_invalid', t('chat.attachment.readFailed')));
}
return;
}
acceptedCount++;
setAttachments(prev => [...prev, result.attachment]);
}
};
const handleSendMessage = async (text?: string) => {
if (pendingSendRef.current) return;
@@ -622,7 +669,7 @@ const Conversations = ({
const trimmed = sendDecision.trimmedText;
if (
sendDecision.blockReason === 'empty_input' ||
(sendDecision.blockReason === 'empty_input' && attachments.length === 0) ||
sendDecision.blockReason === 'missing_thread' ||
sendDecision.blockReason === 'composer_blocked'
) {
@@ -636,7 +683,10 @@ const Conversations = ({
setSendAdvisory(null);
}
if (!sendDecision.shouldSend) {
if (
!sendDecision.shouldSend &&
!(sendDecision.blockReason === 'empty_input' && attachments.length > 0)
) {
const blockedFeedback = getComposerBlockedSendFeedback(sendDecision.blockReason);
if (blockedFeedback) {
setSendError(chatSendError(blockedFeedback.error.code, blockedFeedback.error.message));
@@ -648,11 +698,20 @@ const Conversations = ({
if (!sendingThreadId) return;
pendingSendRef.current = sendingThreadId;
setPendingSendingThreadId(sendingThreadId);
const pendingAttachments = attachments.slice();
const messageText = buildMessageWithAttachments(trimmed, pendingAttachments);
const userMessage: ThreadMessage = {
id: `msg_${globalThis.crypto.randomUUID()}`,
content: trimmed,
type: 'text',
extraMetadata: {},
extraMetadata:
pendingAttachments.length > 0
? {
attachmentCount: pendingAttachments.length,
attachmentNames: pendingAttachments.map(a => a.file.name),
attachmentDataUris: pendingAttachments.map(a => a.dataUri),
}
: {},
sender: 'user',
createdAt: new Date().toISOString(),
};
@@ -677,7 +736,9 @@ const Conversations = ({
return;
}
setInputValue('');
setAttachments([]);
setSendError(null);
setAttachError(null);
// Silence timer: fires only if 600s pass without ANY inference progress
// (tool call, tool result, iteration start, subagent event, text delta).
// The effect below rearms this timer whenever `inferenceStatusByThread`
@@ -696,7 +757,7 @@ const Conversations = ({
try {
await chatSend({
threadId: sendingThreadId,
message: trimmed,
message: messageText,
model: CHAT_MODEL_ID,
profileId: selectedAgentProfileId,
locale: uiLocale,
@@ -1548,13 +1609,43 @@ const Conversations = ({
)}
</div>
) : (
<div className="rounded-2xl px-4 py-2.5 bg-primary-500 text-white rounded-br-md break-words overflow-hidden">
<BubbleMarkdown content={msg.content} tone="user" />
{latestVisibleMessage?.id === msg.id && (
<p className="mt-1 text-[10px] text-white/60">
{formatRelativeTime(msg.createdAt)}
</p>
)}
<div className="flex flex-col items-end gap-1">
{(() => {
const dataUris = Array.isArray(msg.extraMetadata?.attachmentDataUris)
? (msg.extraMetadata.attachmentDataUris as string[])
: parseMessageImages(msg.content ?? '').dataUris;
const hasImages = dataUris.length > 0;
const showTime = latestVisibleMessage?.id === msg.id;
return (
<>
{hasImages && (
<div className="flex flex-wrap gap-1.5 justify-end">
{dataUris.map((uri, i) => (
<img
key={i}
src={uri}
alt=""
className="max-w-[200px] max-h-[200px] rounded-2xl object-cover"
/>
))}
</div>
)}
{(msg.content || showTime) && (
<div className="rounded-2xl px-4 py-2.5 bg-primary-500 text-white rounded-br-md break-words overflow-hidden">
{msg.content && (
<BubbleMarkdown content={msg.content} tone="user" />
)}
{showTime && (
<p
className={`${msg.content ? 'mt-1' : ''} text-[10px] text-white/60`}>
{formatRelativeTime(msg.createdAt)}
</p>
)}
</div>
)}
</>
);
})()}
</div>
)}
<button
@@ -1884,6 +1975,19 @@ const Conversations = ({
</div>
)}
{attachError && (
<div className="flex items-center justify-between mb-2">
<p className="text-xs text-coral-500" data-chat-send-error-code={attachError.code}>
{attachError.message}
</p>
<button
onClick={() => setAttachError(null)}
className="text-xs text-stone-500 dark:text-neutral-400 hover:text-stone-700 dark:hover:text-neutral-200 transition-colors ml-2">
{t('common.dismiss')}
</button>
</div>
)}
{sendError && (
<div className="flex items-center justify-between mb-2">
<p className="text-xs text-coral-500" data-chat-send-error-code={sendError.code}>
@@ -1944,33 +2048,72 @@ const Conversations = ({
</div>
) : inputMode === 'text' ? (
<div className="flex items-end gap-3">
<div className="relative flex flex-1 items-center justify-center rounded-xl border border-stone-200 dark:border-neutral-800 bg-white dark:bg-neutral-900 transition-all focus-within:border-primary-500/50 focus-within:ring-1 focus-within:ring-primary-500/50">
<div
aria-hidden
className="pointer-events-none absolute inset-0 overflow-hidden whitespace-pre-wrap break-words px-4 py-2.5 text-sm leading-normal font-sans">
<span className="invisible">{inputValue}</span>
<span className="text-stone-500 dark:text-neutral-400/50">
{inlineCompletionSuffix}
</span>
</div>
<textarea
ref={textInputRef}
value={inputValue}
onChange={e => setInputValue(e.target.value)}
onCompositionStart={() => {
isComposingTextRef.current = true;
}}
onCompositionEnd={() => {
isComposingTextRef.current = false;
}}
onKeyDown={handleInputKeyDown}
placeholder={t('chat.typeMessage')}
rows={1}
{/* Hidden file input for image attachment */}
<input
ref={fileInputRef}
type="file"
accept={ALLOWED_IMAGE_MIME_TYPES.join(',')}
multiple
className="hidden"
onChange={e => {
void handleAttachFiles(e.target.files);
e.target.value = '';
}}
/>
<div className="relative flex flex-1 flex-col rounded-xl border border-stone-200 dark:border-neutral-800 bg-white dark:bg-neutral-900 transition-all focus-within:border-primary-500/50 focus-within:ring-1 focus-within:ring-primary-500/50">
<AttachmentPreview
attachments={attachments}
onRemove={id => setAttachments(prev => prev.filter(a => a.id !== id))}
disabled={composerInteractionBlocked || isSending}
className="relative z-10 w-full resize-none border-0 bg-transparent pl-4 pr-10 py-2.5 text-sm leading-normal whitespace-pre-wrap break-words font-sans text-stone-900 dark:text-neutral-100 placeholder:text-stone-400 dark:placeholder:text-neutral-500 outline-none focus:outline-none focus-visible:outline-none focus:ring-0 focus-visible:ring-0 max-h-32 disabled:opacity-50 disabled:cursor-not-allowed"
/>
{/* Voice input mic hidden per #717 (inputMode='voice' path retained). */}
<div className="relative flex items-center justify-center">
<div
aria-hidden
className="pointer-events-none absolute inset-0 overflow-hidden whitespace-pre-wrap break-words px-4 py-2.5 text-sm leading-normal font-sans">
<span className="invisible">{inputValue}</span>
<span className="text-stone-500 dark:text-neutral-400/50">
{inlineCompletionSuffix}
</span>
</div>
<textarea
ref={textInputRef}
value={inputValue}
onChange={e => setInputValue(e.target.value)}
onCompositionStart={() => {
isComposingTextRef.current = true;
}}
onCompositionEnd={() => {
isComposingTextRef.current = false;
}}
onKeyDown={handleInputKeyDown}
placeholder={t('chat.typeMessage')}
rows={1}
disabled={composerInteractionBlocked || isSending}
className="relative z-10 w-full resize-none border-0 bg-transparent pl-4 pr-10 py-2.5 text-sm leading-normal whitespace-pre-wrap break-words font-sans text-stone-900 dark:text-neutral-100 placeholder:text-stone-400 dark:placeholder:text-neutral-500 outline-none focus:outline-none focus-visible:outline-none focus:ring-0 focus-visible:ring-0 max-h-32 disabled:opacity-50 disabled:cursor-not-allowed"
/>
{/* Voice input mic hidden per #717 (inputMode='voice' path retained). */}
</div>
</div>
<button
type="button"
aria-label={t('chat.attachment.attach')}
title={t('chat.attachment.attach')}
onClick={() => fileInputRef.current?.click()}
disabled={
composerInteractionBlocked ||
isSending ||
attachments.length >= ATTACHMENT_MAX_IMAGES
}
className="w-10 h-10 flex items-center justify-center rounded-full border border-stone-200 dark:border-neutral-800 bg-white dark:bg-neutral-900 text-stone-500 dark:text-neutral-400 hover:text-primary-500 dark:hover:text-primary-400 hover:border-primary-300 dark:hover:border-primary-700 transition-colors flex-shrink-0 disabled:opacity-40 disabled:cursor-not-allowed">
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M15.172 7l-6.586 6.586a2 2 0 102.828 2.828l6.414-6.586a4 4 0 00-5.656-5.656l-6.415 6.585a6 6 0 108.486 8.486L20.5 13"
/>
</svg>
</button>
<button
type="button"
aria-label={t('mic.startRecording')}
@@ -2000,7 +2143,11 @@ const Conversations = ({
onClick={() => {
void handleSendMessage();
}}
disabled={!inputValue.trim() || composerInteractionBlocked || isSending}
disabled={
(!inputValue.trim() && attachments.length === 0) ||
composerInteractionBlocked ||
isSending
}
className="w-10 h-10 flex items-center justify-center rounded-full bg-primary-500 hover:bg-primary-600 text-white disabled:opacity-40 disabled:cursor-not-allowed transition-colors flex-shrink-0">
{isSending ? (
<svg className="w-4 h-4 animate-spin" fill="none" viewBox="0 0 24 24">
@@ -0,0 +1,402 @@
/**
* Attachment feature tests for Conversations.tsx — covers the new lines added
* for multimodal image attachments: handleAttachFiles, error display,
* attachment-only sends, and user bubble image rendering.
*/
import { combineReducers, configureStore } from '@reduxjs/toolkit';
import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
import { Provider } from 'react-redux';
import { MemoryRouter } from 'react-router-dom';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import agentProfileReducer from '../../store/agentProfileSlice';
import chatRuntimeReducer from '../../store/chatRuntimeSlice';
import socketReducer from '../../store/socketSlice';
import threadReducer from '../../store/threadSlice';
import type { Thread } from '../../types/thread';
// ── Hoisted mock state ──────────────────────────────────────────────────────
const { mockGetThreads, mockGetThreadMessages, mockUseUsageState } = vi.hoisted(() => ({
mockGetThreads: vi.fn().mockResolvedValue({ threads: [], count: 0 }),
mockGetThreadMessages: vi.fn().mockResolvedValue({ messages: [], count: 0 }),
mockUseUsageState: vi.fn(() => ({
teamUsage: null,
currentPlan: null,
currentTier: 'FREE' as const,
isFreeTier: true,
usagePct: 0,
isNearLimit: false,
isAtLimit: false,
isBudgetExhausted: false,
shouldShowBudgetCompletedMessage: false,
isLoading: false,
refresh: vi.fn(),
})),
}));
// ── Module mocks ────────────────────────────────────────────────────────────
vi.mock('../../services/chatService', () => ({
chatCancel: vi.fn(),
chatSend: vi.fn().mockResolvedValue(undefined),
subscribeChatEvents: vi.fn(() => () => {}),
useRustChat: vi.fn(() => true),
}));
vi.mock('../../services/api/threadApi', () => ({
threadApi: {
createNewThread: vi.fn().mockResolvedValue({ id: 'new-thread', labels: [] }),
getThreads: mockGetThreads,
getThreadMessages: mockGetThreadMessages,
getTurnState: vi.fn().mockResolvedValue(null),
getTaskBoard: vi
.fn()
.mockResolvedValue({ threadId: 't-1', cards: [], updatedAt: '2026-05-04T10:00:00Z' }),
putTaskBoard: vi
.fn()
.mockResolvedValue({ threadId: 't-1', cards: [], updatedAt: '2026-05-04T10:00:00Z' }),
appendMessage: vi.fn().mockResolvedValue({}),
deleteThread: vi.fn().mockResolvedValue({ deleted: true }),
generateTitleIfNeeded: vi.fn().mockResolvedValue({}),
updateMessage: vi.fn().mockResolvedValue({}),
purge: vi.fn().mockResolvedValue({}),
updateLabels: vi.fn().mockResolvedValue({}),
updateTitle: vi.fn().mockResolvedValue({}),
persistReaction: vi.fn().mockResolvedValue({}),
},
}));
vi.mock('../../services/api/agentProfilesApi', () => ({
agentProfilesApi: {
list: vi
.fn()
.mockResolvedValue({
activeProfileId: 'default',
profiles: [
{
id: 'default',
name: 'Default',
description: 'Default',
agentId: 'orchestrator',
builtIn: true,
},
],
}),
select: vi.fn().mockResolvedValue({ activeProfileId: 'default', profiles: [] }),
upsert: vi.fn().mockResolvedValue({ activeProfileId: 'default', profiles: [] }),
delete: vi.fn().mockResolvedValue({ activeProfileId: 'default', profiles: [] }),
},
}));
vi.mock('../../hooks/useUsageState', () => ({ useUsageState: mockUseUsageState }));
vi.mock('../../store/socketSelectors', () => ({
selectSocketStatus: (state: { socket?: { byUser?: Record<string, { status: string }> } }) =>
state.socket?.byUser?.__pending__?.status ?? 'disconnected',
}));
vi.mock('../../hooks/useStickToBottom', () => ({
useStickToBottom: vi.fn(() => ({ containerRef: { current: null }, endRef: { current: null } })),
}));
vi.mock('../../features/autocomplete/useAutocompleteSkillStatus', () => ({
useAutocompleteSkillStatus: vi.fn(() => ({ status: 'idle', skills: [] })),
}));
vi.mock('../../utils/openUrl', () => ({ openUrl: vi.fn() }));
vi.mock('../../lib/coreState/store', () => ({
getCoreStateSnapshot: vi.fn(() => ({
isBootstrapping: false,
isReady: true,
snapshot: {
auth: { isAuthenticated: false, userId: null, user: null, profileId: null },
sessionToken: null,
currentUser: null,
onboardingCompleted: true,
chatOnboardingCompleted: true,
analyticsEnabled: false,
localState: {},
runtime: {},
},
})),
isWelcomeLocked: vi.fn(() => false),
setCoreStateSnapshot: vi.fn(),
}));
// ── Helpers ─────────────────────────────────────────────────────────────────
function buildStore(preload: Record<string, unknown> = {}) {
return configureStore({
reducer: combineReducers({
thread: threadReducer,
socket: socketReducer,
chatRuntime: chatRuntimeReducer,
agentProfiles: agentProfileReducer,
}),
preloadedState: preload as never,
});
}
function makeThread(overrides: Partial<Thread> = {}): Thread {
return {
id: 't-1',
title: 'Test thread',
chatId: null,
isActive: false,
messageCount: 0,
lastMessageAt: '2026-01-01T00:00:00.000Z',
createdAt: '2026-01-01T00:00:00.000Z',
labels: [],
...overrides,
};
}
function socketState(status: 'connected' | 'disconnected') {
return {
byUser: { __pending__: { status, socketId: status === 'connected' ? 'socket-1' : null } },
};
}
function makeFile(name: string, type: string, size = 1024): File {
const blob = new Blob([new Uint8Array(size)], { type });
return new File([blob], name, { type });
}
async function renderWithSelectedThread() {
const thread = makeThread({ id: 'attach-thread', title: 'Attach Thread' });
mockGetThreads.mockResolvedValue({ threads: [thread], count: 1 });
mockGetThreadMessages.mockResolvedValue({ messages: [], count: 0 });
const store = buildStore({
thread: {
threads: [thread],
selectedThreadId: thread.id,
activeThreadId: null,
welcomeThreadId: null,
messagesByThreadId: { [thread.id]: [] },
messages: [],
isLoadingThreads: false,
isLoadingMessages: false,
messagesError: null,
},
socket: socketState('connected'),
});
const { default: Conversations } = await import('../Conversations');
render(
<Provider store={store}>
<MemoryRouter initialEntries={['/conversations']}>
<Conversations />
</MemoryRouter>
</Provider>
);
const textarea = await screen.findByPlaceholderText('Type a message...');
return { store, textarea, thread };
}
// ── Tests ────────────────────────────────────────────────────────────────────
describe('Conversations — attachment feature', () => {
beforeEach(() => {
vi.clearAllMocks();
mockGetThreads.mockResolvedValue({ threads: [], count: 0 });
mockGetThreadMessages.mockResolvedValue({ messages: [], count: 0 });
mockUseUsageState.mockReturnValue({
teamUsage: null,
currentPlan: null,
currentTier: 'FREE' as const,
isFreeTier: true,
usagePct: 0,
isNearLimit: false,
isAtLimit: false,
isBudgetExhausted: false,
shouldShowBudgetCompletedMessage: false,
isLoading: false,
refresh: vi.fn(),
});
});
it('renders the paperclip button in the composer', async () => {
await renderWithSelectedThread();
expect(screen.getByTitle('Attach image')).toBeInTheDocument();
});
it('shows attachment chip after selecting a valid image file', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
expect(fileInput).not.toBeNull();
const file = makeFile('photo.png', 'image/png', 512);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(screen.getByText('photo.png')).toBeInTheDocument();
});
});
it('shows too-many error when selecting more than 4 images', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const files = Array.from({ length: 5 }, (_, i) => makeFile(`img${i}.png`, 'image/png', 512));
await act(async () => {
fireEvent.change(fileInput, { target: { files } });
});
await waitFor(() => {
expect(screen.getByText(/Maximum 4 images/i)).toBeInTheDocument();
});
});
it('shows unsupported type error for non-image files', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const file = makeFile('doc.pdf', 'application/pdf', 512);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(screen.getByText(/Unsupported file type/i)).toBeInTheDocument();
});
});
it('shows too-large error for oversized files', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const file = makeFile('big.png', 'image/png', 8 * 1024 * 1024 + 1);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(screen.getByText(/8 MB/i)).toBeInTheDocument();
});
});
it('removes chip when × button is clicked', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const file = makeFile('remove-me.png', 'image/png', 512);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(screen.getByText('remove-me.png')).toBeInTheDocument();
});
const removeBtn = screen.getByRole('button', { name: /remove remove-me\.png/i });
await act(async () => {
fireEvent.click(removeBtn);
});
await waitFor(() => {
expect(screen.queryByText('remove-me.png')).not.toBeInTheDocument();
});
});
it('enables send button when attachment is present with no text', async () => {
await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const file = makeFile('only.png', 'image/png', 512);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(screen.getByRole('button', { name: 'Send message' })).not.toBeDisabled();
});
});
it('clears attachments and calls chatSend after sending with attachment + text', async () => {
const { chatSend } = await import('../../services/chatService');
const { textarea } = await renderWithSelectedThread();
const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement;
const file = makeFile('send.png', 'image/png', 512);
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
expect(screen.getByText('send.png')).toBeInTheDocument();
});
await act(async () => {
fireEvent.change(textarea, { target: { value: 'describe this' } });
});
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: 'Send message' }));
});
await waitFor(() => {
expect(chatSend).toHaveBeenCalled();
expect(screen.queryByText('send.png')).not.toBeInTheDocument();
});
});
it('renders image thumbnails in user message bubble from extraMetadata', async () => {
const thread = makeThread({ id: 'img-thread', title: 'Img Thread' });
const dataUri = 'data:image/png;base64,abc123';
const message = {
id: 'msg-1',
content: 'look at this',
type: 'text' as const,
sender: 'user' as const,
createdAt: new Date().toISOString(),
extraMetadata: { attachmentDataUris: [dataUri] },
};
mockGetThreads.mockResolvedValue({ threads: [thread], count: 1 });
mockGetThreadMessages.mockResolvedValue({ messages: [message], count: 1 });
const store = buildStore({
thread: {
threads: [thread],
selectedThreadId: thread.id,
activeThreadId: null,
welcomeThreadId: null,
messagesByThreadId: { [thread.id]: [message] },
messages: [message],
isLoadingThreads: false,
isLoadingMessages: false,
messagesError: null,
},
socket: socketState('connected'),
});
const { default: Conversations } = await import('../Conversations');
render(
<Provider store={store}>
<MemoryRouter>
<Conversations />
</MemoryRouter>
</Provider>
);
await waitFor(() => {
const img = document.querySelector(`img[src="${dataUri}"]`);
expect(img).not.toBeNull();
});
});
});
+393
View File
@@ -4,25 +4,45 @@
//! expose generated capability tools without adding a bespoke Rust type
//! for each tool and without handing the model a broad raw bridge.
use std::collections::BTreeSet;
use std::sync::Arc;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCategory, ToolResult, ToolScope};
#[derive(Debug, Clone)]
pub struct GeneratedToolDefinition {
/// Stable tool name exposed to the model.
pub name: String,
/// Human-readable tool description.
pub description: String,
/// JSON schema for tool arguments.
pub parameters_schema: Value,
/// Permission required to execute this tool.
pub permission_level: PermissionLevel,
/// Tool category used for agent/tool scoping.
pub category: ToolCategory,
/// Execution surface where the tool is available.
pub scope: ToolScope,
/// Adapter responsible for executing the generated tool.
pub adapter_id: String,
/// Provider that produced this generated tool.
pub provider_id: Option<String>,
/// Provider-scoped capability id for policy and revocation.
pub capability_id: Option<String>,
/// Digest of the source capability definition.
pub source_digest: Option<String>,
/// Declared runtime risk for policy and approval.
pub risk: Option<GeneratedToolRisk>,
/// Optional policy namespace/surface label.
pub policy_surface: Option<String>,
}
impl GeneratedToolDefinition {
/// Build a generated tool definition with legacy-safe defaults.
pub fn new(
name: impl Into<String>,
description: impl Into<String>,
@@ -37,14 +57,78 @@ impl GeneratedToolDefinition {
category: ToolCategory::Skill,
scope: ToolScope::All,
adapter_id: adapter_id.into(),
provider_id: None,
capability_id: None,
source_digest: None,
risk: None,
policy_surface: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum GeneratedToolRisk {
/// Read-only capability.
Read,
/// Local or internal write capability.
Write,
/// Externally observable write capability.
ExternalWrite,
/// Code or command execution capability.
Execute,
/// High-risk or destructive capability.
Dangerous,
}
impl GeneratedToolRisk {
fn is_external_effect(self) -> bool {
matches!(self, Self::ExternalWrite | Self::Execute | Self::Dangerous)
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct GeneratedToolAdmissionConfig {
/// Whether provenance fields are required for admission.
pub enforce_provenance: bool,
/// Provider ids allowed to register generated tools.
///
/// Values are normalized with the same provider-id rules used for
/// generated tool definitions before admission checks run.
pub trusted_providers: BTreeSet<String>,
/// Provider ids blocked from registration.
///
/// Values are normalized with the same provider-id rules used for
/// generated tool definitions before admission checks run.
pub disabled_providers: BTreeSet<String>,
/// Capability ids blocked from registration.
pub disabled_capabilities: BTreeSet<String>,
/// Existing tool names reserved before this admission pass.
pub existing_tool_names: BTreeSet<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GeneratedToolAdmissionRejection {
/// Tool name rejected during admission.
pub tool_name: String,
/// Human-readable rejection reason.
pub reason: String,
}
#[derive(Debug, Clone)]
pub struct GeneratedToolAdmissionReport {
/// Definitions accepted for registration.
pub admitted: Vec<GeneratedToolDefinition>,
/// Definitions rejected before registration.
pub rejected: Vec<GeneratedToolAdmissionRejection>,
}
#[async_trait]
pub trait GeneratedToolAdapter: Send + Sync {
/// Stable adapter id matched against generated definitions.
fn id(&self) -> &str;
/// Execute a generated tool definition with validated arguments.
async fn execute(
&self,
definition: &GeneratedToolDefinition,
@@ -52,12 +136,14 @@ pub trait GeneratedToolAdapter: Send + Sync {
) -> anyhow::Result<ToolResult>;
}
/// Executable wrapper around a generated tool definition and adapter.
pub struct GeneratedTool {
definition: GeneratedToolDefinition,
adapter: Arc<dyn GeneratedToolAdapter>,
}
impl GeneratedTool {
/// Create a generated tool wrapper after validation.
pub fn new(
mut definition: GeneratedToolDefinition,
adapter: Arc<dyn GeneratedToolAdapter>,
@@ -90,6 +176,7 @@ impl GeneratedTool {
})
}
/// Borrow the normalized generated tool definition.
pub fn definition(&self) -> &GeneratedToolDefinition {
&self.definition
}
@@ -124,8 +211,16 @@ impl Tool for GeneratedTool {
fn category(&self) -> ToolCategory {
self.definition.category
}
fn external_effect(&self) -> bool {
self.definition
.risk
.map(GeneratedToolRisk::is_external_effect)
.unwrap_or(false)
}
}
/// Convert generated definitions into boxed tool trait objects.
pub fn generated_tools_from_definitions(
definitions: Vec<GeneratedToolDefinition>,
adapter: Arc<dyn GeneratedToolAdapter>,
@@ -139,10 +234,52 @@ pub fn generated_tools_from_definitions(
.collect()
}
/// Admit generated tool definitions according to provenance policy.
pub fn admit_generated_tool_definitions(
definitions: Vec<GeneratedToolDefinition>,
config: &GeneratedToolAdmissionConfig,
) -> GeneratedToolAdmissionReport {
let mut seen = config.existing_tool_names.clone();
let mut admitted = Vec::new();
let mut rejected = Vec::new();
for mut definition in definitions {
normalize_definition(&mut definition);
let tool_name = definition.name.clone();
match validate_admission(&definition, config, &mut seen) {
Ok(()) => {
log::debug!(
"[generated_tools] admission accepted tool_name={} provider_id={:?} capability_id={:?}",
definition.name,
definition.provider_id,
definition.capability_id
);
admitted.push(definition);
}
Err(reason) => {
log::debug!(
"[generated_tools] admission rejected tool_name={} provider_id={:?} capability_id={:?} reason={}",
tool_name,
definition.provider_id,
definition.capability_id,
reason
);
rejected.push(GeneratedToolAdmissionRejection { tool_name, reason });
}
}
}
GeneratedToolAdmissionReport { admitted, rejected }
}
fn normalize_definition(definition: &mut GeneratedToolDefinition) {
definition.name = definition.name.trim().to_string();
definition.description = definition.description.trim().to_string();
definition.adapter_id = definition.adapter_id.trim().to_string();
definition.provider_id = normalize_optional_provider_id(definition.provider_id.take());
definition.capability_id = trim_option(definition.capability_id.take());
definition.source_digest = trim_option(definition.source_digest.take());
definition.policy_surface = trim_option(definition.policy_surface.take());
}
fn validate_definition(definition: &GeneratedToolDefinition) -> anyhow::Result<()> {
@@ -161,6 +298,129 @@ fn validate_definition(definition: &GeneratedToolDefinition) -> anyhow::Result<(
Ok(())
}
fn validate_admission(
definition: &GeneratedToolDefinition,
config: &GeneratedToolAdmissionConfig,
seen: &mut BTreeSet<String>,
) -> Result<(), String> {
validate_definition(definition).map_err(|err| err.to_string())?;
if !is_safe_generated_tool_name(&definition.name) {
return Err(format!(
"generated tool `{}` name contains unsupported characters",
definition.name
));
}
if !seen.insert(definition.name.clone()) {
return Err(format!("duplicate generated tool `{}`", definition.name));
}
if !config.enforce_provenance {
return Ok(());
}
let provider_id = definition
.provider_id
.as_deref()
.ok_or_else(|| format!("generated tool `{}` missing provider_id", definition.name))?;
if normalize_provider_id(provider_id).is_none() {
return Err(format!(
"generated tool `{}` has invalid provider_id `{provider_id}`",
definition.name
));
}
if normalize_provider_set(&config.disabled_providers).contains(provider_id) {
return Err(format!(
"generated tool `{}` provider `{provider_id}` is disabled",
definition.name
));
}
if !normalize_provider_set(&config.trusted_providers).contains(provider_id) {
return Err(format!(
"generated tool `{}` provider `{provider_id}` is not trusted",
definition.name
));
}
let capability_id = definition
.capability_id
.as_deref()
.ok_or_else(|| format!("generated tool `{}` missing capability_id", definition.name))?;
if config.disabled_capabilities.contains(capability_id) {
return Err(format!(
"generated tool `{}` capability `{capability_id}` is disabled",
definition.name
));
}
if definition.risk.is_none() {
return Err(format!(
"generated tool `{}` missing risk metadata",
definition.name
));
}
if definition.source_digest.is_none() {
return Err(format!(
"generated tool `{}` missing source_digest",
definition.name
));
}
Ok(())
}
fn trim_option(value: Option<String>) -> Option<String> {
value
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
}
fn normalize_optional_provider_id(value: Option<String>) -> Option<String> {
trim_option(value).map(|value| normalize_provider_id(&value).unwrap_or(value))
}
fn normalize_provider_set(values: &BTreeSet<String>) -> BTreeSet<String> {
values
.iter()
.filter_map(|value| normalize_provider_id(value))
.collect()
}
fn normalize_provider_id(value: &str) -> Option<String> {
let normalized = value.trim().to_ascii_lowercase();
if normalized.is_empty() {
return None;
}
let valid = normalized
.chars()
.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, '-' | '_' | '.'));
if !valid {
return None;
}
let starts_or_ends_with_sep = normalized
.chars()
.next()
.zip(normalized.chars().last())
.map(|(first, last)| is_provider_separator(first) || is_provider_separator(last))
.unwrap_or(true);
if starts_or_ends_with_sep {
return None;
}
Some(normalized)
}
fn is_provider_separator(ch: char) -> bool {
matches!(ch, '-' | '_' | '.')
}
fn is_safe_generated_tool_name(name: &str) -> bool {
let trimmed = name.trim();
!trimmed.is_empty()
&& !trimmed.starts_with(['.', '-', '_'])
&& !trimmed.ends_with(['.', '-', '_'])
&& trimmed.chars().all(|ch| {
ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, '.' | '-' | '_')
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -204,9 +464,21 @@ mod tests {
"echo-adapter",
);
definition.permission_level = PermissionLevel::Write;
definition.provider_id = Some("trusted.runtime".into());
definition.capability_id = Some("updates.send".into());
definition.source_digest = Some("sha256:abc".into());
definition.risk = Some(GeneratedToolRisk::ExternalWrite);
definition
}
fn admission_config() -> GeneratedToolAdmissionConfig {
GeneratedToolAdmissionConfig {
enforce_provenance: true,
trusted_providers: BTreeSet::from(["trusted.runtime".to_string()]),
..Default::default()
}
}
#[tokio::test]
async fn generated_tool_executes_through_adapter() {
let tool = GeneratedTool::new(sample_definition(), Arc::new(EchoAdapter)).unwrap();
@@ -268,5 +540,126 @@ mod tests {
assert_eq!(tool.name(), "send_update");
assert_eq!(tool.description(), "Send a scoped update.");
assert_eq!(tool.definition().adapter_id, "echo-adapter");
assert_eq!(
tool.definition().provider_id.as_deref(),
Some("trusted.runtime")
);
}
#[test]
fn admission_allows_trusted_generated_tool() {
let report =
admit_generated_tool_definitions(vec![sample_definition()], &admission_config());
assert_eq!(report.admitted.len(), 1);
assert!(report.rejected.is_empty());
}
#[test]
fn admission_normalizes_provider_ids_before_policy_checks() {
let mut definition = sample_definition();
definition.provider_id = Some(" Trusted.Runtime ".into());
let config = GeneratedToolAdmissionConfig {
enforce_provenance: true,
trusted_providers: BTreeSet::from(["TRUSTED.RUNTIME".to_string()]),
..Default::default()
};
let report = admit_generated_tool_definitions(vec![definition], &config);
assert_eq!(report.admitted.len(), 1);
assert!(report.rejected.is_empty());
assert_eq!(
report.admitted[0].provider_id.as_deref(),
Some("trusted.runtime")
);
}
#[test]
fn admission_rejects_invalid_provider_ids_when_enforced() {
let mut definition = sample_definition();
definition.provider_id = Some("bad/provider".into());
let report = admit_generated_tool_definitions(vec![definition], &admission_config());
assert!(report.admitted.is_empty());
assert!(report.rejected[0].reason.contains("invalid provider_id"));
}
#[test]
fn admission_disabled_preserves_legacy_generated_tools() {
let mut definition = sample_definition();
definition.provider_id = None;
definition.capability_id = None;
definition.source_digest = None;
definition.risk = None;
let report = admit_generated_tool_definitions(
vec![definition],
&GeneratedToolAdmissionConfig::default(),
);
assert_eq!(report.admitted.len(), 1);
assert!(report.rejected.is_empty());
}
#[test]
fn admission_rejects_untrusted_provider() {
let mut definition = sample_definition();
definition.provider_id = Some("other.runtime".into());
let report = admit_generated_tool_definitions(vec![definition], &admission_config());
assert!(report.admitted.is_empty());
assert!(report.rejected[0].reason.contains("not trusted"));
}
#[test]
fn admission_rejects_duplicate_tool_names() {
let report = admit_generated_tool_definitions(
vec![sample_definition(), sample_definition()],
&admission_config(),
);
assert_eq!(report.admitted.len(), 1);
assert!(report.rejected[0].reason.contains("duplicate"));
}
#[test]
fn admission_rejects_missing_risk_when_enforced() {
let mut definition = sample_definition();
definition.risk = None;
let report = admit_generated_tool_definitions(vec![definition], &admission_config());
assert!(report.admitted.is_empty());
assert!(report.rejected[0].reason.contains("missing risk"));
}
#[test]
fn admission_rejects_unsafe_names() {
let mut definition = sample_definition();
definition.name = "Bad Tool".into();
let report = admit_generated_tool_definitions(vec![definition], &admission_config());
assert!(report.admitted.is_empty());
assert!(report.rejected[0].reason.contains("unsupported characters"));
}
#[tokio::test]
async fn generated_tool_marks_external_risk_as_external_effect() {
let tool = GeneratedTool::new(sample_definition(), Arc::new(EchoAdapter)).unwrap();
assert!(tool.external_effect());
}
#[tokio::test]
async fn generated_tool_marks_execute_risk_as_external_effect() {
let mut definition = sample_definition();
definition.risk = Some(GeneratedToolRisk::Execute);
let tool = GeneratedTool::new(definition, Arc::new(EchoAdapter)).unwrap();
assert!(tool.external_effect());
}
}