mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
Co-authored-by: M3gA-Mind <megamind@mahadao.com>
This commit is contained in:
oxoxDev
@@ -30,6 +30,9 @@
|
||||
# VITE_BACKEND_URL=https://staging-api.tinyhumans.ai
|
||||
# [optional] Consumer first-session UX in the desktop/web app (default off). See docs/plans/consumer-first-session-spec.md
|
||||
# VITE_CONSUMER_FIRST_SESSION=true
|
||||
# [optional] Chat attachments (image/video/document) master kill-switch. Enabled
|
||||
# by default; per-message modalities gated on the model's vision tier at runtime.
|
||||
# VITE_CHAT_ATTACHMENTS=false
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Authentication (for skills OAuth proxy and debug scripts)
|
||||
|
||||
@@ -80,6 +80,11 @@ VITE_DEV_FORCE_ONBOARDING=false
|
||||
# [optional] Consumer first-session + home IA experiments (default off). See docs/plans/consumer-first-session-spec.md
|
||||
# VITE_CONSUMER_FIRST_SESSION=true
|
||||
|
||||
# [optional] Chat attachments (image / video / document) master kill-switch.
|
||||
# Enabled by default; per-message modalities are gated on the model's vision
|
||||
# tier at runtime. Set to false to hard-disable the attach button for a build.
|
||||
# VITE_CHAT_ATTACHMENTS=false
|
||||
|
||||
# [optional] Client-side timeout for skill callTool/triggerSync (seconds; default 120, max 3600).
|
||||
# Should match OPENHUMAN_TOOL_TIMEOUT_SECS on the core when set.
|
||||
# VITE_TOOL_TIMEOUT_SECS=
|
||||
|
||||
@@ -28,6 +28,22 @@ export default function AttachmentPreview({
|
||||
alt={attachment.file.name}
|
||||
className="w-8 h-8 rounded object-cover flex-shrink-0"
|
||||
/>
|
||||
) : attachment.kind === 'video' ? (
|
||||
<div className="relative w-8 h-8 flex-shrink-0">
|
||||
<img
|
||||
src={attachment.previewUri ?? attachment.dataUri}
|
||||
alt={attachment.file.name}
|
||||
className="w-8 h-8 rounded object-cover"
|
||||
/>
|
||||
<span className="absolute inset-0 flex items-center justify-center">
|
||||
<svg
|
||||
className="w-3.5 h-3.5 text-white drop-shadow"
|
||||
fill="currentColor"
|
||||
viewBox="0 0 24 24">
|
||||
<path d="M8 5v14l11-7z" />
|
||||
</svg>
|
||||
</span>
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
aria-hidden
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useEffect } from 'react';
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
import type { ChatSendError } from '../../chat/chatSendError';
|
||||
import type { Attachment } from '../../lib/attachments';
|
||||
@@ -32,7 +32,7 @@ export interface ChatComposerProps {
|
||||
*/
|
||||
allowParallelSend?: boolean;
|
||||
attachments: Attachment[];
|
||||
onAttachFiles: (files: FileList | null) => Promise<void>;
|
||||
onAttachFiles: (files: FileList | File[] | null) => Promise<void>;
|
||||
onRemoveAttachment: (id: string) => void;
|
||||
attachError: ChatSendError | null;
|
||||
onSwitchToMicCloud: () => void;
|
||||
@@ -85,6 +85,7 @@ export default function ChatComposer({
|
||||
headerSlots = [],
|
||||
}: ChatComposerProps) {
|
||||
const { t } = useT();
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
|
||||
// While a turn streams (`allowParallelSend`) the composer stays usable for a
|
||||
// queued follow-up / parallel branch, so the in-flight `isSending` spinner
|
||||
@@ -106,6 +107,57 @@ export default function ChatComposer({
|
||||
const hasTypedContent = inputValue.trim().length > 0 || attachments.length > 0;
|
||||
const showStopButton = isSending && !!onStopGeneration && !hasTypedContent;
|
||||
|
||||
// Attachment ingest is blocked while the feature is off, the composer is
|
||||
// locked, or the budget is full — drag-drop and paste honour the same gate as
|
||||
// the [+] button so they can't bypass it.
|
||||
const attachDisabled =
|
||||
!attachmentsEnabled ||
|
||||
composerInteractionBlocked ||
|
||||
isSending ||
|
||||
attachments.length >= maxAttachments;
|
||||
|
||||
// Drag-and-drop: route dropped files through the same handler as the picker.
|
||||
// We always `preventDefault` for *file* drags so the browser/CEF never
|
||||
// navigates away to the dropped file — even when ingest is disabled — and only
|
||||
// attach (and show the overlay) when ingest is actually allowed.
|
||||
const isFileDrag = (e: React.DragEvent) =>
|
||||
Array.from(e.dataTransfer?.types ?? []).includes('Files');
|
||||
const handleDragOver = (e: React.DragEvent) => {
|
||||
if (!isFileDrag(e)) return;
|
||||
e.preventDefault();
|
||||
if (!attachDisabled) setIsDragging(true);
|
||||
};
|
||||
const handleDragLeave = (e: React.DragEvent) => {
|
||||
// Ignore leave events that bubble while the cursor is still over a child.
|
||||
if (e.currentTarget.contains(e.relatedTarget as Node | null)) return;
|
||||
setIsDragging(false);
|
||||
};
|
||||
const handleDrop = (e: React.DragEvent) => {
|
||||
if (!isFileDrag(e)) return;
|
||||
// Stop the default file-navigation before any early-return below.
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
if (attachDisabled) return;
|
||||
const files = e.dataTransfer?.files;
|
||||
if (!files || files.length === 0) return;
|
||||
void onAttachFiles(files);
|
||||
};
|
||||
|
||||
// Clipboard paste: pull image/video files out of the paste payload (e.g. a
|
||||
// screenshot copied to the clipboard) and attach them; leave plain-text paste
|
||||
// untouched so normal typing still works.
|
||||
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
|
||||
if (attachDisabled) return;
|
||||
const items = Array.from(e.clipboardData?.items ?? []);
|
||||
const files = items
|
||||
.filter(item => item.kind === 'file' && /^(image|video)\//.test(item.type))
|
||||
.map(item => item.getAsFile())
|
||||
.filter((file): file is File => file !== null);
|
||||
if (files.length === 0) return;
|
||||
e.preventDefault();
|
||||
void onAttachFiles(files);
|
||||
};
|
||||
|
||||
// Auto-resize textarea: grow with content, cap at COMPOSER_MAX_HEIGHT, then scroll.
|
||||
useEffect(() => {
|
||||
const ta = textInputRef.current;
|
||||
@@ -123,7 +175,17 @@ export default function ChatComposer({
|
||||
{headerSlots}
|
||||
|
||||
{/* The input box — only this carries the focus-within highlight. */}
|
||||
<div className="relative flex flex-col rounded-2xl border border-line bg-surface transition-all focus-within:border-primary-500/50 focus-within:ring-1 focus-within:ring-primary-500/50">
|
||||
<div
|
||||
className="relative flex flex-col rounded-2xl border border-line bg-surface transition-all focus-within:border-primary-500/50 focus-within:ring-1 focus-within:ring-primary-500/50"
|
||||
onDragOver={handleDragOver}
|
||||
onDragLeave={handleDragLeave}
|
||||
onDrop={handleDrop}>
|
||||
{/* Drag-and-drop overlay: shown while a file drag hovers the composer. */}
|
||||
{isDragging && (
|
||||
<div className="pointer-events-none absolute inset-0 z-20 flex items-center justify-center rounded-2xl border-2 border-dashed border-primary-500 bg-surface/90 text-sm font-medium text-primary-600">
|
||||
{t('chat.attachment.dropToAttach')}
|
||||
</div>
|
||||
)}
|
||||
{/* Hidden file input for attachment (gated — see attachmentsEnabled). */}
|
||||
{attachmentsEnabled && (
|
||||
<input
|
||||
@@ -201,6 +263,7 @@ export default function ChatComposer({
|
||||
isComposingTextRef.current = false;
|
||||
}}
|
||||
onKeyDown={handleInputKeyDown}
|
||||
onPaste={attachmentsEnabled ? handlePaste : undefined}
|
||||
placeholder={allowParallelSend ? t('chat.followupHint') : t('chat.typeMessage')}
|
||||
rows={1}
|
||||
disabled={textareaDisabled}
|
||||
|
||||
@@ -54,6 +54,26 @@ describe('AttachmentPreview', () => {
|
||||
expect(screen.queryByAltText('doc.pdf')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders a video poster thumbnail with a play overlay', () => {
|
||||
const file = new File([new Uint8Array(64)], 'clip.mp4', { type: 'video/mp4' });
|
||||
const att = makeAttachment({
|
||||
kind: 'video',
|
||||
file,
|
||||
dataUri: 'data:image/jpeg;base64,poster',
|
||||
previewUri: 'data:image/jpeg;base64,poster',
|
||||
mimeType: 'video/mp4',
|
||||
frames: ['data:image/jpeg;base64,poster'],
|
||||
});
|
||||
const { container } = render(<AttachmentPreview attachments={[att]} onRemove={vi.fn()} />);
|
||||
const img = screen.getByAltText('clip.mp4') as HTMLImageElement;
|
||||
expect(img.src).toBe('data:image/jpeg;base64,poster');
|
||||
expect(screen.getByText('clip.mp4')).toBeInTheDocument();
|
||||
// Explicitly assert the play overlay (the ▶ triangle drawn over the poster)
|
||||
// is rendered — without this the test would still pass if the overlay
|
||||
// disappeared, which is exactly the regression the title promises to catch.
|
||||
expect(container.querySelector('path[d="M8 5v14l11-7z"]')).not.toBeNull();
|
||||
});
|
||||
|
||||
it('calls onRemove with the attachment id when × is clicked', () => {
|
||||
const onRemove = vi.fn();
|
||||
const att = makeAttachment({ id: 'att-42' });
|
||||
|
||||
@@ -177,6 +177,72 @@ describe('ChatComposer', () => {
|
||||
expect(onSwitchToMicCloud).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
describe('drag-and-drop', () => {
|
||||
function makeVideoFile() {
|
||||
return new File([new Uint8Array(8)], 'clip.mp4', { type: 'video/mp4' });
|
||||
}
|
||||
|
||||
it('shows the drop overlay while a file drag is over the composer', () => {
|
||||
renderComposer();
|
||||
const textarea = screen.getByRole('textbox');
|
||||
const box = textarea.closest('div.rounded-2xl') as HTMLElement;
|
||||
fireEvent.dragOver(box, { dataTransfer: { types: ['Files'] } });
|
||||
expect(screen.getByText('chat.attachment.dropToAttach')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('routes dropped files through onAttachFiles', () => {
|
||||
const onAttachFiles = vi.fn().mockResolvedValue(undefined);
|
||||
renderComposer({ onAttachFiles });
|
||||
const textarea = screen.getByRole('textbox');
|
||||
const box = textarea.closest('div.rounded-2xl') as HTMLElement;
|
||||
const file = makeVideoFile();
|
||||
fireEvent.drop(box, { dataTransfer: { files: [file], types: ['Files'] } });
|
||||
expect(onAttachFiles).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not handle drops when attachments are disabled', () => {
|
||||
const onAttachFiles = vi.fn().mockResolvedValue(undefined);
|
||||
renderComposer({ onAttachFiles, attachmentsEnabled: false });
|
||||
const textarea = screen.getByRole('textbox');
|
||||
const box = textarea.closest('div.rounded-2xl') as HTMLElement;
|
||||
fireEvent.drop(box, { dataTransfer: { files: [makeVideoFile()], types: ['Files'] } });
|
||||
expect(onAttachFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('prevents browser file-navigation on drop even when disabled', () => {
|
||||
// fireEvent returns false when the event default was prevented.
|
||||
renderComposer({ attachmentsEnabled: false });
|
||||
const textarea = screen.getByRole('textbox');
|
||||
const box = textarea.closest('div.rounded-2xl') as HTMLElement;
|
||||
const notPrevented = fireEvent.drop(box, {
|
||||
dataTransfer: { files: [makeVideoFile()], types: ['Files'] },
|
||||
});
|
||||
expect(notPrevented).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('clipboard paste', () => {
|
||||
it('attaches image files pasted from the clipboard', () => {
|
||||
const onAttachFiles = vi.fn().mockResolvedValue(undefined);
|
||||
renderComposer({ onAttachFiles });
|
||||
const file = new File([new Uint8Array(4)], 'pasted.png', { type: 'image/png' });
|
||||
fireEvent.paste(screen.getByRole('textbox'), {
|
||||
clipboardData: { items: [{ kind: 'file', type: 'image/png', getAsFile: () => file }] },
|
||||
});
|
||||
expect(onAttachFiles).toHaveBeenCalledTimes(1);
|
||||
expect(onAttachFiles).toHaveBeenCalledWith([file]);
|
||||
});
|
||||
|
||||
it('ignores plain-text paste (no media items)', () => {
|
||||
const onAttachFiles = vi.fn().mockResolvedValue(undefined);
|
||||
renderComposer({ onAttachFiles });
|
||||
fireEvent.paste(screen.getByRole('textbox'), {
|
||||
clipboardData: { items: [{ kind: 'string', type: 'text/plain', getAsFile: () => null }] },
|
||||
});
|
||||
expect(onAttachFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('follow-up / parallel mode (allowParallelSend during a streaming turn)', () => {
|
||||
it('keeps the textarea editable even while an in-flight turn is sending', () => {
|
||||
renderComposer({
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import {
|
||||
ALLOWED_ATTACHMENT_MIME_TYPES,
|
||||
@@ -6,12 +6,19 @@ import {
|
||||
ATTACHMENT_MAX_FILE_SIZE_BYTES,
|
||||
ATTACHMENT_MAX_IMAGES,
|
||||
ATTACHMENT_MAX_SIZE_BYTES,
|
||||
ATTACHMENT_MAX_VIDEO_SIZE_BYTES,
|
||||
attachmentKindForMime,
|
||||
buildMessageWithAttachments,
|
||||
extractVideoFrames,
|
||||
fileToDataUri,
|
||||
formatFileSize,
|
||||
imageMarkerCost,
|
||||
isAllowedMimeType,
|
||||
isVideoMimeType,
|
||||
parseMessageImages,
|
||||
validateAndReadFile,
|
||||
VIDEO_FRAME_COUNT,
|
||||
videoFrameExtractor,
|
||||
} from './attachments';
|
||||
|
||||
function makeFile(name: string, type: string, size = 1024): File {
|
||||
@@ -188,11 +195,211 @@ describe('validateAndReadFile', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('video attachments', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('classifies video MIME types as the video kind', () => {
|
||||
expect(isVideoMimeType('video/mp4')).toBe(true);
|
||||
expect(isVideoMimeType('video/quicktime')).toBe(true);
|
||||
expect(isVideoMimeType('video/webm')).toBe(true);
|
||||
expect(isVideoMimeType('image/png')).toBe(false);
|
||||
expect(attachmentKindForMime('video/mp4')).toBe('video');
|
||||
});
|
||||
|
||||
it('costs 1 image marker per image and VIDEO_FRAME_COUNT per video', () => {
|
||||
expect(imageMarkerCost('image')).toBe(1);
|
||||
expect(imageMarkerCost('video')).toBe(VIDEO_FRAME_COUNT);
|
||||
expect(imageMarkerCost('file')).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects video when allowImages is false (non-vision model)', async () => {
|
||||
const result = await validateAndReadFile(makeFile('clip.mp4', 'video/mp4', 8), 0, 0, false);
|
||||
expect('error' in result && result.error.code).toBe('video_not_supported');
|
||||
});
|
||||
|
||||
it('rejects video over the video size limit', async () => {
|
||||
const big = makeFile('big.mp4', 'video/mp4', ATTACHMENT_MAX_VIDEO_SIZE_BYTES + 1);
|
||||
const result = await validateAndReadFile(big, 0, 0, true);
|
||||
expect('error' in result && result.error.code).toBe('too_large');
|
||||
});
|
||||
|
||||
it('rejects video when its frames would exceed the shared image-marker budget', async () => {
|
||||
// Budget full of images already → no room for a video's frames.
|
||||
const result = await validateAndReadFile(
|
||||
makeFile('clip.mp4', 'video/mp4', 8),
|
||||
ATTACHMENT_MAX_IMAGES,
|
||||
0,
|
||||
true
|
||||
);
|
||||
expect('error' in result && result.error.code).toBe('too_many');
|
||||
// Reported as the image budget (frames are images to the model).
|
||||
expect('error' in result && result.error.code === 'too_many' && result.error.kind).toBe(
|
||||
'image'
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a video that partially overflows the budget (images + frames)', async () => {
|
||||
// One image used (1 marker); a video costs VIDEO_FRAME_COUNT more. With the
|
||||
// default budget of 4, 1 + 2 = 3 fits, but seed close to the cap to overflow.
|
||||
const used = ATTACHMENT_MAX_IMAGES - (VIDEO_FRAME_COUNT - 1); // leaves < cost
|
||||
const result = await validateAndReadFile(makeFile('c.mp4', 'video/mp4', 8), used, 0, true);
|
||||
expect('error' in result && result.error.code).toBe('too_many');
|
||||
});
|
||||
|
||||
it('samples frames and returns a video attachment within budget', async () => {
|
||||
const frames = ['data:image/jpeg;base64,f1', 'data:image/jpeg;base64,f2'];
|
||||
vi.spyOn(videoFrameExtractor, 'extract').mockResolvedValue(frames);
|
||||
const file = makeFile('clip.mp4', 'video/mp4', 4096);
|
||||
const result = await validateAndReadFile(file, 0, 0, true);
|
||||
expect('attachment' in result).toBe(true);
|
||||
if ('attachment' in result) {
|
||||
expect(result.attachment.kind).toBe('video');
|
||||
expect(result.attachment.frames).toEqual(frames);
|
||||
expect(result.attachment.previewUri).toBe(frames[0]);
|
||||
expect(result.attachment.file).toBe(file);
|
||||
}
|
||||
});
|
||||
|
||||
it('fails when no frames could be extracted', async () => {
|
||||
vi.spyOn(videoFrameExtractor, 'extract').mockResolvedValue([]);
|
||||
const result = await validateAndReadFile(makeFile('clip.mp4', 'video/mp4', 8), 0, 0, true);
|
||||
expect('error' in result && result.error.code).toBe('read_failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractVideoFrames (DOM-mocked)', () => {
|
||||
// jsdom has no video codec, so stub <video>/<canvas> + URL to exercise the
|
||||
// real decode/seek/paint path deterministically.
|
||||
let realCreateObjectURL: typeof URL.createObjectURL;
|
||||
let realRevokeObjectURL: typeof URL.revokeObjectURL;
|
||||
|
||||
function makeFakeVideo(duration: number) {
|
||||
const listeners: Record<string, Array<() => void>> = {};
|
||||
let ct = 0;
|
||||
return {
|
||||
muted: false,
|
||||
preload: '',
|
||||
_src: '',
|
||||
duration,
|
||||
videoWidth: 320,
|
||||
videoHeight: 240,
|
||||
onloadedmetadata: null as null | (() => void),
|
||||
onerror: null as null | (() => void),
|
||||
set src(v: string) {
|
||||
this._src = v;
|
||||
// Fire metadata asynchronously after the impl assigns the handler.
|
||||
void Promise.resolve().then(() => this.onloadedmetadata?.());
|
||||
},
|
||||
get src() {
|
||||
return this._src;
|
||||
},
|
||||
get currentTime() {
|
||||
return ct;
|
||||
},
|
||||
set currentTime(v: number) {
|
||||
ct = v;
|
||||
(listeners['seeked'] ?? []).forEach(cb => cb());
|
||||
},
|
||||
addEventListener(ev: string, cb: () => void) {
|
||||
(listeners[ev] ??= []).push(cb);
|
||||
},
|
||||
removeEventListener(ev: string, cb: () => void) {
|
||||
listeners[ev] = (listeners[ev] ?? []).filter(f => f !== cb);
|
||||
},
|
||||
removeAttribute() {},
|
||||
};
|
||||
}
|
||||
|
||||
function makeFakeCanvas() {
|
||||
return {
|
||||
width: 0,
|
||||
height: 0,
|
||||
getContext: () => ({ drawImage() {} }),
|
||||
toDataURL: () => 'data:image/jpeg;base64,FRAME',
|
||||
};
|
||||
}
|
||||
|
||||
function installDom(duration: number) {
|
||||
const realCreate = document.createElement.bind(document);
|
||||
vi.spyOn(document, 'createElement').mockImplementation((tag: string) => {
|
||||
if (tag === 'video') return makeFakeVideo(duration) as unknown as HTMLElement;
|
||||
if (tag === 'canvas') return makeFakeCanvas() as unknown as HTMLElement;
|
||||
return realCreate(tag as keyof HTMLElementTagNameMap);
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
realCreateObjectURL = URL.createObjectURL;
|
||||
realRevokeObjectURL = URL.revokeObjectURL;
|
||||
URL.createObjectURL = vi.fn(() => 'blob:fake');
|
||||
URL.revokeObjectURL = vi.fn();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
URL.createObjectURL = realCreateObjectURL;
|
||||
URL.revokeObjectURL = realRevokeObjectURL;
|
||||
});
|
||||
|
||||
it('samples one frame per seek point for a normal clip', async () => {
|
||||
installDom(10);
|
||||
const frames = await extractVideoFrames(makeFile('c.mp4', 'video/mp4', 8), 2);
|
||||
expect(frames).toEqual(['data:image/jpeg;base64,FRAME', 'data:image/jpeg;base64,FRAME']);
|
||||
});
|
||||
|
||||
it('handles a zero-duration clip without hanging (seek short-circuits)', async () => {
|
||||
installDom(0);
|
||||
const frames = await extractVideoFrames(makeFile('z.mp4', 'video/mp4', 8), 2);
|
||||
expect(frames.length).toBe(2);
|
||||
});
|
||||
|
||||
it('propagates a read_failed error when the video fails to decode', async () => {
|
||||
const realCreate = document.createElement.bind(document);
|
||||
vi.spyOn(document, 'createElement').mockImplementation((tag: string) => {
|
||||
if (tag === 'video') {
|
||||
const v = makeFakeVideo(10);
|
||||
// Override src to fire onerror instead of onloadedmetadata.
|
||||
Object.defineProperty(v, 'src', {
|
||||
set(value: string) {
|
||||
this._src = value;
|
||||
void Promise.resolve().then(() => this.onerror?.());
|
||||
},
|
||||
get() {
|
||||
return this._src;
|
||||
},
|
||||
});
|
||||
return v as unknown as HTMLElement;
|
||||
}
|
||||
if (tag === 'canvas') return makeFakeCanvas() as unknown as HTMLElement;
|
||||
return realCreate(tag as keyof HTMLElementTagNameMap);
|
||||
});
|
||||
const result = await validateAndReadFile(makeFile('bad.mp4', 'video/mp4', 8), 0, 0, true);
|
||||
expect('error' in result && result.error.code).toBe('read_failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildMessageWithAttachments', () => {
|
||||
it('returns text unchanged when no attachments', () => {
|
||||
expect(buildMessageWithAttachments('hello', [])).toBe('hello');
|
||||
});
|
||||
|
||||
it('expands a video into one IMAGE marker per sampled frame', () => {
|
||||
const video = makeAttachment({
|
||||
kind: 'video',
|
||||
file: makeFile('clip.mp4', 'video/mp4'),
|
||||
mimeType: 'video/mp4',
|
||||
dataUri: 'data:image/jpeg;base64,f1',
|
||||
previewUri: 'data:image/jpeg;base64,f1',
|
||||
frames: ['data:image/jpeg;base64,f1', 'data:image/jpeg;base64,f2'],
|
||||
});
|
||||
const result = buildMessageWithAttachments('watch', [video]);
|
||||
expect(result).toBe(
|
||||
'watch [IMAGE:data:image/jpeg;base64,f1] [IMAGE:data:image/jpeg;base64,f2]'
|
||||
);
|
||||
});
|
||||
|
||||
it('appends IMAGE markers after the text', () => {
|
||||
const a = makeAttachment({ dataUri: 'data:image/png;base64,abc' });
|
||||
const result = buildMessageWithAttachments('describe this', [a]);
|
||||
|
||||
+193
-12
@@ -33,12 +33,25 @@ export const ALLOWED_FILE_MIME_TYPES = [
|
||||
] as const;
|
||||
|
||||
export type AllowedFileMimeType = (typeof ALLOWED_FILE_MIME_TYPES)[number];
|
||||
export type AllowedAttachmentMimeType = AllowedImageMimeType | AllowedFileMimeType;
|
||||
export type AttachmentKind = 'image' | 'file';
|
||||
|
||||
// Video formats accepted by the composer. The original video is never sent to
|
||||
// the provider — instead we sample a few still frames client-side and forward
|
||||
// them through the existing `[IMAGE:]` vision path (see `extractVideoFrames` +
|
||||
// `buildMessageWithAttachments`). So a vision-capable tier is required, the same
|
||||
// gate as images; audio and motion between frames are not conveyed.
|
||||
export const ALLOWED_VIDEO_MIME_TYPES = ['video/mp4', 'video/quicktime', 'video/webm'] as const;
|
||||
|
||||
export type AllowedVideoMimeType = (typeof ALLOWED_VIDEO_MIME_TYPES)[number];
|
||||
export type AllowedAttachmentMimeType =
|
||||
| AllowedImageMimeType
|
||||
| AllowedFileMimeType
|
||||
| AllowedVideoMimeType;
|
||||
export type AttachmentKind = 'image' | 'file' | 'video';
|
||||
|
||||
export const ALLOWED_ATTACHMENT_MIME_TYPES = [
|
||||
...ALLOWED_IMAGE_MIME_TYPES,
|
||||
...ALLOWED_FILE_MIME_TYPES,
|
||||
...ALLOWED_VIDEO_MIME_TYPES,
|
||||
] as const;
|
||||
|
||||
// Document formats the backend actually text-extracts (PDF via pdf_extract;
|
||||
@@ -52,11 +65,21 @@ export const ALLOWED_ATTACHMENT_MIME_TYPES = [
|
||||
// picks, not at the dialog.
|
||||
const EXTRACTABLE_FILE_MIME_TYPES = ['application/pdf', 'text/plain', 'text/markdown'] as const;
|
||||
|
||||
// Shared image-marker budget per message. Images cost 1 marker each; a video
|
||||
// costs VIDEO_FRAME_COUNT markers (its sampled frames). Mirrors the core default
|
||||
// `multimodal.max_images` (src/openhuman/config/schema/tools/multimodal.rs) — the
|
||||
// core counts every `[IMAGE:]` marker (frames included) and errors on overflow,
|
||||
// so the composer must budget images + video frames against this single cap.
|
||||
export const ATTACHMENT_MAX_IMAGES = 4;
|
||||
export const ATTACHMENT_MAX_FILES = 4;
|
||||
export const ATTACHMENT_MAX_IMAGE_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
|
||||
export const ATTACHMENT_MAX_FILE_SIZE_BYTES = 16 * 1024 * 1024; // 16 MB
|
||||
export const ATTACHMENT_MAX_VIDEO_SIZE_BYTES = 50 * 1024 * 1024; // 50 MB
|
||||
export const ATTACHMENT_MAX_SIZE_BYTES = ATTACHMENT_MAX_IMAGE_SIZE_BYTES;
|
||||
// Still frames sampled from a video and forwarded as `[IMAGE:]` markers. 2 keeps
|
||||
// a clip within the 4-marker budget alongside other attachments (e.g. 1 video +
|
||||
// 2 images, or 2 videos).
|
||||
export const VIDEO_FRAME_COUNT = 2;
|
||||
|
||||
export interface Attachment {
|
||||
id: string;
|
||||
@@ -68,6 +91,10 @@ export interface Attachment {
|
||||
originalSizeBytes: number;
|
||||
payloadSizeBytes: number;
|
||||
compressed: boolean;
|
||||
// Only set for `kind: 'video'`: the still frames sampled from the clip,
|
||||
// expanded into `[IMAGE:]` markers at send time. The chip itself shows a
|
||||
// single poster (the first frame) via `previewUri`.
|
||||
frames?: string[];
|
||||
}
|
||||
|
||||
export type AttachmentError =
|
||||
@@ -75,12 +102,17 @@ export type AttachmentError =
|
||||
| { code: 'too_large'; sizeBytes: number; maxBytes: number }
|
||||
| { code: 'too_many'; kind: AttachmentKind; max: number }
|
||||
| { code: 'image_not_supported' }
|
||||
| { code: 'video_not_supported' }
|
||||
| { code: 'read_failed'; reason: string };
|
||||
|
||||
export function isAllowedMimeType(mime: string): mime is AllowedImageMimeType {
|
||||
return (ALLOWED_IMAGE_MIME_TYPES as readonly string[]).includes(mime);
|
||||
}
|
||||
|
||||
export function isVideoMimeType(mime: string): mime is AllowedVideoMimeType {
|
||||
return (ALLOWED_VIDEO_MIME_TYPES as readonly string[]).includes(mime);
|
||||
}
|
||||
|
||||
export function isAllowedAttachmentMimeType(mime: string): mime is AllowedAttachmentMimeType {
|
||||
return (ALLOWED_ATTACHMENT_MIME_TYPES as readonly string[]).includes(mime);
|
||||
}
|
||||
@@ -92,6 +124,7 @@ export function isAllowedAttachmentMimeType(mime: string): mime is AllowedAttach
|
||||
*/
|
||||
export type SupportedAttachmentMimeType =
|
||||
| AllowedImageMimeType
|
||||
| AllowedVideoMimeType
|
||||
| (typeof EXTRACTABLE_FILE_MIME_TYPES)[number];
|
||||
|
||||
/**
|
||||
@@ -104,12 +137,15 @@ export type SupportedAttachmentMimeType =
|
||||
export function isSupportedAttachmentMimeType(mime: string): mime is SupportedAttachmentMimeType {
|
||||
return (
|
||||
(ALLOWED_IMAGE_MIME_TYPES as readonly string[]).includes(mime) ||
|
||||
(ALLOWED_VIDEO_MIME_TYPES as readonly string[]).includes(mime) ||
|
||||
(EXTRACTABLE_FILE_MIME_TYPES as readonly string[]).includes(mime)
|
||||
);
|
||||
}
|
||||
|
||||
export function attachmentKindForMime(mime: AllowedAttachmentMimeType): AttachmentKind {
|
||||
return isAllowedMimeType(mime) ? 'image' : 'file';
|
||||
if (isAllowedMimeType(mime)) return 'image';
|
||||
if (isVideoMimeType(mime)) return 'video';
|
||||
return 'file';
|
||||
}
|
||||
|
||||
export function fileToDataUri(file: Blob): Promise<string> {
|
||||
@@ -180,12 +216,114 @@ async function buildAttachmentDataUri(
|
||||
return { dataUri, payloadSizeBytes: file.size, compressed: false };
|
||||
}
|
||||
|
||||
/** Evenly spread `count` sample points across the 0.1–0.9 span of the clip. */
|
||||
function sampleFractions(count: number): number[] {
|
||||
if (count <= 1) return [0.1];
|
||||
const fractions: number[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
fractions.push(0.1 + (0.8 * i) / (count - 1));
|
||||
}
|
||||
return fractions;
|
||||
}
|
||||
|
||||
function seekVideo(video: HTMLVideoElement, time: number): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Assigning currentTime to (approximately) its current value is a no-op and
|
||||
// never fires `seeked` (HTML spec) — which would hang for zero-length clips
|
||||
// or a first frame already at t=0. Short-circuit those.
|
||||
if (Math.abs(video.currentTime - time) < 0.01) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const cleanup = () => {
|
||||
video.removeEventListener('seeked', onSeeked);
|
||||
video.removeEventListener('error', onError);
|
||||
};
|
||||
const onSeeked = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
reject(new Error('video seek failed'));
|
||||
};
|
||||
video.addEventListener('seeked', onSeeked);
|
||||
video.addEventListener('error', onError);
|
||||
video.currentTime = time;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample `count` still frames from a video file as JPEG data URIs by decoding it
|
||||
* in a detached `<video>` element and painting each seek point onto a `<canvas>`.
|
||||
* The full clip is never uploaded — only these frames ride the `[IMAGE:]` vision
|
||||
* path. Throws if the browser can't decode the file (the caller maps that to a
|
||||
* `read_failed` error). Requires a real codec-capable runtime (CEF/Chromium);
|
||||
* jsdom can't decode video, so unit tests stub {@link videoFrameExtractor}.
|
||||
*/
|
||||
async function extractVideoFramesImpl(
|
||||
file: File,
|
||||
count: number = VIDEO_FRAME_COUNT
|
||||
): Promise<string[]> {
|
||||
const url = URL.createObjectURL(file);
|
||||
const video = document.createElement('video');
|
||||
video.muted = true;
|
||||
video.preload = 'auto';
|
||||
video.src = url;
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
video.onloadedmetadata = () => resolve();
|
||||
video.onerror = () => reject(new Error('video metadata load failed'));
|
||||
});
|
||||
const duration = Number.isFinite(video.duration) && video.duration > 0 ? video.duration : 0;
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = video.videoWidth || 320;
|
||||
canvas.height = video.videoHeight || 240;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) throw new Error('canvas 2d context unavailable');
|
||||
|
||||
const frames: string[] = [];
|
||||
for (const fraction of sampleFractions(count)) {
|
||||
const target = duration ? Math.min(duration * fraction, Math.max(duration - 0.05, 0)) : 0;
|
||||
await seekVideo(video, target);
|
||||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
frames.push(canvas.toDataURL('image/jpeg', 0.7));
|
||||
}
|
||||
debug('[chat:attachments] video_frames name=%s count=%d', file.name, frames.length);
|
||||
return frames;
|
||||
} finally {
|
||||
URL.revokeObjectURL(url);
|
||||
video.removeAttribute('src');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Indirection seam so unit tests can stub frame extraction (jsdom has no video
|
||||
* decoder). Production calls `extract` directly.
|
||||
*/
|
||||
export const videoFrameExtractor = { extract: extractVideoFramesImpl };
|
||||
|
||||
export function extractVideoFrames(file: File, count?: number): Promise<string[]> {
|
||||
return videoFrameExtractor.extract(file, count);
|
||||
}
|
||||
|
||||
/** Image-marker cost of an attachment kind (video = its sampled frames). */
|
||||
export function imageMarkerCost(kind: AttachmentKind): number {
|
||||
if (kind === 'image') return 1;
|
||||
if (kind === 'video') return VIDEO_FRAME_COUNT;
|
||||
return 0;
|
||||
}
|
||||
|
||||
export async function validateAndReadFile(
|
||||
file: File,
|
||||
existingCount: number,
|
||||
// Image-marker slots already consumed this message: 1 per image + VIDEO_FRAME_COUNT
|
||||
// per video. Images and videos share one budget (ATTACHMENT_MAX_IMAGES) because
|
||||
// the core counts every `[IMAGE:]` marker — frames included — and rejects overflow.
|
||||
existingImageMarkers: number,
|
||||
existingFileCount = 0,
|
||||
// When `false` (the active chat model isn't vision-capable), image files are
|
||||
// rejected; documents (PDF/Word/etc.) still flow. Defaults `true` so non-chat
|
||||
// When `false` (the active chat model isn't vision-capable), image AND video
|
||||
// files are rejected (video is conveyed as sampled frames through the vision
|
||||
// path); documents (PDF/Word/etc.) still flow. Defaults `true` so non-chat
|
||||
// callers are unaffected.
|
||||
allowImages = true
|
||||
): Promise<{ attachment: Attachment } | { error: AttachmentError }> {
|
||||
@@ -197,19 +335,55 @@ export async function validateAndReadFile(
|
||||
if (!allowImages && kind === 'image') {
|
||||
return { error: { code: 'image_not_supported' } };
|
||||
}
|
||||
const maxCount = kind === 'image' ? ATTACHMENT_MAX_IMAGES : ATTACHMENT_MAX_FILES;
|
||||
const count = kind === 'image' ? existingCount : existingFileCount;
|
||||
if (count >= maxCount) {
|
||||
return { error: { code: 'too_many', kind, max: maxCount } };
|
||||
if (!allowImages && kind === 'video') {
|
||||
return { error: { code: 'video_not_supported' } };
|
||||
}
|
||||
|
||||
if (kind === 'file') {
|
||||
if (existingFileCount >= ATTACHMENT_MAX_FILES) {
|
||||
return { error: { code: 'too_many', kind: 'file', max: ATTACHMENT_MAX_FILES } };
|
||||
}
|
||||
} else {
|
||||
// image or video: budget against the shared image-marker cap.
|
||||
if (existingImageMarkers + imageMarkerCost(kind) > ATTACHMENT_MAX_IMAGES) {
|
||||
return { error: { code: 'too_many', kind: 'image', max: ATTACHMENT_MAX_IMAGES } };
|
||||
}
|
||||
}
|
||||
|
||||
const maxBytes =
|
||||
kind === 'image' ? ATTACHMENT_MAX_IMAGE_SIZE_BYTES : ATTACHMENT_MAX_FILE_SIZE_BYTES;
|
||||
kind === 'image'
|
||||
? ATTACHMENT_MAX_IMAGE_SIZE_BYTES
|
||||
: kind === 'video'
|
||||
? ATTACHMENT_MAX_VIDEO_SIZE_BYTES
|
||||
: ATTACHMENT_MAX_FILE_SIZE_BYTES;
|
||||
if (file.size > maxBytes) {
|
||||
return { error: { code: 'too_large', sizeBytes: file.size, maxBytes } };
|
||||
}
|
||||
|
||||
try {
|
||||
if (kind === 'video') {
|
||||
const frames = await videoFrameExtractor.extract(file);
|
||||
if (frames.length === 0) {
|
||||
return { error: { code: 'read_failed', reason: 'no frames extracted' } };
|
||||
}
|
||||
return {
|
||||
attachment: {
|
||||
id: globalThis.crypto.randomUUID(),
|
||||
kind: 'video',
|
||||
file,
|
||||
// The clip is represented to the agent by its frames, not a data URI;
|
||||
// the poster (first frame) drives the chip thumbnail.
|
||||
dataUri: frames[0],
|
||||
previewUri: frames[0],
|
||||
mimeType: file.type,
|
||||
originalSizeBytes: file.size,
|
||||
payloadSizeBytes: file.size,
|
||||
compressed: false,
|
||||
frames,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const { dataUri, payloadSizeBytes, compressed } = await buildAttachmentDataUri(file, file.type);
|
||||
const previewUri = kind === 'image' ? await fileToDataUri(file) : undefined;
|
||||
return {
|
||||
@@ -241,7 +415,14 @@ export async function validateAndReadFile(
|
||||
export function buildMessageWithAttachments(text: string, attachments: Attachment[]): string {
|
||||
if (attachments.length === 0) return text;
|
||||
const markers = attachments
|
||||
.map(a => (a.kind === 'image' ? `[IMAGE:${a.dataUri}]` : `[FILE:${a.dataUri}]`))
|
||||
.map(a => {
|
||||
if (a.kind === 'image') return `[IMAGE:${a.dataUri}]`;
|
||||
if (a.kind === 'file') return `[FILE:${a.dataUri}]`;
|
||||
// Video: forward each sampled still as its own image marker so the agent
|
||||
// "sees" the clip through the existing vision path.
|
||||
return (a.frames ?? []).map(frame => `[IMAGE:${frame}]`).join(' ');
|
||||
})
|
||||
.filter(marker => marker.length > 0)
|
||||
.join(' ');
|
||||
return text.trim() ? `${text.trim()} ${markers}` : markers;
|
||||
}
|
||||
|
||||
@@ -2216,6 +2216,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': 'إزالة {name}',
|
||||
'chat.attachment.tooMany': 'الحد الأقصى {max} صور لكل رسالة',
|
||||
'chat.attachment.tooManyFiles': 'الحد الأقصى {max} ملفات لكل رسالة',
|
||||
'chat.attachment.tooManyVideos': 'الحد الأقصى {max} مقاطع فيديو لكل رسالة',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'لا يمكن لهذا النموذج قراءة مقاطع الفيديو. يمكنك استخدام مستوى التفكير من OpenHuman لإرفاق الفيديو.',
|
||||
'chat.attachment.dropToAttach': 'أفلت الملفات لإرفاقها',
|
||||
'chat.attachment.tooLarge': 'حجم الصورة يتجاوز الحد المسموح {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'نوع ملف غير مدعوم. استخدم صورة (PNG أو JPEG أو WebP أو GIF أو BMP) أو ملف PDF أو TXT أو Markdown.',
|
||||
|
||||
@@ -2268,6 +2268,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': '{name} সরান',
|
||||
'chat.attachment.tooMany': 'প্রতি বার্তায় সর্বোচ্চ {max}টি ছবি',
|
||||
'chat.attachment.tooManyFiles': 'প্রতি বার্তায় সর্বোচ্চ {max}টি ফাইল',
|
||||
'chat.attachment.tooManyVideos': 'প্রতি বার্তায় সর্বোচ্চ {max}টি ভিডিও',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'এই মডেল ভিডিও পড়তে পারে না। আপনার ভিডিও সংযুক্ত করতে আপনি OpenHuman যুক্তি স্তর ব্যবহার করতে পারেন।',
|
||||
'chat.attachment.dropToAttach': 'সংযুক্ত করতে ফাইল ছেড়ে দিন',
|
||||
'chat.attachment.tooLarge': 'ছবি {max} আকারের সীমা অতিক্রম করেছে',
|
||||
'chat.attachment.unsupportedType':
|
||||
'অসমর্থিত ফাইল প্রকার। একটি ছবি (PNG, JPEG, WebP, GIF, BMP) অথবা একটি PDF, TXT, বা Markdown ফাইল ব্যবহার করুন।',
|
||||
|
||||
@@ -2321,6 +2321,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': '{name} entfernen',
|
||||
'chat.attachment.tooMany': 'Maximal {max} Bilder pro Nachricht',
|
||||
'chat.attachment.tooManyFiles': 'Maximal {max} Dateien pro Nachricht',
|
||||
'chat.attachment.tooManyVideos': 'Maximal {max} Videos pro Nachricht',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'Dieses Modell kann keine Videos lesen. Du kannst die OpenHuman-Stufe „Denkmodus“ verwenden, um dein Video anzuhängen.',
|
||||
'chat.attachment.dropToAttach': 'Dateien zum Anhängen ablegen',
|
||||
'chat.attachment.tooLarge': 'Bild überschreitet die Größenbeschränkung von {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Nicht unterstützter Dateityp. Verwenden Sie ein Bild (PNG, JPEG, WebP, GIF, BMP) oder eine PDF-, TXT- oder Markdown-Datei.',
|
||||
|
||||
@@ -2666,6 +2666,10 @@ const en: TranslationMap = {
|
||||
'chat.attachment.remove': 'Remove {name}',
|
||||
'chat.attachment.tooMany': 'Maximum {max} images per message',
|
||||
'chat.attachment.tooManyFiles': 'Maximum {max} files per message',
|
||||
'chat.attachment.tooManyVideos': 'Maximum {max} videos per message',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'This model can’t read videos. You can use the OpenHuman Reasoning tier to attach your video.',
|
||||
'chat.attachment.dropToAttach': 'Drop files to attach',
|
||||
'chat.attachment.tooLarge': 'Image exceeds {max} size limit',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Unsupported file type. Use an image (PNG, JPEG, WebP, GIF, BMP) or a PDF, TXT, or Markdown file.',
|
||||
|
||||
@@ -2308,6 +2308,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': 'Eliminar {name}',
|
||||
'chat.attachment.tooMany': 'Máximo {max} imágenes por mensaje',
|
||||
'chat.attachment.tooManyFiles': 'Máximo {max} archivos por mensaje',
|
||||
'chat.attachment.tooManyVideos': 'Máximo {max} vídeos por mensaje',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'Este modelo no puede leer vídeos. Puedes usar el nivel Razonamiento de OpenHuman para adjuntar tu vídeo.',
|
||||
'chat.attachment.dropToAttach': 'Suelta los archivos para adjuntar',
|
||||
'chat.attachment.tooLarge': 'La imagen supera el límite de tamaño de {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Tipo de archivo no compatible. Use una imagen (PNG, JPEG, WebP, GIF, BMP) o un archivo PDF, TXT o Markdown.',
|
||||
|
||||
@@ -2321,6 +2321,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': 'Supprimer {name}',
|
||||
'chat.attachment.tooMany': 'Maximum {max} images par message',
|
||||
'chat.attachment.tooManyFiles': 'Maximum {max} fichiers par message',
|
||||
'chat.attachment.tooManyVideos': 'Maximum {max} vidéos par message',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'Ce modèle ne peut pas lire les vidéos. Tu peux utiliser le niveau Raisonnement d’OpenHuman pour joindre ta vidéo.',
|
||||
'chat.attachment.dropToAttach': 'Déposez les fichiers pour les joindre',
|
||||
'chat.attachment.tooLarge': "L'image dépasse la taille limite de {max}",
|
||||
'chat.attachment.unsupportedType':
|
||||
'Type de fichier non pris en charge. Utilise une image (PNG, JPEG, WebP, GIF, BMP) ou un fichier PDF, TXT ou Markdown.',
|
||||
|
||||
@@ -2265,6 +2265,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': '{name} हटाएं',
|
||||
'chat.attachment.tooMany': 'प्रति संदेश अधिकतम {max} छवियां',
|
||||
'chat.attachment.tooManyFiles': 'प्रति संदेश अधिकतम {max} फ़ाइलें',
|
||||
'chat.attachment.tooManyVideos': 'प्रति संदेश अधिकतम {max} वीडियो',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'यह मॉडल वीडियो नहीं पढ़ सकता। अपना वीडियो संलग्न करने के लिए आप OpenHuman तर्क टियर का उपयोग कर सकते हैं।',
|
||||
'chat.attachment.dropToAttach': 'संलग्न करने के लिए फ़ाइलें छोड़ें',
|
||||
'chat.attachment.tooLarge': 'छवि {max} आकार सीमा से अधिक है',
|
||||
'chat.attachment.unsupportedType':
|
||||
'असमर्थित फ़ाइल प्रकार। कोई छवि (PNG, JPEG, WebP, GIF, BMP) या PDF, TXT, या Markdown फ़ाइल का उपयोग करें।',
|
||||
|
||||
@@ -2269,6 +2269,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': 'Hapus {name}',
|
||||
'chat.attachment.tooMany': 'Maksimal {max} gambar per pesan',
|
||||
'chat.attachment.tooManyFiles': 'Maksimal {max} file per pesan',
|
||||
'chat.attachment.tooManyVideos': 'Maksimal {max} video per pesan',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'Model ini tidak dapat membaca video. Anda dapat menggunakan tingkat Penalaran OpenHuman untuk melampirkan video Anda.',
|
||||
'chat.attachment.dropToAttach': 'Lepaskan file untuk melampirkan',
|
||||
'chat.attachment.tooLarge': 'Gambar melebihi batas ukuran {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Jenis file tidak didukung. Gunakan gambar (PNG, JPEG, WebP, GIF, BMP) atau file PDF, TXT, atau Markdown.',
|
||||
|
||||
@@ -2302,6 +2302,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': 'Rimuovi {name}',
|
||||
'chat.attachment.tooMany': 'Massimo {max} immagini per messaggio',
|
||||
'chat.attachment.tooManyFiles': 'Massimo {max} file per messaggio',
|
||||
'chat.attachment.tooManyVideos': 'Massimo {max} video per messaggio',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'Questo modello non può leggere i video. Puoi usare il livello Ragionamento di OpenHuman per allegare il tuo video.',
|
||||
'chat.attachment.dropToAttach': 'Rilascia i file per allegarli',
|
||||
'chat.attachment.tooLarge': "L'immagine supera il limite di dimensione di {max}",
|
||||
'chat.attachment.unsupportedType':
|
||||
'Tipo di file non supportato. Usa un file immagine (PNG, JPEG, WebP, GIF, BMP) oppure PDF, TXT o Markdown.',
|
||||
|
||||
@@ -2242,6 +2242,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': '{name} 제거',
|
||||
'chat.attachment.tooMany': '메시지당 최대 {max}개 이미지',
|
||||
'chat.attachment.tooManyFiles': '메시지당 최대 {max}개 파일',
|
||||
'chat.attachment.tooManyVideos': '메시지당 최대 {max}개 동영상',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'이 모델은 동영상을 읽을 수 없습니다. 동영상을 첨부하려면 OpenHuman 추론 등급을 사용할 수 있습니다.',
|
||||
'chat.attachment.dropToAttach': '파일을 놓아 첨부',
|
||||
'chat.attachment.tooLarge': '이미지가 {max} 크기 제한을 초과합니다',
|
||||
'chat.attachment.unsupportedType':
|
||||
'지원되지 않는 파일 형식입니다. 이미지(PNG, JPEG, WebP, GIF, BMP) 또는 PDF, TXT, Markdown 파일을 사용하세요.',
|
||||
|
||||
@@ -2288,6 +2288,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': 'Usuń {name}',
|
||||
'chat.attachment.tooMany': 'Maksymalnie {max} obrazów na wiadomość',
|
||||
'chat.attachment.tooManyFiles': 'Maksymalnie {max} plików na wiadomość',
|
||||
'chat.attachment.tooManyVideos': 'Maksymalnie {max} filmów na wiadomość',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'Ten model nie odczytuje filmów. Możesz użyć poziomu Rozumowanie OpenHuman, aby dołączyć film.',
|
||||
'chat.attachment.dropToAttach': 'Upuść pliki, aby je dołączyć',
|
||||
'chat.attachment.tooLarge': 'Obraz przekracza limit rozmiaru {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Nieobsługiwany typ pliku. Użyj obrazu (PNG, JPEG, WebP, GIF, BMP) lub pliku PDF, TXT albo Markdown.',
|
||||
|
||||
@@ -2310,6 +2310,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': 'Remover {name}',
|
||||
'chat.attachment.tooMany': 'Máximo de {max} imagens por mensagem',
|
||||
'chat.attachment.tooManyFiles': 'Máximo de {max} arquivos por mensagem',
|
||||
'chat.attachment.tooManyVideos': 'Máximo de {max} vídeos por mensagem',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'Este modelo não consegue ler vídeos. Você pode usar o nível Raciocínio do OpenHuman para anexar seu vídeo.',
|
||||
'chat.attachment.dropToAttach': 'Solte os arquivos para anexar',
|
||||
'chat.attachment.tooLarge': 'A imagem excede o limite de tamanho de {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Tipo de arquivo não suportado. Use uma imagem (PNG, JPEG, WebP, GIF, BMP) ou um arquivo PDF, TXT ou Markdown.',
|
||||
|
||||
@@ -2284,6 +2284,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': 'Удалить {name}',
|
||||
'chat.attachment.tooMany': 'Максимум {max} изображений на сообщение',
|
||||
'chat.attachment.tooManyFiles': 'Максимум {max} файлов на сообщение',
|
||||
'chat.attachment.tooManyVideos': 'Максимум {max} видео на сообщение',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'Эта модель не может читать видео. Вы можете использовать уровень «Рассуждение» от OpenHuman, чтобы прикрепить видео.',
|
||||
'chat.attachment.dropToAttach': 'Перетащите файлы, чтобы прикрепить',
|
||||
'chat.attachment.tooLarge': 'Изображение превышает ограничение размера {max}',
|
||||
'chat.attachment.unsupportedType':
|
||||
'Неподдерживаемый тип файла. Используйте изображение (PNG, JPEG, WebP, GIF, BMP) или файл PDF, TXT либо Markdown.',
|
||||
|
||||
@@ -2145,6 +2145,10 @@ const messages: TranslationMap = {
|
||||
'chat.attachment.remove': '移除 {name}',
|
||||
'chat.attachment.tooMany': '每条消息最多 {max} 张图片',
|
||||
'chat.attachment.tooManyFiles': '每条消息最多 {max} 个文件',
|
||||
'chat.attachment.tooManyVideos': '每条消息最多 {max} 个视频',
|
||||
'chat.attachment.videoNotSupported':
|
||||
'此模型无法读取视频。您可以使用 OpenHuman 推理层级来附加视频。',
|
||||
'chat.attachment.dropToAttach': '拖放文件以添加',
|
||||
'chat.attachment.tooLarge': '图片超过 {max} 大小限制',
|
||||
'chat.attachment.unsupportedType':
|
||||
'不支持的文件类型。请使用图片（PNG、JPEG、WebP、GIF、BMP）或 PDF、TXT、Markdown 文件。',
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
ATTACHMENT_MAX_FILES,
|
||||
ATTACHMENT_MAX_IMAGES,
|
||||
buildMessageWithAttachments,
|
||||
imageMarkerCost,
|
||||
parseMessageImages,
|
||||
validateAndReadFile,
|
||||
} from '../lib/attachments';
|
||||
@@ -930,17 +931,23 @@ const Conversations = ({
|
||||
return true;
|
||||
};
|
||||
|
||||
const handleAttachFiles = async (files: FileList | null) => {
|
||||
const handleAttachFiles = async (files: FileList | File[] | null) => {
|
||||
if (!files) return;
|
||||
let acceptedImageCount = attachments.filter(attachment => attachment.kind === 'image').length;
|
||||
let acceptedFileCount = attachments.filter(attachment => attachment.kind === 'file').length;
|
||||
// Images and videos share one image-marker budget (video = its frames), so
|
||||
// track consumed markers rather than per-kind counts.
|
||||
let acceptedImageMarkers = attachments.reduce(
|
||||
(sum, attachment) => sum + imageMarkerCost(attachment.kind),
|
||||
0
|
||||
);
|
||||
for (const file of Array.from(files)) {
|
||||
const result = await validateAndReadFile(
|
||||
file,
|
||||
acceptedImageCount,
|
||||
acceptedImageMarkers,
|
||||
acceptedFileCount,
|
||||
// Allow the image when the active model is vision-capable OR a vision
|
||||
// sub-agent can take it (orchestrator delegates the image onward).
|
||||
// Allow images AND video when the active model is vision-capable OR a
|
||||
// vision sub-agent can take it (orchestrator delegates the image/frames
|
||||
// onward). Video is sampled into still frames that ride the same path.
|
||||
modelSupportsVision || visionDelegateAvailable
|
||||
);
|
||||
if ('error' in result) {
|
||||
@@ -949,9 +956,14 @@ const Conversations = ({
|
||||
setAttachError(
|
||||
chatSendError('attachment_invalid', t('chat.attachment.imageNotSupported'))
|
||||
);
|
||||
} else if (error.code === 'video_not_supported') {
|
||||
setAttachError(
|
||||
chatSendError('attachment_invalid', t('chat.attachment.videoNotSupported'))
|
||||
);
|
||||
} else if (error.code === 'too_many') {
|
||||
// image/video share the image-marker budget → tooMany; files separate.
|
||||
const key =
|
||||
error.kind === 'image' ? 'chat.attachment.tooMany' : 'chat.attachment.tooManyFiles';
|
||||
error.kind === 'file' ? 'chat.attachment.tooManyFiles' : 'chat.attachment.tooMany';
|
||||
setAttachError(
|
||||
chatSendError('attachment_invalid', t(key).replace('{max}', String(error.max)))
|
||||
);
|
||||
@@ -970,10 +982,10 @@ const Conversations = ({
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (result.attachment.kind === 'image') {
|
||||
acceptedImageCount++;
|
||||
} else {
|
||||
if (result.attachment.kind === 'file') {
|
||||
acceptedFileCount++;
|
||||
} else {
|
||||
acceptedImageMarkers += imageMarkerCost(result.attachment.kind);
|
||||
}
|
||||
setAttachments(prev => [...prev, result.attachment]);
|
||||
}
|
||||
@@ -1051,6 +1063,11 @@ const Conversations = ({
|
||||
attachmentDataUris: pendingAttachments
|
||||
.filter(a => a.kind === 'image')
|
||||
.map(a => a.previewUri ?? a.dataUri),
|
||||
// Poster (first frame) per attachment, index-aligned with
|
||||
// attachmentKinds — only video entries carry one; others null.
|
||||
attachmentPosters: pendingAttachments.map(a =>
|
||||
a.kind === 'video' ? (a.previewUri ?? a.dataUri) : null
|
||||
),
|
||||
attachmentCompressed: pendingAttachments.map(a => a.compressed),
|
||||
}
|
||||
: {},
|
||||
@@ -1170,6 +1187,11 @@ const Conversations = ({
|
||||
attachmentDataUris: pendingAttachments
|
||||
.filter(a => a.kind === 'image')
|
||||
.map(a => a.previewUri ?? a.dataUri),
|
||||
// Poster (first frame) per attachment, index-aligned with
|
||||
// attachmentKinds — only video entries carry one; others null.
|
||||
attachmentPosters: pendingAttachments.map(a =>
|
||||
a.kind === 'video' ? (a.previewUri ?? a.dataUri) : null
|
||||
),
|
||||
attachmentCompressed: pendingAttachments.map(a => a.compressed),
|
||||
parallelBranch: true,
|
||||
}
|
||||
@@ -1250,6 +1272,11 @@ const Conversations = ({
|
||||
attachmentDataUris: pendingAttachments
|
||||
.filter(a => a.kind === 'image')
|
||||
.map(a => a.previewUri ?? a.dataUri),
|
||||
// Poster (first frame) per attachment, index-aligned with
|
||||
// attachmentKinds — only video entries carry one; others null.
|
||||
attachmentPosters: pendingAttachments.map(a =>
|
||||
a.kind === 'video' ? (a.previewUri ?? a.dataUri) : null
|
||||
),
|
||||
attachmentCompressed: pendingAttachments.map(a => a.compressed),
|
||||
}
|
||||
: {},
|
||||
@@ -2281,6 +2308,18 @@ const Conversations = ({
|
||||
const fileNames = kinds
|
||||
.map((k, i) => (k === 'file' ? names[i] : null))
|
||||
.filter((n): n is string => Boolean(n));
|
||||
const posters = Array.isArray(msg.extraMetadata?.attachmentPosters)
|
||||
? (msg.extraMetadata.attachmentPosters as (string | null)[])
|
||||
: [];
|
||||
const videoItems = kinds
|
||||
.map((k, i) =>
|
||||
k === 'video'
|
||||
? { name: names[i] ?? '', poster: posters[i] ?? null }
|
||||
: null
|
||||
)
|
||||
.filter((v): v is { name: string; poster: string | null } =>
|
||||
Boolean(v)
|
||||
);
|
||||
const showTime = latestVisibleMessage?.id === msg.id;
|
||||
return (
|
||||
<>
|
||||
@@ -2296,6 +2335,47 @@ const Conversations = ({
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{videoItems.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5 justify-end">
|
||||
{videoItems.map((video, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="relative flex items-center gap-2 rounded-lg border border-line bg-surface-muted px-2.5 py-1.5 text-xs text-content-secondary max-w-[220px]">
|
||||
{video.poster ? (
|
||||
<div className="relative w-10 h-10 flex-shrink-0">
|
||||
<img
|
||||
src={video.poster}
|
||||
alt=""
|
||||
className="w-10 h-10 rounded object-cover"
|
||||
/>
|
||||
<span className="absolute inset-0 flex items-center justify-center">
|
||||
<svg
|
||||
className="w-4 h-4 text-white drop-shadow"
|
||||
fill="currentColor"
|
||||
viewBox="0 0 24 24">
|
||||
<path d="M8 5v14l11-7z" />
|
||||
</svg>
|
||||
</span>
|
||||
</div>
|
||||
) : (
|
||||
<svg
|
||||
className="w-4 h-4 flex-shrink-0 text-content-muted"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24">
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={1.8}
|
||||
d="M15 10l4.553-2.276A1 1 0 0121 8.618v6.764a1 1 0 01-1.447.894L15 14M5 6h8a2 2 0 012 2v8a2 2 0 01-2 2H5a2 2 0 01-2-2V8a2 2 0 012-2z"
|
||||
/>
|
||||
</svg>
|
||||
)}
|
||||
<span className="truncate font-medium">{video.name}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{fileNames.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5 justify-end">
|
||||
{fileNames.map((name, i) => (
|
||||
|
||||
@@ -638,6 +638,63 @@ describe('Conversations — attachment feature', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('renders a video poster chip in the user bubble from attachmentKinds/Posters', async () => {
|
||||
const thread = makeThread({ id: 'video-thread', title: 'Video Thread' });
|
||||
const message = {
|
||||
id: 'msg-video-1',
|
||||
content: 'whats in this clip',
|
||||
type: 'text' as const,
|
||||
sender: 'user' as const,
|
||||
createdAt: new Date().toISOString(),
|
||||
extraMetadata: {
|
||||
attachmentCount: 1,
|
||||
attachmentKinds: ['video'],
|
||||
attachmentNames: ['demo.mp4'],
|
||||
attachmentPosters: ['data:image/jpeg;base64,poster'],
|
||||
},
|
||||
};
|
||||
|
||||
mockGetThreads.mockResolvedValue({ threads: [thread], count: 1 });
|
||||
mockGetThreadMessages.mockResolvedValue({ messages: [message], count: 1 });
|
||||
|
||||
const store = buildStore({
|
||||
thread: {
|
||||
threads: [thread],
|
||||
selectedThreadId: thread.id,
|
||||
activeThreadIds: {},
|
||||
welcomeThreadId: null,
|
||||
messagesByThreadId: { [thread.id]: [message] },
|
||||
messages: [message],
|
||||
isLoadingThreads: false,
|
||||
isLoadingMessages: false,
|
||||
messagesError: null,
|
||||
},
|
||||
socket: socketState('connected'),
|
||||
});
|
||||
|
||||
const { default: Conversations } = await import('../Conversations');
|
||||
|
||||
render(
|
||||
<Provider store={store}>
|
||||
<MemoryRouter>
|
||||
<SidebarSlotProvider>
|
||||
<SidebarSlotOutlet />
|
||||
<Conversations />
|
||||
</SidebarSlotProvider>
|
||||
</MemoryRouter>
|
||||
</Provider>
|
||||
);
|
||||
|
||||
// The video attachment surfaces as a filename chip with its poster <img>.
|
||||
await waitFor(() => {
|
||||
expect(document.body.textContent).toContain('demo.mp4');
|
||||
});
|
||||
const poster = Array.from(document.querySelectorAll('img')).find(
|
||||
img => (img as HTMLImageElement).src === 'data:image/jpeg;base64,poster'
|
||||
);
|
||||
expect(poster).toBeTruthy();
|
||||
});
|
||||
|
||||
it('strips raw IMAGE/FILE markers from a legacy message with no extraMetadata', async () => {
|
||||
const writeText = vi.fn().mockResolvedValue(undefined);
|
||||
Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } });
|
||||
|
||||
@@ -91,11 +91,13 @@ export const CONSUMER_FIRST_SESSION_ENABLED =
|
||||
import.meta.env.VITE_CONSUMER_FIRST_SESSION === 'true';
|
||||
|
||||
/**
|
||||
* Chat multimodal attachments (image + supported file markers). Disabled by
|
||||
* default — the attach affordance and file-picker path are off. Opt in for a
|
||||
* build by setting `VITE_CHAT_ATTACHMENTS=true`.
|
||||
* Master kill-switch for chat multimodal attachments (image / video / document).
|
||||
* Enabled by default; the actual affordance is gated on the resolved model's
|
||||
* capability tier at the call site (`Conversations.tsx`) — images and video need
|
||||
* a vision-capable tier, documents flow on any model. Set
|
||||
* `VITE_CHAT_ATTACHMENTS=false` to hard-disable the whole feature for a build.
|
||||
*/
|
||||
export const CHAT_ATTACHMENTS_ENABLED = import.meta.env.VITE_CHAT_ATTACHMENTS === 'true';
|
||||
export const CHAT_ATTACHMENTS_ENABLED = import.meta.env.VITE_CHAT_ATTACHMENTS !== 'false';
|
||||
|
||||
export const SKILLS_GITHUB_REPO =
|
||||
import.meta.env.VITE_SKILLS_GITHUB_REPO || 'tinyhumansai/openhuman-skills';
|
||||
|
||||
@@ -187,6 +187,8 @@ Canonical mapping of every product feature to its test source(s). Drives gap-fil
|
||||
| 4.2.6 | Background-activity panel (chat-header Background tasks button) | VU+WD | `app/src/pages/conversations/hooks/useBackgroundActivity.test.ts`, `app/src/pages/conversations/components/__tests__/BackgroundActivityRows.test.tsx`, `app/test/e2e/specs/chat-background-activity-panel.spec.ts` | ✅ | View-only panel surfacing this chat's async sub-agents + global cron jobs, subconscious/heartbeat status, and memory syncing; freshness-only "Syncing now" labeling; E2E opens the panel and asserts its sections render and close |
|
||||
| 4.2.7 | Plan-mode review (Approve / Reject / Send-feedback before execute) | RU+RI+VU | `src/openhuman/plan_review/gate.rs`, `src/openhuman/plan_review/tool.rs`, `src/openhuman/plan_review/schemas.rs`, `tests/json_rpc_e2e.rs`, `app/src/pages/conversations/components/PlanReviewCard.test.tsx`, `app/src/pages/__tests__/Conversations.render.test.tsx` | ✅ | Interactive turns call `request_plan_review`, which parks the LIVE turn on the in-memory `PlanReviewGate` (oneshot) until the user decides; `plan_review_request` socket event drives `PlanReviewCard`, which resolves via `openhuman.plan_review_decide` (approve resumes-and-executes / reject resumes-and-stops / revise resumes-with-feedback). RU covers gate park/resolve/timeout + tool auto-approve + parking; RI covers the decide RPC; VU covers the card + provider wiring. WD E2E (agent-driven park flow) tracked as follow-up |
|
||||
|
||||
| 4.2.8 | Composer attachments (image / video / document; drag-drop + paste) | VU | `app/src/lib/attachments.test.ts`, `app/src/components/chat/__tests__/ChatComposer.test.tsx`, `app/src/pages/__tests__/Conversations.attachments.test.tsx` | 🟡 | Attach affordance gated on the resolved vision tier (images/video need vision; documents flow on any model); video is sampled into still frames client-side and forwarded through the existing `[IMAGE:]` vision path; drag-drop + clipboard-paste reuse the picker ingest. VU covers MIME/kind/limits/marker building + drag-drop + paste; real video decode and the frames→vision round-trip are manual-smoke only (jsdom has no video codec). WD E2E is a follow-up |
|
||||
|
||||
### 4.3 Tool Invocation
|
||||
|
||||
| ID | Feature | Layer | Test path(s) | Status | Notes |
|
||||
|
||||
Reference in New Issue
Block a user