diff --git a/.env.example b/.env.example index 826cc2e4c..1e6b053d2 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,9 @@ # VITE_BACKEND_URL=https://staging-api.tinyhumans.ai # [optional] Consumer first-session UX in the desktop/web app (default off). See docs/plans/consumer-first-session-spec.md # VITE_CONSUMER_FIRST_SESSION=true +# [optional] Chat attachments (image/video/document) master kill-switch. Enabled +# by default; per-message modalities gated on the model's vision tier at runtime. +# VITE_CHAT_ATTACHMENTS=false # --------------------------------------------------------------------------- # Authentication (for skills OAuth proxy and debug scripts) diff --git a/app/.env.example b/app/.env.example index d0b490328..716b1e03d 100644 --- a/app/.env.example +++ b/app/.env.example @@ -80,6 +80,11 @@ VITE_DEV_FORCE_ONBOARDING=false # [optional] Consumer first-session + home IA experiments (default off). See docs/plans/consumer-first-session-spec.md # VITE_CONSUMER_FIRST_SESSION=true +# [optional] Chat attachments (image / video / document) master kill-switch. +# Enabled by default; per-message modalities are gated on the model's vision +# tier at runtime. Set to false to hard-disable the attach button for a build. +# VITE_CHAT_ATTACHMENTS=false + # [optional] Client-side timeout for skill callTool/triggerSync (seconds; default 120, max 3600). # Should match OPENHUMAN_TOOL_TIMEOUT_SECS on the core when set. # VITE_TOOL_TIMEOUT_SECS= diff --git a/app/src/components/chat/AttachmentPreview.tsx b/app/src/components/chat/AttachmentPreview.tsx index a6ecd386b..9767935c6 100644 --- a/app/src/components/chat/AttachmentPreview.tsx +++ b/app/src/components/chat/AttachmentPreview.tsx @@ -28,6 +28,22 @@ export default function AttachmentPreview({ alt={attachment.file.name} className="w-8 h-8 rounded object-cover flex-shrink-0" /> + ) : attachment.kind === 'video' ? ( +
+ {attachment.file.name} + + + + + +
) : (
Promise; + onAttachFiles: (files: FileList | File[] | null) => Promise; onRemoveAttachment: (id: string) => void; attachError: ChatSendError | null; onSwitchToMicCloud: () => void; @@ -85,6 +85,7 @@ export default function ChatComposer({ headerSlots = [], }: ChatComposerProps) { const { t } = useT(); + const [isDragging, setIsDragging] = useState(false); // While a turn streams (`allowParallelSend`) the composer stays usable for a // queued follow-up / parallel branch, so the in-flight `isSending` spinner @@ -106,6 +107,57 @@ export default function ChatComposer({ const hasTypedContent = inputValue.trim().length > 0 || attachments.length > 0; const showStopButton = isSending && !!onStopGeneration && !hasTypedContent; + // Attachment ingest is blocked while the feature is off, the composer is + // locked, or the budget is full — drag-drop and paste honour the same gate as + // the [+] button so they can't bypass it. + const attachDisabled = + !attachmentsEnabled || + composerInteractionBlocked || + isSending || + attachments.length >= maxAttachments; + + // Drag-and-drop: route dropped files through the same handler as the picker. + // We always `preventDefault` for *file* drags so the browser/CEF never + // navigates away to the dropped file — even when ingest is disabled — and only + // attach (and show the overlay) when ingest is actually allowed. + const isFileDrag = (e: React.DragEvent) => + Array.from(e.dataTransfer?.types ?? []).includes('Files'); + const handleDragOver = (e: React.DragEvent) => { + if (!isFileDrag(e)) return; + e.preventDefault(); + if (!attachDisabled) setIsDragging(true); + }; + const handleDragLeave = (e: React.DragEvent) => { + // Ignore leave events that bubble while the cursor is still over a child. + if (e.currentTarget.contains(e.relatedTarget as Node | null)) return; + setIsDragging(false); + }; + const handleDrop = (e: React.DragEvent) => { + if (!isFileDrag(e)) return; + // Stop the default file-navigation before any early-return below. + e.preventDefault(); + setIsDragging(false); + if (attachDisabled) return; + const files = e.dataTransfer?.files; + if (!files || files.length === 0) return; + void onAttachFiles(files); + }; + + // Clipboard paste: pull image/video files out of the paste payload (e.g. a + // screenshot copied to the clipboard) and attach them; leave plain-text paste + // untouched so normal typing still works. + const handlePaste = (e: React.ClipboardEvent) => { + if (attachDisabled) return; + const items = Array.from(e.clipboardData?.items ?? []); + const files = items + .filter(item => item.kind === 'file' && /^(image|video)\//.test(item.type)) + .map(item => item.getAsFile()) + .filter((file): file is File => file !== null); + if (files.length === 0) return; + e.preventDefault(); + void onAttachFiles(files); + }; + // Auto-resize textarea: grow with content, cap at COMPOSER_MAX_HEIGHT, then scroll. useEffect(() => { const ta = textInputRef.current; @@ -123,7 +175,17 @@ export default function ChatComposer({ {headerSlots} {/* The input box — only this carries the focus-within highlight. */} -
+
+ {/* Drag-and-drop overlay: shown while a file drag hovers the composer. */} + {isDragging && ( +
+ {t('chat.attachment.dropToAttach')} +
+ )} {/* Hidden file input for attachment (gated — see attachmentsEnabled). */} {attachmentsEnabled && ( { expect(screen.queryByAltText('doc.pdf')).not.toBeInTheDocument(); }); + it('renders a video poster thumbnail with a play overlay', () => { + const file = new File([new Uint8Array(64)], 'clip.mp4', { type: 'video/mp4' }); + const att = makeAttachment({ + kind: 'video', + file, + dataUri: 'data:image/jpeg;base64,poster', + previewUri: 'data:image/jpeg;base64,poster', + mimeType: 'video/mp4', + frames: ['data:image/jpeg;base64,poster'], + }); + const { container } = render(); + const img = screen.getByAltText('clip.mp4') as HTMLImageElement; + expect(img.src).toBe('data:image/jpeg;base64,poster'); + expect(screen.getByText('clip.mp4')).toBeInTheDocument(); + // Explicitly assert the play overlay (the ▶ triangle drawn over the poster) + // is rendered — without this the test would still pass if the overlay + // disappeared, which is exactly the regression the title promises to catch. + expect(container.querySelector('path[d="M8 5v14l11-7z"]')).not.toBeNull(); + }); + it('calls onRemove with the attachment id when × is clicked', () => { const onRemove = vi.fn(); const att = makeAttachment({ id: 'att-42' }); diff --git a/app/src/components/chat/__tests__/ChatComposer.test.tsx b/app/src/components/chat/__tests__/ChatComposer.test.tsx index cb6f074c3..c8084e3fd 100644 --- a/app/src/components/chat/__tests__/ChatComposer.test.tsx +++ b/app/src/components/chat/__tests__/ChatComposer.test.tsx @@ -177,6 +177,72 @@ describe('ChatComposer', () => { expect(onSwitchToMicCloud).toHaveBeenCalledTimes(1); }); + describe('drag-and-drop', () => { + function makeVideoFile() { + return new File([new Uint8Array(8)], 'clip.mp4', { type: 'video/mp4' }); + } + + it('shows the drop overlay while a file drag is over the composer', () => { + renderComposer(); + const textarea = screen.getByRole('textbox'); + const box = textarea.closest('div.rounded-2xl') as HTMLElement; + fireEvent.dragOver(box, { dataTransfer: { types: ['Files'] } }); + expect(screen.getByText('chat.attachment.dropToAttach')).toBeInTheDocument(); + }); + + it('routes dropped files through onAttachFiles', () => { + const onAttachFiles = vi.fn().mockResolvedValue(undefined); + renderComposer({ onAttachFiles }); + const textarea = screen.getByRole('textbox'); + const box = textarea.closest('div.rounded-2xl') as HTMLElement; + const file = makeVideoFile(); + fireEvent.drop(box, { dataTransfer: { files: [file], types: ['Files'] } }); + expect(onAttachFiles).toHaveBeenCalledTimes(1); + }); + + it('does not handle drops when attachments are disabled', () => { + const onAttachFiles = vi.fn().mockResolvedValue(undefined); + renderComposer({ onAttachFiles, attachmentsEnabled: false }); + const textarea = screen.getByRole('textbox'); + const box = textarea.closest('div.rounded-2xl') as HTMLElement; + fireEvent.drop(box, { dataTransfer: { files: [makeVideoFile()], types: ['Files'] } }); + expect(onAttachFiles).not.toHaveBeenCalled(); + }); + + it('prevents browser file-navigation on drop even when disabled', () => { + // fireEvent returns false when the event default was prevented. + renderComposer({ attachmentsEnabled: false }); + const textarea = screen.getByRole('textbox'); + const box = textarea.closest('div.rounded-2xl') as HTMLElement; + const notPrevented = fireEvent.drop(box, { + dataTransfer: { files: [makeVideoFile()], types: ['Files'] }, + }); + expect(notPrevented).toBe(false); + }); + }); + + describe('clipboard paste', () => { + it('attaches image files pasted from the clipboard', () => { + const onAttachFiles = vi.fn().mockResolvedValue(undefined); + renderComposer({ onAttachFiles }); + const file = new File([new Uint8Array(4)], 'pasted.png', { type: 'image/png' }); + fireEvent.paste(screen.getByRole('textbox'), { + clipboardData: { items: [{ kind: 'file', type: 'image/png', getAsFile: () => file }] }, + }); + expect(onAttachFiles).toHaveBeenCalledTimes(1); + expect(onAttachFiles).toHaveBeenCalledWith([file]); + }); + + it('ignores plain-text paste (no media items)', () => { + const onAttachFiles = vi.fn().mockResolvedValue(undefined); + renderComposer({ onAttachFiles }); + fireEvent.paste(screen.getByRole('textbox'), { + clipboardData: { items: [{ kind: 'string', type: 'text/plain', getAsFile: () => null }] }, + }); + expect(onAttachFiles).not.toHaveBeenCalled(); + }); + }); + describe('follow-up / parallel mode (allowParallelSend during a streaming turn)', () => { it('keeps the textarea editable even while an in-flight turn is sending', () => { renderComposer({ diff --git a/app/src/lib/attachments.test.ts b/app/src/lib/attachments.test.ts index 4b67e1378..b758d27ae 100644 --- a/app/src/lib/attachments.test.ts +++ b/app/src/lib/attachments.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ALLOWED_ATTACHMENT_MIME_TYPES, @@ -6,12 +6,19 @@ import { ATTACHMENT_MAX_FILE_SIZE_BYTES, ATTACHMENT_MAX_IMAGES, ATTACHMENT_MAX_SIZE_BYTES, + ATTACHMENT_MAX_VIDEO_SIZE_BYTES, + attachmentKindForMime, buildMessageWithAttachments, + extractVideoFrames, fileToDataUri, formatFileSize, + imageMarkerCost, isAllowedMimeType, + isVideoMimeType, parseMessageImages, validateAndReadFile, + VIDEO_FRAME_COUNT, + videoFrameExtractor, } from './attachments'; function makeFile(name: string, type: string, size = 1024): File { @@ -188,11 +195,211 @@ describe('validateAndReadFile', () => { }); }); +describe('video attachments', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('classifies video MIME types as the video kind', () => { + expect(isVideoMimeType('video/mp4')).toBe(true); + expect(isVideoMimeType('video/quicktime')).toBe(true); + expect(isVideoMimeType('video/webm')).toBe(true); + expect(isVideoMimeType('image/png')).toBe(false); + expect(attachmentKindForMime('video/mp4')).toBe('video'); + }); + + it('costs 1 image marker per image and VIDEO_FRAME_COUNT per video', () => { + expect(imageMarkerCost('image')).toBe(1); + expect(imageMarkerCost('video')).toBe(VIDEO_FRAME_COUNT); + expect(imageMarkerCost('file')).toBe(0); + }); + + it('rejects video when allowImages is false (non-vision model)', async () => { + const result = await validateAndReadFile(makeFile('clip.mp4', 'video/mp4', 8), 0, 0, false); + expect('error' in result && result.error.code).toBe('video_not_supported'); + }); + + it('rejects video over the video size limit', async () => { + const big = makeFile('big.mp4', 'video/mp4', ATTACHMENT_MAX_VIDEO_SIZE_BYTES + 1); + const result = await validateAndReadFile(big, 0, 0, true); + expect('error' in result && result.error.code).toBe('too_large'); + }); + + it('rejects video when its frames would exceed the shared image-marker budget', async () => { + // Budget full of images already → no room for a video's frames. + const result = await validateAndReadFile( + makeFile('clip.mp4', 'video/mp4', 8), + ATTACHMENT_MAX_IMAGES, + 0, + true + ); + expect('error' in result && result.error.code).toBe('too_many'); + // Reported as the image budget (frames are images to the model). + expect('error' in result && result.error.code === 'too_many' && result.error.kind).toBe( + 'image' + ); + }); + + it('rejects a video that partially overflows the budget (images + frames)', async () => { + // One image used (1 marker); a video costs VIDEO_FRAME_COUNT more. With the + // default budget of 4, 1 + 2 = 3 fits, but seed close to the cap to overflow. + const used = ATTACHMENT_MAX_IMAGES - (VIDEO_FRAME_COUNT - 1); // leaves < cost + const result = await validateAndReadFile(makeFile('c.mp4', 'video/mp4', 8), used, 0, true); + expect('error' in result && result.error.code).toBe('too_many'); + }); + + it('samples frames and returns a video attachment within budget', async () => { + const frames = ['data:image/jpeg;base64,f1', 'data:image/jpeg;base64,f2']; + vi.spyOn(videoFrameExtractor, 'extract').mockResolvedValue(frames); + const file = makeFile('clip.mp4', 'video/mp4', 4096); + const result = await validateAndReadFile(file, 0, 0, true); + expect('attachment' in result).toBe(true); + if ('attachment' in result) { + expect(result.attachment.kind).toBe('video'); + expect(result.attachment.frames).toEqual(frames); + expect(result.attachment.previewUri).toBe(frames[0]); + expect(result.attachment.file).toBe(file); + } + }); + + it('fails when no frames could be extracted', async () => { + vi.spyOn(videoFrameExtractor, 'extract').mockResolvedValue([]); + const result = await validateAndReadFile(makeFile('clip.mp4', 'video/mp4', 8), 0, 0, true); + expect('error' in result && result.error.code).toBe('read_failed'); + }); +}); + +describe('extractVideoFrames (DOM-mocked)', () => { + // jsdom has no video codec, so stub
)} + {videoItems.length > 0 && ( +
+ {videoItems.map((video, i) => ( +
+ {video.poster ? ( +
+ + + + + + +
+ ) : ( + + + + )} + {video.name} +
+ ))} +
+ )} {fileNames.length > 0 && (
{fileNames.map((name, i) => ( diff --git a/app/src/pages/__tests__/Conversations.attachments.test.tsx b/app/src/pages/__tests__/Conversations.attachments.test.tsx index 5a026260a..e24e569ce 100644 --- a/app/src/pages/__tests__/Conversations.attachments.test.tsx +++ b/app/src/pages/__tests__/Conversations.attachments.test.tsx @@ -638,6 +638,63 @@ describe('Conversations — attachment feature', () => { }); }); + it('renders a video poster chip in the user bubble from attachmentKinds/Posters', async () => { + const thread = makeThread({ id: 'video-thread', title: 'Video Thread' }); + const message = { + id: 'msg-video-1', + content: 'whats in this clip', + type: 'text' as const, + sender: 'user' as const, + createdAt: new Date().toISOString(), + extraMetadata: { + attachmentCount: 1, + attachmentKinds: ['video'], + attachmentNames: ['demo.mp4'], + attachmentPosters: ['data:image/jpeg;base64,poster'], + }, + }; + + mockGetThreads.mockResolvedValue({ threads: [thread], count: 1 }); + mockGetThreadMessages.mockResolvedValue({ messages: [message], count: 1 }); + + const store = buildStore({ + thread: { + threads: [thread], + selectedThreadId: thread.id, + activeThreadIds: {}, + welcomeThreadId: null, + messagesByThreadId: { [thread.id]: [message] }, + messages: [message], + isLoadingThreads: false, + isLoadingMessages: false, + messagesError: null, + }, + socket: socketState('connected'), + }); + + const { default: Conversations } = await import('../Conversations'); + + render( + + + + + + + + + ); + + // The video attachment surfaces as a filename chip with its poster . + await waitFor(() => { + expect(document.body.textContent).toContain('demo.mp4'); + }); + const poster = Array.from(document.querySelectorAll('img')).find( + img => (img as HTMLImageElement).src === 'data:image/jpeg;base64,poster' + ); + expect(poster).toBeTruthy(); + }); + it('strips raw IMAGE/FILE markers from a legacy message with no extraMetadata', async () => { const writeText = vi.fn().mockResolvedValue(undefined); Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } }); diff --git a/app/src/utils/config.ts b/app/src/utils/config.ts index 269e6e2b7..6557e9a44 100644 --- a/app/src/utils/config.ts +++ b/app/src/utils/config.ts @@ -91,11 +91,13 @@ export const CONSUMER_FIRST_SESSION_ENABLED = import.meta.env.VITE_CONSUMER_FIRST_SESSION === 'true'; /** - * Chat multimodal attachments (image + supported file markers). Disabled by - * default — the attach affordance and file-picker path are off. Opt in for a - * build by setting `VITE_CHAT_ATTACHMENTS=true`. + * Master kill-switch for chat multimodal attachments (image / video / document). + * Enabled by default; the actual affordance is gated on the resolved model's + * capability tier at the call site (`Conversations.tsx`) — images and video need + * a vision-capable tier, documents flow on any model. Set + * `VITE_CHAT_ATTACHMENTS=false` to hard-disable the whole feature for a build. */ -export const CHAT_ATTACHMENTS_ENABLED = import.meta.env.VITE_CHAT_ATTACHMENTS === 'true'; +export const CHAT_ATTACHMENTS_ENABLED = import.meta.env.VITE_CHAT_ATTACHMENTS !== 'false'; export const SKILLS_GITHUB_REPO = import.meta.env.VITE_SKILLS_GITHUB_REPO || 'tinyhumansai/openhuman-skills'; diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index 795312d19..1ff3db5f3 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -187,6 +187,8 @@ Canonical mapping of every product feature to its test source(s). Drives gap-fil | 4.2.6 | Background-activity panel (chat-header Background tasks button) | VU+WD | `app/src/pages/conversations/hooks/useBackgroundActivity.test.ts`, `app/src/pages/conversations/components/__tests__/BackgroundActivityRows.test.tsx`, `app/test/e2e/specs/chat-background-activity-panel.spec.ts` | ✅ | View-only panel surfacing this chat's async sub-agents + global cron jobs, subconscious/heartbeat status, and memory syncing; freshness-only "Syncing now" labeling; E2E opens the panel and asserts its sections render and close | | 4.2.7 | Plan-mode review (Approve / Reject / Send-feedback before execute) | RU+RI+VU | `src/openhuman/plan_review/gate.rs`, `src/openhuman/plan_review/tool.rs`, `src/openhuman/plan_review/schemas.rs`, `tests/json_rpc_e2e.rs`, `app/src/pages/conversations/components/PlanReviewCard.test.tsx`, `app/src/pages/__tests__/Conversations.render.test.tsx` | ✅ | Interactive turns call `request_plan_review`, which parks the LIVE turn on the in-memory `PlanReviewGate` (oneshot) until the user decides; `plan_review_request` socket event drives `PlanReviewCard`, which resolves via `openhuman.plan_review_decide` (approve resumes-and-executes / reject resumes-and-stops / revise resumes-with-feedback). RU covers gate park/resolve/timeout + tool auto-approve + parking; RI covers the decide RPC; VU covers the card + provider wiring. WD E2E (agent-driven park flow) tracked as follow-up | +| 4.2.8 | Composer attachments (image / video / document; drag-drop + paste) | VU | `app/src/lib/attachments.test.ts`, `app/src/components/chat/__tests__/ChatComposer.test.tsx`, `app/src/pages/__tests__/Conversations.attachments.test.tsx` | 🟡 | Attach affordance gated on the resolved vision tier (images/video need vision; documents flow on any model); video is sampled into still frames client-side and forwarded through the existing `[IMAGE:]` vision path; drag-drop + clipboard-paste reuse the picker ingest. VU covers MIME/kind/limits/marker building + drag-drop + paste; real video decode and the frames→vision round-trip are manual-smoke only (jsdom has no video codec). WD E2E is a follow-up | + ### 4.3 Tool Invocation | ID | Feature | Layer | Test path(s) | Status | Notes |