From 43e9f0a75a2e4a3972575f1f8ff56c670811b671 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 18 Apr 2026 16:48:45 +0000 Subject: [PATCH] Add image-guided studio avatar generation Co-authored-by: Luke The Dev --- src/app/api/studio-world/route.ts | 89 ++++++++- .../preview/StudioWorldPreview.tsx | 37 +++- .../studio-world/preview/scene-utils.ts | 84 +++++++++ .../screens/StudioWorldScreen.tsx | 162 +++++++++++++++- src/lib/studio-world/generator.ts | 176 +++++++++++++++++- src/lib/studio-world/image-analysis.ts | 152 +++++++++++++++ src/lib/studio-world/store.ts | 67 +++++++ src/lib/studio-world/types.ts | 27 ++- tests/unit/studioWorldRoute.test.ts | 78 ++++++++ 9 files changed, 858 insertions(+), 14 deletions(-) create mode 100644 src/lib/studio-world/image-analysis.ts diff --git a/src/app/api/studio-world/route.ts b/src/app/api/studio-world/route.ts index 969c4f9..e65ef2c 100644 --- a/src/app/api/studio-world/route.ts +++ b/src/app/api/studio-world/route.ts @@ -12,6 +12,7 @@ import { import { buildOfficeMapFromStudioProject } from "@/lib/studio-world/office"; import { createStudioProject, + createStudioSourceImage, deleteStudioProject, getStudioProject, listStudioProjects, @@ -27,6 +28,7 @@ export const runtime = "nodejs"; const WORKSPACE_ID = "default"; const OFFICE_ID = "studio-world"; +const MAX_IMAGE_UPLOAD_BYTES = 8 * 1024 * 1024; const isRecord = (value: unknown): value is Record => Boolean(value && typeof value === "object" && !Array.isArray(value)); @@ -49,17 +51,43 @@ const parseGenerationInput = (value: unknown): StudioGenerationInput | null => { if (!isRecord(value)) return null; const name = asString(value.name) || "Untitled Studio World"; const prompt = asString(value.prompt); - if (!prompt) return null; + const sourceImage = isRecord(value.sourceImage) + ? { + id: asString(value.sourceImage.id), + fileName: asString(value.sourceImage.fileName), + mimeType: asString(value.sourceImage.mimeType), + width: + typeof value.sourceImage.width === "number" && Number.isFinite(value.sourceImage.width) + ? value.sourceImage.width + : 0, + height: + typeof value.sourceImage.height === "number" && Number.isFinite(value.sourceImage.height) + ? value.sourceImage.height + : 0, + sizeBytes: + typeof value.sourceImage.sizeBytes === "number" && + Number.isFinite(value.sourceImage.sizeBytes) + ? value.sourceImage.sizeBytes + : 0, + storagePath: asString(value.sourceImage.storagePath), + dataUrl: asString(value.sourceImage.dataUrl), + palette: Array.isArray(value.sourceImage.palette) + ? value.sourceImage.palette.filter((entry): entry is string => typeof entry === "string") + : [], + } + : null; + if (!prompt && !sourceImage) return null; const rawSeed = value.seed; const seed = typeof rawSeed === "number" && Number.isFinite(rawSeed) ? rawSeed : null; return { name, - prompt, + prompt: prompt || "Image-guided avatar generation", style: parseStyle(value.style), scale: parseScale(value.scale), focus: parseFocus(value.focus), seed, + sourceImage, }; }; @@ -164,6 +192,63 @@ export async function GET(request: Request) { export async function POST(request: Request) { try { + const contentType = + typeof request.headers?.get === "function" + ? request.headers.get("content-type") ?? "" + : ""; + if (contentType.includes("multipart/form-data")) { + const contentLengthHeader = request.headers.get("content-length"); + if (contentLengthHeader !== null) { + const contentLength = Number(contentLengthHeader); + if (!Number.isNaN(contentLength) && contentLength > MAX_IMAGE_UPLOAD_BYTES + 2048) { + return NextResponse.json( + { + error: `Image upload exceeds the ${MAX_IMAGE_UPLOAD_BYTES} byte limit.`, + }, + { status: 413 }, + ); + } + } + const formData = await request.formData(); + const image = formData.get("image"); + if ( + image === null || + typeof image !== "object" || + typeof (image as File).arrayBuffer !== "function" + ) { + return NextResponse.json({ error: "image file is required." }, { status: 400 }); + } + const imageFile = image as File; + const mimeType = imageFile.type.trim().toLowerCase(); + if (mimeType !== "image/png" && mimeType !== "image/jpeg" && mimeType !== "image/webp") { + return NextResponse.json( + { error: "Only PNG, JPEG, and WEBP uploads are supported." }, + { status: 400 }, + ); + } + const arrayBuffer = await imageFile.arrayBuffer(); + if (arrayBuffer.byteLength <= 0) { + return NextResponse.json({ error: "Image upload is empty." }, { status: 400 }); + } + if (arrayBuffer.byteLength > MAX_IMAGE_UPLOAD_BYTES) { + return NextResponse.json( + { + error: `Image upload exceeds the ${MAX_IMAGE_UPLOAD_BYTES} byte limit.`, + }, + { status: 413 }, + ); + } + const requestedFileName = asString(formData.get("fileName")); + const sourceImage = createStudioSourceImage({ + fileName: requestedFileName || imageFile.name || "studio-reference", + mimeType, + buffer: Buffer.from(arrayBuffer), + }); + return NextResponse.json( + { sourceImage }, + { headers: { "Cache-Control": "no-store" } }, + ); + } const rawBody = await request.text(); if (!rawBody.trim()) { return NextResponse.json({ error: "Invalid request payload." }, { status: 400 }); diff --git a/src/features/studio-world/preview/StudioWorldPreview.tsx b/src/features/studio-world/preview/StudioWorldPreview.tsx index 8923a12..130d492 100644 --- a/src/features/studio-world/preview/StudioWorldPreview.tsx +++ b/src/features/studio-world/preview/StudioWorldPreview.tsx @@ -1,12 +1,18 @@ "use client"; +import Image from "next/image"; import { Environment, OrbitControls } from "@react-three/drei"; import { Canvas, useFrame } from "@react-three/fiber"; import { useMemo, useRef } from "react"; import * as THREE from "three"; import type { Group } from "three"; -import type { StudioWorldAssetDraft, StudioWorldDraft } from "@/lib/studio-world/types"; +import type { + StudioProjectRecord, + StudioSourceImageRecord, + StudioWorldAssetDraft, + StudioWorldDraft, +} from "@/lib/studio-world/types"; import { buildAssetGeometry, buildAssetMaterial, buildGlowMaterial } from "@/features/studio-world/preview/scene-utils"; type AssetMeshProps = { @@ -105,9 +111,15 @@ const SceneContents = ({ sceneDraft }: { sceneDraft: StudioWorldDraft }) => { type StudioWorldPreviewProps = { sceneDraft: StudioWorldDraft; + referenceImage?: StudioSourceImageRecord | null; + project?: Pick | null; }; -export function StudioWorldPreview({ sceneDraft }: StudioWorldPreviewProps) { +export function StudioWorldPreview({ + sceneDraft, + referenceImage = null, + project = null, +}: StudioWorldPreviewProps) { return (
+ {referenceImage ? ( +
+ {referenceImage.fileName} +
+
+ Reference +
+
{referenceImage.fileName}
+
+ {project?.mode === "image_avatar" ? "Image-guided avatar proxy." : "Reference image attached."} +
+
+
+ ) : null} ); } diff --git a/src/features/studio-world/preview/scene-utils.ts b/src/features/studio-world/preview/scene-utils.ts index fa1cd74..20eeef9 100644 --- a/src/features/studio-world/preview/scene-utils.ts +++ b/src/features/studio-world/preview/scene-utils.ts @@ -41,6 +41,24 @@ export const buildAssetGeometry = (kind: StudioWorldAssetDraft["kind"]) => { if (kind === "portal") { return new THREE.TorusGeometry(0.6, 0.16, 16, 32); } + if (kind === "avatar_head") { + return new THREE.SphereGeometry(0.9, 24, 24); + } + if (kind === "avatar_hair") { + return new THREE.ConeGeometry(1, 1.4, 14); + } + if (kind === "avatar_torso") { + return new THREE.BoxGeometry(1, 1, 1); + } + if (kind === "avatar_limb") { + return new THREE.CapsuleGeometry(0.22, 1.1, 8, 14); + } + if (kind === "avatar_accessory") { + return new THREE.BoxGeometry(1, 0.28, 0.12); + } + if (kind === "avatar_orb") { + return new THREE.OctahedronGeometry(0.82, 0); + } return new THREE.BoxGeometry(1, 1, 1); }; @@ -59,6 +77,72 @@ const createAssetMesh = (asset: StudioWorldAssetDraft) => { return mesh; } + if (asset.kind === "avatar_head") { + const mesh = new THREE.Mesh(buildAssetGeometry(asset.kind), material); + mesh.scale.set(asset.scale[0], asset.scale[1], asset.scale[2]); + return mesh; + } + + if (asset.kind === "avatar_hair") { + const mesh = new THREE.Mesh(buildAssetGeometry(asset.kind), material); + mesh.scale.set(asset.scale[0], asset.scale[1], asset.scale[2]); + return mesh; + } + + if (asset.kind === "avatar_torso") { + const mesh = new THREE.Mesh(buildAssetGeometry(asset.kind), material); + mesh.scale.set(asset.scale[0], asset.scale[1], asset.scale[2]); + return mesh; + } + + if (asset.kind === "avatar_limb") { + const mesh = new THREE.Mesh(buildAssetGeometry(asset.kind), material); + mesh.scale.set(asset.scale[0], asset.scale[1], asset.scale[2]); + return mesh; + } + + if (asset.kind === "avatar_accessory") { + const group = new THREE.Group(); + const bar = new THREE.Mesh(buildAssetGeometry(asset.kind), material); + const leftLens = new THREE.Mesh( + new THREE.CylinderGeometry(0.26, 0.26, 0.1, 22), + new THREE.MeshStandardMaterial({ + color: asset.color, + emissive: asset.emissive ?? asset.color, + emissiveIntensity: 1, + roughness: 0.15, + metalness: 0.42, + }), + ); + const rightLens = leftLens.clone(); + leftLens.rotation.z = Math.PI / 2; + rightLens.rotation.z = Math.PI / 2; + leftLens.position.x = -0.36; + rightLens.position.x = 0.36; + group.add(bar, leftLens, rightLens); + group.scale.set(asset.scale[0], asset.scale[1], asset.scale[2]); + return group; + } + + if (asset.kind === "avatar_orb") { + const group = new THREE.Group(); + const core = new THREE.Mesh(buildAssetGeometry(asset.kind), material); + const halo = new THREE.Mesh( + new THREE.TorusGeometry(0.75, 0.06, 12, 30), + new THREE.MeshStandardMaterial({ + color: asset.emissive ?? asset.color, + emissive: asset.emissive ?? asset.color, + emissiveIntensity: 1.2, + roughness: 0.12, + metalness: 0.24, + }), + ); + halo.rotation.x = Math.PI / 3; + group.add(core, halo); + group.scale.set(asset.scale[0], asset.scale[1], asset.scale[2]); + return group; + } + if (asset.kind === "arch") { const group = new THREE.Group(); const legGeometry = new THREE.BoxGeometry(0.24, 1, 0.24); diff --git a/src/features/studio-world/screens/StudioWorldScreen.tsx b/src/features/studio-world/screens/StudioWorldScreen.tsx index 1b5fd41..65000c3 100644 --- a/src/features/studio-world/screens/StudioWorldScreen.tsx +++ b/src/features/studio-world/screens/StudioWorldScreen.tsx @@ -1,12 +1,14 @@ "use client"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import Image from "next/image"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { HeaderBar } from "@/features/agents/components/HeaderBar"; import { exportStudioProjectGlb } from "@/features/studio-world/export/exportGlb"; import { StudioWorldPreview } from "@/features/studio-world/preview/StudioWorldPreview"; import type { StudioProjectRecord, + StudioSourceImageRecord, StudioWorldFocus, StudioWorldScale, StudioWorldStyle, @@ -39,6 +41,7 @@ type ExportManifestResponse = { type StudioWorldResponse = { projects?: StudioProjectRecord[]; project?: StudioProjectRecord; + sourceImage?: StudioSourceImageRecord; office?: { workspaceId: string; officeId: string; @@ -86,6 +89,9 @@ export function StudioWorldScreen() { const [scale, setScale] = useState("medium"); const [focus, setFocus] = useState("world"); const [seed, setSeed] = useState(""); + const [uploadedImage, setUploadedImage] = useState(null); + const [uploadingImage, setUploadingImage] = useState(false); + const uploadInputRef = useRef(null); const selectedProject = useMemo( () => projects.find((entry) => entry.id === selectedProjectId) ?? projects[0] ?? null, @@ -115,9 +121,46 @@ export function StudioWorldScreen() { void refreshProjects(); }, [refreshProjects]); + useEffect(() => { + if (!selectedProject) return; + setUploadedImage(selectedProject.sourceImages[0] ?? null); + }, [selectedProject]); + + const handleImageUpload = async (file: File) => { + setUploadingImage(true); + setStatusLine("Uploading reference image."); + try { + const formData = new FormData(); + formData.append("image", file); + const response = await fetch("/api/studio-world", { + method: "POST", + body: formData, + }); + const body = (await response.json()) as StudioWorldResponse; + if (!response.ok || !body.sourceImage) { + throw new Error(body.error || "Failed to upload image."); + } + setUploadedImage(body.sourceImage); + setError(null); + setStatusLine(`Uploaded ${body.sourceImage.fileName}.`); + } catch (uploadError) { + setError(uploadError instanceof Error ? uploadError.message : "Failed to upload image."); + setStatusLine(null); + } finally { + setUploadingImage(false); + if (uploadInputRef.current) { + uploadInputRef.current.value = ""; + } + } + }; + const handleGenerate = async () => { setBusy(true); - setStatusLine("Generating clean-room studio draft."); + setStatusLine( + uploadedImage + ? "Generating image-guided 3D proxy." + : "Generating clean-room studio draft.", + ); try { const parsedSeed = seed.trim() ? Number.parseInt(seed.trim(), 10) : null; const response = await fetch("/api/studio-world", { @@ -132,6 +175,7 @@ export function StudioWorldScreen() { scale, focus, seed: Number.isFinite(parsedSeed) ? parsedSeed : null, + sourceImage: uploadedImage, }, }), }); @@ -271,12 +315,74 @@ export function StudioWorldScreen() {
Claw3D Studio
-

Generate 3D worlds, assets, and motion.

+

Generate 3D worlds, assets, motion, and image-guided avatars.

- This clean-room workflow is inspired by modern world-model pipelines, but implemented as Claw3D-native - tooling and data contracts. + This clean-room workflow is inspired by tools like Meshy, but implemented as Claw3D-native tooling and + data contracts.

+
+
+
+
Reference image
+
+ Upload a PNG, JPEG, or WEBP to generate a stylized 3D avatar proxy inspired by that image. +
+
+ +
+ { + const file = event.target.files?.[0]; + if (!file) return; + void handleImageUpload(file); + }} + /> + {uploadedImage ? ( +
+
+ {uploadedImage.fileName} +
+
+
{uploadedImage.fileName}
+
+ {uploadedImage.width} x {uploadedImage.height} • {Math.round(uploadedImage.sizeBytes / 1024)} KB +
+
+ {uploadedImage.palette.map((color) => ( +
+ + + {color} + +
+ ))} +
+
+
+ ) : null} +
- - + {uploadedImage ? ( + + ) : null}
@@ -434,11 +559,32 @@ export function StudioWorldScreen() { {project.scale} + + {project.mode === "image_avatar" ? "image avatar" : "text scene"} +
Updated {formatTimestamp(project.updatedAt)}.
+ {project.sourceImages[0] ? ( +
+ {project.sourceImages[0].fileName} +
+
{project.sourceImages[0].fileName}
+
+ {project.sourceImages[0].width} x {project.sourceImages[0].height} +
+
+
+ ) : null}