diff --git a/.env.example b/.env.example index 8842418..46c55a9 100644 --- a/.env.example +++ b/.env.example @@ -48,3 +48,10 @@ DEBUG=true # ELEVENLABS_API_KEY= # ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM # ELEVENLABS_MODEL_ID=eleven_flash_v2_5 + +# Optional: AI image -> 3D reconstruction for office assets +# Uses an OpenAI-compatible chat completions endpoint with image input and +# json_schema structured output. +# OPENAI_API_KEY= +# OPENAI_BASE_URL=https://api.openai.com/v1 +# OPENAI_3D_MODEL=gpt-4o-mini diff --git a/src/app/api/office/picture-model/route.ts b/src/app/api/office/picture-model/route.ts new file mode 100644 index 0000000..6a86d1b --- /dev/null +++ b/src/app/api/office/picture-model/route.ts @@ -0,0 +1,78 @@ +import { NextResponse } from "next/server"; +import { + generatePictureModelFromImage, + MAX_PICTURE_MODEL_UPLOAD_BYTES, +} from "@/lib/office/pictureModelGeneration"; + +export const runtime = "nodejs"; + +export async function POST(request: Request) { + try { + const MULTIPART_OVERHEAD_ALLOWANCE = 1024; + const contentLengthHeader = request.headers.get("content-length"); + if (contentLengthHeader !== null) { + const contentLength = Number(contentLengthHeader); + if ( + !Number.isNaN(contentLength) && + contentLength > MAX_PICTURE_MODEL_UPLOAD_BYTES + MULTIPART_OVERHEAD_ALLOWANCE + ) { + return NextResponse.json( + { + error: `Image upload exceeds the ${MAX_PICTURE_MODEL_UPLOAD_BYTES} byte limit.`, + }, + { status: 413 }, + ); + } + } + + const formData = await request.formData(); + const image = formData.get("image"); + if ( + image === null || + typeof image !== "object" || + typeof (image as File).arrayBuffer !== "function" + ) { + return NextResponse.json( + { error: "image file is required." }, + { status: 400 }, + ); + } + const imageFile = image as File; + const arrayBuffer = await imageFile.arrayBuffer(); + const byteLength = arrayBuffer.byteLength; + if (byteLength <= 0) { + return NextResponse.json({ error: "Image upload is empty." }, { status: 400 }); + } + if (byteLength > MAX_PICTURE_MODEL_UPLOAD_BYTES) { + return NextResponse.json( + { + error: `Image upload exceeds the ${MAX_PICTURE_MODEL_UPLOAD_BYTES} byte limit.`, + }, + { status: 413 }, + ); + } + + if (!imageFile.type.startsWith("image/")) { + return NextResponse.json( + { error: "Only image uploads are supported." }, + { status: 400 }, + ); + } + + const base64 = Buffer.from(arrayBuffer).toString("base64"); + const imageDataUrl = `data:${imageFile.type};base64,${base64}`; + const result = await generatePictureModelFromImage({ + imageDataUrl, + fileName: imageFile.name, + mimeType: imageFile.type, + }); + + return NextResponse.json(result); + } catch (error) { + const message = + error instanceof Error + ? error.message + : "Failed to generate the 3D model from the uploaded image."; + return NextResponse.json({ error: message }, { status: 500 }); + } +} diff --git a/src/features/retro-office/RetroOffice3D.tsx b/src/features/retro-office/RetroOffice3D.tsx index 2e1c41e..0c3a490 100644 --- a/src/features/retro-office/RetroOffice3D.tsx +++ b/src/features/retro-office/RetroOffice3D.tsx @@ -4996,13 +4996,29 @@ export function RetroOffice3D({ setPictureDraftStatus("processing"); setPictureDraftError(null); try { - const asset = await createPictureAssetFromFile(file); - setPictureDraft(asset); + const previewAsset = await createPictureAssetFromFile(file); + const formData = new FormData(); + formData.set("image", file); + formData.set("previewDataUrl", previewAsset.imageDataUrl); + const response = await fetch("/api/office/picture-model", { + method: "POST", + body: formData, + }); + const payload = (await response.json().catch(() => null)) as + | { + asset?: PicturePropAsset; + error?: string; + } + | null; + if (!response.ok || !payload?.asset) { + throw new Error(payload?.error || "AI 3D reconstruction failed."); + } + setPictureDraft(payload.asset); } catch (error) { setPictureDraftError( error instanceof Error ? error.message - : "Picture processing failed.", + : "Picture model generation failed.", ); } finally { setPictureDraftStatus("idle"); @@ -7130,7 +7146,7 @@ export function RetroOffice3D({ {selectedPictureAsset ? (
- Picture to GLB + AI Model to GLB
- Picture Lab + Image to 3D Lab
- Upload any picture, stylize it into a matte office prop, then export the same model as a GLB. + Upload a picture and let AI reconstruct it as a simplified 3D office asset, then export that model as a GLB.
{pictureDraft ? (
@@ -7266,7 +7282,7 @@ export function RetroOffice3D({ : "border-amber-900/20 bg-[#16100a] text-amber-400/40" }`} > - Place in Office + Generate and Place
diff --git a/src/features/retro-office/core/pictureAsset.ts b/src/features/retro-office/core/pictureAsset.ts index 84a2c58..22d8082 100644 --- a/src/features/retro-office/core/pictureAsset.ts +++ b/src/features/retro-office/core/pictureAsset.ts @@ -3,7 +3,11 @@ import * as THREE from "three"; import { GLTFExporter } from "three/examples/jsm/exporters/GLTFExporter.js"; import { SCALE } from "@/features/retro-office/core/constants"; -import type { PicturePropAsset } from "@/features/retro-office/core/types"; +import type { + Picture3dPrimitive, + Picture3dRecipe, + PicturePropAsset, +} from "@/features/retro-office/core/types"; export const PICTURE_PROP_TYPE = "picture_prop"; @@ -12,7 +16,11 @@ const MIN_PIXEL_WIDTH = 20; const MAX_PIXEL_WIDTH = 44; const MIN_PIXEL_HEIGHT = 20; const MAX_PIXEL_HEIGHT = 44; -const PICTURE_PROP_DEPTH_UNITS = 24; +const PICTURE_PROP_DEPTH_UNITS = 30; +const DEFAULT_PALETTE = { + accentColor: "#d97706", + dominantColor: "#7c5c3b", +} as const; const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value)); @@ -108,11 +116,7 @@ export const derivePicturePalette = (rgba: ArrayLike) => { } if (visiblePixels === 0 || buckets.size === 0) { - return { - accentColor: "#d97706", - dominantColor: "#7c5c3b", - frameColor: "#24170d", - }; + return { ...DEFAULT_PALETTE }; } const dominantBucket = @@ -160,7 +164,7 @@ const loadImageElement = (src: string) => image.src = src; }); -const renderCoverImage = ( +const renderPreviewImage = ( context: CanvasRenderingContext2D, image: CanvasImageSource, targetWidth: number, @@ -219,16 +223,95 @@ export const getPicturePropGlbFileName = (asset: PicturePropAsset) => export const resolvePicturePropFootprint = (aspectRatio: number) => { const safeAspect = clamp(Number.isFinite(aspectRatio) ? aspectRatio : 1, 0.65, 1.9); - const widthUnits = Math.round(36 + (safeAspect - 0.65) * 16); + const widthUnits = Math.round(44 + (safeAspect - 0.65) * 18); return { depthUnits: PICTURE_PROP_DEPTH_UNITS, - widthUnits: clamp(widthUnits, 34, 58), + widthUnits: clamp(widthUnits, 40, 66), + }; +}; + +export const buildFallbackGeneratedModel = ( + palette: Pick, + aspectRatio: number, +): Picture3dRecipe => { + const safeAspect = clamp(aspectRatio, 0.65, 1.9); + const width = clamp(1.1 + (safeAspect - 1) * 0.22, 0.86, 1.42); + const depth = 0.72; + const towerWidth = clamp(width * 0.28, 0.22, 0.34); + const headWidth = clamp(width * 0.5, 0.32, 0.58); + const highlightWidth = clamp(width * 0.14, 0.08, 0.18); + return { + title: "AI office sculpture", + summary: + "retro office collectible with chunky low-poly forms, matte materials, soft bevels, and furniture-like proportions.", + footprintMeters: { + width, + depth, + height: 1.66, + }, + primitives: [ + { + kind: "box", + size: [width, 0.22, depth], + position: [0, 0.11, 0], + material: { + color: palette.accentColor, + roughness: 0.82, + }, + }, + { + kind: "box", + size: [width * 0.74, 0.92, depth * 0.7], + position: [0, 0.68, -0.02], + material: { + color: palette.dominantColor, + roughness: 0.76, + }, + }, + { + kind: "box", + size: [headWidth, 0.36, depth * 0.44], + position: [0, 1.3, 0.04], + material: { + color: mixColors(palette.dominantColor, "#f6efe1", 0.3), + roughness: 0.74, + }, + }, + { + kind: "box", + size: [towerWidth, 0.54, depth * 0.44], + position: [-(width * 0.22), 0.86, 0.12], + material: { + color: mixColors(palette.dominantColor, "#0f0a06", 0.72), + roughness: 0.82, + }, + }, + { + kind: "box", + size: [towerWidth, 0.62, depth * 0.36], + position: [width * 0.22, 0.78, -0.08], + material: { + color: mixColors(mixColors(palette.dominantColor, "#0f0a06", 0.72), palette.dominantColor, 0.24), + roughness: 0.78, + }, + }, + { + kind: "box", + size: [highlightWidth, 0.68, depth * 0.8], + position: [width * 0.34, 0.78, 0.05], + material: { + color: mixColors(palette.accentColor, "#f8d34d", 0.24), + roughness: 0.66, + metalness: 0.1, + }, + }, + ], }; }; export const createPictureAssetFromFile = async (file: File) => { if (!file.type.startsWith("image/")) { - throw new Error("Only image uploads are supported for picture props."); + throw new Error("Only image uploads are supported for 3D generation."); } const objectUrl = URL.createObjectURL(file); @@ -254,7 +337,7 @@ export const createPictureAssetFromFile = async (file: File) => { throw new Error("Could not create an image processing context."); } sourceContext.imageSmoothingEnabled = true; - renderCoverImage(sourceContext, image, pixelWidth, pixelHeight); + renderPreviewImage(sourceContext, image, pixelWidth, pixelHeight); const previewScale = Math.max( 1, @@ -279,7 +362,6 @@ export const createPictureAssetFromFile = async (file: File) => { const palette = derivePicturePalette( sourceContext.getImageData(0, 0, pixelWidth, pixelHeight).data, ); - return { ...palette, aspectRatio, @@ -287,91 +369,146 @@ export const createPictureAssetFromFile = async (file: File) => { imageDataUrl: previewCanvas.toDataURL("image/webp", 0.86), pixelHeight, pixelWidth, + provider: "preview", + model: "local-fallback", + summary: + "Local preview generated. Upload will request an AI low-poly office asset recipe.", + recipe: buildFallbackGeneratedModel(palette, aspectRatio), } satisfies PicturePropAsset; } finally { URL.revokeObjectURL(objectUrl); } }; -type PicturePropDimensions = { - artHeight: number; - artWidth: number; - backdropDepth: number; - backdropHeight: number; - backdropWidth: number; - baseDepth: number; - baseHeight: number; - baseWidth: number; - braceLength: number; - braceTilt: number; - frameDepth: number; - frameHeight: number; - frameInset: number; - frameThickness: number; - frameWidth: number; - supportHeight: number; - supportWidth: number; +const clampPrimitiveSize = (value: number, fallback: number, min: number, max: number) => + clamp(Number.isFinite(value) ? value : fallback, min, max); + +const clampPrimitivePosition = (value: number, fallback = 0, min = -1.5, max = 1.8) => + clamp(Number.isFinite(value) ? value : fallback, min, max); + +const clampPrimitiveRotation = (value: number, fallback = 0) => + clamp(Number.isFinite(value) ? value : fallback, -Math.PI, Math.PI); + +const normalizeHexColor = (value: string | undefined, fallback: string) => { + const raw = value?.trim() ?? ""; + if (!/^#([0-9a-f]{3}|[0-9a-f]{6})$/i.test(raw)) { + return fallback; + } + if (raw.length === 4) { + return `#${raw[1]}${raw[1]}${raw[2]}${raw[2]}${raw[3]}${raw[3]}`.toLowerCase(); + } + return raw.toLowerCase(); }; -export const resolvePicturePropDimensions = (params: { - aspectRatio: number; - footprintDepth: number; - footprintWidth: number; -}): PicturePropDimensions => { - const safeAspect = clamp(Number.isFinite(params.aspectRatio) ? params.aspectRatio : 1, 0.65, 1.9); - let artWidth = clamp(params.footprintWidth * 0.78, 0.48, 1.16); - let artHeight = artWidth / safeAspect; - if (artHeight > 1.08) { - artHeight = 1.08; - artWidth = artHeight * safeAspect; +export const sanitizeGeneratedPrimitive = ( + primitive: Picture3dPrimitive, + palette: Pick, + index: number, +): Picture3dPrimitive => { + const fallbackColor = + index === 0 + ? palette.dominantColor + : index % 3 === 0 + ? palette.accentColor + : mixColors(palette.dominantColor, "#0f0a06", 0.72); + const material = { + color: normalizeHexColor(primitive.material?.color, fallbackColor), + metalness: clamp(primitive.material?.metalness ?? 0.08, 0, 0.35), + roughness: clamp(primitive.material?.roughness ?? 0.76, 0.4, 1), + }; + const rotation = primitive.rotation + ? ([ + clampPrimitiveRotation(primitive.rotation[0] ?? Number.NaN), + clampPrimitiveRotation(primitive.rotation[1] ?? Number.NaN), + clampPrimitiveRotation(primitive.rotation[2] ?? Number.NaN), + ] as [number, number, number]) + : undefined; + const position = [ + clampPrimitivePosition(primitive.position[0] ?? Number.NaN), + clampPrimitivePosition(primitive.position[1] ?? Number.NaN, 0.2, 0, 2.2), + clampPrimitivePosition(primitive.position[2] ?? Number.NaN), + ] as [number, number, number]; + if (primitive.kind === "cylinder") { + return { + kind: "cylinder", + height: clampPrimitiveSize(primitive.height, 0.46, 0.08, 2.2), + radiusTop: clampPrimitiveSize(primitive.radiusTop, 0.18, 0.04, 0.8), + radiusBottom: clampPrimitiveSize(primitive.radiusBottom, 0.18, 0.04, 0.8), + radialSegments: Math.round(clampPrimitiveSize(primitive.radialSegments ?? 16, 16, 8, 24)), + position, + ...(rotation ? { rotation } : {}), + material, + }; } - if (artHeight < 0.56) { - artHeight = 0.56; - artWidth = artHeight * safeAspect; + if (primitive.kind === "sphere") { + return { + kind: "sphere", + radius: clampPrimitiveSize(primitive.radius, 0.24, 0.04, 0.8), + widthSegments: Math.round(clampPrimitiveSize(primitive.widthSegments ?? 16, 16, 8, 24)), + heightSegments: Math.round(clampPrimitiveSize(primitive.heightSegments ?? 16, 16, 8, 24)), + position, + ...(rotation ? { rotation } : {}), + material, + }; } - const frameThickness = clamp(artWidth * 0.085, 0.045, 0.085); - const frameInset = frameThickness * 0.68; - const frameWidth = artWidth + frameThickness * 2; - const frameHeight = artHeight + frameThickness * 2; - const frameDepth = 0.08; - const baseHeight = 0.08; - const baseWidth = clamp(frameWidth * 0.72, 0.34, params.footprintWidth * 0.92); - const baseDepth = clamp(params.footprintDepth * 0.72, 0.2, 0.34); - const supportHeight = clamp(frameHeight * 0.84, 0.56, 1.08); - const supportWidth = clamp(frameWidth * 0.12, 0.05, 0.09); return { - artHeight, - artWidth, - backdropDepth: 0.03, - backdropHeight: frameHeight * 0.96, - backdropWidth: frameWidth * 0.96, - baseDepth, - baseHeight, - baseWidth, - braceLength: clamp(frameHeight * 0.55, 0.38, 0.62), - braceTilt: 0.68, - frameDepth, - frameHeight, - frameInset, - frameThickness, - frameWidth, - supportHeight, - supportWidth, + kind: "box", + size: [ + clampPrimitiveSize(primitive.size[0] ?? Number.NaN, 0.46, 0.08, 1.8), + clampPrimitiveSize(primitive.size[1] ?? Number.NaN, 0.46, 0.08, 2.2), + clampPrimitiveSize(primitive.size[2] ?? Number.NaN, 0.46, 0.08, 1.8), + ], + position, + ...(rotation ? { rotation } : {}), + material, }; }; -const applyPictureTextureStyle = (texture: THREE.Texture) => { - texture.colorSpace = THREE.SRGBColorSpace; - texture.magFilter = THREE.NearestFilter; - texture.minFilter = THREE.NearestFilter; - texture.generateMipmaps = false; - texture.needsUpdate = true; - return texture; +export const sanitizeGeneratedModel = ( + model: Picture3dRecipe | null | undefined, + asset: Pick< + PicturePropAsset, + | "accentColor" + | "aspectRatio" + | "dominantColor" + | "fileName" + | "recipe" + >, +): Picture3dRecipe => { + const fallback = buildFallbackGeneratedModel( + { + accentColor: asset.accentColor, + dominantColor: asset.dominantColor, + }, + asset.aspectRatio, + ); + if (!model) return fallback; + const primitives: Picture3dPrimitive[] = Array.isArray(model.primitives) + ? model.primitives + .slice(0, 16) + .map((primitive: Picture3dPrimitive, index: number) => + sanitizeGeneratedPrimitive( + primitive, + { + accentColor: asset.accentColor, + dominantColor: asset.dominantColor, + }, + index, + ), + ) + : fallback.primitives; + return { + title: model.title?.trim() || sanitizeBaseFileName(asset.fileName), + summary: model.summary?.trim() || fallback.summary, + footprintMeters: { + width: clamp(model.footprintMeters?.width ?? fallback.footprintMeters.width, 0.6, 1.8), + depth: clamp(model.footprintMeters?.depth ?? fallback.footprintMeters.depth, 0.4, 1.4), + height: clamp(model.footprintMeters?.height ?? fallback.footprintMeters.height, 0.8, 2.1), + }, + primitives: primitives.length > 0 ? primitives : fallback.primitives, + }; }; -export const createStyledPictureTexture = (texture: THREE.Texture) => - applyPictureTextureStyle(texture.clone()); - const createStandardMaterial = ( color: string, overrides: Partial = {}, @@ -379,135 +516,70 @@ const createStandardMaterial = ( new THREE.MeshStandardMaterial({ color, metalness: 0.08, - roughness: 0.72, + roughness: 0.76, ...overrides, }); -type BuildPicturePropGroupParams = { - asset: PicturePropAsset; - footprintDepth: number; - footprintWidth: number; - texture: THREE.Texture; +const createPrimitiveGeometry = (primitive: Picture3dPrimitive) => { + switch (primitive.kind) { + case "sphere": + return new THREE.SphereGeometry( + clamp(primitive.radius, 0.05, 0.9), + primitive.widthSegments ?? 18, + primitive.heightSegments ?? 18, + ); + case "cylinder": + return new THREE.CylinderGeometry( + clamp(primitive.radiusTop, 0.04, 0.8), + clamp(primitive.radiusBottom, 0.04, 0.8), + clamp(primitive.height, 0.08, 2.2), + primitive.radialSegments ?? 18, + ); + case "box": + default: + return new THREE.BoxGeometry( + clamp(primitive.size[0], 0.08, 1.8), + clamp(primitive.size[1], 0.08, 2.2), + clamp(primitive.size[2], 0.08, 1.8), + ); + } }; -export const buildPicturePropGroup = ({ - asset, - footprintDepth, - footprintWidth, - texture, -}: BuildPicturePropGroupParams) => { - const dominantShadow = mixColors(asset.dominantColor, "#111827", 0.42); - const accentShadow = mixColors(asset.accentColor, "#111827", 0.28); - const dims = resolvePicturePropDimensions({ - aspectRatio: asset.aspectRatio, - footprintDepth, - footprintWidth, - }); +export const buildPicturePropGroup = (asset: PicturePropAsset) => { + const model = sanitizeGeneratedModel(asset.recipe, asset); + const footprint = resolvePicturePropFootprint(asset.aspectRatio); + const widthScale = (footprint.widthUnits * SCALE) / Math.max(model.footprintMeters.width, 0.1); + const depthScale = (footprint.depthUnits * SCALE) / Math.max(model.footprintMeters.depth, 0.1); + const heightScale = 1.55 / Math.max(model.footprintMeters.height, 0.1); + const uniformScale = clamp(Math.min(widthScale, depthScale, heightScale), 0.4, 1.6); const group = new THREE.Group(); - const frameCenterY = dims.baseHeight + dims.frameHeight * 0.5 + 0.18; - const applySharedFlags = (mesh: THREE.Mesh) => { + for (const primitive of model.primitives) { + const geometry = createPrimitiveGeometry(primitive); + const material = createStandardMaterial(primitive.material.color, { + metalness: primitive.material.metalness ?? 0.08, + roughness: primitive.material.roughness ?? 0.76, + }); + const mesh = new THREE.Mesh(geometry, material); + mesh.position.set( + primitive.position[0] * uniformScale, + primitive.position[1] * uniformScale, + primitive.position[2] * uniformScale, + ); + mesh.rotation.set( + primitive.rotation?.[0] ?? 0, + primitive.rotation?.[1] ?? 0, + primitive.rotation?.[2] ?? 0, + ); mesh.castShadow = true; mesh.receiveShadow = true; - return mesh; - }; - - const base = applySharedFlags( - new THREE.Mesh( - new THREE.BoxGeometry(dims.baseWidth, dims.baseHeight, dims.baseDepth), - createStandardMaterial(asset.accentColor, { roughness: 0.78 }), - ), - ); - base.position.set(0, dims.baseHeight * 0.5, 0); - group.add(base); - - const support = applySharedFlags( - new THREE.Mesh( - new THREE.BoxGeometry(dims.supportWidth, dims.supportHeight, 0.07), - createStandardMaterial(dominantShadow, { roughness: 0.82 }), - ), - ); - support.position.set(0, dims.baseHeight + dims.supportHeight * 0.5, -0.02); - group.add(support); - - const brace = applySharedFlags( - new THREE.Mesh( - new THREE.BoxGeometry(dims.supportWidth * 0.8, dims.braceLength, 0.06), - createStandardMaterial(accentShadow, { roughness: 0.76 }), - ), - ); - brace.position.set(0, dims.baseHeight + dims.braceLength * 0.55, -dims.baseDepth * 0.2); - brace.rotation.x = -dims.braceTilt; - group.add(brace); - - const backdrop = applySharedFlags( - new THREE.Mesh( - new THREE.BoxGeometry( - dims.backdropWidth, - dims.backdropHeight, - dims.backdropDepth, - ), - createStandardMaterial(mixColors(asset.dominantColor, "#efe6d8", 0.18), { - roughness: 0.9, - }), - ), - ); - backdrop.position.set(0, frameCenterY, -0.01); - group.add(backdrop); - - const frame = applySharedFlags( - new THREE.Mesh( - new THREE.BoxGeometry(dims.frameWidth, dims.frameHeight, dims.frameDepth), - createStandardMaterial(asset.frameColor, { - metalness: 0.12, - roughness: 0.7, - }), - ), - ); - frame.position.set(0, frameCenterY, 0); - group.add(frame); - - const innerPanel = applySharedFlags( - new THREE.Mesh( - new THREE.BoxGeometry( - dims.frameWidth - dims.frameInset * 2, - dims.frameHeight - dims.frameInset * 2, - dims.frameDepth * 0.52, - ), - createStandardMaterial(mixColors(asset.dominantColor, "#f6efe1", 0.32), { - roughness: 0.92, - }), - ), - ); - innerPanel.position.set(0, frameCenterY, dims.frameDepth * 0.06); - group.add(innerPanel); - - const artPlane = applySharedFlags( - new THREE.Mesh( - new THREE.PlaneGeometry(dims.artWidth, dims.artHeight), - createStandardMaterial("#ffffff", { - map: texture, - metalness: 0.02, - roughness: 0.9, - side: THREE.DoubleSide, - }), - ), - ); - artPlane.position.set(0, frameCenterY, dims.frameDepth * 0.5 + 0.002); - group.add(artPlane); - - const topCap = applySharedFlags( - new THREE.Mesh( - new THREE.BoxGeometry(dims.frameWidth * 0.18, 0.06, dims.frameDepth * 0.78), - createStandardMaterial(accentShadow, { - metalness: 0.16, - roughness: 0.62, - }), - ), - ); - topCap.position.set(0, frameCenterY + dims.frameHeight * 0.5 + 0.01, -0.008); - group.add(topCap); + group.add(mesh); + } + const boundingBox = new THREE.Box3().setFromObject(group); + const center = boundingBox.getCenter(new THREE.Vector3()); + const minY = boundingBox.min.y; + group.position.set(-center.x, Math.max(0, -minY), -center.z); return group; }; @@ -529,21 +601,8 @@ export const buildPicturePropItem = ( }; }; -const loadTexture = async (imageDataUrl: string) => { - const loader = new THREE.TextureLoader(); - const texture = await loader.loadAsync(imageDataUrl); - return applyPictureTextureStyle(texture); -}; - export const exportPictureAssetToGlb = async (asset: PicturePropAsset) => { - const footprint = resolvePicturePropFootprint(asset.aspectRatio); - const texture = await loadTexture(asset.imageDataUrl); - const group = buildPicturePropGroup({ - asset, - footprintDepth: footprint.depthUnits * SCALE, - footprintWidth: footprint.widthUnits * SCALE, - texture, - }); + const group = buildPicturePropGroup(asset); const exporter = new GLTFExporter(); const binary = await new Promise((resolve, reject) => { exporter.parse( @@ -553,18 +612,15 @@ export const exportPictureAssetToGlb = async (asset: PicturePropAsset) => { resolve(result); return; } - reject(new Error("Picture prop export did not return a binary GLB.")); + reject(new Error("3D generation export did not return a binary GLB.")); }, (error) => { reject( - error instanceof Error - ? error - : new Error("Picture prop export failed."), + error instanceof Error ? error : new Error("3D generation export failed."), ); }, { binary: true, - maxTextureSize: 1024, onlyVisible: false, }, ); diff --git a/src/features/retro-office/core/types.ts b/src/features/retro-office/core/types.ts index d02d785..9e97596 100644 --- a/src/features/retro-office/core/types.ts +++ b/src/features/retro-office/core/types.ts @@ -64,15 +64,67 @@ export type RenderAgent = SceneActor & { janitorPauseUntil?: number; }; +export type Picture3dMaterial = { + color: string; + roughness?: number; + metalness?: number; +}; + +export type Picture3dPrimitiveBase = { + material: Picture3dMaterial; + position: [number, number, number]; + rotation?: [number, number, number]; +}; + +export type Picture3dBoxPrimitive = Picture3dPrimitiveBase & { + kind: "box"; + size: [number, number, number]; +}; + +export type Picture3dCylinderPrimitive = Picture3dPrimitiveBase & { + kind: "cylinder"; + radiusBottom: number; + radiusTop: number; + height: number; + radialSegments?: number; +}; + +export type Picture3dSpherePrimitive = Picture3dPrimitiveBase & { + kind: "sphere"; + radius: number; + widthSegments?: number; + heightSegments?: number; +}; + +export type Picture3dPrimitive = + | Picture3dBoxPrimitive + | Picture3dCylinderPrimitive + | Picture3dSpherePrimitive; + +export type Picture3dRecipe = { + title: string; + summary: string; + footprintMeters: { + width: number; + depth: number; + height: number; + }; + primitives: Picture3dPrimitive[]; +}; + export type PicturePropAsset = { fileName: string; imageDataUrl: string; + sourceImageDataUrl?: string; aspectRatio: number; dominantColor: string; accentColor: string; - frameColor: string; pixelWidth: number; pixelHeight: number; + provider: string; + model: string; + summary: string; + recipe: Picture3dRecipe; }; export type FurnitureItem = { diff --git a/src/features/retro-office/objects/pictureProps.tsx b/src/features/retro-office/objects/pictureProps.tsx index a0b7ef7..3b45a42 100644 --- a/src/features/retro-office/objects/pictureProps.tsx +++ b/src/features/retro-office/objects/pictureProps.tsx @@ -1,6 +1,5 @@ "use client"; -import { useTexture } from "@react-three/drei"; import { useEffect, useMemo } from "react"; import * as THREE from "three"; import { SCALE } from "@/features/retro-office/core/constants"; @@ -12,7 +11,6 @@ import { import { buildPicturePropGroup, PICTURE_PROP_TYPE, - createStyledPictureTexture, resolvePicturePropFootprint, } from "@/features/retro-office/core/pictureAsset"; import type { FurnitureItem } from "@/features/retro-office/core/types"; @@ -36,9 +34,6 @@ const disposeObject3D = (object: THREE.Object3D) => { }); }; -const EMPTY_TEXTURE_DATA_URL = - "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=="; - const applyHighlight = ({ editMode, isHovered, @@ -72,17 +67,10 @@ const applyHighlight = ({ const buildStyledObject = ( item: FurnitureItem, - texture: THREE.Texture, overrideAsset = item.pictureAsset, ) => { if (!overrideAsset) return null; - const { width, height } = getItemBaseSize(item); - return buildPicturePropGroup({ - asset: overrideAsset, - footprintDepth: height * SCALE, - footprintWidth: width * SCALE, - texture: createStyledPictureTexture(texture), - }); + return buildPicturePropGroup(overrideAsset); }; export function PicturePropModel({ @@ -96,11 +84,9 @@ export function PicturePropModel({ onClick, }: InteractiveFurnitureModelProps) { const asset = item.pictureAsset; - const imageDataUrl = asset?.imageDataUrl ?? EMPTY_TEXTURE_DATA_URL; - const sourceTexture = useTexture(imageDataUrl); const modelObject = useMemo( - () => buildStyledObject(item, sourceTexture), - [item, sourceTexture], + () => buildStyledObject(item), + [item], ); const [wx, , wz] = toWorld(item.x, item.y); const { width, height } = getItemBaseSize(item); @@ -163,7 +149,6 @@ export function PicturePropGhost({ asset: NonNullable; position: [number, number, number]; }) { - const sourceTexture = useTexture(asset.imageDataUrl); const footprint = resolvePicturePropFootprint(asset.aspectRatio); const ghostItem = useMemo( () => ({ @@ -178,8 +163,8 @@ export function PicturePropGhost({ [asset, footprint.depthUnits, footprint.widthUnits], ); const modelObject = useMemo( - () => buildStyledObject(ghostItem, sourceTexture, asset), - [asset, ghostItem, sourceTexture], + () => buildStyledObject(ghostItem, asset), + [asset, ghostItem], ); const pivotX = footprint.widthUnits * SCALE * 0.5; const pivotZ = footprint.depthUnits * SCALE * 0.5; diff --git a/src/lib/office/pictureModelGeneration.ts b/src/lib/office/pictureModelGeneration.ts new file mode 100644 index 0000000..6195d95 --- /dev/null +++ b/src/lib/office/pictureModelGeneration.ts @@ -0,0 +1,178 @@ +import type { Picture3dRecipe } from "@/features/retro-office/core/types"; + +const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"; +const DEFAULT_PICTURE_3D_MODEL = "gpt-4o-mini"; +export const MAX_PICTURE_MODEL_UPLOAD_BYTES = 12 * 1024 * 1024; + +type GeneratePictureModelParams = { + imageDataUrl: string; + fileName?: string; + mimeType?: string; +}; + +const generatedPrimitiveSchema = { + type: "object", + additionalProperties: false, + properties: { + kind: { + type: "string", + enum: ["box", "cylinder", "sphere"], + }, + position: { + type: "array", + minItems: 3, + maxItems: 3, + items: { type: "number" }, + }, + rotation: { + type: "array", + minItems: 3, + maxItems: 3, + items: { type: "number" }, + }, + material: { + type: "object", + additionalProperties: false, + properties: { + color: { type: "string" }, + roughness: { type: "number" }, + metalness: { type: "number" }, + }, + required: ["color"], + }, + size: { + type: "array", + minItems: 3, + maxItems: 3, + items: { type: "number" }, + }, + radiusTop: { type: "number" }, + radiusBottom: { type: "number" }, + height: { type: "number" }, + radius: { type: "number" }, + radialSegments: { type: "number" }, + widthSegments: { type: "number" }, + heightSegments: { type: "number" }, + }, + required: ["kind", "position", "material"], +} as const; + +const generatedModelSchema = { + name: "picture_to_3d_office_asset", + schema: { + type: "object", + additionalProperties: false, + properties: { + title: { type: "string" }, + summary: { type: "string" }, + footprintMeters: { + type: "object", + additionalProperties: false, + properties: { + width: { type: "number" }, + depth: { type: "number" }, + height: { type: "number" }, + }, + required: ["width", "depth", "height"], + }, + primitives: { + type: "array", + minItems: 3, + maxItems: 16, + items: generatedPrimitiveSchema, + }, + }, + required: ["title", "summary", "footprintMeters", "primitives"], + }, + strict: true, +} as const; + +const isRecord = (value: unknown): value is Record => + Boolean(value && typeof value === "object" && !Array.isArray(value)); + +const extractStructuredOutput = (payload: unknown): Picture3dRecipe | null => { + if (!isRecord(payload)) return null; + const choices = Array.isArray(payload.choices) ? payload.choices : []; + const firstChoice = choices[0]; + if (!isRecord(firstChoice)) return null; + const message = isRecord(firstChoice.message) ? firstChoice.message : null; + const content = message?.content; + if (typeof content === "string" && content.trim()) { + try { + return JSON.parse(content) as Picture3dRecipe; + } catch { + return null; + } + } + const parsed = message && "parsed" in message ? message.parsed : null; + return isRecord(parsed) ? (parsed as Picture3dRecipe) : null; +}; + +export const generatePictureModelFromImage = async ({ + imageDataUrl, +}: GeneratePictureModelParams): Promise => { + const apiKey = process.env.OPENAI_API_KEY?.trim(); + if (!apiKey) { + throw new Error("Missing OPENAI_API_KEY for AI 3D generation."); + } + + const baseUrl = + process.env.OPENAI_BASE_URL?.trim() || DEFAULT_OPENAI_BASE_URL; + const model = + process.env.OPENAI_PICTURE_3D_MODEL?.trim() || DEFAULT_PICTURE_3D_MODEL; + + const response = await fetch( + `${baseUrl.replace(/\/$/, "")}/chat/completions`, + { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + cache: "no-store", + body: JSON.stringify({ + model, + response_format: { + type: "json_schema", + json_schema: generatedModelSchema, + }, + messages: [ + { + role: "system", + content: + "You convert a reference image into a compact low-poly office sculpture recipe. Output only structured JSON. Use 3-16 simple primitives. Match a matte retro office furniture style with chunky shapes, no thin details, and plausible freestanding balance. Return only boxes, cylinders, and spheres.", + }, + { + role: "user", + content: [ + { + type: "text", + text: "Recreate the uploaded image as a stylized 3D object that feels like it belongs next to the office furniture and avatars in a retro Three.js office. Use simple primitives, strong silhouette, and no textures. Keep all dimensions normalized to roughly desk-scale collectible proportions and prefer grounded, freestanding forms.", + }, + { + type: "image_url", + image_url: { + url: imageDataUrl, + }, + }, + ], + }, + ], + }), + }, + ); + + if (!response.ok) { + const detail = (await response.text().catch(() => "")).trim(); + throw new Error(detail || "AI picture-to-3D generation failed."); + } + + const payload = (await response.json()) as unknown; + const parsed = extractStructuredOutput(payload); + if (!parsed) { + throw new Error( + "AI picture-to-3D generation returned invalid structured output.", + ); + } + return parsed; +}; diff --git a/tests/unit/pictureAsset.test.ts b/tests/unit/pictureAsset.test.ts index f02eeac..a3feca5 100644 --- a/tests/unit/pictureAsset.test.ts +++ b/tests/unit/pictureAsset.test.ts @@ -1,9 +1,56 @@ import { describe, expect, it } from "vitest"; +import * as THREE from "three"; import { + buildPicturePropGroup, + buildPicturePropItem, derivePicturePalette, - resolvePicturePropDimensions, resolvePicturePropFootprint, } from "@/features/retro-office/core/pictureAsset"; +import type { PicturePropAsset } from "@/features/retro-office/core/types"; + +const demoAsset: PicturePropAsset = { + fileName: "photo.png", + imageDataUrl: "data:image/png;base64,abc", + aspectRatio: 1.1, + dominantColor: "#774433", + accentColor: "#335577", + pixelWidth: 32, + pixelHeight: 28, + provider: "openai", + model: "gpt-4o-mini", + summary: "Chunky desk collectible inspired by the uploaded reference.", + recipe: { + title: "Retro Desk Figure", + summary: "Layered low-poly character silhouette.", + footprintMeters: { + width: 0.84, + depth: 0.46, + height: 1.24, + }, + primitives: [ + { + kind: "box", + size: [0.78, 0.2, 0.44], + position: [0, 0.1, 0], + material: { color: "#774433", roughness: 0.82, metalness: 0.06 }, + }, + { + kind: "cylinder", + radiusTop: 0.14, + radiusBottom: 0.18, + height: 0.78, + position: [0, 0.59, 0], + material: { color: "#335577", roughness: 0.74, metalness: 0.08 }, + }, + { + kind: "sphere", + radius: 0.22, + position: [0, 1.08, 0.02], + material: { color: "#d9c4aa", roughness: 0.7, metalness: 0.04 }, + }, + ], + }, +}; describe("derivePicturePalette", () => { it("builds stable dominant and accent colors from visible pixels", () => { @@ -28,45 +75,37 @@ describe("derivePicturePalette", () => { frameColor: "#3c191b", }); }); - - it("falls back to the default palette for fully transparent images", () => { - const pixels = new Uint8ClampedArray([0, 0, 0, 0, 255, 255, 255, 0]); - - expect(derivePicturePalette(pixels)).toEqual({ - accentColor: "#d97706", - dominantColor: "#7c5c3b", - frameColor: "#24170d", - }); - }); }); describe("resolvePicturePropFootprint", () => { it("scales width with aspect ratio and clamps the result", () => { expect(resolvePicturePropFootprint(0.4)).toEqual({ - depthUnits: 24, - widthUnits: 36, + depthUnits: 30, + widthUnits: 44, }); expect(resolvePicturePropFootprint(1.5)).toEqual({ - depthUnits: 24, - widthUnits: 50, - }); - expect(resolvePicturePropFootprint(3)).toEqual({ - depthUnits: 24, - widthUnits: 56, + depthUnits: 30, + widthUnits: 59, }); }); }); -describe("resolvePicturePropDimensions", () => { - it("keeps portrait props within the scene height budget", () => { - const dims = resolvePicturePropDimensions({ - aspectRatio: 0.68, - footprintDepth: 24 * 0.018, - footprintWidth: 34 * 0.018, - }); +describe("buildPicturePropGroup", () => { + it("builds a freestanding object whose base sits on the floor", () => { + const group = buildPicturePropGroup(demoAsset); + expect(group.children.length).toBe(3); + const bounds = new THREE.Box3().setFromObject(group); + expect(bounds.min.y).toBeGreaterThanOrEqual(-0.000001); + }); +}); - expect(dims.artHeight).toBeLessThanOrEqual(1.08); - expect(dims.baseDepth).toBeGreaterThan(0.19); - expect(dims.frameWidth).toBeGreaterThan(dims.artWidth); +describe("buildPicturePropItem", () => { + it("stores the AI recipe on the furniture item", () => { + const item = buildPicturePropItem(demoAsset, "item-1", 100, 120); + + expect(item.pictureAsset?.recipe.title).toBe("Retro Desk Figure"); + expect(item.type).toBe("picture_prop"); + expect(item.w).toBeGreaterThan(0); + expect(item.h).toBeGreaterThan(0); }); }); diff --git a/tests/unit/pictureModelRoute.test.ts b/tests/unit/pictureModelRoute.test.ts new file mode 100644 index 0000000..d4e3c79 --- /dev/null +++ b/tests/unit/pictureModelRoute.test.ts @@ -0,0 +1,128 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/office/pictureModelGeneration", () => ({ + MAX_PICTURE_MODEL_UPLOAD_BYTES: 8 * 1024 * 1024, + generatePictureModelFromImage: vi.fn().mockResolvedValue({ + asset: { + accentColor: "#f59e0b", + aspectRatio: 1, + dominantColor: "#7c5c3b", + fileName: "demo.png", + imageDataUrl: "data:image/webp;base64,abc", + model: "gpt-4o-mini", + pixelHeight: 32, + pixelWidth: 32, + provider: "openai-compatible", + recipe: { + footprintMeters: { + depth: 0.6, + height: 1.2, + width: 0.72, + }, + primitives: [ + { + kind: "box", + material: { + color: "#7c5c3b", + metalness: 0.08, + roughness: 0.78, + }, + position: [0, 0.4, 0], + size: [0.72, 0.8, 0.32], + }, + ], + summary: "Chunky desk sculpture.", + title: "Desk sculpture", + }, + summary: "Chunky desk sculpture.", + }, + }), +})); + +const { POST } = await import("@/app/api/office/picture-model/route"); +const { MAX_PICTURE_MODEL_UPLOAD_BYTES } = await import( + "@/lib/office/pictureModelGeneration" +); + +function makeImageFile(byteLength: number, type = "image/png") { + return { + arrayBuffer: () => Promise.resolve(new ArrayBuffer(byteLength)), + name: "demo.png", + type, + }; +} + +function mockRequest(opts: { + contentLength?: string; + imageFile?: ReturnType | null; +}): Request { + const headersMap = new Map(); + if (opts.contentLength !== undefined) { + headersMap.set("content-length", opts.contentLength); + } + + const image = opts.imageFile ?? null; + const fakeFormData = { + get: (key: string) => (key === "image" ? image : null), + }; + + return { + headers: { get: (name: string) => headersMap.get(name) ?? null }, + formData: () => Promise.resolve(fakeFormData), + } as unknown as Request; +} + +describe("POST /api/office/picture-model", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("returns 413 for obviously oversized uploads", async () => { + const request = mockRequest({ + contentLength: String(MAX_PICTURE_MODEL_UPLOAD_BYTES + 4096), + imageFile: makeImageFile(1024), + }); + + const response = await POST(request); + + expect(response.status).toBe(413); + }); + + it("returns 400 when the upload is missing", async () => { + const response = await POST(mockRequest({ imageFile: null })); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ + error: expect.stringMatching(/image file is required/i), + }); + }); + + it("returns 400 for unsupported mime types", async () => { + const response = await POST( + mockRequest({ imageFile: makeImageFile(1024, "application/pdf") }), + ); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ + error: expect.stringMatching(/only image uploads/i), + }); + }); + + it("returns generated asset payload for valid uploads", async () => { + const response = await POST( + mockRequest({ imageFile: makeImageFile(2048, "image/png") }), + ); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + asset: { + fileName: "demo.png", + pixelWidth: 32, + recipe: { + primitives: expect.any(Array), + title: "Desk sculpture", + }, + }, + }); + }); +});