From a3a506595101723ba372f44b9f3d6600a95f89eb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 19 Apr 2026 04:41:12 +0000 Subject: [PATCH] Add studio AI worker scaffold Co-authored-by: Luke The Dev --- package.json | 1 + server/studio-ai-worker.js | 501 +++++++++++++++++++++++++++++++++++++ 2 files changed, 502 insertions(+) create mode 100644 server/studio-ai-worker.js diff --git a/package.json b/package.json index c16d970..648f7d3 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "dev": "node server/index.js --dev", "dev:https": "node server/index.js --dev --https", + "studio-ai-worker": "node server/studio-ai-worker.js", "hermes-adapter": "node server/hermes-gateway-adapter.js", "demo-gateway": "node server/demo-gateway-adapter.js", "build": "next build", diff --git a/server/studio-ai-worker.js b/server/studio-ai-worker.js new file mode 100644 index 0000000..7a0bf1d --- /dev/null +++ b/server/studio-ai-worker.js @@ -0,0 +1,501 @@ +const fs = require("node:fs"); +const http = require("node:http"); +const os = require("node:os"); +const path = require("node:path"); +const { randomUUID } = require("node:crypto"); + +const THREE = require("three"); + +const DEFAULT_PORT = Number.parseInt(process.env.CLAW3D_STUDIO_PROVIDER_PORT || "3333", 10); +const DEFAULT_HOST = process.env.CLAW3D_STUDIO_PROVIDER_HOST || "127.0.0.1"; +const TASK_TIMEOUT_MS = 1_200; + +const respondJson = (res, statusCode, body) => { + res.writeHead(statusCode, { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET,POST,OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, Authorization", + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + }); + res.end(JSON.stringify(body)); +}; + +const respondFile = (res, statusCode, filePath, contentType) => { + res.writeHead(statusCode, { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "no-store", + "Content-Type": contentType, + }); + fs.createReadStream(filePath).pipe(res); +}; + +const readRequestBody = (req) => + new Promise((resolve, reject) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + req.on("error", reject); + req.on("end", () => resolve(Buffer.concat(chunks))); + }); + +const ensureDirectory = (dirPath) => { + if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }); + } +}; + +const resolveStateDir = () => { + const override = process.env.OPENCLAW_STATE_DIR?.trim(); + if (override) { + return path.resolve(override); + } + return path.join(os.homedir(), ".openclaw"); +}; + +const resolveWorkerDir = () => { + const dir = path.join(resolveStateDir(), "claw3d", "studio-ai-worker"); + ensureDirectory(dir); + return dir; +}; + +const hex = (value) => value.toString(16).padStart(2, "0"); +const rgbToHex = (red, green, blue) => `#${hex(red)}${hex(green)}${hex(blue)}`; + +const clamp = (value, min, max) => Math.min(max, Math.max(min, value)); + +const parsePngSize = (buffer) => { + if (buffer.length < 24) return { width: 1024, height: 1024 }; + const signature = buffer.subarray(0, 8); + const expected = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); + if (!signature.equals(expected)) return { width: 1024, height: 1024 }; + return { + width: buffer.readUInt32BE(16), + height: buffer.readUInt32BE(20), + }; +}; + +const sampleIntensityGrid = (buffer, cells = 20) => { + const grid = []; + const length = Math.max(buffer.length, 3); + for (let row = 0; row < cells; row += 1) { + const values = []; + for (let col = 0; col < cells; col += 1) { + const normalizedIndex = (row * cells + col) / Math.max(cells * cells - 1, 1); + const offset = Math.min(Math.floor(normalizedIndex * (length - 3)), length - 3); + const red = buffer[offset] ?? 0; + const green = buffer[offset + 1] ?? red; + const blue = buffer[offset + 2] ?? green; + const luminance = (red * 0.2126 + green * 0.7152 + blue * 0.0722) / 255; + values.push(clamp(Math.round(luminance * 1000) / 1000, 0, 1)); + } + grid.push(values); + } + return grid; +}; + +const sampleColorGrid = (buffer, cells = 20) => { + const grid = []; + const length = Math.max(buffer.length, 3); + for (let row = 0; row < cells; row += 1) { + const values = []; + for (let col = 0; col < cells; col += 1) { + const normalizedIndex = (row * cells + col) / Math.max(cells * cells - 1, 1); + const offset = Math.min(Math.floor(normalizedIndex * (length - 3)), length - 3); + values.push([ + buffer[offset] ?? 0, + buffer[offset + 1] ?? buffer[offset] ?? 0, + buffer[offset + 2] ?? buffer[offset + 1] ?? 0, + ]); + } + grid.push(values); + } + return grid; +}; + +const samplePalette = (buffer) => { + const buckets = new Map(); + const step = Math.max(3, Math.floor(buffer.length / 1800)); + for (let index = 0; index + 2 < buffer.length; index += step) { + const red = buffer[index] ?? 0; + const green = buffer[index + 1] ?? 0; + const blue = buffer[index + 2] ?? 0; + const key = `${red >> 4}:${green >> 4}:${blue >> 4}`; + const current = buckets.get(key); + if (current) { + current.count += 1; + current.red += red; + current.green += green; + current.blue += blue; + } else { + buckets.set(key, { count: 1, red, green, blue }); + } + } + const colors = Array.from(buckets.values()) + .sort((left, right) => right.count - left.count) + .slice(0, 4) + .map((bucket) => + rgbToHex( + Math.round(bucket.red / bucket.count), + Math.round(bucket.green / bucket.count), + Math.round(bucket.blue / bucket.count), + ), + ); + while (colors.length < 4) { + colors.push(colors[colors.length - 1] || "#8b5cf6"); + } + return colors; +}; + +const decodeImageUrl = async (imageUrl) => { + const trimmed = typeof imageUrl === "string" ? imageUrl.trim() : ""; + if (!trimmed) { + throw new Error("image_url is required."); + } + if (trimmed.startsWith("data:")) { + const idx = trimmed.indexOf(","); + if (idx === -1) throw new Error("Invalid data URI."); + const meta = trimmed.slice(5, idx); + const mimeType = meta.split(";")[0] || "image/png"; + return { + buffer: Buffer.from(trimmed.slice(idx + 1), "base64"), + mimeType, + }; + } + const response = await fetch(trimmed); + if (!response.ok) { + throw new Error(`Failed to download image_url (${response.status}).`); + } + return { + buffer: Buffer.from(await response.arrayBuffer()), + mimeType: response.headers.get("content-type") || "image/png", + }; +}; + +const createHeightfieldScene = async (params) => { + const { GLTFExporter } = await import("three/examples/jsm/exporters/GLTFExporter.js"); + const { + colorGrid, + intensityGrid, + palette, + width, + height, + } = params; + const rows = intensityGrid.length; + const cols = intensityGrid[0]?.length ?? 0; + + const scene = new THREE.Scene(); + + const panelWidth = clamp((width / Math.max(height, 1)) * 5.8, 2.8, 7.2); + const panelHeight = clamp((height / Math.max(width, 1)) * 7.2, 4.2, 8.4); + + const group = new THREE.Group(); + group.name = "self_hosted_image_to_3d"; + + const base = new THREE.Mesh( + new THREE.BoxGeometry(panelWidth * 1.18, 0.26, panelHeight * 1.18), + new THREE.MeshStandardMaterial({ color: palette[3] || "#222" }), + ); + base.position.set(0, -0.35, 0); + group.add(base); + + const frame = new THREE.Mesh( + new THREE.BoxGeometry(panelWidth * 1.02, panelHeight * 1.02, 0.18), + new THREE.MeshStandardMaterial({ color: palette[1] || "#666" }), + ); + frame.position.set(0, panelHeight * 0.5, -0.12); + group.add(frame); + + const plane = new THREE.PlaneGeometry(panelWidth, panelHeight, cols - 1, rows - 1); + const position = plane.attributes.position; + const textureData = new Uint8Array(rows * cols * 3); + + for (let row = 0; row < rows; row += 1) { + for (let col = 0; col < cols; col += 1) { + const idx = row * cols + col; + const intensity = intensityGrid[row]?.[col] ?? 0.5; + const displacement = intensity * 0.9; + position.setZ(idx, displacement); + const rgb = colorGrid[row]?.[col] ?? [127, 127, 127]; + textureData[idx * 3] = rgb[0]; + textureData[idx * 3 + 1] = rgb[1]; + textureData[idx * 3 + 2] = rgb[2]; + } + } + position.needsUpdate = true; + plane.computeVertexNormals(); + + const texture = new THREE.DataTexture(textureData, cols, rows, THREE.RGBFormat); + texture.needsUpdate = true; + texture.flipY = true; + + const mesh = new THREE.Mesh( + plane, + new THREE.MeshStandardMaterial({ + map: texture, + roughness: 0.84, + metalness: 0.02, + side: THREE.DoubleSide, + }), + ); + mesh.position.set(0, panelHeight * 0.5, 0.02); + group.add(mesh); + + const accent = new THREE.Mesh( + new THREE.OctahedronGeometry(0.62, 0), + new THREE.MeshStandardMaterial({ + color: palette[2] || palette[0] || "#fff", + emissive: palette[2] || "#000", + emissiveIntensity: 0.7, + roughness: 0.18, + metalness: 0.22, + }), + ); + accent.position.set(panelWidth * 0.42, panelHeight * 0.82, -0.8); + group.add(accent); + + scene.add(group); + + const exporter = new GLTFExporter(); + const glb = await new Promise((resolve, reject) => { + exporter.parse( + scene, + (result) => { + if (result instanceof ArrayBuffer) { + resolve(Buffer.from(result)); + return; + } + reject(new Error("Expected binary GLB output.")); + }, + (error) => reject(error instanceof Error ? error : new Error(String(error))), + { binary: true, onlyVisible: true, trs: false }, + ); + }); + + return glb; +}; + +const createTaskStore = () => { + const tasks = new Map(); + const rootDir = resolveWorkerDir(); + + const getTaskDir = (taskId) => { + const dir = path.join(rootDir, taskId); + ensureDirectory(dir); + return dir; + }; + + const toTaskObject = (task, baseUrl) => ({ + id: task.id, + type: "image-to-3d", + model_urls: task.modelPath + ? { + glb: `${baseUrl}/openapi/v1/image-to-3d/${task.id}/output/model.glb`, + } + : {}, + thumbnail_url: task.thumbnailPath + ? `${baseUrl}/openapi/v1/image-to-3d/${task.id}/output/thumbnail.png` + : "", + progress: task.progress, + created_at: task.createdAt, + started_at: task.startedAt, + finished_at: task.finishedAt, + status: task.status, + texture_urls: [], + task_error: { + message: task.errorMessage || "", + }, + }); + + const createTask = async (params, baseUrl) => { + const taskId = randomUUID(); + const taskDir = getTaskDir(taskId); + const sourceImagePath = path.join(taskDir, "source.png"); + fs.writeFileSync(sourceImagePath, params.buffer); + + const size = parsePngSize(params.buffer); + const palette = samplePalette(params.buffer); + const intensityGrid = sampleIntensityGrid(params.buffer, 20); + const colorGrid = sampleColorGrid(params.buffer, 20); + + const task = { + id: taskId, + status: "PENDING", + progress: 0, + createdAt: Date.now(), + startedAt: 0, + finishedAt: 0, + modelPath: null, + thumbnailPath: sourceImagePath, + errorMessage: "", + sourceImagePath, + palette, + size, + intensityGrid, + colorGrid, + }; + tasks.set(taskId, task); + + setTimeout(async () => { + task.status = "IN_PROGRESS"; + task.progress = 18; + task.startedAt = Date.now(); + try { + const glb = await createHeightfieldScene({ + colorGrid, + intensityGrid, + palette, + width: size.width, + height: size.height, + }); + const modelPath = path.join(taskDir, "model.glb"); + fs.writeFileSync(modelPath, glb); + task.modelPath = modelPath; + task.thumbnailPath = sourceImagePath; + task.progress = 100; + task.status = "SUCCEEDED"; + task.finishedAt = Date.now(); + } catch (error) { + task.status = "FAILED"; + task.progress = 100; + task.finishedAt = Date.now(); + task.errorMessage = error instanceof Error ? error.message : String(error); + } + }, TASK_TIMEOUT_MS); + + return { result: taskId, task: toTaskObject(task, baseUrl) }; + }; + + return { + createTask, + getTask(taskId, baseUrl) { + const task = tasks.get(taskId); + if (!task) return null; + return toTaskObject(task, baseUrl); + }, + getTaskFile(taskId, kind) { + const task = tasks.get(taskId); + if (!task) return null; + if (kind === "thumbnail") return task.thumbnailPath; + if (kind === "model") return task.modelPath; + return null; + }, + }; +}; + +const createStudioAiWorkerServer = (params = {}) => { + const host = params.host || DEFAULT_HOST; + const port = Number.isFinite(params.port) ? params.port : DEFAULT_PORT; + const taskStore = createTaskStore(); + + const server = http.createServer(async (req, res) => { + if (!req.url) { + respondJson(res, 404, { error: "Not found." }); + return; + } + if (req.method === "OPTIONS") { + res.writeHead(204, { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET,POST,OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, Authorization", + }); + res.end(); + return; + } + + const url = new URL(req.url, `http://${host}:${port}`); + const pathname = url.pathname; + const baseUrl = `http://${host}:${port}`; + + try { + if (req.method === "GET" && pathname === "/health") { + respondJson(res, 200, { ok: true, service: "studio-ai-worker" }); + return; + } + + if (req.method === "POST" && pathname === "/openapi/v1/image-to-3d") { + const rawBody = await readRequestBody(req); + const body = JSON.parse(rawBody.toString("utf8")); + const { buffer } = await decodeImageUrl(body.image_url); + const created = await taskStore.createTask({ buffer }, baseUrl); + respondJson(res, 200, { result: created.result }); + return; + } + + const taskMatch = pathname.match(/^\/openapi\/v1\/image-to-3d\/([^/]+)$/); + if (req.method === "GET" && taskMatch) { + const task = taskStore.getTask(taskMatch[1], baseUrl); + if (!task) { + respondJson(res, 404, { error: "Task not found." }); + return; + } + respondJson(res, 200, task); + return; + } + + const modelMatch = pathname.match(/^\/openapi\/v1\/image-to-3d\/([^/]+)\/output\/model\.glb$/); + if (req.method === "GET" && modelMatch) { + const filePath = taskStore.getTaskFile(modelMatch[1], "model"); + if (!filePath || !fs.existsSync(filePath)) { + respondJson(res, 404, { error: "Model not ready." }); + return; + } + respondFile(res, 200, filePath, "model/gltf-binary"); + return; + } + + const thumbnailMatch = pathname.match(/^\/openapi\/v1\/image-to-3d\/([^/]+)\/output\/thumbnail\.png$/); + if (req.method === "GET" && thumbnailMatch) { + const filePath = taskStore.getTaskFile(thumbnailMatch[1], "thumbnail"); + if (!filePath || !fs.existsSync(filePath)) { + respondJson(res, 404, { error: "Thumbnail not ready." }); + return; + } + respondFile(res, 200, filePath, "image/png"); + return; + } + + respondJson(res, 404, { error: "Not found." }); + } catch (error) { + respondJson(res, 500, { + error: error instanceof Error ? error.message : "Worker failure.", + }); + } + }); + + return { + server, + start() { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { + server.off("error", reject); + resolve(); + }); + }); + }, + close() { + return new Promise((resolve) => { + server.close(() => resolve()); + }); + }, + host, + port, + }; +}; + +async function main() { + const worker = createStudioAiWorkerServer(); + await worker.start(); + console.info(`[studio-ai-worker] Listening on http://${worker.host}:${worker.port}`); +} + +if (require.main === module) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} + +module.exports = { + createStudioAiWorkerServer, +};