3D engine that AI agents drive
Render jobs draw scenes that place catalog assets (part of #373)
Nguyen Xuan Anebcc682c81cd
diff --git a/changes/320.md b/changes/320.mdindex 94f978ef..ad6a00ad 100644--- a/changes/320.md+++ b/changes/320.md@@ -1 +1 @@-- Place catalog GLB assets inside recipe scenes with a node `asset` field (issue #320): terrain, lights, procedural meshes and many imported models share one scene. Copies of an asset share geometry and materials and draw as one batch, materials override by source name, a placed clip plays from its own start time and seeks to the same frame, and `scene_inspect` reports each placement's id, bounds and clips. Render jobs do not draw placed assets yet; `docs/mcp.md` gives the command that bundles a Poly Haven glTF folder into a GLB.+- Place catalog GLB assets inside recipe scenes with a node `asset` field (issue #320): terrain, lights, procedural meshes and many imported models share one scene. Copies of an asset share geometry and materials and draw as one batch, materials override by source name, a placed clip plays from its own start time and seeks to the same frame, and `scene_inspect` reports each placement's id, bounds and clips. Render jobs draw placed assets too; `docs/mcp.md` gives the command that bundles a Poly Haven glTF folder into a GLB.diff --git a/changes/373.md b/changes/373.mdnew file mode 100644index 00000000..14a0cd44--- /dev/null+++ b/changes/373.md@@ -0,0 +1 @@+- `scene_render_submit` renders scenes that place catalog assets as a still, stills or MP4 (issue #373): the job page loads the bytes of every placed asset by content ID the way `scene_look` does, and `settings.motion.start` plays placed clips on scene time without authored motion.diff --git a/docs/agent-scene.md b/docs/agent-scene.mdindex b8813ed4..125cf4d3 100644--- a/docs/agent-scene.md+++ b/docs/agent-scene.md@@ -254,7 +254,7 @@ A node with an `asset` field places a GLB from the `--assets` catalog inside the- `clip` plays that clip, held at its first frame until scene time `start`, then at `speed`, looping unless `loop` is false. Each placement that plays a clip gets a decoded copy of its own (its geometry is not shared), because a pose is state of one decoded asset. A seek gives the same frame as playing to that time. `set-motion` tracks still move the node itself, so a walking figure is a clip plus a motion track on the same node.- Bounds: 512 placements, 16 different assets, 16 placements playing a clip, and `maxNodes` for the generated parts. Placed geometry does not count against `maxVertices`; each asset keeps the decoder limits of the catalog.- `scene_inspect` adds `assets`: per placement the asset id, part count, world bounds, clips (index, name, duration), material names and the playing clip.-- Not yet: render jobs refuse a scene with placed assets (`scene_look` draws them), and a scene created without any placed asset cannot add the first one with `scene_apply`.+- Not yet: a scene created without any placed asset cannot add the first one with `scene_apply`. Render jobs draw placed assets and play their clips on scene time (`settings.motion.start`).### Relations between partsdiff --git a/scripts/agent-render-jobs.mjs b/scripts/agent-render-jobs.mjsindex 42e963f4..293142bb 100644--- a/scripts/agent-render-jobs.mjs+++ b/scripts/agent-render-jobs.mjs@@ -3,6 +3,7 @@ import { randomUUID, createHash } from 'node:crypto';import { isAbsolute, join, resolve } from 'node:path';import { RENDER_PASS_NAMES, VIDEO_ENCODERS, AUDIO_FILE_PATTERN, MAXIMUM_VIDEO_SECONDS, MAXIMUM_JOB_SIDE, MAXIMUM_JOB_PIXELS } from '../src/agent-scene-capabilities.js';+import { recipeAssetNeeds } from '../src/agent-scene-assets.js';import { normalizeFilmTimeline, filmTimelineDuration } from '../src/film-timeline.js';import { OIDN_DEFAULT_SAMPLES, PATH_TRACER_DENOISERS, PATH_TRACER_JOB_LIMITS } from '../src/path-tracer-capabilities.js';@@ -108,16 +109,18 @@ function settingsFrom(input, clips, recipe) {if (input.audio !== undefined) result.audio = audioFrom(input.audio, input.kind);if (input.motion !== undefined) {objectKeys(input.motion, ['start'], 'motion');- if (!['67js.scene', '67js.asset'].includes(recipe?.format) || !recipe.motion) {- throw new RangeError('Motion settings require a recipe with authored motion.');+ // Clips of placed assets play on scene time, so a scene that places one needs no authored motion.+ const placesClips = recipe?.format === '67js.scene' && recipeAssetNeeds(recipe).some(need => need.copies > 1);+ if (!['67js.scene', '67js.asset'].includes(recipe?.format) || (!recipe.motion && !placesClips)) {+ throw new RangeError('Motion settings require a recipe with authored motion or a placed clip.');}- const motionDuration = boundedNumber(recipe.motion.duration, 0.1, 30, 'motion duration');- if (recipe.motion.loop !== undefined && typeof recipe.motion.loop !== 'boolean') {+ const motionDuration = recipe.motion ? boundedNumber(recipe.motion.duration, 0.1, 30, 'motion duration') : undefined;+ if (recipe.motion?.loop !== undefined && typeof recipe.motion.loop !== 'boolean') {throw new TypeError('Motion loop must be boolean.');}const start = boundedNumber(input.motion.start === undefined ? 0 : input.motion.start, 0, 3600, 'motion start');boundedNumber(start + lastSampleTime, 0, 3600, 'motion end');- if (!recipe.motion.loop && start + lastSampleTime > motionDuration) {+ if (recipe.motion && !recipe.motion.loop && start + lastSampleTime > motionDuration) {throw new RangeError('Motion range exceeds the authored duration.');}result.motion = { start };diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjsindex a6af7f77..f795fa18 100644--- a/scripts/agent-runtime.mjs+++ b/scripts/agent-runtime.mjs@@ -373,7 +373,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browawait jobPage.goto(origin);await jobPage.waitForFunction(() => Boolean(window.__agentPreview));const input = { sessionId: jobId, recipe: snapshot.recipe, memoryBudgetMiB: memory, backend,- ...(snapshot.assetBytes ? { data: snapshot.assetBytes.toString('base64'), ...(await motionSources([snapshot.recipe.motion])) } : {}) };+ ...(snapshot.assetBytes ? { data: snapshot.assetBytes.toString('base64'), ...(await motionSources([snapshot.recipe.motion])) } : await placedAssetData(snapshot.recipe)) };const created = await jobPage.evaluate(({ method, input }) => window.__agentPreview[method](input),{ method: snapshot.assetBytes ? 'createAsset' : 'create', input });const width = settings.kind === 'video' ? Math.floor(created.render.width / 2) * 2 : created.render.width;@@ -853,9 +853,6 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow}const snapshot = await invoke('snapshot', { sessionId, expectedRevision });const assetBytes = snapshot.recipe.format === '67js.asset' ? await (await catalog()).read(snapshot.recipe.assetId) : undefined;- if (recipeAssetNeeds(snapshot.recipe).length) {- throw new Error('Render jobs do not yet render scenes that place catalog assets. Use scene_look for stills of this scene.');- }phase = 'rendering';try {return await controller.submit({ sourceRevision: snapshot.sourceRevision,diff --git a/scripts/smoke-agent-scene-assets.mjs b/scripts/smoke-agent-scene-assets.mjsindex 7ba95e95..bb457d36 100644--- a/scripts/smoke-agent-scene-assets.mjs+++ b/scripts/smoke-agent-scene-assets.mjs@@ -1,19 +1,24 @@// Through MCP on the CPU backend: a recipe with terrain, two different catalog GLBs and ten copies of one of them,// one of which plays a clip. Checks the draw batches, the inspected assets and that a seek reproduces its frame.import assert from 'node:assert/strict';-import { mkdir, readFile, writeFile } from 'node:fs/promises';+import { spawnSync } from 'node:child_process';+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';import { constants, setPriority } from 'node:os';import { resolve } from 'node:path';+import { setTimeout as delay } from 'node:timers/promises';import { Client } from '@modelcontextprotocol/client';import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';import { createTexturedRigFixture } from './agent-asset-fixture.mjs';setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);+await rm('artifacts/scene-assets-jobs', { recursive: true, force: true });+await mkdir('artifacts/scene-assets-jobs', { recursive: true });await mkdir('artifacts/scene-assets-proof', { recursive: true });await writeFile('artifacts/scene-assets-proof/wall-e.glb', await readFile('public/wall-e.glb'));await writeFile('artifacts/scene-assets-proof/textured-rig.glb', await createTexturedRigFixture());const transport = new StdioClientTransport({ command: process.execPath,- args: [resolve('scripts/mcp-server.mjs'), '--backend', 'cpu', '--memory', '192', '--assets', resolve('artifacts/scene-assets-proof')],+ args: [resolve('scripts/mcp-server.mjs'), '--backend', 'cpu', '--memory', '192', '--assets', resolve('artifacts/scene-assets-proof'),+ '--jobs', resolve('artifacts/scene-assets-jobs')],stderr: 'pipe', cwd: process.cwd(),});let diagnostics = '';@@ -32,6 +37,24 @@ async function look(sessionId, label, motionTime) {await writeFile(`artifacts/scene-assets-${label}.png`, bytes);return bytes;}+const MINIMUM_PARITY_DECIBELS = 30;+const ffmpeg = process.platform === 'win32' ? resolve(process.env.USERPROFILE, '.local/bin/ffmpeg.exe') : 'ffmpeg';+const ffprobe = process.platform === 'win32' ? resolve(process.env.USERPROFILE, '.local/bin/ffprobe.exe') : 'ffprobe';+async function renderJob(sessionId, settings) {+ const { jobId } = (await call('scene_render_submit', { sessionId, expectedRevision: 0, settings })).structuredContent;+ for (let attempt = 0; attempt < 1200; attempt++) {+ const record = (await call('scene_render_status', { jobId })).structuredContent;+ if (record.state === 'completed') return (await call('scene_render_result', { jobId })).structuredContent.artifacts[0].path;+ assert.ok(!['failed', 'cancelled'].includes(record.state), `render job ${record.state}: ${record.reason}`);+ await delay(100);+ }+ throw new Error('Render job did not finish.');+}+function psnr(firstPath, secondPath) {+ const compare = spawnSync(ffmpeg, ['-hide_banner', '-i', firstPath, '-i', secondPath, '-lavfi', 'psnr', '-f', 'null', '-'], { encoding: 'utf8' });+ const average = /average:(\S+)/.exec(compare.stderr)?.[1];+ return average === 'inf' ? Infinity : Number(average);+}try {await client.connect(transport, { timeout: 180000 });assert.equal((await client.listTools()).tools.length, 22);@@ -64,10 +87,17 @@ try {assert.notDeepEqual(first, later, 'the placed clip changes the frame');assert.deepEqual(await look(sessionId, 'later-again', 1.1), later, 'a seek reproduces its frame');assert.deepEqual(await look(sessionId, 'start-again', 0), first);+ // A render job places the same assets and plays the placed clip on scene time, so it matches scene_look.+ const stillPath = await renderJob(sessionId, { kind: 'image', width: 768, height: 512, motion: { start: 1.1 } });+ const parity = psnr(stillPath, 'artifacts/scene-assets-later.png');+ assert.ok(parity >= MINIMUM_PARITY_DECIBELS, `the job still matches scene_look at ${parity} dB, below ${MINIMUM_PARITY_DECIBELS} dB`);+ const videoPath = await renderJob(sessionId, { kind: 'video', width: 320, height: 214, fps: 8, duration: 1, motion: { start: 0 } });+ const probe = spawnSync(ffprobe, ['-v', 'error', '-count_frames', '-show_entries', 'stream=nb_read_frames', '-of', 'json', videoPath], { encoding: 'utf8' });+ assert.equal(JSON.parse(probe.stdout).streams[0].nb_read_frames, '8');const exported = (await call('scene_export_recipe', { sessionId })).structuredContent;assert.equal(exported.recipe.nodes.length, 12);await call('scene_dispose', { sessionId });- const report = { backend: 'cpu', placements: 11, drawBatches: batches, seekExact: true };+ const report = { backend: 'cpu', placements: 11, drawBatches: batches, seekExact: true, jobStillParityDecibels: Number.isFinite(parity) ? parity : 'identical', jobVideoFrames: 8 };await writeFile('artifacts/scene-assets-cpu.json', JSON.stringify(report, null, 2));console.log(JSON.stringify(report));} catch (error) {diff --git a/tests/agent-render-jobs.test.js b/tests/agent-render-jobs.test.jsindex d55e05d1..741935b7 100644--- a/tests/agent-render-jobs.test.js+++ b/tests/agent-render-jobs.test.js@@ -76,6 +76,22 @@ test('motion job ranges use immutable recipe duration, loops and actual sample tawait reopened.close();});+test('a scene that places a clip takes motion settings without authored motion, one that places a still asset does not', async context => {+ const root = await temporary(context);+ const controller = await createAgentRenderJobs({ root, limits, run: writeImage });+ const id = 'a'.repeat(64);+ const input = request();+ input.settings = { kind: 'image', width: 64, height: 64, motion: { start: 2 } };+ input.snapshot.recipe = { format: '67js.scene', nodes: [{ id: 'walker', asset: { id, clip: { index: 0 } } }] };+ const submitted = await controller.submit(input);+ assert.equal((await terminal(controller, submitted.jobId)).state, 'completed');+ assert.equal(controller.status(submitted.jobId).settings.motion.start, 2);+ input.snapshot.recipe = { format: '67js.scene', nodes: [{ id: 'prop', asset: { id } }] };+ await assert.rejects(controller.submit(input), /authored motion or a placed clip/);+ await controller.release(submitted.jobId);+ await controller.close();+});+test('real files publish after success with copied snapshots, hashes and restart persistence', async context => {const root = await temporary(context);const started = deferred();