Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Render jobs draw scenes that place catalog assets (part of #373)

Nguyen Xuan Anebcc682c81cd
diff --git a/changes/320.md b/changes/320.md
index 94f978ef..ad6a00ad 100644
--- a/changes/320.md
+++ b/changes/320.md
@@ -1 +1 @@
-- Place catalog GLB assets inside recipe scenes with a node `asset` field (issue #320): terrain, lights, procedural meshes and many imported models share one scene. Copies of an asset share geometry and materials and draw as one batch, materials override by source name, a placed clip plays from its own start time and seeks to the same frame, and `scene_inspect` reports each placement's id, bounds and clips. Render jobs do not draw placed assets yet; `docs/mcp.md` gives the command that bundles a Poly Haven glTF folder into a GLB.
+- Place catalog GLB assets inside recipe scenes with a node `asset` field (issue #320): terrain, lights, procedural meshes and many imported models share one scene. Copies of an asset share geometry and materials and draw as one batch, materials override by source name, a placed clip plays from its own start time and seeks to the same frame, and `scene_inspect` reports each placement's id, bounds and clips. Render jobs draw placed assets too; `docs/mcp.md` gives the command that bundles a Poly Haven glTF folder into a GLB.
diff --git a/changes/373.md b/changes/373.md
new file mode 100644
index 00000000..14a0cd44
--- /dev/null
+++ b/changes/373.md
@@ -0,0 +1 @@
+- `scene_render_submit` renders scenes that place catalog assets as a still, stills or MP4 (issue #373): the job page loads the bytes of every placed asset by content ID the way `scene_look` does, and `settings.motion.start` plays placed clips on scene time without authored motion.
diff --git a/docs/agent-scene.md b/docs/agent-scene.md
index b8813ed4..125cf4d3 100644
--- a/docs/agent-scene.md
+++ b/docs/agent-scene.md
@@ -254,7 +254,7 @@ A node with an `asset` field places a GLB from the `--assets` catalog inside the
- `clip` plays that clip, held at its first frame until scene time `start`, then at `speed`, looping unless `loop` is false. Each placement that plays a clip gets a decoded copy of its own (its geometry is not shared), because a pose is state of one decoded asset. A seek gives the same frame as playing to that time. `set-motion` tracks still move the node itself, so a walking figure is a clip plus a motion track on the same node.
- Bounds: 512 placements, 16 different assets, 16 placements playing a clip, and `maxNodes` for the generated parts. Placed geometry does not count against `maxVertices`; each asset keeps the decoder limits of the catalog.
- `scene_inspect` adds `assets`: per placement the asset id, part count, world bounds, clips (index, name, duration), material names and the playing clip.
-- Not yet: render jobs refuse a scene with placed assets (`scene_look` draws them), and a scene created without any placed asset cannot add the first one with `scene_apply`.
+- Not yet: a scene created without any placed asset cannot add the first one with `scene_apply`. Render jobs draw placed assets and play their clips on scene time (`settings.motion.start`).
### Relations between parts
diff --git a/scripts/agent-render-jobs.mjs b/scripts/agent-render-jobs.mjs
index 42e963f4..293142bb 100644
--- a/scripts/agent-render-jobs.mjs
+++ b/scripts/agent-render-jobs.mjs
@@ -3,6 +3,7 @@ import { randomUUID, createHash } from 'node:crypto';
import { isAbsolute, join, resolve } from 'node:path';
import { RENDER_PASS_NAMES, VIDEO_ENCODERS, AUDIO_FILE_PATTERN, MAXIMUM_VIDEO_SECONDS, MAXIMUM_JOB_SIDE, MAXIMUM_JOB_PIXELS } from '../src/agent-scene-capabilities.js';
+import { recipeAssetNeeds } from '../src/agent-scene-assets.js';
import { normalizeFilmTimeline, filmTimelineDuration } from '../src/film-timeline.js';
import { OIDN_DEFAULT_SAMPLES, PATH_TRACER_DENOISERS, PATH_TRACER_JOB_LIMITS } from '../src/path-tracer-capabilities.js';
@@ -108,16 +109,18 @@ function settingsFrom(input, clips, recipe) {
if (input.audio !== undefined) result.audio = audioFrom(input.audio, input.kind);
if (input.motion !== undefined) {
objectKeys(input.motion, ['start'], 'motion');
- if (!['67js.scene', '67js.asset'].includes(recipe?.format) || !recipe.motion) {
- throw new RangeError('Motion settings require a recipe with authored motion.');
+ // Clips of placed assets play on scene time, so a scene that places one needs no authored motion.
+ const placesClips = recipe?.format === '67js.scene' && recipeAssetNeeds(recipe).some(need => need.copies > 1);
+ if (!['67js.scene', '67js.asset'].includes(recipe?.format) || (!recipe.motion && !placesClips)) {
+ throw new RangeError('Motion settings require a recipe with authored motion or a placed clip.');
}
- const motionDuration = boundedNumber(recipe.motion.duration, 0.1, 30, 'motion duration');
- if (recipe.motion.loop !== undefined && typeof recipe.motion.loop !== 'boolean') {
+ const motionDuration = recipe.motion ? boundedNumber(recipe.motion.duration, 0.1, 30, 'motion duration') : undefined;
+ if (recipe.motion?.loop !== undefined && typeof recipe.motion.loop !== 'boolean') {
throw new TypeError('Motion loop must be boolean.');
}
const start = boundedNumber(input.motion.start === undefined ? 0 : input.motion.start, 0, 3600, 'motion start');
boundedNumber(start + lastSampleTime, 0, 3600, 'motion end');
- if (!recipe.motion.loop && start + lastSampleTime > motionDuration) {
+ if (recipe.motion && !recipe.motion.loop && start + lastSampleTime > motionDuration) {
throw new RangeError('Motion range exceeds the authored duration.');
}
result.motion = { start };
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index a6af7f77..f795fa18 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -373,7 +373,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
await jobPage.goto(origin);
await jobPage.waitForFunction(() => Boolean(window.__agentPreview));
const input = { sessionId: jobId, recipe: snapshot.recipe, memoryBudgetMiB: memory, backend,
- ...(snapshot.assetBytes ? { data: snapshot.assetBytes.toString('base64'), ...(await motionSources([snapshot.recipe.motion])) } : {}) };
+ ...(snapshot.assetBytes ? { data: snapshot.assetBytes.toString('base64'), ...(await motionSources([snapshot.recipe.motion])) } : await placedAssetData(snapshot.recipe)) };
const created = await jobPage.evaluate(({ method, input }) => window.__agentPreview[method](input),
{ method: snapshot.assetBytes ? 'createAsset' : 'create', input });
const width = settings.kind === 'video' ? Math.floor(created.render.width / 2) * 2 : created.render.width;
@@ -853,9 +853,6 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
}
const snapshot = await invoke('snapshot', { sessionId, expectedRevision });
const assetBytes = snapshot.recipe.format === '67js.asset' ? await (await catalog()).read(snapshot.recipe.assetId) : undefined;
- if (recipeAssetNeeds(snapshot.recipe).length) {
- throw new Error('Render jobs do not yet render scenes that place catalog assets. Use scene_look for stills of this scene.');
- }
phase = 'rendering';
try {
return await controller.submit({ sourceRevision: snapshot.sourceRevision,
diff --git a/scripts/smoke-agent-scene-assets.mjs b/scripts/smoke-agent-scene-assets.mjs
index 7ba95e95..bb457d36 100644
--- a/scripts/smoke-agent-scene-assets.mjs
+++ b/scripts/smoke-agent-scene-assets.mjs
@@ -1,19 +1,24 @@
// Through MCP on the CPU backend: a recipe with terrain, two different catalog GLBs and ten copies of one of them,
// one of which plays a clip. Checks the draw batches, the inspected assets and that a seek reproduces its frame.
import assert from 'node:assert/strict';
-import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { spawnSync } from 'node:child_process';
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { constants, setPriority } from 'node:os';
import { resolve } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
import { Client } from '@modelcontextprotocol/client';
import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';
import { createTexturedRigFixture } from './agent-asset-fixture.mjs';
setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);
+await rm('artifacts/scene-assets-jobs', { recursive: true, force: true });
+await mkdir('artifacts/scene-assets-jobs', { recursive: true });
await mkdir('artifacts/scene-assets-proof', { recursive: true });
await writeFile('artifacts/scene-assets-proof/wall-e.glb', await readFile('public/wall-e.glb'));
await writeFile('artifacts/scene-assets-proof/textured-rig.glb', await createTexturedRigFixture());
const transport = new StdioClientTransport({ command: process.execPath,
- args: [resolve('scripts/mcp-server.mjs'), '--backend', 'cpu', '--memory', '192', '--assets', resolve('artifacts/scene-assets-proof')],
+ args: [resolve('scripts/mcp-server.mjs'), '--backend', 'cpu', '--memory', '192', '--assets', resolve('artifacts/scene-assets-proof'),
+ '--jobs', resolve('artifacts/scene-assets-jobs')],
stderr: 'pipe', cwd: process.cwd(),
});
let diagnostics = '';
@@ -32,6 +37,24 @@ async function look(sessionId, label, motionTime) {
await writeFile(`artifacts/scene-assets-${label}.png`, bytes);
return bytes;
}
+const MINIMUM_PARITY_DECIBELS = 30;
+const ffmpeg = process.platform === 'win32' ? resolve(process.env.USERPROFILE, '.local/bin/ffmpeg.exe') : 'ffmpeg';
+const ffprobe = process.platform === 'win32' ? resolve(process.env.USERPROFILE, '.local/bin/ffprobe.exe') : 'ffprobe';
+async function renderJob(sessionId, settings) {
+ const { jobId } = (await call('scene_render_submit', { sessionId, expectedRevision: 0, settings })).structuredContent;
+ for (let attempt = 0; attempt < 1200; attempt++) {
+ const record = (await call('scene_render_status', { jobId })).structuredContent;
+ if (record.state === 'completed') return (await call('scene_render_result', { jobId })).structuredContent.artifacts[0].path;
+ assert.ok(!['failed', 'cancelled'].includes(record.state), `render job ${record.state}: ${record.reason}`);
+ await delay(100);
+ }
+ throw new Error('Render job did not finish.');
+}
+function psnr(firstPath, secondPath) {
+ const compare = spawnSync(ffmpeg, ['-hide_banner', '-i', firstPath, '-i', secondPath, '-lavfi', 'psnr', '-f', 'null', '-'], { encoding: 'utf8' });
+ const average = /average:(\S+)/.exec(compare.stderr)?.[1];
+ return average === 'inf' ? Infinity : Number(average);
+}
try {
await client.connect(transport, { timeout: 180000 });
assert.equal((await client.listTools()).tools.length, 22);
@@ -64,10 +87,17 @@ try {
assert.notDeepEqual(first, later, 'the placed clip changes the frame');
assert.deepEqual(await look(sessionId, 'later-again', 1.1), later, 'a seek reproduces its frame');
assert.deepEqual(await look(sessionId, 'start-again', 0), first);
+ // A render job places the same assets and plays the placed clip on scene time, so it matches scene_look.
+ const stillPath = await renderJob(sessionId, { kind: 'image', width: 768, height: 512, motion: { start: 1.1 } });
+ const parity = psnr(stillPath, 'artifacts/scene-assets-later.png');
+ assert.ok(parity >= MINIMUM_PARITY_DECIBELS, `the job still matches scene_look at ${parity} dB, below ${MINIMUM_PARITY_DECIBELS} dB`);
+ const videoPath = await renderJob(sessionId, { kind: 'video', width: 320, height: 214, fps: 8, duration: 1, motion: { start: 0 } });
+ const probe = spawnSync(ffprobe, ['-v', 'error', '-count_frames', '-show_entries', 'stream=nb_read_frames', '-of', 'json', videoPath], { encoding: 'utf8' });
+ assert.equal(JSON.parse(probe.stdout).streams[0].nb_read_frames, '8');
const exported = (await call('scene_export_recipe', { sessionId })).structuredContent;
assert.equal(exported.recipe.nodes.length, 12);
await call('scene_dispose', { sessionId });
- const report = { backend: 'cpu', placements: 11, drawBatches: batches, seekExact: true };
+ const report = { backend: 'cpu', placements: 11, drawBatches: batches, seekExact: true, jobStillParityDecibels: Number.isFinite(parity) ? parity : 'identical', jobVideoFrames: 8 };
await writeFile('artifacts/scene-assets-cpu.json', JSON.stringify(report, null, 2));
console.log(JSON.stringify(report));
} catch (error) {
diff --git a/tests/agent-render-jobs.test.js b/tests/agent-render-jobs.test.js
index d55e05d1..741935b7 100644
--- a/tests/agent-render-jobs.test.js
+++ b/tests/agent-render-jobs.test.js
@@ -76,6 +76,22 @@ test('motion job ranges use immutable recipe duration, loops and actual sample t
await reopened.close();
});
+test('a scene that places a clip takes motion settings without authored motion, one that places a still asset does not', async context => {
+ const root = await temporary(context);
+ const controller = await createAgentRenderJobs({ root, limits, run: writeImage });
+ const id = 'a'.repeat(64);
+ const input = request();
+ input.settings = { kind: 'image', width: 64, height: 64, motion: { start: 2 } };
+ input.snapshot.recipe = { format: '67js.scene', nodes: [{ id: 'walker', asset: { id, clip: { index: 0 } } }] };
+ const submitted = await controller.submit(input);
+ assert.equal((await terminal(controller, submitted.jobId)).state, 'completed');
+ assert.equal(controller.status(submitted.jobId).settings.motion.start, 2);
+ input.snapshot.recipe = { format: '67js.scene', nodes: [{ id: 'prop', asset: { id } }] };
+ await assert.rejects(controller.submit(input), /authored motion or a placed clip/);
+ await controller.release(submitted.jobId);
+ await controller.close();
+});
+
test('real files publish after success with copied snapshots, hashes and restart persistence', async context => {
const root = await temporary(context);
const started = deferred();