3D engine that AI agents drive
render-media: render a scene.json world along a camera key track (closes #554)
Nguyen Xuan An9c718df06ff0
diff --git a/changes/554.md b/changes/554.mdnew file mode 100644index 00000000..21ca57cf--- /dev/null+++ b/changes/554.md@@ -0,0 +1 @@+- Render a `scene.json` world along a camera track with `render-media` (issue #554). `--scene file --config scene.json` loads the same GLB, sky, renderer options, exposure and frame effects the showcase page reads, headless, and `--camera keys.json` (`{ "fps": 24, "frames": [[eye, target], ...] }`, or `createCameraPath` keys) drives the camera one frame per output frame through the existing bounded frame queue and encoder. Asset paths such as `/showcase/bistro/scene.glb` resolve under `--root`, then next to the config. The config's `renderer.memoryBudgetMiB` is the default budget (up to 8192 MiB with `--memory`). On an RTX 3060 the Bistro walk renders 120 frames at 960x540 in 13 s including load, 9.2 fps, against about 0.7 fps for the hidden-pane showcase capture.diff --git a/file.html b/file.htmlnew file mode 100644index 00000000..972ef551--- /dev/null+++ b/file.html@@ -0,0 +1,12 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="UTF-8" />+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />+ <title>Scene file render · 67js</title>+ </head>+ <body style="margin:0;background:#000">+ <canvas id="file-viewport"></canvas>+ <script type="module" src="/src/file-scene.js"></script>+ </body>+</html>diff --git a/scripts/file-scene.mjs b/scripts/file-scene.mjsnew file mode 100644index 00000000..561354c6--- /dev/null+++ b/scripts/file-scene.mjs@@ -0,0 +1,53 @@+import { readFileSync, existsSync, statSync } from 'node:fs';+import { resolve, dirname, join, basename } from 'node:path';++const ASSET_FIELDS = ['glb', 'hdri'];+const MAXIMUM_KEY_FILE_BYTES = 64 * 1024 ** 2;++/** Reads camera keys: { fps, frames: [[eye, target], ...] } or the keys createCameraPath takes. */+export function readCameraKeys(path) {+ const file = resolve(path);+ if (!existsSync(file) || statSync(file).size > MAXIMUM_KEY_FILE_BYTES) throw new Error('The camera keys file is missing or larger than 64 MiB.');+ const keys = JSON.parse(readFileSync(file, 'utf8'));+ if (Array.isArray(keys.frames)) {+ if (!keys.frames.length) throw new Error('Camera keys need at least one frame.');+ const valid = pose => Array.isArray(pose) && pose.length === 2 && pose.every(vector => Array.isArray(vector) && vector.length === 3 && vector.every(Number.isFinite));+ if (!keys.frames.every(valid)) throw new Error('Every camera frame must be [[eyeX, eyeY, eyeZ], [targetX, targetY, targetZ]].');+ const fps = keys.fps ?? 24;+ if (!Number.isFinite(fps) || fps < 1 || fps > 240) throw new Error('Camera keys fps must be between 1 and 240.');+ return { ...keys, fps, duration: keys.frames.length / fps };+ }+ if (!keys.eye || !Number.isFinite(keys.duration)) throw new Error('Camera keys need either frames or createCameraPath keys (duration, eye, target).');+ return keys;+}++/**+ * Finds a scene.json asset on disk. A path like /showcase/bistro/scene.glb is relative to a served public folder, so it is tried+ * under --root, then under the config's folder, then by file name next to the config.+ */+export function resolveAssetPath(assetPath, { configDirectory, root }) {+ const relative = assetPath.replace(/^\/+/, '');+ const candidates = [root && join(root, relative), join(configDirectory, relative), join(configDirectory, basename(relative))].filter(Boolean);+ const found = candidates.find(candidate => existsSync(candidate) && statSync(candidate).isFile());+ if (!found) throw new Error(`Scene asset ${assetPath} was not found. Looked in: ${candidates.join(', ')}. Use --root to name the folder it is relative to.`);+ return found;+}++/** Reads scene.json and swaps each asset path for a /__file/asset/<n>/<name> URL the export script answers from disk. */+export function prepareFileScene({ configPath, root }) {+ const file = resolve(configPath);+ const description = JSON.parse(readFileSync(file, 'utf8'));+ const configDirectory = dirname(file);+ const assets = [];+ const route = assetPath => {+ if (/^[a-z]+:/i.test(assetPath)) throw new Error(`Remote asset URLs are not allowed: ${assetPath}`);+ const diskPath = resolveAssetPath(assetPath, { configDirectory, root: root ? resolve(root) : null });+ assets.push(diskPath);+ return `/__file/asset/${assets.length - 1}/${encodeURIComponent(basename(diskPath))}`;+ };+ for (const field of ASSET_FIELDS) {+ if (typeof description[field] !== 'string') throw new Error(`scene.json needs a "${field}" path.`);+ description[field] = route(description[field]);+ }+ return { description, assets };+}diff --git a/scripts/render-media.mjs b/scripts/render-media.mjsindex 0ce2e3dd..243e6829 100644--- a/scripts/render-media.mjs+++ b/scripts/render-media.mjs@@ -1,18 +1,19 @@import { chromium } from 'playwright';import { spawn } from 'node:child_process';import { mkdir, writeFile, stat } from 'node:fs/promises';-import { existsSync } from 'node:fs';+import { existsSync, readFileSync } from 'node:fs';import { createServer } from 'node:net';import { fileURLToPath } from 'node:url';import { resolve, dirname, basename } from 'node:path';import { FILM_DURATION } from '../src/film.js';import { RENDER_PRESETS } from '../src/budget.js';import { createFrameEncoder } from './frame-encoder.mjs';+import { prepareFileScene, readCameraKeys } from './file-scene.mjs';import { DEFAULT_PAGE_ENCODER_QUANTIZER, createFrameReceiver, installFrameReaderInPage, startFrameServer, startVideoEncoderInPage } from './gpu-frame-reader.mjs';function argumentsFrom(values) {const result = {};- const allowed = new Set(['kind', 'backend', 'scene', 'preset', 'width', 'height', 'fps', 'duration', 'time', 'memory', 'assembly', 'output', 'audio', 'encoder', 'start', 'aspect', 'asset', 'capture', 'jpeg-quality']);+ const allowed = new Set(['kind', 'backend', 'scene', 'preset', 'width', 'height', 'fps', 'duration', 'time', 'memory', 'assembly', 'output', 'audio', 'encoder', 'start', 'aspect', 'asset', 'config', 'camera', 'root', 'capture', 'jpeg-quality']);for (let index = 0; index < values.length; index += 2) {if (!values[index].startsWith('--') || values[index + 1] === undefined) throw new Error('Every option needs a value.');const name = values[index].slice(2);@@ -30,8 +31,11 @@ export function parseRenderOptions(values) {const presetName = options.preset ?? (backend === 'cpu' ? 'cpu' : 'balanced');const preset = RENDER_PRESETS[presetName];const aspect = options.aspect ?? 'landscape';- if (!preset || !['image', 'video', 'stills'].includes(kind) || !['gpu', 'cpu'].includes(backend) || !['film', 'nature', 'object', 'motion', 'view2d', 'assets', 'modeling'].includes(scene) || !['landscape', 'portrait', 'square'].includes(aspect)) throw new Error('Unknown render kind, backend, scene, aspect or preset.');+ if (!preset || !['image', 'video', 'stills'].includes(kind) || !['gpu', 'cpu'].includes(backend) || !['film', 'nature', 'object', 'motion', 'view2d', 'assets', 'modeling', 'file'].includes(scene) || !['landscape', 'portrait', 'square'].includes(aspect)) throw new Error('Unknown render kind, backend, scene, aspect or preset.');if (options.asset && scene !== 'assets') throw new Error('The asset file option requires --scene assets.');+ if (scene === 'file' && !options.config) throw new Error('--scene file needs --config <scene.json>.');+ if ((options.config || options.camera || options.root) && scene !== 'file') throw new Error('--config, --camera and --root require --scene file.');+ const cameraKeys = options.camera ? readCameraKeys(options.camera) : null;const numberOption = (name, fallback, minimum, maximum) => {const value = Number(options[name] ?? fallback);if (!Number.isFinite(value) || value < minimum || value > maximum) throw new Error(`${name} must be between ${minimum} and ${maximum}.`);@@ -47,11 +51,13 @@ export function parseRenderOptions(values) {const width = Math.round(numberOption('width', defaultWidth, 64, 4096));const height = Math.round(numberOption('height', defaultHeight, 64, 4096));if (kind === 'video' && (width % 2 || height % 2)) throw new Error('Video dimensions must be even for yuv420p output.');- const fps = numberOption('fps', preset.fps, 1, 60);+ const fps = numberOption('fps', cameraKeys?.fps ?? preset.fps, 1, 60);const duration = numberOption('duration', scene === 'film' ? FILM_DURATION : ['motion', 'modeling'].includes(scene) ? 12 : 32, 0.1, 240);const time = numberOption('time', scene === 'assets' ? 0 : 3, 0, 3600);const start = numberOption('start', 0, 0, 3600);- const memory = numberOption('memory', preset.memoryBudgetMiB, 32, 1024);+ // A scene file carries its own budget (the Bistro needs about 1.4 GiB for textures), so it is the default and the cap is higher.+ const sceneFileBudget = scene === 'file' ? JSON.parse(readFileSync(resolve(options.config), 'utf8')).renderer?.memoryBudgetMiB : undefined;+ const memory = numberOption('memory', sceneFileBudget ?? preset.memoryBudgetMiB, 32, scene === 'file' ? 8192 : 1024);const assembly = numberOption('assembly', 0, 0, 1);const captureFormat = options.capture ?? (kind === 'video' ? 'auto' : 'png');if (!['auto', 'webcodecs', 'raw', 'jpeg', 'png'].includes(captureFormat)) throw new Error('Capture must be auto, webcodecs, raw, jpeg or png.');@@ -62,7 +68,7 @@ export function parseRenderOptions(values) {const encoderBackend = options.encoder ?? (backend === 'cpu' ? 'cpu' : 'auto');if (!['auto', 'cpu', 'gpu'].includes(encoderBackend)) throw new Error('Encoder must be auto, cpu or gpu.');const audio = options.audio ?? (scene === 'film' ? 'artifacts/critical-score.wav' : null);- return { options, kind, backend, scene, presetName, preset, aspect, width, height, fps, duration, time, start, memory, assembly, output, encoderBackend, audio, captureFormat, jpegQuality };+ return { cameraKeys, options, kind, backend, scene, presetName, preset, aspect, width, height, fps, duration, time, start, memory, assembly, output, encoderBackend, audio, captureFormat, jpegQuality };}export async function selectEncoderBackend(requested, probe) {@@ -159,8 +165,20 @@ export async function renderMedia(values = process.argv.slice(2)) {page.on('pageerror', (error) => errors.push(error.message));const route = scene === 'object' ? 'object.html' : `${scene}.html`;const query = new URLSearchParams({ export: '1', width: String(width), height: String(height), aspect, memory: String(memory), samples: String(backend === 'cpu' ? 1 : preset.samples) });+ if (scene === 'file') {+ const world = prepareFileScene({ configPath: options.config, root: options.root });+ await page.route('**/__file/**', async (request) => {+ const { pathname } = new URL(request.request().url());+ if (pathname === '/__file/config.json') await request.fulfill({ json: world.description });+ else if (pathname === '/__file/camera.json' && plan.cameraKeys) await request.fulfill({ json: plan.cameraKeys });+ else if (pathname.startsWith('/__file/asset/')) await request.fulfill({ path: world.assets[Number(pathname.split('/')[3])] });+ else await request.fulfill({ status: 404, body: 'not found' });+ });+ }await page.goto(`http://127.0.0.1:${serverPort}/${route}?${query}`);- await page.waitForFunction((mode) => window.__scene?.renderer || (mode === 'film' ? window.__filmReady : mode === 'nature' ? window.__nature?.renderer : mode === 'object' ? window.__studio?.renderer : false), scene, { timeout: 60000 });+ await page.waitForFunction((mode) => window.__fileSceneError || window.__scene?.renderer || (mode === 'film' ? window.__filmReady : mode === 'nature' ? window.__nature?.renderer : mode === 'object' ? window.__studio?.renderer : false), scene, { timeout: 60000 });+ const fileSceneError = await page.evaluate(() => window.__fileSceneError);+ if (fileSceneError) throw new Error(`Scene file failed to load: ${fileSceneError}`);if (options.asset) {await page.locator('#file').setInputFiles(resolve(options.asset));await page.waitForFunction(() => document.querySelector('#notice').hidden || document.querySelector('#notice').textContent.startsWith('⚠'));diff --git a/skills/render-67js-media/SKILL.md b/skills/render-67js-media/SKILL.mdindex 99489a9c..8bff5717 100644--- a/skills/render-67js-media/SKILL.md+++ b/skills/render-67js-media/SKILL.md@@ -19,6 +19,17 @@ For imported character motion from `0.8.0`, read `docs/agent-character-motion.mdFor an offline path-traced still with **Cook** (glass refraction, soft multi-bounce indirect light, subsurface random walk, many lights), read `docs/path-tracer.md` and submit an `image` job with `settings.renderer: { type: "pathTracer" }`. It is slow, bounded and offline: read `scene_capabilities` `pathTracer` for its limits and what it skips, and keep rasterizer jobs for video and passes.+## Render a scene.json world along a camera track++To film a world the showcase page shows (GLB, sky, renderer options, exposure, effects), plan the path and render it headless:++```powershell+# keys.json: { "fps": 24, "frames": [[[x, y, z], [tx, ty, tz]], ...] } (one eye and target per frame; createCameraPath keys also work)+bash ~/.agents/tools/gpu-lock.sh node scripts/render-media.mjs --kind video --scene file --config public/showcase/bistro/scene.json --camera keys.json --width 960 --height 540 --duration 5 --output artifacts/walk.mp4+```++`--root <folder>` names the folder that `/showcase/...` paths in the config are relative to; without it they resolve next to the config. `--duration` and `--start` pick a slice of the track (default: all of it, at the track's fps). Check one frame first with `--kind image --time 2.5`.+## Choose and render1. Check the available GPU memory before a GPU job on a shared machine.diff --git a/src/camera-track.js b/src/camera-track.jsnew file mode 100644index 00000000..fa3dd493--- /dev/null+++ b/src/camera-track.js@@ -0,0 +1,37 @@+import { createCameraPath } from './camera-path.js';++function lerpVector(first, second, amount) {+ return first.map((value, axis) => value + (second[axis] - value) * amount);+}++/**+ * Turns camera keys into a function from seconds to a camera pose. Keys are either { fps, frames: [[eye, target], ...] },+ * sampled with a linear blend between neighbouring frames, or the keys createCameraPath takes (duration, eye, target, fov...).+ */+export function createCameraSampler(keys) {+ if (Array.isArray(keys.frames)) {+ const { frames } = keys;+ const framesPerSecond = keys.fps ?? 24;+ return {+ duration: frames.length / framesPerSecond,+ sample(seconds) {+ const position = Math.max(0, Math.min(frames.length - 1, seconds * framesPerSecond));+ const before = Math.floor(position);+ const after = Math.min(frames.length - 1, before + 1);+ const amount = position - before;+ return {+ eye: lerpVector(frames[before][0], frames[after][0], amount),+ target: lerpVector(frames[before][1], frames[after][1], amount),+ };+ },+ };+ }+ const path = createCameraPath({ loop: false, ...keys });+ return {+ duration: keys.duration,+ sample(seconds) {+ const pose = path.sample(Math.min(seconds, keys.duration));+ return { eye: Array.from(pose.eye), target: Array.from(pose.target), up: Array.from(pose.up), fov: pose.fov };+ },+ };+}diff --git a/src/file-scene.js b/src/file-scene.jsnew file mode 100644index 00000000..7472542d--- /dev/null+++ b/src/file-scene.js@@ -0,0 +1,49 @@+import { Renderer67 } from './renderer.js';+import { createSharedTextures, loadGltf } from './gltf.js';+import { loadHdri } from './hdri.js';+import { createCascadedShadows } from './cascaded-shadows.js';+import { applyWaterReflection, attachWaterReflection } from './water-reflection.js';+import { createCameraSampler } from './camera-track.js';++// Headless twin of the showcase page for scripts/render-media.mjs (--scene file). The export script serves the scene.json+// at /__file/config.json with every asset path already pointing at a file it can read, and an optional camera track at+// /__file/camera.json. The renderer options, sky, exposure and frame effects are read exactly as src/showcase.js reads them.++async function readJson(url) {+ const response = await fetch(url);+ if (!response.ok) throw new Error(`${url} is not available (${response.status})`);+ return response.json();+}++async function start() {+ const description = await readJson('/__file/config.json');+ const cameraResponse = await fetch('/__file/camera.json');+ const camera = cameraResponse.ok ? createCameraSampler(await cameraResponse.json()) : null;+ if (description.glbRest?.length || description.hdriPreview) throw new Error('Staged scene.json files (glbRest, hdriPreview) are not supported by --scene file yet.');+ const [model, map] = await Promise.all([+ fetch(description.glb).then(response => response.blob()),+ loadHdri(description.hdri, description.hdriOptions ?? {}),+ ]);+ const asset = await loadGltf(model, { baseUrl: new URL('.', new URL(description.glb, location.href)).href, sharedTextures: createSharedTextures(), ...(description.gltfOptions ?? {}) });+ const rendererOptions = description.renderer ?? {};+ const shadowCascades = rendererOptions.shadowCascades ? createCascadedShadows(rendererOptions.shadowCascades) : undefined;+ const reflection = description.waterReflection ? applyWaterReflection(asset.parts) : { parts: asset.parts, surface: null };+ const renderer = new Renderer67(document.querySelector('#file-viewport'), reflection.parts, {+ ground: false, hdr: true, samples: 4, memoryBudgetMiB: 384,+ ...rendererOptions,+ shadowCascades,+ environment: { ...(rendererOptions.environment ?? {}), map },+ });+ attachWaterReflection(renderer, reflection.surface, { updateEvery: 1 });+ const frame = { exposure: description.exposure ?? 1, ...(description.frame ?? {}) };+ const renderAt = seconds => {+ const pose = camera ? camera.sample(seconds) : { eye: description.eye, target: description.target };+ renderer.render({ ...frame, ...pose });+ };+ window.__scene = { renderer, duration: camera?.duration, renderAt, pause() {} };+}++start().catch(error => {+ window.__fileSceneError = error.message;+ throw error;+});diff --git a/tests/file-scene-render.test.js b/tests/file-scene-render.test.jsnew file mode 100644index 00000000..52d94c29--- /dev/null+++ b/tests/file-scene-render.test.js@@ -0,0 +1,49 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import { mkdtemp, writeFile, mkdir } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { parseRenderOptions } from '../scripts/render-media.mjs';+import { prepareFileScene, readCameraKeys } from '../scripts/file-scene.mjs';+import { createCameraSampler } from '../src/camera-track.js';++async function writeWorld() {+ const folder = await mkdtemp(join(tmpdir(), 'file-scene-'));+ await mkdir(join(folder, 'public/world'), { recursive: true });+ await writeFile(join(folder, 'public/world/scene.glb'), 'glb');+ await writeFile(join(folder, 'public/world/sky.hdr'), 'hdr');+ const config = join(folder, 'scene.json');+ await writeFile(config, JSON.stringify({ glb: '/world/scene.glb', hdri: '/world/sky.hdr', renderer: { memoryBudgetMiB: 2048 } }));+ const keys = join(folder, 'keys.json');+ await writeFile(keys, JSON.stringify({ fps: 30, frames: [[[0, 0, 0], [0, 0, -1]], [[0, 0, 3], [0, 0, 2]]] }));+ return { folder, config, keys };+}++test('a scene file resolves its served-root asset paths under --root and swaps them for export URLs', async () => {+ const { folder, config } = await writeWorld();+ const { description, assets } = prepareFileScene({ configPath: config, root: join(folder, 'public') });+ assert.equal(description.glb, '/__file/asset/0/scene.glb');+ assert.equal(description.hdri, '/__file/asset/1/sky.hdr');+ assert.equal(assets.length, 2);+ assert.throws(() => prepareFileScene({ configPath: config }), /was not found/);+});++test('render options take fps from the camera keys and the memory budget from the scene file', async () => {+ const { config, keys } = await writeWorld();+ const plan = parseRenderOptions(['--kind', 'video', '--scene', 'file', '--config', config, '--camera', keys]);+ assert.equal(plan.fps, 30);+ assert.equal(plan.memory, 2048);+ assert.equal(plan.cameraKeys.duration, 2 / 30);+ assert.throws(() => parseRenderOptions(['--scene', 'file']), /--config/);+ assert.throws(() => parseRenderOptions(['--config', config]), /--scene file/);+});++test('camera keys validate and sample by time', async () => {+ const { keys } = await writeWorld();+ const track = createCameraSampler(readCameraKeys(keys));+ assert.deepEqual(track.sample(0).eye, [0, 0, 0]);+ assert.deepEqual(track.sample(1 / 60).eye, [0, 0, 1.5]);+ assert.deepEqual(track.sample(5).eye, [0, 0, 3]);+ await writeFile(keys, JSON.stringify({ frames: [[[0, 0], [1, 1, 1]]] }));+ assert.throws(() => readCameraKeys(keys), /Every camera frame/);+});