Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

render-media: render a scene.json world along a camera key track (closes #554)

Nguyen Xuan An9c718df06ff0
diff --git a/changes/554.md b/changes/554.md
new file mode 100644
index 00000000..21ca57cf
--- /dev/null
+++ b/changes/554.md
@@ -0,0 +1 @@
+- Render a `scene.json` world along a camera track with `render-media` (issue #554). `--scene file --config scene.json` loads the same GLB, sky, renderer options, exposure and frame effects the showcase page reads, headless, and `--camera keys.json` (`{ "fps": 24, "frames": [[eye, target], ...] }`, or `createCameraPath` keys) drives the camera one frame per output frame through the existing bounded frame queue and encoder. Asset paths such as `/showcase/bistro/scene.glb` resolve under `--root`, then next to the config. The config's `renderer.memoryBudgetMiB` is the default budget (up to 8192 MiB with `--memory`). On an RTX 3060 the Bistro walk renders 120 frames at 960x540 in 13 s including load, 9.2 fps, against about 0.7 fps for the hidden-pane showcase capture.
diff --git a/file.html b/file.html
new file mode 100644
index 00000000..972ef551
--- /dev/null
+++ b/file.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <title>Scene file render · 67js</title>
+ </head>
+ <body style="margin:0;background:#000">
+ <canvas id="file-viewport"></canvas>
+ <script type="module" src="/src/file-scene.js"></script>
+ </body>
+</html>
diff --git a/scripts/file-scene.mjs b/scripts/file-scene.mjs
new file mode 100644
index 00000000..561354c6
--- /dev/null
+++ b/scripts/file-scene.mjs
@@ -0,0 +1,53 @@
+import { readFileSync, existsSync, statSync } from 'node:fs';
+import { resolve, dirname, join, basename } from 'node:path';
+
+const ASSET_FIELDS = ['glb', 'hdri'];
+const MAXIMUM_KEY_FILE_BYTES = 64 * 1024 ** 2;
+
+/** Reads camera keys: { fps, frames: [[eye, target], ...] } or the keys createCameraPath takes. */
+export function readCameraKeys(path) {
+ const file = resolve(path);
+ if (!existsSync(file) || statSync(file).size > MAXIMUM_KEY_FILE_BYTES) throw new Error('The camera keys file is missing or larger than 64 MiB.');
+ const keys = JSON.parse(readFileSync(file, 'utf8'));
+ if (Array.isArray(keys.frames)) {
+ if (!keys.frames.length) throw new Error('Camera keys need at least one frame.');
+ const valid = pose => Array.isArray(pose) && pose.length === 2 && pose.every(vector => Array.isArray(vector) && vector.length === 3 && vector.every(Number.isFinite));
+ if (!keys.frames.every(valid)) throw new Error('Every camera frame must be [[eyeX, eyeY, eyeZ], [targetX, targetY, targetZ]].');
+ const fps = keys.fps ?? 24;
+ if (!Number.isFinite(fps) || fps < 1 || fps > 240) throw new Error('Camera keys fps must be between 1 and 240.');
+ return { ...keys, fps, duration: keys.frames.length / fps };
+ }
+ if (!keys.eye || !Number.isFinite(keys.duration)) throw new Error('Camera keys need either frames or createCameraPath keys (duration, eye, target).');
+ return keys;
+}
+
+/**
+ * Finds a scene.json asset on disk. A path like /showcase/bistro/scene.glb is relative to a served public folder, so it is tried
+ * under --root, then under the config's folder, then by file name next to the config.
+ */
+export function resolveAssetPath(assetPath, { configDirectory, root }) {
+ const relative = assetPath.replace(/^\/+/, '');
+ const candidates = [root && join(root, relative), join(configDirectory, relative), join(configDirectory, basename(relative))].filter(Boolean);
+ const found = candidates.find(candidate => existsSync(candidate) && statSync(candidate).isFile());
+ if (!found) throw new Error(`Scene asset ${assetPath} was not found. Looked in: ${candidates.join(', ')}. Use --root to name the folder it is relative to.`);
+ return found;
+}
+
+/** Reads scene.json and swaps each asset path for a /__file/asset/<n>/<name> URL the export script answers from disk. */
+export function prepareFileScene({ configPath, root }) {
+ const file = resolve(configPath);
+ const description = JSON.parse(readFileSync(file, 'utf8'));
+ const configDirectory = dirname(file);
+ const assets = [];
+ const route = assetPath => {
+ if (/^[a-z]+:/i.test(assetPath)) throw new Error(`Remote asset URLs are not allowed: ${assetPath}`);
+ const diskPath = resolveAssetPath(assetPath, { configDirectory, root: root ? resolve(root) : null });
+ assets.push(diskPath);
+ return `/__file/asset/${assets.length - 1}/${encodeURIComponent(basename(diskPath))}`;
+ };
+ for (const field of ASSET_FIELDS) {
+ if (typeof description[field] !== 'string') throw new Error(`scene.json needs a "${field}" path.`);
+ description[field] = route(description[field]);
+ }
+ return { description, assets };
+}
diff --git a/scripts/render-media.mjs b/scripts/render-media.mjs
index 0ce2e3dd..243e6829 100644
--- a/scripts/render-media.mjs
+++ b/scripts/render-media.mjs
@@ -1,18 +1,19 @@
import { chromium } from 'playwright';
import { spawn } from 'node:child_process';
import { mkdir, writeFile, stat } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
+import { existsSync, readFileSync } from 'node:fs';
import { createServer } from 'node:net';
import { fileURLToPath } from 'node:url';
import { resolve, dirname, basename } from 'node:path';
import { FILM_DURATION } from '../src/film.js';
import { RENDER_PRESETS } from '../src/budget.js';
import { createFrameEncoder } from './frame-encoder.mjs';
+import { prepareFileScene, readCameraKeys } from './file-scene.mjs';
import { DEFAULT_PAGE_ENCODER_QUANTIZER, createFrameReceiver, installFrameReaderInPage, startFrameServer, startVideoEncoderInPage } from './gpu-frame-reader.mjs';
function argumentsFrom(values) {
const result = {};
- const allowed = new Set(['kind', 'backend', 'scene', 'preset', 'width', 'height', 'fps', 'duration', 'time', 'memory', 'assembly', 'output', 'audio', 'encoder', 'start', 'aspect', 'asset', 'capture', 'jpeg-quality']);
+ const allowed = new Set(['kind', 'backend', 'scene', 'preset', 'width', 'height', 'fps', 'duration', 'time', 'memory', 'assembly', 'output', 'audio', 'encoder', 'start', 'aspect', 'asset', 'config', 'camera', 'root', 'capture', 'jpeg-quality']);
for (let index = 0; index < values.length; index += 2) {
if (!values[index].startsWith('--') || values[index + 1] === undefined) throw new Error('Every option needs a value.');
const name = values[index].slice(2);
@@ -30,8 +31,11 @@ export function parseRenderOptions(values) {
const presetName = options.preset ?? (backend === 'cpu' ? 'cpu' : 'balanced');
const preset = RENDER_PRESETS[presetName];
const aspect = options.aspect ?? 'landscape';
- if (!preset || !['image', 'video', 'stills'].includes(kind) || !['gpu', 'cpu'].includes(backend) || !['film', 'nature', 'object', 'motion', 'view2d', 'assets', 'modeling'].includes(scene) || !['landscape', 'portrait', 'square'].includes(aspect)) throw new Error('Unknown render kind, backend, scene, aspect or preset.');
+ if (!preset || !['image', 'video', 'stills'].includes(kind) || !['gpu', 'cpu'].includes(backend) || !['film', 'nature', 'object', 'motion', 'view2d', 'assets', 'modeling', 'file'].includes(scene) || !['landscape', 'portrait', 'square'].includes(aspect)) throw new Error('Unknown render kind, backend, scene, aspect or preset.');
if (options.asset && scene !== 'assets') throw new Error('The asset file option requires --scene assets.');
+ if (scene === 'file' && !options.config) throw new Error('--scene file needs --config <scene.json>.');
+ if ((options.config || options.camera || options.root) && scene !== 'file') throw new Error('--config, --camera and --root require --scene file.');
+ const cameraKeys = options.camera ? readCameraKeys(options.camera) : null;
const numberOption = (name, fallback, minimum, maximum) => {
const value = Number(options[name] ?? fallback);
if (!Number.isFinite(value) || value < minimum || value > maximum) throw new Error(`${name} must be between ${minimum} and ${maximum}.`);
@@ -47,11 +51,13 @@ export function parseRenderOptions(values) {
const width = Math.round(numberOption('width', defaultWidth, 64, 4096));
const height = Math.round(numberOption('height', defaultHeight, 64, 4096));
if (kind === 'video' && (width % 2 || height % 2)) throw new Error('Video dimensions must be even for yuv420p output.');
- const fps = numberOption('fps', preset.fps, 1, 60);
+ const fps = numberOption('fps', cameraKeys?.fps ?? preset.fps, 1, 60);
const duration = numberOption('duration', scene === 'film' ? FILM_DURATION : ['motion', 'modeling'].includes(scene) ? 12 : 32, 0.1, 240);
const time = numberOption('time', scene === 'assets' ? 0 : 3, 0, 3600);
const start = numberOption('start', 0, 0, 3600);
- const memory = numberOption('memory', preset.memoryBudgetMiB, 32, 1024);
+ // A scene file carries its own budget (the Bistro needs about 1.4 GiB for textures), so it is the default and the cap is higher.
+ const sceneFileBudget = scene === 'file' ? JSON.parse(readFileSync(resolve(options.config), 'utf8')).renderer?.memoryBudgetMiB : undefined;
+ const memory = numberOption('memory', sceneFileBudget ?? preset.memoryBudgetMiB, 32, scene === 'file' ? 8192 : 1024);
const assembly = numberOption('assembly', 0, 0, 1);
const captureFormat = options.capture ?? (kind === 'video' ? 'auto' : 'png');
if (!['auto', 'webcodecs', 'raw', 'jpeg', 'png'].includes(captureFormat)) throw new Error('Capture must be auto, webcodecs, raw, jpeg or png.');
@@ -62,7 +68,7 @@ export function parseRenderOptions(values) {
const encoderBackend = options.encoder ?? (backend === 'cpu' ? 'cpu' : 'auto');
if (!['auto', 'cpu', 'gpu'].includes(encoderBackend)) throw new Error('Encoder must be auto, cpu or gpu.');
const audio = options.audio ?? (scene === 'film' ? 'artifacts/critical-score.wav' : null);
- return { options, kind, backend, scene, presetName, preset, aspect, width, height, fps, duration, time, start, memory, assembly, output, encoderBackend, audio, captureFormat, jpegQuality };
+ return { cameraKeys, options, kind, backend, scene, presetName, preset, aspect, width, height, fps, duration, time, start, memory, assembly, output, encoderBackend, audio, captureFormat, jpegQuality };
}
export async function selectEncoderBackend(requested, probe) {
@@ -159,8 +165,20 @@ export async function renderMedia(values = process.argv.slice(2)) {
page.on('pageerror', (error) => errors.push(error.message));
const route = scene === 'object' ? 'object.html' : `${scene}.html`;
const query = new URLSearchParams({ export: '1', width: String(width), height: String(height), aspect, memory: String(memory), samples: String(backend === 'cpu' ? 1 : preset.samples) });
+ if (scene === 'file') {
+ const world = prepareFileScene({ configPath: options.config, root: options.root });
+ await page.route('**/__file/**', async (request) => {
+ const { pathname } = new URL(request.request().url());
+ if (pathname === '/__file/config.json') await request.fulfill({ json: world.description });
+ else if (pathname === '/__file/camera.json' && plan.cameraKeys) await request.fulfill({ json: plan.cameraKeys });
+ else if (pathname.startsWith('/__file/asset/')) await request.fulfill({ path: world.assets[Number(pathname.split('/')[3])] });
+ else await request.fulfill({ status: 404, body: 'not found' });
+ });
+ }
await page.goto(`http://127.0.0.1:${serverPort}/${route}?${query}`);
- await page.waitForFunction((mode) => window.__scene?.renderer || (mode === 'film' ? window.__filmReady : mode === 'nature' ? window.__nature?.renderer : mode === 'object' ? window.__studio?.renderer : false), scene, { timeout: 60000 });
+ await page.waitForFunction((mode) => window.__fileSceneError || window.__scene?.renderer || (mode === 'film' ? window.__filmReady : mode === 'nature' ? window.__nature?.renderer : mode === 'object' ? window.__studio?.renderer : false), scene, { timeout: 60000 });
+ const fileSceneError = await page.evaluate(() => window.__fileSceneError);
+ if (fileSceneError) throw new Error(`Scene file failed to load: ${fileSceneError}`);
if (options.asset) {
await page.locator('#file').setInputFiles(resolve(options.asset));
await page.waitForFunction(() => document.querySelector('#notice').hidden || document.querySelector('#notice').textContent.startsWith('⚠'));
diff --git a/skills/render-67js-media/SKILL.md b/skills/render-67js-media/SKILL.md
index 99489a9c..8bff5717 100644
--- a/skills/render-67js-media/SKILL.md
+++ b/skills/render-67js-media/SKILL.md
@@ -19,6 +19,17 @@ For imported character motion from `0.8.0`, read `docs/agent-character-motion.md
For an offline path-traced still with **Cook** (glass refraction, soft multi-bounce indirect light, subsurface random walk, many lights), read `docs/path-tracer.md` and submit an `image` job with `settings.renderer: { type: "pathTracer" }`. It is slow, bounded and offline: read `scene_capabilities` `pathTracer` for its limits and what it skips, and keep rasterizer jobs for video and passes.
+## Render a scene.json world along a camera track
+
+To film a world the showcase page shows (GLB, sky, renderer options, exposure, effects), plan the path and render it headless:
+
+```powershell
+# keys.json: { "fps": 24, "frames": [[[x, y, z], [tx, ty, tz]], ...] } (one eye and target per frame; createCameraPath keys also work)
+bash ~/.agents/tools/gpu-lock.sh node scripts/render-media.mjs --kind video --scene file --config public/showcase/bistro/scene.json --camera keys.json --width 960 --height 540 --duration 5 --output artifacts/walk.mp4
+```
+
+`--root <folder>` names the folder that `/showcase/...` paths in the config are relative to; without it they resolve next to the config. `--duration` and `--start` pick a slice of the track (default: all of it, at the track's fps). Check one frame first with `--kind image --time 2.5`.
+
## Choose and render
1. Check the available GPU memory before a GPU job on a shared machine.
diff --git a/src/camera-track.js b/src/camera-track.js
new file mode 100644
index 00000000..fa3dd493
--- /dev/null
+++ b/src/camera-track.js
@@ -0,0 +1,37 @@
+import { createCameraPath } from './camera-path.js';
+
+function lerpVector(first, second, amount) {
+ return first.map((value, axis) => value + (second[axis] - value) * amount);
+}
+
+/**
+ * Turns camera keys into a function from seconds to a camera pose. Keys are either { fps, frames: [[eye, target], ...] },
+ * sampled with a linear blend between neighbouring frames, or the keys createCameraPath takes (duration, eye, target, fov...).
+ */
+export function createCameraSampler(keys) {
+ if (Array.isArray(keys.frames)) {
+ const { frames } = keys;
+ const framesPerSecond = keys.fps ?? 24;
+ return {
+ duration: frames.length / framesPerSecond,
+ sample(seconds) {
+ const position = Math.max(0, Math.min(frames.length - 1, seconds * framesPerSecond));
+ const before = Math.floor(position);
+ const after = Math.min(frames.length - 1, before + 1);
+ const amount = position - before;
+ return {
+ eye: lerpVector(frames[before][0], frames[after][0], amount),
+ target: lerpVector(frames[before][1], frames[after][1], amount),
+ };
+ },
+ };
+ }
+ const path = createCameraPath({ loop: false, ...keys });
+ return {
+ duration: keys.duration,
+ sample(seconds) {
+ const pose = path.sample(Math.min(seconds, keys.duration));
+ return { eye: Array.from(pose.eye), target: Array.from(pose.target), up: Array.from(pose.up), fov: pose.fov };
+ },
+ };
+}
diff --git a/src/file-scene.js b/src/file-scene.js
new file mode 100644
index 00000000..7472542d
--- /dev/null
+++ b/src/file-scene.js
@@ -0,0 +1,49 @@
+import { Renderer67 } from './renderer.js';
+import { createSharedTextures, loadGltf } from './gltf.js';
+import { loadHdri } from './hdri.js';
+import { createCascadedShadows } from './cascaded-shadows.js';
+import { applyWaterReflection, attachWaterReflection } from './water-reflection.js';
+import { createCameraSampler } from './camera-track.js';
+
+// Headless twin of the showcase page for scripts/render-media.mjs (--scene file). The export script serves the scene.json
+// at /__file/config.json with every asset path already pointing at a file it can read, and an optional camera track at
+// /__file/camera.json. The renderer options, sky, exposure and frame effects are read exactly as src/showcase.js reads them.
+
+async function readJson(url) {
+ const response = await fetch(url);
+ if (!response.ok) throw new Error(`${url} is not available (${response.status})`);
+ return response.json();
+}
+
+async function start() {
+ const description = await readJson('/__file/config.json');
+ const cameraResponse = await fetch('/__file/camera.json');
+ const camera = cameraResponse.ok ? createCameraSampler(await cameraResponse.json()) : null;
+ if (description.glbRest?.length || description.hdriPreview) throw new Error('Staged scene.json files (glbRest, hdriPreview) are not supported by --scene file yet.');
+ const [model, map] = await Promise.all([
+ fetch(description.glb).then(response => response.blob()),
+ loadHdri(description.hdri, description.hdriOptions ?? {}),
+ ]);
+ const asset = await loadGltf(model, { baseUrl: new URL('.', new URL(description.glb, location.href)).href, sharedTextures: createSharedTextures(), ...(description.gltfOptions ?? {}) });
+ const rendererOptions = description.renderer ?? {};
+ const shadowCascades = rendererOptions.shadowCascades ? createCascadedShadows(rendererOptions.shadowCascades) : undefined;
+ const reflection = description.waterReflection ? applyWaterReflection(asset.parts) : { parts: asset.parts, surface: null };
+ const renderer = new Renderer67(document.querySelector('#file-viewport'), reflection.parts, {
+ ground: false, hdr: true, samples: 4, memoryBudgetMiB: 384,
+ ...rendererOptions,
+ shadowCascades,
+ environment: { ...(rendererOptions.environment ?? {}), map },
+ });
+ attachWaterReflection(renderer, reflection.surface, { updateEvery: 1 });
+ const frame = { exposure: description.exposure ?? 1, ...(description.frame ?? {}) };
+ const renderAt = seconds => {
+ const pose = camera ? camera.sample(seconds) : { eye: description.eye, target: description.target };
+ renderer.render({ ...frame, ...pose });
+ };
+ window.__scene = { renderer, duration: camera?.duration, renderAt, pause() {} };
+}
+
+start().catch(error => {
+ window.__fileSceneError = error.message;
+ throw error;
+});
diff --git a/tests/file-scene-render.test.js b/tests/file-scene-render.test.js
new file mode 100644
index 00000000..52d94c29
--- /dev/null
+++ b/tests/file-scene-render.test.js
@@ -0,0 +1,49 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, writeFile, mkdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { parseRenderOptions } from '../scripts/render-media.mjs';
+import { prepareFileScene, readCameraKeys } from '../scripts/file-scene.mjs';
+import { createCameraSampler } from '../src/camera-track.js';
+
+async function writeWorld() {
+ const folder = await mkdtemp(join(tmpdir(), 'file-scene-'));
+ await mkdir(join(folder, 'public/world'), { recursive: true });
+ await writeFile(join(folder, 'public/world/scene.glb'), 'glb');
+ await writeFile(join(folder, 'public/world/sky.hdr'), 'hdr');
+ const config = join(folder, 'scene.json');
+ await writeFile(config, JSON.stringify({ glb: '/world/scene.glb', hdri: '/world/sky.hdr', renderer: { memoryBudgetMiB: 2048 } }));
+ const keys = join(folder, 'keys.json');
+ await writeFile(keys, JSON.stringify({ fps: 30, frames: [[[0, 0, 0], [0, 0, -1]], [[0, 0, 3], [0, 0, 2]]] }));
+ return { folder, config, keys };
+}
+
+test('a scene file resolves its served-root asset paths under --root and swaps them for export URLs', async () => {
+ const { folder, config } = await writeWorld();
+ const { description, assets } = prepareFileScene({ configPath: config, root: join(folder, 'public') });
+ assert.equal(description.glb, '/__file/asset/0/scene.glb');
+ assert.equal(description.hdri, '/__file/asset/1/sky.hdr');
+ assert.equal(assets.length, 2);
+ assert.throws(() => prepareFileScene({ configPath: config }), /was not found/);
+});
+
+test('render options take fps from the camera keys and the memory budget from the scene file', async () => {
+ const { config, keys } = await writeWorld();
+ const plan = parseRenderOptions(['--kind', 'video', '--scene', 'file', '--config', config, '--camera', keys]);
+ assert.equal(plan.fps, 30);
+ assert.equal(plan.memory, 2048);
+ assert.equal(plan.cameraKeys.duration, 2 / 30);
+ assert.throws(() => parseRenderOptions(['--scene', 'file']), /--config/);
+ assert.throws(() => parseRenderOptions(['--config', config]), /--scene file/);
+});
+
+test('camera keys validate and sample by time', async () => {
+ const { keys } = await writeWorld();
+ const track = createCameraSampler(readCameraKeys(keys));
+ assert.deepEqual(track.sample(0).eye, [0, 0, 0]);
+ assert.deepEqual(track.sample(1 / 60).eye, [0, 0, 1.5]);
+ assert.deepEqual(track.sample(5).eye, [0, 0, 3]);
+ await writeFile(keys, JSON.stringify({ frames: [[[0, 0], [1, 1, 1]]] }));
+ assert.throws(() => readCameraKeys(keys), /Every camera frame/);
+});