Skip to content

3D engine that AI agents drive

Play the glTF clip in scene-file renders (#2308)

Nguyen Xuan Anee02fc4dcfc1
diff --git a/changes/2308.md b/changes/2308.md
new file mode 100644
index 00000000..9610d506
--- /dev/null
+++ b/changes/2308.md
@@ -0,0 +1 @@
+- `render-media --scene file` plays the first animation clip of the scene's GLB on a loop, so a rigged character no longer renders frozen in its bind pose; a GLB without a clip renders as before (#2308)
diff --git a/docs/gltf.md b/docs/gltf.md
index bd17fcfc..5151fb02 100644
--- a/docs/gltf.md
+++ b/docs/gltf.md
@@ -110,6 +110,8 @@ A file over `maxBytes` (256 MiB by default) fails before decoding with an error
`scripts/render-media.mjs --scene file` and `scripts/render-file-native.mjs` serve a GLB above 32 MiB from a local HTTP server that streams it from disk. Handing a file that size to the page through a Playwright route copies it through the browser protocol as one message and closed the whole browser tab (a 225 MB file did, with nothing else loading).
+A scene file whose GLB has an animation clip plays the first clip on a loop in `scripts/render-media.mjs --scene file`, the way the showcase page does: the pose at a frame time is the clip at that time modulo the clip length. Staged files (`glbRest`) are not posed, and `scripts/render-file-native.mjs` still traces the bind pose. A GLB without a clip renders as before.
+
## Loading a large scene without a long freeze
Building a renderer on a big asset blocks the page while the driver links the shader program, uploads textures and packs batches.
diff --git a/src/file-scene-clip.js b/src/file-scene-clip.js
new file mode 100644
index 00000000..0c94c29c
--- /dev/null
+++ b/src/file-scene-clip.js
@@ -0,0 +1,26 @@
+import { loopedClipTime } from './hero-lab/loop/clip-time.js';
+
+// Plays the first animation clip of a scene file's GLB, the way the showcase page does, so a headless frame shows the pose the
+// page shows instead of the bind pose. The clip loops: the time inside it is the frame time modulo the clip length.
+//
+// The renderer holds the parts of the scene as it was built: water reflection and dense levels of detail swap some parts for
+// copies, one per original part and in the same order. A copy shares the original's matrix and skin arrays, so posing the asset
+// moves it too, but `renderer.updateParts` only knows the copies. `sceneParts` is the list the asset's parts were joined into
+// (its order is the one `rendererParts` follows), `rendererParts` is the list the renderer was built from and `posedParts` are
+// the parts the asset itself poses (its own parts, before the staged files were joined in).
+export function createFileSceneClipPlayer({ asset, posedParts, sceneParts, rendererParts }) {
+ const clip = asset.animations?.[0];
+ if (!clip) return null;
+ if (sceneParts.length !== rendererParts.length) {
+ throw new Error(`The renderer parts (${rendererParts.length}) must match the scene parts (${sceneParts.length}) one for one.`);
+ }
+ const posed = new Set(posedParts);
+ const partsToUpdate = rendererParts.filter((part, index) => posed.has(sceneParts[index]));
+ return {
+ clip,
+ poseAt(renderer, seconds) {
+ asset.applyAnimation(loopedClipTime(seconds, clip.duration), 0);
+ renderer.updateParts(partsToUpdate, { shadows: true });
+ },
+ };
+}
diff --git a/src/file-scene.js b/src/file-scene.js
index 161ad5b9..13968932 100644
--- a/src/file-scene.js
+++ b/src/file-scene.js
@@ -6,6 +6,7 @@ import { applyWaterReflection, attachWaterReflection } from './water-reflection.
import { createCameraSampler } from './camera-track.js';
import { inWholeWorldOrder, readPartRanks } from './showcase-download.js';
import { withinDrawnLightLimits } from './drawn-light-lists.js';
+import { createFileSceneClipPlayer } from './file-scene-clip.js';
import { partsForRendererOptions, withSceneFileRendererDefaults } from './dense-lods.js';
// Headless twin of the showcase page for scripts/render-media.mjs (--scene file). The export script serves the scene.json
@@ -54,12 +55,14 @@ async function start() {
const sharedTextures = createSharedTextures();
const gltfOptions = description.gltfOptions ?? {};
const asset = await loadGltf(model, { baseUrl: new URL('.', new URL(description.glb, location.href)).href, sharedTextures, ...gltfOptions });
+ const assetOwnParts = asset.parts;
asset.parts = await loadStagedParts(description, asset, model, sharedTextures);
const rendererOptions = withSceneFileRendererDefaults(description.renderer);
const shadowCascades = rendererOptions.shadowCascades ? createCascadedShadows(rendererOptions.shadowCascades) : undefined;
const reflection = description.waterReflection ? applyWaterReflection(asset.parts) : { parts: asset.parts, surface: null };
const viewport = document.querySelector('#file-viewport');
- const renderer = new CrucibleRenderer(viewport, partsForRendererOptions(reflection.parts, viewport, rendererOptions), {
+ const rendererParts = partsForRendererOptions(reflection.parts, viewport, rendererOptions);
+ const renderer = new CrucibleRenderer(viewport, rendererParts, {
ground: false, hdr: true, samples: 4, memoryBudgetMiB: 384,
...rendererOptions,
shadowCascades,
@@ -67,7 +70,10 @@ async function start() {
});
attachWaterReflection(renderer, reflection.surface, { updateEvery: 1 });
const frame = { exposure: description.exposure ?? 1, ...(description.frame ?? {}) };
+ // A GLB with an animation clip plays its first clip on a loop, as the showcase page does; one without a clip is drawn as before.
+ const clipPlayer = createFileSceneClipPlayer({ asset, posedParts: assetOwnParts, sceneParts: asset.parts, rendererParts });
const renderAt = seconds => {
+ clipPlayer?.poseAt(renderer, seconds);
const pose = camera ? camera.sample(seconds) : { eye: description.eye, target: description.target };
renderer.render({ ...frame, ...pose });
};
diff --git a/tests/file-scene-clip.test.js b/tests/file-scene-clip.test.js
new file mode 100644
index 00000000..d641b46c
--- /dev/null
+++ b/tests/file-scene-clip.test.js
@@ -0,0 +1,81 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { parseGltf } from '../src/gltf.js';
+import { createFileSceneClipPlayer } from '../src/file-scene-clip.js';
+
+async function loadRobot() {
+ const bytes = await readFile(new URL('fixtures/RobotExpressive.glb', import.meta.url));
+ return parseGltf(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), { maxVertices: 500000 });
+}
+
+// What the page does before it builds the renderer: copies of the parts (dense levels of detail and water reflection do this)
+// that share the originals' matrix and skin arrays.
+function rendererCopiesOf(sceneParts) {
+ return sceneParts.map(part => ({ ...part }));
+}
+
+function recordingRenderer() {
+ const calls = [];
+ return { calls, updateParts: (parts, options) => calls.push({ parts, options }) };
+}
+
+function poseOf(parts) {
+ return parts.map(part => ({ matrix: [...part.matrix], palette: part.skin ? [...part.skin.jointMatrices] : null }));
+}
+
+async function playerForRobot({ stagedPart = null } = {}) {
+ const asset = await loadRobot();
+ const posedParts = [...asset.parts];
+ const sceneParts = stagedPart ? [...posedParts, stagedPart] : posedParts;
+ const rendererParts = rendererCopiesOf(sceneParts);
+ const player = createFileSceneClipPlayer({ asset, posedParts, sceneParts, rendererParts });
+ return { asset, player, rendererParts, posedParts };
+}
+
+test('the first clip poses the renderer parts: the pose at t=0 differs from the pose at half the clip', async () => {
+ const { player, rendererParts } = await playerForRobot();
+ const renderer = recordingRenderer();
+ player.poseAt(renderer, 0);
+ const atStart = poseOf(rendererParts);
+ player.poseAt(renderer, player.clip.duration / 2);
+ const atMiddle = poseOf(rendererParts);
+ assert.notDeepEqual(atMiddle, atStart);
+ assert.equal(renderer.calls.length, 2);
+ assert.deepEqual(renderer.calls[0].options, { shadows: true });
+});
+
+test('the clip loops: a time past its length poses like the same time inside it', async () => {
+ const { player, rendererParts } = await playerForRobot();
+ const renderer = recordingRenderer();
+ player.poseAt(renderer, 0.4);
+ const inside = poseOf(rendererParts);
+ player.poseAt(renderer, 0.4 + 3 * player.clip.duration);
+ assert.deepEqual(poseOf(rendererParts), inside);
+});
+
+test('only the renderer copies of the posed parts go to updateParts, not parts that came from another file', async () => {
+ const stagedPart = { name: 'staged', matrix: new Float32Array(16), geometry: {}, material: {} };
+ const { player, rendererParts, posedParts } = await playerForRobot({ stagedPart });
+ const renderer = recordingRenderer();
+ player.poseAt(renderer, 0.2);
+ const updated = renderer.calls[0].parts;
+ assert.equal(updated.length, posedParts.length);
+ assert.ok(updated.every((part, index) => part === rendererParts[index]));
+ assert.ok(!updated.includes(rendererParts.at(-1)));
+});
+
+test('a GLB without a clip gets no player, so its frames are drawn as before', async () => {
+ const asset = await loadRobot();
+ const withoutClip = { ...asset, animations: [] };
+ assert.equal(createFileSceneClipPlayer({ asset: withoutClip, posedParts: asset.parts, sceneParts: asset.parts, rendererParts: asset.parts }), null);
+ assert.equal(createFileSceneClipPlayer({ asset: { parts: [] }, posedParts: [], sceneParts: [], rendererParts: [] }), null);
+});
+
+test('renderer parts that do not match the scene parts one for one are refused by count', async () => {
+ const asset = await loadRobot();
+ assert.throws(
+ () => createFileSceneClipPlayer({ asset, posedParts: asset.parts, sceneParts: asset.parts, rendererParts: asset.parts.slice(1) }),
+ /must match the scene parts/,
+ );
+});