3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Play every animation of a GLB with clip index "all" (#2281)
Nguyen Xuan An24193f64d71b
diff --git a/changes/2281.md b/changes/2281.mdnew file mode 100644index 00000000..4ca94753--- /dev/null+++ b/changes/2281.md@@ -0,0 +1 @@+- A placed asset takes `clip: { index: "all" }` to play every animation of a GLB together, and `scene_inspect` warns when a single clip leaves out animations that move other nodes (#2281)diff --git a/changes/2281.size.md b/changes/2281.size.mdnew file mode 100644index 00000000..44918054--- /dev/null+++ b/changes/2281.size.md@@ -0,0 +1 @@+- goal:gltf-box: pose animations accept clip "all" (61 bytes gzip), so a GLB with one animation per object plays all of its animations; the only other way is a hidden merged clip appended to the asset's clip listdiff --git a/docs/agent-scene.md b/docs/agent-scene.mdindex bebfa0e2..f7e57ebb 100644--- a/docs/agent-scene.md+++ b/docs/agent-scene.md@@ -312,6 +312,7 @@ A node with an `asset` field places a GLB from the `--assets` catalog inside the- `flatten` flattens every material of the placement in one call, so an imported asset with many materials needs no override per name: `{ "unlit": true }` draws them without lighting, `{ "color": [0.8, 0.8, 0.8] }` paints them one linear colour, and both together give a flat unlit silhouette for a sketch-to-3D composite. It drops normal, metallic-roughness, occlusion and emissive maps and any glow; `color` also drops the base map, and `unlit` alone keeps it. Other placements in the scene are unchanged, and a named override in `materials` still wins over `flatten`. To flatten the whole scene, give each placement the same `flatten`.- `emissionMask` on an override (and on a recipe material) makes the glow follow a ramp along one axis: `{ "axis": "y", "from": 0.2, "to": 0.5 }` multiplies `emission` by a smooth step that is 0 at `from` and 1 at `to` (a `from` above `to` flips it), so a part glows only at the end that would cool last while its texture and shading stay. The axis and the numbers are metres in the part's own space (the coordinates its mesh stores, at the placement's scale), so read them from the bounds in `scene_inspect`. Without `emissionMask` nothing changes and the shader pays nothing. The rasterizer applies it; the path tracers and GLB export ignore it (the path tracer says so in its notes). Errors name the field and what to pass, such as `emissionMask.axis` needing `x`, `y` or `z`.- Placements of the same asset share geometry and materials, so they draw as one instanced batch per material (`drawBatches` in the render status stays flat as copies are added). Skins and morphs keep working.+- `clip.index` is a clip number or `"all"`. A GLB that holds one animation per object plays only the object of the clip you name, so `"all"` plays every clip of the asset together on one timeline as long as the longest clip; a shorter clip holds its last frame. `scene_inspect` warns (`clip-leaves-other-animations`) when a number leaves out clips that move other nodes; a Blender export keeps one animation per object only with `export_animation_mode='ACTIVE_ACTIONS'`.- `clip` plays that clip, held at its first frame until scene time `start`, then at `speed`, looping unless `loop` is false. Each placement that plays a clip gets a decoded copy of its own (its geometry is not shared), because a pose is state of one decoded asset. A seek gives the same frame as playing to that time. `set-motion` tracks still move the node itself, so a walking figure is a clip plus a motion track on the same node.- Bounds: 512 placements, 16 different assets, 16 placements playing a clip, and `maxNodes` for the generated parts. Placed geometry does not count against `maxVertices`; each asset keeps the decoder limits of the catalog.- `scene_inspect` adds `assets`: per placement the asset id, part count, world bounds, clips (index, name, duration), material names and the playing clip.diff --git a/src/agent-scene-assets.js b/src/agent-scene-assets.jsindex e06dcadf..30d31ba2 100644--- a/src/agent-scene-assets.js+++ b/src/agent-scene-assets.js@@ -30,6 +30,8 @@ const ASSET_ID = /^[a-f0-9]{64}$/;const PLACEMENT_FIELDS = ['id', 'materials', 'flatten', 'clip', 'drive', 'crowd'];const FLATTEN_FIELDS = ['color', 'unlit'];const CLIP_FIELDS = ['index', 'start', 'speed', 'loop'];+// `clip.index: 'all'` plays every clip of the asset on one timeline (the same word the pose animation takes in gltf.js).+const ALL_CLIPS = 'all';const COLOR_FIELDS = ['color', 'emissiveColor'];const UNIT_FIELDS = ['metallic', 'roughness', 'opacity'];const OVERRIDE_FIELDS = ['name', 'id', 'tag', ...COLOR_FIELDS, ...UNIT_FIELDS, 'emission', 'emissionMask'];@@ -141,7 +143,9 @@ export function normalizeAssetPlacement(value) {if (value.clip !== undefined) {knownKeys(value.clip, CLIP_FIELDS, 'asset.clip');const { index, start = 0, speed = 1, loop = true } = value.clip;- if (!Number.isInteger(index) || index < 0) throw invalidField('asset.clip.index', index, 'a whole number of 0 or more, such as 0');+ if (index !== ALL_CLIPS && (!Number.isInteger(index) || index < 0)) {+ throw invalidField('asset.clip.index', index, `a whole number of 0 or more, such as 0, or "${ALL_CLIPS}" to play every clip of the asset together`);+ }if (!Number.isFinite(start) || start < 0 || start > 3600) throw invalidField('asset.clip.start', start, 'seconds from 0 to 3600, such as 0');if (!Number.isFinite(speed) || speed <= 0 || speed > 16) throw invalidField('asset.clip.speed', speed, 'a number above 0 and at most 16, such as 1');if (typeof loop !== 'boolean') throw invalidField('asset.clip.loop', loop, 'true or false', TypeError);@@ -303,8 +307,9 @@ export function expandAssetPlacements(recipe, assets, limits, outputCache = newif (!asset) {throw new Error(`Asset ${assetId.slice(0, 12)} copy ${copy} is not loaded. The host loads it before createAgentScene or apply.`);}- if (clip && !asset.animations[clip.index]) {- throw invalidField(`nodes[${recipe.nodes.indexOf(node)}].asset.clip.index`, clip.index, `a clip index from 0 to ${asset.animations.length - 1}, since asset ${assetId.slice(0, 12)} has ${asset.animations.length} clips; scene_inspect lists them under assets`);+ if (clip && !playedClips(asset, clip).length) {+ const range = asset.animations.length ? `a clip index from 0 to ${asset.animations.length - 1}, or "${ALL_CLIPS}"` : 'no clip index, since the asset has no clips; drop the clip field';+ throw invalidField(`nodes[${recipe.nodes.indexOf(node)}].asset.clip.index`, clip.index, `${range}, since asset ${assetId.slice(0, 12)} has ${asset.animations.length} clips; scene_inspect lists them under assets`);}if (crowd) {const expanded = crowdCode().expandCrowd(recipe, node, asset, copy, outputCache, runtime, limits);@@ -407,7 +412,47 @@ export function clipTimeAt(clip, duration, time) {export function placedClipEnds(runtime) {return runtime.placements.filter(record => record.clip)- .map(record => record.clip.start + record.asset.animations[record.clip.index].duration / record.clip.speed);+ .map(record => record.clip.start + clipDuration(record.asset, record.clip) / record.clip.speed);+}++/** The clips a placement plays: the one at its index, or every clip of the asset for `index: 'all'`. */+function playedClips(asset, clip) {+ if (clip.index === ALL_CLIPS) return asset.animations;+ return asset.animations[clip.index] ? [asset.animations[clip.index]] : [];+}++/** Seconds one pass of a placement's clip takes at speed 1: the longest clip when it plays them all. */+function clipDuration(asset, clip) {+ return Math.max(0, ...playedClips(asset, clip).map(candidate => candidate.duration));+}++/**+ * Diagnostics for a placement that plays one clip of an asset whose other clips move other nodes: the file probably holds+ * one animation per object (Blender exports one action at a time by default), and the rest never play.+ */+export function placedClipDiagnostics(runtime) {+ const problems = [];+ for (const record of runtime.placements) {+ if (!record.clip || record.clip.index === ALL_CLIPS || record.asset.animations.length < 2) continue;+ const played = animatedNodeNames(record.asset.animations[record.clip.index]);+ const skipped = new Set();+ record.asset.animations.forEach((candidate, index) => {+ if (index === record.clip.index) return;+ const names = animatedNodeNames(candidate);+ if (names.size && [...names].every(name => !played.has(name))) skipped.add(index);+ });+ if (!skipped.size) continue;+ problems.push({+ kind: 'clip-leaves-other-animations', severity: 'warning', target: record.node,+ message: `Only clip ${record.clip.index} plays, and clip${skipped.size > 1 ? 's' : ''} ${[...skipped].join(', ')} of this asset move${skipped.size > 1 ? '' : 's'} other nodes, so those nodes stay still.`,+ fix: `Set clip.index to "${ALL_CLIPS}" to play every clip together. If the file came from Blender, export it with export_animation_mode='ACTIVE_ACTIONS' so each object keeps its own animation.`,+ });+ }+ return problems;+}++function animatedNodeNames(animation) {+ return new Set(animation.tracks.map(track => track.name.slice(0, track.name.lastIndexOf('.'))));}/**@@ -431,7 +476,7 @@ export function stageAssetClips(runtime, graph, time) {record.drivePose = driven.pose;transaction = record.asset.stagePose({ nodeOverrides: driven.nodeOverrides });} else {- const duration = record.asset.animations[record.clip.index].duration;+ const duration = clipDuration(record.asset, record.clip);transaction = record.asset.stagePose({ animation: { clip: record.clip.index, time: clipTimeAt(record.clip, duration, time) }, nodeOverrides: [] });}transactions.push(transaction);diff --git a/src/agent-scene.js b/src/agent-scene.jsindex 23b1568f..ff2df485 100644--- a/src/agent-scene.js+++ b/src/agent-scene.js@@ -2411,7 +2411,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {return result;});const resident = geometryStats(ownedGeometryRecords());- const diagnostics = [...recipeDiagnostics(state.recipe), ...closedRoomProblems(), ...staleBakeProblems(), ...(state.bakeSkipped ? [agentSceneModule('bake').bakeSkippedDiagnostic(state.bakeSkipped)] : []),...coplanarProblems(), ...(agentSceneModuleLoaded('unsupportedPart') ? agentSceneModule('unsupportedPart').unsupportedPartProblems(state.parts, state.recipe, revision) : []), ...(state.particleSampler?.diagnostics() ?? [])];+ const diagnostics = [...recipeDiagnostics(state.recipe), ...closedRoomProblems(), ...staleBakeProblems(), ...(state.bakeSkipped ? [agentSceneModule('bake').bakeSkippedDiagnostic(state.bakeSkipped)] : []),...coplanarProblems(), ...(agentSceneModuleLoaded('unsupportedPart') ? agentSceneModule('unsupportedPart').unsupportedPartProblems(state.parts, state.recipe, revision) : []), ...(state.particleSampler?.diagnostics() ?? []), ...(state.assetRuntime.animators.length ? agentSceneModule('assets').placedClipDiagnostics(state.assetRuntime) : [])];return {revision, topologyRevision,...(state.recipe.motion?.ik?.length ? { motionIk: agentSceneModule('motion').compileAgentMotion(state.recipe.motion, state.recipe).sweepIk() } : {}),diff --git a/src/gltf.js b/src/gltf.jsindex 82587da3..469d99b5 100644--- a/src/gltf.js+++ b/src/gltf.js@@ -14,6 +14,9 @@ import { textureDescriptorInfo } from './renderer.js';import { createSkinPose, matrixDeterminant, updateSkinPose, validateSkinGeometry, validateSkinPose } from './skinning.js';import { MAX_STORED_MORPH_TARGETS, createMorphPose, validateMorphGeometry, validateMorphPose, validateMorphTransform } from './morph.js';+/** The `clip` of a pose animation that plays every clip of the asset together. */+export const ALL_CLIPS = 'all';+// The defaults fit one large exported scene (a 240 MB GLB of 8 million vertices with 8K textures) and still bound a hostile file.const DEFAULT_LIMITS = Object.freeze({maxBytes: 256 * 1024 * 1024,@@ -1536,6 +1539,10 @@ export async function parseGltf(data, options = {}) {if (animation === null) {return;}+ if (animation.clip === ALL_CLIPS) {+ for (const clip of gltf.animations) applyClip(clip, animation.time);+ return;+ }applyClip(gltf.animations[animation.clip], animation.time);}@@ -1608,8 +1615,10 @@ export async function parseGltf(data, options = {}) {const animation = input.animation ?? null;if (animation !== null) {knownPoseFields(animation, ['clip', 'time'], 'pose animation');- const clip = gltf.animations[animation.clip];- if (!Number.isInteger(animation.clip) || !clip || !Number.isFinite(animation.time) || animation.time < 0 || animation.time > clip.duration) {+ // 'all' plays every clip at once; its length is the longest clip, and a shorter clip holds its last frame.+ const playsAll = animation.clip === ALL_CLIPS && gltf.animations.length > 0;+ const clip = playsAll ? { duration: Math.max(...gltf.animations.map(candidate => candidate.duration)) } : gltf.animations[animation.clip];+ if (!playsAll && !Number.isInteger(animation.clip) || !clip || !Number.isFinite(animation.time) || animation.time < 0 || animation.time > clip.duration) {throw new RangeError('Pose animation needs an existing clip and time within its duration.');}}diff --git a/tests/agent-scene-assets.test.js b/tests/agent-scene-assets.test.jsindex 0f3825ae..9c2166dd 100644--- a/tests/agent-scene-assets.test.js+++ b/tests/agent-scene-assets.test.js@@ -162,6 +162,70 @@ test('a placed clip gives the scene a timeline: render settings play it with noassert.deepEqual(createAgentScene(recipe([]), {}).placedClipEnds(), []);});+// A file with one animation per object, as a Blender export of several objects writes it: Bend moves the Tip joint, Move the+// Root joint, Slide the whole character, and Slide is the longest.+async function onePerObjectAsset() {+ return parseGltf(skinnedStripAsset((document, append) => {+ const times = append(new Float32Array([0, 3]), 'SCALAR');+ const translations = append(new Float32Array([1, 2, 0, 5, 2, 0]), 'VEC3');+ document.animations.push({ name: 'Slide', samplers: [{ input: times, output: translations, interpolation: 'LINEAR' }], channels: [{ sampler: 0, target: { node: 0, path: 'translation' } }] });+ }));+}++async function onePerObjectAssets(copies) {+ const assets = new Map();+ for (let copy = 0; copy < copies; copy++) assets.set(assetCopyKey(ASSET_ID, copy), await onePerObjectAsset());+ return assets;+}++test('clip index "all" plays every animation of the asset together (#2281)', async () => {+ const nodes = [+ { id: 'first', asset: { id: ASSET_ID, clip: { index: 0 } } },+ { id: 'every', asset: { id: ASSET_ID, clip: { index: 'all' } } },+ { id: 'still', asset: { id: ASSET_ID } },+ ];+ const scene = createAgentScene(recipe(nodes), {}, { assets: await onePerObjectAssets(3) });+ const joints = (id, joint) => Array.from(scene.parts.find(part => part.name === `${id}/0`).skin.jointMatrices.slice(joint * 16, joint * 16 + 16));+ const partX = id => scene.parts.find(part => part.name === `${id}/0`).matrix[12];+ const restRoot = joints('still', 0);+ const restTip = joints('still', 1);+ scene.sampleMotion(1.5);+ assert.deepEqual(joints('first', 0), restRoot, 'index 0 leaves the Root joint where it was');+ assert.equal(partX('first'), partX('still'), 'index 0 leaves the character where it was');+ assert.notDeepEqual(joints('first', 1), restTip, 'index 0 bends the Tip');+ assert.notDeepEqual(joints('every', 0), restRoot, 'all moves the Root joint');+ assert.notDeepEqual(joints('every', 1), restTip, 'all bends the Tip');+ assert.notDeepEqual(partX('every'), partX('still'), 'all slides the character');+ assert.deepEqual(scene.placedClipEnds(), [2, 3], 'the timeline is as long as the longest animation');+ const seeked = joints('every', 0);+ scene.sampleMotion(2);+ const bendEnd = joints('every', 1).slice(0, 12);+ scene.sampleMotion(2.5);+ assert.deepEqual(joints('every', 1).slice(0, 12), bendEnd, 'a clip shorter than the timeline holds its last frame');+ scene.sampleMotion(0);+ scene.sampleMotion(1.5);+ assert.deepEqual(joints('every', 0), seeked, 'a seek gives the same frame');+});++test('clip index is "all" or a clip of the asset (#2281)', async () => {+ const placed = index => recipe([{ id: 'walker', asset: { id: ASSET_ID, clip: { index } } }]);+ const assets = await onePerObjectAssets(2);+ assert.doesNotThrow(() => createAgentScene(placed('all'), {}, { assets }));+ assert.throws(() => createAgentScene(placed('some'), {}, { assets }), /asset\.clip\.index: got "some"; expected a whole number of 0 or more, such as 0, or "all"/);+ assert.throws(() => createAgentScene(placed(-1), {}, { assets }), /"all"/);+ assert.throws(() => createAgentScene(placed(3), {}, { assets }), /from 0 to 2, or "all"/);+});++test('a single clip of a file whose other animations move other nodes warns (#2281)', async () => {+ const one = async index => createAgentScene(recipe([{ id: 'walker', asset: { id: ASSET_ID, clip: { index } } }]), {}, { assets: new Map([[assetCopyKey(ASSET_ID, 1), await onePerObjectAsset()]]) });+ const warned = (await one(0)).inspect().diagnostics.filter(entry => entry.kind === 'clip-leaves-other-animations');+ assert.equal(warned.length, 1);+ assert.equal(warned[0].target, 'walker');+ assert.match(warned[0].message, /clips 1, 2 of this asset move other nodes/);+ assert.match(warned[0].fix, /"all".*export_animation_mode='ACTIVE_ACTIONS'/);+ assert.equal((await one('all')).inspect().diagnostics, undefined, 'all plays every clip, so nothing is left out');+});+test('an empty motion on a scene that places a clip says how to play the clip', async () => {const nodes = [{ id: 'walker', asset: { id: ASSET_ID, clip: { index: 0 } } }];const scene = createAgentScene(recipe(nodes), {}, { assets: await loadedAssets(2) });