3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
import test from 'node:test';
import assert from 'node:assert/strict';
import { parseGltf } from '../src/gltf.js';
import { createAgentScene, loadAllAgentSceneModules } from '../src/agent-scene.js';
import { assetCopyKey, recipeAssetNeeds } from '../src/agent-scene-assets.js';
import { gameAssetIds } from '../src/game-assets.js';
import { groupParts } from '../src/renderer.js';
import { normalizeJobSettings } from '../scripts/agent-render-jobs.mjs';
import { skinnedStripAsset } from './fixtures/skinned-strip.js';
const ASSET_ID = 'ab'.repeat(32);
const OTHER_ID = 'cd'.repeat(32);
await loadAllAgentSceneModules();
async function strip(withName = true) {
return parseGltf(skinnedStripAsset(document => {
if (withName) document.materials[0].name = 'Skin';
}));
}
async function loadedAssets(copies = 1) {
const assets = new Map();
for (const id of [ASSET_ID, OTHER_ID]) {
for (let copy = 0; copy < copies; copy++) assets.set(assetCopyKey(id, copy), await strip());
}
return assets;
}
function recipe(nodes) {
return {
format: 'crucible.scene', schemaVersion: 1,
geometries: [{ id: 'ground', type: 'box' }],
materials: [{ id: 'grass', color: [0.2, 0.5, 0.2] }],
nodes: [{ id: 'floor', geometry: 'ground', material: 'grass', scale: [20, 0.1, 20] }, ...nodes],
};
}
test('ten placements of one asset share geometry, material and a draw batch', async () => {
const nodes = Array.from({ length: 10 }, (_, index) => ({ id: `tree-${index}`, translation: [index * 2, 0, 0], asset: { id: ASSET_ID } }));
nodes.push({ id: 'other', asset: { id: OTHER_ID } });
const scene = createAgentScene(recipe(nodes), {}, { assets: await loadedAssets() });
const placed = scene.parts.filter(part => part.name.startsWith('tree-'));
assert.equal(placed.length, 10);
assert.equal(new Set(placed.map(part => part.geometry)).size, 1);
assert.equal(new Set(placed.map(part => part.material)).size, 1);
assert.equal(new Set(placed.map(part => part.skin)).size, 1);
assert.equal(placed[3].matrix[12], 6 + 1, 'the placement translation composes with the asset part matrix');
const batches = groupParts(scene.parts);
assert.equal(batches.length, 3, 'ground, ten copies, and the other asset');
assert.equal(batches.find(batch => batch.length === 10).length, 10);
const report = scene.inspect({ includeBounds: true });
assert.equal(report.assets.length, 11);
assert.equal(report.assets[0].assetId, ASSET_ID);
assert.deepEqual(report.assets[0].clips.map(clip => clip.name), ['Bend', 'Move']);
assert.deepEqual(report.assets[0].materials, ['Skin']);
assert.ok(report.assets[0].bounds.max[1] > report.assets[0].bounds.min[1]);
assert.equal(report.nodes.length, 12, 'generated part nodes stay out of the node list');
assert.equal(scene.exportRecipe().nodes.length, 12);
assert.equal(scene.exportRecipe().geometries.length, 1);
});
test('material overrides name the source material and stay shared between placements', async () => {
const nodes = [
{ id: 'red-a', asset: { id: ASSET_ID, materials: [{ name: 'Skin', color: [1, 0, 0] }] } },
{ id: 'red-b', asset: { id: ASSET_ID, materials: [{ name: 'Skin', color: [1, 0, 0] }] } },
{ id: 'plain', asset: { id: ASSET_ID } },
];
const scene = createAgentScene(recipe(nodes), {}, { assets: await loadedAssets() });
const material = id => scene.parts.find(part => part.name === `${id}/0`).material;
assert.deepEqual(Array.from(material('red-a').color), [1, 0, 0]);
assert.equal(material('red-a'), material('red-b'));
assert.notEqual(material('red-a'), material('plain'));
assert.throws(() => createAgentScene(recipe([{ id: 'bad', asset: { id: ASSET_ID, materials: [{ name: 'Nope', color: [1, 0, 0] }] } }]), {}, { assets: new Map([[assetCopyKey(ASSET_ID, 0), null]]) }), /not loaded/);
await assert.rejects(async () => createAgentScene(recipe([{ id: 'bad', asset: { id: ASSET_ID, materials: [{ name: 'Nope', color: [1, 0, 0] }] } }]), {}, { assets: await loadedAssets() }), /^RangeError: nodes\[1\]\.asset\.materials\[0\]\.name: got "Nope"; expected one of "Skin"/);
});
test('recipeAssetNeeds skips a malformed node list, null node and non-list npcs (#2623)', () => {
const asset = { id: ASSET_ID };
assert.deepEqual(recipeAssetNeeds({ nodes: {} }), []);
assert.deepEqual(recipeAssetNeeds({ nodes: [null] }), []);
assert.deepEqual(recipeAssetNeeds({ game: { npcs: {} } }), []);
assert.deepEqual(recipeAssetNeeds({ nodes: { asset } }), []);
assert.deepEqual(recipeAssetNeeds({ game: { npcs: { asset } } }), []);
assert.deepEqual(recipeAssetNeeds({ nodes: [{ asset }], game: { npcs: { asset } } }), [{ id: ASSET_ID, copies: 1 }]);
assert.deepEqual(gameAssetIds({ game: { npcs: {} } }), []);
assert.deepEqual(gameAssetIds({ game: { npcs: [null] } }), []);
});
test('a placed clip plays on its own time offset and a seek gives the same frame', async () => {
const nodes = [
{ id: 'walker', translation: [3, 0, 0], asset: { id: ASSET_ID, clip: { index: 0, start: 0.5 } } },
{ id: 'late', asset: { id: ASSET_ID, clip: { index: 0, start: 2 } } },
{ id: 'still', asset: { id: ASSET_ID } },
];
const needs = recipeAssetNeeds(recipe(nodes));
assert.deepEqual(needs, [{ id: ASSET_ID, copies: 3 }]);
const scene = createAgentScene(recipe(nodes), {}, { assets: await loadedAssets(3) });
const joints = id => Array.from(scene.parts.find(part => part.name === `${id}/0`).skin.jointMatrices);
const restJoints = joints('still');
assert.deepEqual(joints('walker'), restJoints, 'before its start the clip holds its first frame');
scene.sampleMotion(1);
const walkerAtOne = joints('walker');
assert.notDeepEqual(walkerAtOne, restJoints);
assert.deepEqual(joints('late'), restJoints, 'the later placement has not started');
assert.deepEqual(joints('still'), restJoints);
scene.sampleMotion(0);
scene.sampleMotion(1);
assert.deepEqual(joints('walker'), walkerAtOne);
assert.equal(scene.parts.find(part => part.name === 'walker/0').matrix[12], 3 + 1);
});
test('placements are bounded and need their assets', async () => {
const many = Array.from({ length: 513 }, (_, index) => ({ id: `n${index}`, asset: { id: ASSET_ID } }));
assert.throws(() => createAgentScene(recipe(many), {}, { assets: new Map() }), /at most 512/);
assert.throws(() => createAgentScene(recipe([{ id: 'x', asset: { id: ASSET_ID } }])), /places catalog assets/);
assert.throws(() => createAgentScene(recipe([{ id: 'x', geometry: 'ground', material: 'grass', asset: { id: ASSET_ID } }]), {}, { assets: new Map() }), /group/);
assert.throws(() => createAgentScene(recipe([{ id: 'x', asset: { id: 'short' } }]), {}, { assets: new Map() }), /64 hexadecimal/);
});
test('apply can add and remove placed assets', async () => {
const scene = createAgentScene(recipe([]), {}, { assets: await loadedAssets() });
scene.apply([{ type: 'add-node', node: { id: 'late-tree', translation: [2, 0, 0], asset: { id: ASSET_ID } } }], { expectedRevision: 0 });
assert.equal(scene.parts.filter(part => part.name.startsWith('late-tree')).length, 1);
scene.apply([{ type: 'set-node', id: 'late-tree', translation: [5, 0, 0] }], { expectedRevision: 1 });
assert.equal(scene.parts.find(part => part.name === 'late-tree/0').matrix[12], 6);
scene.apply([{ type: 'remove-node', id: 'late-tree' }], { expectedRevision: 2 });
assert.equal(scene.parts.filter(part => part.name.startsWith('late-tree')).length, 0);
});
test('a scene created without assets places its first catalog asset, then a second instance, through apply', async () => {
const scene = createAgentScene(recipe([]), {}, {});
assert.throws(() => scene.apply([{ type: 'add-node', node: { id: 'boat', asset: { id: ASSET_ID } } }], { expectedRevision: scene.revision }), /places catalog assets/);
assert.equal(scene.exportRecipe().nodes.length, 1, 'the refused edit leaves the scene alone');
scene.provideAssets(await loadedAssets());
scene.apply([{ type: 'add-node', node: { id: 'boat', translation: [3, 0, 0], asset: { id: ASSET_ID } } }], { expectedRevision: scene.revision });
scene.apply([{ type: 'add-node', node: { id: 'boat-2', translation: [-3, 0, 0], asset: { id: ASSET_ID } } }], { expectedRevision: scene.revision });
const placed = scene.inspect().assets;
assert.deepEqual(placed.map(placement => placement.assetId), [ASSET_ID, ASSET_ID]);
});
test('an asset that is not in the pool is refused by id, and the 16 distinct asset bound still applies', async () => {
const scene = createAgentScene(recipe([]), {}, {});
scene.provideAssets(new Map());
assert.throws(() => scene.apply([{ type: 'add-node', node: { id: 'boat', asset: { id: ASSET_ID } } }], { expectedRevision: scene.revision }), /Asset abababababab copy 0 is not loaded/);
const many = Array.from({ length: 17 }, (_, index) => ({ type: 'add-node', node: { id: `asset-${index}`, asset: { id: index.toString(16).padStart(2, '0').repeat(32) } } }));
const pool = new Map();
for (const operation of many) pool.set(assetCopyKey(operation.node.asset.id, 0), await strip());
scene.provideAssets(pool);
assert.throws(() => scene.apply(many, { expectedRevision: scene.revision }), /at most 512 assets, 16 different ones/);
});
test('a placed clip gives the scene a timeline: render settings play it with no motion track (#2261)', async () => {
const nodes = [{ id: 'walker', asset: { id: ASSET_ID, clip: { index: 0, start: 0.5, speed: 2 } } }, { id: 'still', asset: { id: ASSET_ID } }];
const scene = createAgentScene(recipe(nodes), {}, { assets: await loadedAssets(3) });
assert.equal(scene.exportRecipe().motion, undefined, 'the recipe has no motion track');
const clipLength = scene.inspect().assets[0].clips[0].duration;
const clipEnds = scene.placedClipEnds();
assert.deepEqual(clipEnds, [0.5 + clipLength / 2]);
const joints = id => Array.from(scene.parts.find(part => part.name === `${id}/0`).skin.jointMatrices);
const first = joints('walker');
const settings = { kind: 'video', width: 64, height: 64, fps: 4 };
const clips = clipEnds.map(duration => ({ duration }));
const sceneRecipe = scene.exportRecipe();
const played = normalizeJobSettings({ ...settings, animation: { clip: 0, start: 0 } }, clips, sceneRecipe);
assert.deepEqual(played.motion, { start: 0 });
assert.equal(played.animation, undefined);
assert.equal(played.duration, clipEnds[0], 'a video with no duration runs as long as the placed clip');
scene.sampleMotion(played.motion.start + 0.75);
const later = joints('walker');
assert.notDeepEqual(later, first, 'the clip pose changes over the render time');
assert.throws(() => normalizeJobSettings({ ...settings, animation: { clip: 0 }, motion: { start: 0 } }, clips, sceneRecipe), /either motion or animation/);
assert.throws(() => normalizeJobSettings({ ...settings, animation: { clip: 0 } }, [], sceneRecipe), /this scene has none/);
assert.equal(normalizeJobSettings(settings, clips, sceneRecipe).duration, 1, 'a scene that plays no clip keeps the one second default');
assert.deepEqual(createAgentScene(recipe([]), {}).placedClipEnds(), []);
});
// A file with one animation per object, as a Blender export of several objects writes it: Bend moves the Tip joint, Move the
// Root joint, Slide the whole character, and Slide is the longest.
async function onePerObjectAsset() {
return parseGltf(skinnedStripAsset((document, append) => {
const times = append(new Float32Array([0, 3]), 'SCALAR');
const translations = append(new Float32Array([1, 2, 0, 5, 2, 0]), 'VEC3');
document.animations.push({ name: 'Slide', samplers: [{ input: times, output: translations, interpolation: 'LINEAR' }], channels: [{ sampler: 0, target: { node: 0, path: 'translation' } }] });
}));
}
async function onePerObjectAssets(copies) {
const assets = new Map();
for (let copy = 0; copy < copies; copy++) assets.set(assetCopyKey(ASSET_ID, copy), await onePerObjectAsset());
return assets;
}
test('clip index "all" plays every animation of the asset together (#2281)', async () => {
const nodes = [
{ id: 'first', asset: { id: ASSET_ID, clip: { index: 0 } } },
{ id: 'every', asset: { id: ASSET_ID, clip: { index: 'all' } } },
{ id: 'still', asset: { id: ASSET_ID } },
];
const scene = createAgentScene(recipe(nodes), {}, { assets: await onePerObjectAssets(3) });
const joints = (id, joint) => Array.from(scene.parts.find(part => part.name === `${id}/0`).skin.jointMatrices.slice(joint * 16, joint * 16 + 16));
const partX = id => scene.parts.find(part => part.name === `${id}/0`).matrix[12];
const restRoot = joints('still', 0);
const restTip = joints('still', 1);
scene.sampleMotion(1.5);
assert.deepEqual(joints('first', 0), restRoot, 'index 0 leaves the Root joint where it was');
assert.equal(partX('first'), partX('still'), 'index 0 leaves the character where it was');
assert.notDeepEqual(joints('first', 1), restTip, 'index 0 bends the Tip');
assert.notDeepEqual(joints('every', 0), restRoot, 'all moves the Root joint');
assert.notDeepEqual(joints('every', 1), restTip, 'all bends the Tip');
assert.notDeepEqual(partX('every'), partX('still'), 'all slides the character');
assert.deepEqual(scene.placedClipEnds(), [2, 3], 'the timeline is as long as the longest animation');
const seeked = joints('every', 0);
scene.sampleMotion(2);
const bendEnd = joints('every', 1).slice(0, 12);
scene.sampleMotion(2.5);
assert.deepEqual(joints('every', 1).slice(0, 12), bendEnd, 'a clip shorter than the timeline holds its last frame');
scene.sampleMotion(0);
scene.sampleMotion(1.5);
assert.deepEqual(joints('every', 0), seeked, 'a seek gives the same frame');
});
test('clip index is "all" or a clip of the asset (#2281)', async () => {
const placed = index => recipe([{ id: 'walker', asset: { id: ASSET_ID, clip: { index } } }]);
const assets = await onePerObjectAssets(2);
assert.doesNotThrow(() => createAgentScene(placed('all'), {}, { assets }));
assert.throws(() => createAgentScene(placed('some'), {}, { assets }), /asset\.clip\.index: got "some"; expected a whole number of 0 or more, such as 0, or "all"/);
assert.throws(() => createAgentScene(placed(-1), {}, { assets }), /"all"/);
assert.throws(() => createAgentScene(placed(3), {}, { assets }), /from 0 to 2, or "all"/);
});
test('a single clip of a file whose other animations move other nodes warns (#2281)', async () => {
const one = async index => createAgentScene(recipe([{ id: 'walker', asset: { id: ASSET_ID, clip: { index } } }]), {}, { assets: new Map([[assetCopyKey(ASSET_ID, 1), await onePerObjectAsset()]]) });
const warned = (await one(0)).inspect().diagnostics.filter(entry => entry.kind === 'clip-leaves-other-animations');
assert.equal(warned.length, 1);
assert.equal(warned[0].target, 'walker');
assert.match(warned[0].message, /clips 1, 2 of this asset move other nodes/);
assert.match(warned[0].fix, /"all".*export_animation_mode='ACTIVE_ACTIONS'/);
assert.equal((await one('all')).inspect().diagnostics, undefined, 'all plays every clip, so nothing is left out');
});
test('an empty motion on a scene that places a clip says how to play the clip', async () => {
const nodes = [{ id: 'walker', asset: { id: ASSET_ID, clip: { index: 0 } } }];
const scene = createAgentScene(recipe(nodes), {}, { assets: await loadedAssets(2) });
assert.throws(() => scene.apply([{ type: 'set-motion', motion: { duration: 1, tracks: [] } }], { expectedRevision: 0 }), /needs no motion: leave motion out/);
});