3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
import assert from 'node:assert/strict';
import test from 'node:test';
import { cookInstalled, cookModule, cookTest } from './support/cook.js';
import { createNativeSequence } from '../scripts/agent-native-sequence.mjs';
import { traceFrameNative } from '../scripts/agent-native-trace.mjs';
import { createCamera } from '../src/path-tracer.js';
const renderNative = (await cookModule()) ?? {};
const addonIsBuilt = cookInstalled && renderNative.findNativeAddon() !== null;
const skipWithoutAddon = addonIsBuilt ? false : 'the native addon is not built';
const TRACE_OPTIONS = { width: 4, height: 4, samples: 2, maxSeconds: 5, kernel: 'cycles' };
// ---- The frame loop's decisions, with a stand-in for the package ----
function inputWith({ materialByte = 1, instanceCount = 1, camera = 0, geometryByte = 0 } = {}) {
return {
schemaVersion: 2,
kernelParams: new Uint8Array([camera]),
instanceCount,
instances: new Uint8Array(instanceCount * 160),
materialCount: 1,
materials: new Uint8Array([materialByte]),
triangleCount: 1,
triangles: new Uint8Array([geometryByte]),
};
}
function fakeTracer({ failUpdate = false, failRender = null, failCreate = false, withSessions = true } = {}) {
const events = [];
const tracer = {
events,
prepareCyclesInput: input => input,
planNativeSessionUpdate: renderNative.planNativeSessionUpdate,
traceNative(input, options) {
events.push(['one-shot', options.kernel]);
return { rgba: new Float32Array(4), oneShot: true };
},
};
if (withSessions) {
let sessionNumber = 0;
tracer.createNativeSession = input => {
if (failCreate) throw new Error('no adapter');
sessionNumber += 1;
const id = sessionNumber;
let frames = 0;
events.push(['create', id]);
return {
update(next) {
events.push(['update', id]);
if (failUpdate) throw new Error('the card failed part way');
return { accelerationRebuilt: false, kernelRespecialized: false, updateSeconds: 0.5, input: next, ...(next.triangles[0] === 5 ? { changedMeshes: 1 } : {}) };
},
render(frameOptions) {
events.push(['render', id, Array.from(frameOptions.kernelParams)]);
if (failRender) throw failRender;
frames += 1;
return { rgba: new Float32Array(4), pipelineReused: frames > 1, accelerationReused: frames > 1, frameIndex: frames - 1 };
},
close() {
events.push(['close', id]);
},
};
};
}
return tracer;
}
cookTest('the first frame creates a session and a camera-only frame reuses it with its own kernel parameters', () => {
const tracer = fakeTracer();
const sequence = createNativeSequence({ tracer });
const first = sequence.trace(inputWith({ camera: 1 }), TRACE_OPTIONS);
assert.equal(first.route.path, 'new-session');
assert.equal(first.route.reason, 'first frame of the sequence');
const second = sequence.trace(inputWith({ camera: 2 }), TRACE_OPTIONS);
assert.deepEqual(second.route, { path: 'reused', pipelineReused: true, accelerationReused: true, frameIndex: 1 });
assert.deepEqual(tracer.events, [['create', 1], ['render', 1, [1]], ['render', 1, [2]]]);
});
cookTest('a frame with other materials sends the changed tables through update and keeps the session', () => {
const tracer = fakeTracer();
const sequence = createNativeSequence({ tracer });
sequence.trace(inputWith(), TRACE_OPTIONS);
const { route } = sequence.trace(inputWith({ materialByte: 9 }), TRACE_OPTIONS);
assert.equal(route.path, 'updated');
assert.deepEqual(route.changedTables, ['materials']);
assert.equal(route.accelerationRebuilt, false);
assert.equal(route.updateSeconds, 0.5);
assert.deepEqual(tracer.events.map(event => event[0]), ['create', 'render', 'update', 'render']);
});
cookTest('a frame the session cannot take closes it and starts a new one, saying what changed', () => {
const tracer = fakeTracer();
const sequence = createNativeSequence({ tracer });
sequence.trace(inputWith(), TRACE_OPTIONS);
const counted = sequence.trace(inputWith({ instanceCount: 2 }), TRACE_OPTIONS);
assert.equal(counted.route.path, 'new-session');
assert.match(counted.route.reason, /`instanceCount` changes from 1 to 2/);
assert.deepEqual(tracer.events.filter(event => event[0] !== 'render'), [['create', 1], ['close', 1], ['create', 2]]);
});
cookTest('a frame with other triangles goes through update and reports the meshes Cook rebuilt', () => {
const tracer = fakeTracer();
const sequence = createNativeSequence({ tracer });
sequence.trace(inputWith(), TRACE_OPTIONS);
const geometry = sequence.trace(inputWith({ geometryByte: 5 }), TRACE_OPTIONS);
assert.equal(geometry.route.path, 'updated');
assert.deepEqual(geometry.route.changedTables, ['triangles']);
assert.equal(geometry.route.changedMeshes, 1);
const camera = sequence.trace(inputWith({ geometryByte: 5, camera: 1 }), TRACE_OPTIONS);
assert.equal(camera.route.changedMeshes, undefined);
});
cookTest('a failed update, which closes the session by design, falls to a new session for that frame', () => {
const tracer = fakeTracer({ failUpdate: true });
const sequence = createNativeSequence({ tracer });
sequence.trace(inputWith(), TRACE_OPTIONS);
const { route } = sequence.trace(inputWith({ materialByte: 9 }), TRACE_OPTIONS);
assert.equal(route.path, 'new-session');
assert.match(route.reason, /the session update failed \(the card failed part way\)/);
assert.deepEqual(tracer.events.map(event => event[0]), ['create', 'render', 'update', 'close', 'create', 'render']);
const next = sequence.trace(inputWith({ materialByte: 9, camera: 3 }), TRACE_OPTIONS);
assert.equal(next.route.path, 'reused', 'the new session carries the sequence on');
});
cookTest('a package without sessions, or a card that cannot make one, traces each frame one-shot and says why', () => {
const old = createNativeSequence({ tracer: fakeTracer({ withSessions: false }) });
const oldFrame = old.trace(inputWith(), TRACE_OPTIONS);
assert.equal(oldFrame.route.path, 'one-shot');
assert.match(oldFrame.route.reason, /no createNativeSession/);
assert.equal(oldFrame.traced.oneShot, true);
const refusing = createNativeSequence({ tracer: fakeTracer({ failCreate: true }) });
const refusedFrame = refusing.trace(inputWith(), TRACE_OPTIONS);
assert.equal(refusedFrame.route.path, 'one-shot');
assert.match(refusedFrame.route.reason, /first frame of the sequence; creating a session failed \(no adapter\)/);
});
cookTest('a render that fails falls back one-shot and the next frame starts a new session, but a time limit passes through', () => {
const failing = fakeTracer({ failRender: new Error('device lost') });
const sequence = createNativeSequence({ tracer: failing });
const frame = sequence.trace(inputWith(), TRACE_OPTIONS);
assert.equal(frame.route.path, 'one-shot');
assert.match(frame.route.reason, /the session render failed \(device lost\)/);
assert.deepEqual(failing.events.map(event => event[0]), ['create', 'render', 'close', 'one-shot']);
const slow = fakeTracer({ failRender: new Error('the trace passed its time limit') });
assert.throws(() => createNativeSequence({ tracer: slow }).trace(inputWith(), TRACE_OPTIONS), /time limit/);
assert.deepEqual(slow.events.map(event => event[0]), ['create', 'render', 'close']);
});
cookTest('closing frees the session and a closed sequence refuses a frame', () => {
const tracer = fakeTracer();
const sequence = createNativeSequence({ tracer });
sequence.trace(inputWith(), TRACE_OPTIONS);
sequence.close();
sequence.close();
assert.deepEqual(tracer.events.filter(event => event[0] === 'close'), [['close', 1]]);
assert.throws(() => sequence.trace(inputWith(), TRACE_OPTIONS), /sequence is closed/);
assert.throws(() => createNativeSequence({}), { name: 'TypeError', message: /pass `tracer`/ });
});
const emptyExport = {
triangleCount: 0, triangles: 0, ignored: [],
tables: Object.fromEntries(['triangles', 'materials', 'lights', 'atlas', 'environmentColors', 'environmentTables', 'parameters'].map(name => [name, ''])),
};
test('traceFrameNative reports the route of a sequence frame, and a single frame carries none', async () => {
const pixels = 16;
const plane = (count) => new Float32Array(pixels * count);
const traced = { rgba: plane(4), albedo: plane(3), normal: plane(3), depth: plane(1), variance: plane(1), adapter: 'test', backend: 'test' };
const tracer = { traceNative: () => traced };
const route = { path: 'reused', frameIndex: 3 };
const sequenceCalls = [];
const sequence = { trace: (input, options) => { sequenceCalls.push(options.kernel); return { traced, route }; } };
const common = { tracer, exported: emptyExport, width: 4, height: 4, samples: 2, denoise: 'none' };
const viaSequence = await traceFrameNative({ ...common, sequence, kernel: 'cycles' });
assert.deepEqual(viaSequence.report.nativeSession, route);
assert.deepEqual(sequenceCalls, ['cycles']);
const single = await traceFrameNative({ ...common, kernel: 'cycles' });
assert.equal('nativeSession' in single.report, false);
const legacy = await traceFrameNative({ ...common, sequence, kernel: 'legacy' });
assert.equal('nativeSession' in legacy.report, false, 'the legacy kernel has no sessions');
assert.deepEqual(sequenceCalls, ['cycles']);
});
// ---- Identity against the one-shot render: needs the built addon and a Vulkan ray query adapter ----
const UNIT_QUAD = {
positions: new Float32Array([-1, -1, 0, 1, -1, 0, 1, 1, 0, -1, 1, 0]),
normals: new Float32Array([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1]),
uvs: new Float32Array([0, 1, 1, 1, 1, 0, 0, 0]),
colors: new Float32Array(12).fill(1),
indices: new Uint32Array([0, 1, 2, 0, 2, 3]),
};
const IMAGE_OPTIONS = { width: 24, height: 16, samples: 16, bounces: 3, seed: 11 };
const RESULT_PLANES = ['rgba', 'albedo', 'normal', 'depth', 'variance'];
function horizontalQuadMatrix(scale, height, facesUp) {
const upOrDown = facesUp ? 1 : -1;
return [scale, 0, 0, 0, 0, 0, -upOrDown * scale, 0, 0, upOrDown * scale, 0, 0, 0, height, 0, 1];
}
// A particle: a small quad of its own geometry (not shared, as a flattened particle system exports it) that moves with the frame.
function particleGeometry(offsetX) {
const positions = UNIT_QUAD.positions.map((value, index) => {
if (index % 3 === 0) return value * 0.1 + offsetX;
return value * 0.1 + (index % 3 === 1 ? 0.5 : 0);
});
return { ...UNIT_QUAD, positions };
}
function frameOf({ eye = [0, 2.2, 6], floorColor = [0.6, 0.6, 0.6], lampEmission = 5, lampHeight = 2, extraQuad = false, particleX = null }) {
const scene = {
materials: [{ color: floorColor, roughness: 0.8 }, { color: [1, 0.95, 0.9], emission: lampEmission, roughness: 1 }],
instances: [
{ geometry: UNIT_QUAD, matrix: horizontalQuadMatrix(8, 0, true), materialIndex: 0 },
{ geometry: UNIT_QUAD, matrix: horizontalQuadMatrix(2, 3, false), materialOverride: 1 },
{ geometry: UNIT_QUAD, matrix: horizontalQuadMatrix(1, lampHeight, false), materialOverride: 1 },
],
};
if (extraQuad) scene.instances.push({ geometry: UNIT_QUAD, matrix: horizontalQuadMatrix(1, 1, true), materialIndex: 0 });
if (particleX !== null) {
const identity = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
scene.instances.push({ geometry: particleGeometry(particleX), matrix: identity, materialIndex: 0 });
}
const camera = createCamera({ eye, target: [0, 0.5, 0], fov: 45 }, IMAGE_OPTIONS.width, IMAGE_OPTIONS.height);
return renderNative.buildNativeInputV2(scene, camera, IMAGE_OPTIONS);
}
function sameBytes(first, second) {
return RESULT_PLANES.every(plane => Buffer.from(first[plane].buffer, first[plane].byteOffset, first[plane].byteLength)
.equals(Buffer.from(second[plane].buffer, second[plane].byteOffset, second[plane].byteLength)));
}
// Renders every frame through one sequence and one-shot, requires the same bytes, and returns the routes.
function paths(frameSpecs) {
const options = { ...IMAGE_OPTIONS, kernel: 'cycles' };
const sequence = createNativeSequence({ tracer: renderNative });
try {
return frameSpecs.map((spec, index) => {
const input = frameOf(spec);
const { traced, route } = sequence.trace(input, options);
assert.ok(sameBytes(traced, renderNative.traceNative(input, options)), `frame ${index} differs from the one-shot render`);
return { route, traced };
});
} finally {
sequence.close();
}
}
cookTest('a camera orbit through a session equals one-shot frames and builds nothing after frame 1', { skip: skipWithoutAddon }, () => {
const frames = paths([{ eye: [0, 2.2, 6] }, { eye: [1.5, 2.2, 5.5] }, { eye: [3, 2.2, 4.5] }]);
assert.deepEqual(frames.map(frame => frame.route.path), ['new-session', 'reused', 'reused']);
for (const { traced, route } of frames.slice(1)) {
assert.equal(route.pipelineReused, true, 'the kernel is held, not compiled again');
assert.equal(route.accelerationReused, true);
assert.equal(traced.setupSeconds, 0);
assert.equal(traced.accelerationBuildSeconds, 0);
}
});
cookTest('a material, a light and a transform sequence equal one-shot frames and update in place', { skip: skipWithoutAddon }, () => {
const frames = paths([
{ },
{ floorColor: [0.2, 0.5, 0.8], eye: [1, 2.2, 6] },
{ floorColor: [0.2, 0.5, 0.8], lampEmission: 9 },
{ floorColor: [0.2, 0.5, 0.8], lampEmission: 9, lampHeight: 2.4 },
]);
assert.deepEqual(frames.map(frame => frame.route.path), ['new-session', 'updated', 'updated', 'updated']);
assert.deepEqual(frames[1].route.changedTables, ['materials']);
assert.equal(frames[3].route.accelerationRebuilt, true);
});
cookTest('a sequence that changes its instance count mid-way falls back to a new session and carries on identically', { skip: skipWithoutAddon }, () => {
const frames = paths([{ }, { eye: [1, 2.2, 6] }, { extraQuad: true }, { extraQuad: true, eye: [2, 2.2, 5] }]);
assert.deepEqual(frames.map(frame => frame.route.path), ['new-session', 'reused', 'new-session', 'reused']);
assert.match(frames[2].route.reason, /instanceCount/);
});
cookTest('moving particles take the update route on frame 2, rebuild only the changed mesh and equal a fresh trace', { skip: skipWithoutAddon }, () => {
const frames = paths([{ particleX: -0.5 }, { particleX: 0.5 }, { particleX: 0.5, eye: [1, 2.2, 6] }]);
assert.deepEqual(frames.map(frame => frame.route.path), ['new-session', 'updated', 'reused']);
assert.ok(frames[1].route.changedTables.includes('triangles'));
assert.equal(frames[1].route.changedMeshes, 1, 'only the particle mesh gets a new BLAS');
assert.equal(frames[1].route.accelerationRebuilt, true);
});