Skip to content

3D engine that AI agents drive

16.1 KB
Raw Blame
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import test from 'node:test';
import assert from 'node:assert/strict';
import { resolveEffects } from '../src/effects.js';
import { normalizeAgentLook } from '../src/agent-look.js';
import { RENDER_PASS_MODES, installRenderPasses, cryptomatteValue, cryptomatteHashHex } from '../src/render-passes.js';
import { createFragmentShader } from '../src/shaders.js';
import { RENDER_PASS_NAMES } from '../src/agent-scene-capabilities.js';
import { installCameraEffects, depthOfFieldFragment, motionBlurFragment } from '../src/camera-effects.js';
import { createAgentRenderJobs, artifactNames } from '../scripts/agent-render-jobs.mjs';
import { encodeRenderOutput, encodeMultilayerExr } from '../scripts/render-output-encoders.mjs';
import { createAgentScene, loadAllAgentSceneModules } from '../src/agent-scene.js';
import { bakeMaterialGraph } from '../src/material-graph.js';

const limits = { maxOutputBytes: 1024, maxFrameBytes: 128, maxStoreBytes: 1024 ** 2, cleanupMs: 100 };

test('capabilities and the renderer list the same passes in the same order, wireframe last as a composed pass', () => {
  assert.deepEqual([...Object.keys(RENDER_PASS_MODES), 'wireframe'], [...RENDER_PASS_NAMES]);
  assert.deepEqual(Object.values(RENDER_PASS_MODES), [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]);
});

test('the pass branch of the main shader is compiled out unless a pass is requested', () => {
  assert.match(createFragmentShader([]), /#define USES_RENDER_PASS false/);
  assert.match(createFragmentShader(['renderPass']), /#define USES_RENDER_PASS \(renderPassMode > 0\.5 && renderPassMode < 5\.5\)/);
  assert.match(createFragmentShader(['renderPass', 'lightingPass']), /cryptoHash/);
});

test('camera effects default off, validate ranges and survive look normalization', () => {
  const neutral = resolveEffects();
  assert.equal(neutral.depthOfField.blur, 0);
  assert.equal(neutral.motionBlur.shutter, 0);
  const look = normalizeAgentLook({ effects: { depthOfField: { blur: 0.5 }, motionBlur: { shutter: 0.1 } } });
  assert.deepEqual(look.effects.depthOfField, { blur: 0.5, focusDistance: 5 });
  assert.equal(look.effects.motionBlur.shutter, 0.1);
  assert.deepEqual(normalizeAgentLook(look), look);
  for (const effects of [{ depthOfField: { blur: 2 } }, { depthOfField: { focusDistance: 0 } }, { depthOfField: { mystery: 1 } },
    { motionBlur: { shutter: 0.6 } }, { motionBlur: { shutter: 'long' } }, { motionBlur: 1 }]) {
    assert.throws(() => normalizeAgentLook({ effects }), undefined, JSON.stringify(effects));
  }
});

test('installing camera effects adds two stages once and costs nothing until a frame uses them', () => {
  const unusedContext = new Proxy({}, { get() { throw new Error('The GL context was touched.'); } });
  const renderer = { postStages: [], context: unusedContext };
  installCameraEffects(installCameraEffects(renderer));
  assert.deepEqual(renderer.postStages.map(stage => stage.name), ['Depth of field', 'Motion blur']);
  const effects = resolveEffects();
  for (const stage of renderer.postStages) assert.equal(stage.apply(renderer, {}, effects, null), null);
  const withoutPreviousCamera = resolveEffects({ motionBlur: { shutter: 0.2 } });
  assert.equal(renderer.postStages[1].apply({ ...renderer, previousViewProjection: null }, {}, withoutPreviousCamera, null), null);
  const blurred = resolveEffects({ depthOfField: { blur: 1 } });
  assert.throws(() => renderer.postStages[0].apply(renderer, {}, blurred, null), /requires depth:true/);
  for (const source of [depthOfFieldFragment, motionBlurFragment]) assert.doesNotMatch(source, /rand\(|Math\.random|time/);
});

test('render passes need the addon, reject unknown names and bind per-pass uniforms and manifests', () => {
  const calls = [];
  const depthFormats = [];
  const recordedTargets = [];
  const context = {
    VERTEX_SHADER: 1, FRAGMENT_SHADER: 2, COMPILE_STATUS: 3, LINK_STATUS: 4,
    TEXTURE_2D: 5, RGBA16F: 6, RGBA8: 7, RGBA: 8, HALF_FLOAT: 9, UNSIGNED_BYTE: 10,
    TEXTURE_MIN_FILTER: 11, TEXTURE_MAG_FILTER: 12, NEAREST: 13, RENDERBUFFER: 14,
    DEPTH_COMPONENT24: 15, DEPTH_COMPONENT32F: 24, FRAMEBUFFER: 16, COLOR_ATTACHMENT0: 17, DEPTH_ATTACHMENT: 18,
    FRAMEBUFFER_COMPLETE: 19, DEPTH_TEST: 20, CULL_FACE: 21, TEXTURE0: 22, TRIANGLES: 23,
    getExtension: () => ({}),
    createShader: () => ({}),
    shaderSource: () => {},
    compileShader: () => {},
    getShaderParameter: () => true,
    deleteShader: () => {},
    createProgram: () => ({}),
    attachShader: () => {},
    linkProgram: () => {},
    getProgramParameter: () => true,
    getProgramInfoLog: () => '',
    getShaderInfoLog: () => '',
    deleteProgram: () => {},
    getUniformLocation: (_program, name) => name,
    createTexture: () => ({ kind: 'tex' }),
    bindTexture: () => {},
    texImage2D: () => {},
    texParameteri: () => {},
    createRenderbuffer: () => ({ kind: 'rb' }),
    bindRenderbuffer: () => {},
    renderbufferStorage: (_target, format) => depthFormats.push(format),
    createFramebuffer: () => ({ kind: 'fb' }),
    bindFramebuffer: () => {},
    framebufferTexture2D: () => {},
    framebufferRenderbuffer: () => {},
    checkFramebufferStatus: () => 19,
    deleteTexture: () => {},
    deleteRenderbuffer: () => {},
    deleteFramebuffer: () => {},
    viewport: () => {},
    disable: () => {},
    bindVertexArray: () => {},
    useProgram: () => {},
    activeTexture: () => {},
    uniform1i: (loc, val) => calls.push([loc, val]),
    uniform1f: (loc, val) => calls.push([loc, val]),
    uniform1fv: (loc, val) => calls.push([loc, val.length]),
    uniform3f: (loc, x, y, z) => calls.push([loc, [x, y, z]]),
    uniform3fv: (loc, val) => calls.push([loc, val]),
    uniformMatrix4fv: (loc, _transpose, val) => calls.push([loc, val.length]),
    drawArrays: () => calls.push(['drawArrays']),
  };
  const drawn = [];
  const batchOpaque = {
    parts: [
      { name: 'hero', matrix: new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]), material: { id: 'steel', aov: [0.2, 0.4, 0.6], aovMap: { id: 'aov-tex' } } },
      { name: 'floor', matrix: new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]), material: { name: 'stone' } },
    ],
    layerState: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    textures: ['t0', 't1', 't2', 't3', 't4'],
  };
  const batchGlass = {
    parts: [{ name: 'glass', matrix: new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]), material: { transmissionMap: { id: 'trans-tex' } } }],
    layerState: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0.85],
    textures: ['g0', 'g1', 'g2', 'g3', 'g4'],
  };
  const renderer = {
    passSupport: null,
    context,
    canvas: { width: 8, height: 8 },
    benchmark: false,
    program: {},
    viewProjection: new Float32Array(16),
    renderTargets: { record: (name, description) => recordedTargets.push([name, description.format]), release: () => {} },
    requireFeatureNames: () => {},
    drawsInOpaqueLoop: batch => batch === batchOpaque,
    createTexture: desc => desc,
    drawAlphaBatch: (batch, _tex, culled) => drawn.push({ batch, tex4: batch.textures[4], culled }),
    batches: [batchOpaque, batchGlass],
  };
  installRenderPasses(installRenderPasses(renderer));
  assert.ok(renderer.passSupport);
  assert.throws(() => renderer.passSupport.prepare({ pass: 'beauty' }), /Unknown render pass/);
  assert.throws(() => renderer.passSupport.prepare({ pass: 'depth', passDepthRange: 0 }), /depth range/);
  assert.deepEqual(renderer.passSupport.background('depth'), [1, 1, 0, 1]);
  assert.deepEqual(renderer.passSupport.background('objectId'), [0, 0, 0, 1]);
  const manifest = renderer.passSupport.cryptomatteManifest();
  assert.deepEqual(Object.keys(manifest.objects), ['hero', 'floor']);
  assert.deepEqual(Object.keys(manifest.materials), ['steel', 'stone']);
  assert.equal(manifest.objects.hero.value, cryptomatteValue(1));
  assert.equal(manifest.objects.hero.hash, cryptomatteHashHex('hero'));
  assert.equal(manifest.materials.stone.value, cryptomatteValue(2));
  assert.equal(renderer.passSupport.drawsInPass(batchGlass, RENDER_PASS_MODES.albedo), false);
  assert.equal(renderer.passSupport.drawsInPass(batchGlass, RENDER_PASS_MODES.transmission), true);

  assert.equal(renderer.passSupport.prepare({ pass: 'aov', passFloat: true }), RENDER_PASS_MODES.aov);
  assert.ok(renderer.passSupport.framebuffer);
  assert.ok(renderer.passSupport.texture);
  assert.deepEqual(depthFormats, [context.DEPTH_COMPONENT24], 'without reversed-Z a pass keeps the 24-bit depth');

  // With reversed-Z every auxiliary pass draws into the beauty pass's 32-bit float depth, never into 24 bits (issue #934).
  renderer.reversedDepth = true;
  for (const passName of ['albedo', 'objectId', 'normal', 'depth', 'cryptomatte']) {
    depthFormats.length = 0;
    recordedTargets.length = 0;
    renderer.passSupport.releaseTargets(context);
    renderer.passSupport.prepare({ pass: passName });
    assert.deepEqual(depthFormats, [context.DEPTH_COMPONENT32F], passName);
    assert.deepEqual(recordedTargets.find(([name]) => name === 'passDepth'), ['passDepth', 'DEPTH_COMPONENT32F'], passName);
  }
  depthFormats.length = 0;
  renderer.reversedDepth = false;
  renderer.passSupport.prepare({ pass: 'albedo' });
  assert.deepEqual(depthFormats, [context.DEPTH_COMPONENT24], 'a pass target follows the depth mode of the renderer');
  renderer.passSupport.prepare({ pass: 'aov', passFloat: true });
  renderer.passSupport.bindFrame(
    { passFloat: true, previousCamera: { eye: [1, 2, 5], target: [0, 0, 0] } },
    { fogDensity: 0.01, fogColor: [0.3, 0.4, 0.5] },
    { density: 0, color: [0, 0, 0] },
  );
  renderer.passSupport.drawBatch(
    batchOpaque,
    RENDER_PASS_MODES.aov,
    { previousMatrices: { hero: new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1]) } },
    0,
  );
  assert.equal(drawn.at(-1).tex4.id, 'aov-tex');
  assert.equal(batchOpaque.textures[4], 't4');
  renderer.passSupport.drawBatch(
    batchGlass,
    RENDER_PASS_MODES.transmission,
    { objectMotion: { matrices: new Float32Array(32) } },
    0,
  );
  assert.equal(drawn.at(-1).tex4.id, 'trans-tex');
  assert.equal(batchGlass.textures[4], 'g4');
  renderer.passSupport.copyToCanvas();
  assert.equal(renderer.passSupport.prepare({ pass: 'cryptomatte', passFloat: false }), RENDER_PASS_MODES.cryptomatte);
  renderer.passSupport.dispose(context);
});

test('materials accept scalar or RGB aov values and material graphs bake aovMap descriptors', async () => {
  await loadAllAgentSceneModules();
  const scene = createAgentScene({
    format: 'crucible.scene',
    schemaVersion: 1,
    geometries: [{ id: 'cube', type: 'box' }],
    materials: [
      { id: 'scalar-aov', color: [0.5, 0.5, 0.5], aov: 0.75 },
      { id: 'rgb-aov', color: [0.5, 0.5, 0.5], aov: [0.2, 0.4, 0.8] },
    ],
    nodes: [
      { id: 'a', geometry: 'cube', material: 'scalar-aov' },
      { id: 'b', geometry: 'cube', material: 'rgb-aov', translation: [2, 0, 0] },
    ],
  });
  assert.deepEqual(scene.parts[0].material.aov, [0.75, 0.75, 0.75]);
  assert.deepEqual(scene.parts[1].material.aov, [0.2, 0.4, 0.8]);
  assert.equal(scene.parts[0].material.id, 'scalar-aov');
  assert.throws(() => createAgentScene({
    format: 'crucible.scene',
    schemaVersion: 1,
    geometries: [{ id: 'cube', type: 'box' }],
    materials: [{ id: 'bad', aov: [1.5, 0, 0] }],
    nodes: [],
  }), /aov/);
  const baked = bakeMaterialGraph({
    color: [0.5, 0.5, 0.5],
    aov: { checker: { count: 2 } },
  }, { resolution: 64, bump: 1 });
  assert.ok(baked.aovMap);
  assert.equal(baked.aovMap.colorSpace, 'linear');
  assert.equal(baked.aovMap.image.width, 64);
});

test('artifact names and multilayer EXR encoders cover new passes and multilayer jobs', () => {
  assert.deepEqual(artifactNames('image'), ['render.png']);
  assert.deepEqual(artifactNames('image', ['depth', 'objectId']), ['render.png', 'render-depth.png', 'render-objectId.png']);
  assert.deepEqual(artifactNames('image', ['depth', 'position', 'cryptomatte'], 'exr', true), ['render.exr']);
  assert.deepEqual(artifactNames('stills', ['depth', 'uv'], 'exr', true).length, 8);
  assert.equal(artifactNames('stills', ['normal']).length, 16);
  assert.ok(artifactNames('stills', ['normal']).includes('frame-7-normal.png'));
  assert.deepEqual(artifactNames('video'), ['render.mp4']);

  const pixels = new Float32Array([
    1.5, -2.0, 3.25, 1,
    0.25, 0.75, 0.5, 1,
    -0.1, 0.2, 0, 1,
    0.8, 0.4, 1, 1,
  ]);
  for (const pass of ['position', 'velocity', 'uv', 'cryptomatte', 'aov']) {
    const exr = encodeRenderOutput({ format: 'exr', pixels, width: 2, height: 2, pass });
    assert.equal(exr[0], 0x76);
    const png16 = encodeRenderOutput({ format: 'png16', pixels, width: 2, height: 2, pass });
    assert.equal(png16[0], 0x89);
  }
  const multi = encodeMultilayerExr({
    width: 2,
    height: 2,
    beauty: pixels,
    passes: {
      position: pixels,
      velocity: pixels,
      uv: pixels,
      depth: new Float32Array(16),
      objectId: new Float32Array(16),
      cryptomatte: pixels,
      aov: pixels,
    },
    hasAlpha: true,
  });
  const headerText = multi.subarray(0, 512).toString('latin1');
  for (const channelName of ['Combined.R', 'Combined.A', 'position.X', 'position.Z', 'velocity.X', 'uv.U', 'depth.Z', 'objectId.id', 'cryptomatte.R', 'aov.G']) {
    assert.ok(headerText.includes(channelName), `multilayer EXR header includes ${channelName}`);
  }
});

test('a job with passes or multilayer EXR publishes every expected file and rejects bad pass requests', async context => {
  const root = await mkdtemp(join(tmpdir(), 'crucible-render-passes-'));
  context.after(() => rm(root, { recursive: true, force: true }));
  const run = async ({ directory, settings }) => {
    const artifacts = [];
    const mimeType = settings.format === 'exr' ? 'image/x-exr' : 'image/png';
    for (const filename of artifactNames(settings.kind, settings.passes, settings.format, settings.multilayer)) {
      await writeFile(join(directory, `${filename}.partial`), Buffer.from(filename));
      artifacts.push({ temporary: `${filename}.partial`, filename, mimeType });
    }
    return { artifacts, metrics: {} };
  };
  const controller = await createAgentRenderJobs({ root, limits, run });
  const snapshot = { recipe: { format: 'fixture' } };
  const submitted = await controller.submit({ sourceRevision: 0, snapshot, settings: { kind: 'image', width: 64, height: 64, passes: ['depth', 'albedo'] } });
  while (controller.status(submitted.jobId).state !== 'completed') await new Promise(resolve => setTimeout(resolve, 2));
  const record = controller.status(submitted.jobId);
  assert.deepEqual(record.artifacts.map(artifact => artifact.filename), ['render.png', 'render-depth.png', 'render-albedo.png']);
  assert.deepEqual(record.settings.passes, ['depth', 'albedo']);
  await controller.release(submitted.jobId);

  const multiSubmitted = await controller.submit({
    sourceRevision: 0,
    snapshot,
    settings: { kind: 'image', width: 64, height: 64, format: 'exr', multilayer: true, passes: ['position', 'velocity', 'uv', 'cryptomatte', 'aov'] },
  });
  while (controller.status(multiSubmitted.jobId).state !== 'completed') await new Promise(resolve => setTimeout(resolve, 2));
  const multiRecord = controller.status(multiSubmitted.jobId);
  assert.deepEqual(multiRecord.artifacts.map(artifact => artifact.filename), ['render.exr']);
  assert.equal(multiRecord.settings.multilayer, true);
  await controller.release(multiSubmitted.jobId);

  await assert.rejects(controller.submit({ sourceRevision: 0, snapshot, settings: { kind: 'image', width: 64, height: 64, multilayer: true, passes: ['depth'] } }), /multilayer requires format exr/);
  await assert.rejects(controller.submit({ sourceRevision: 0, snapshot, settings: { kind: 'image', width: 64, height: 64, format: 'exr', multilayer: true } }), /multilayer requires format exr and passes/);
  for (const passes of [['mystery'], ['depth', 'depth'], [], 'depth']) {
    await assert.rejects(controller.submit({ sourceRevision: 0, snapshot, settings: { kind: 'image', width: 64, height: 64, passes } }), /pass/i);
  }
  await assert.rejects(controller.submit({ sourceRevision: 0, snapshot, settings: { kind: 'video', width: 64, height: 64, passes: ['depth'] } }), /image or stills/);
  await controller.close();
});