Skip to content

3D engine that AI agents drive

14.2 KB
Raw Blame
// Fit "mesh" volumes in the rasterizer (issue 807): the mesh pass finds how long each pixel's ray spends inside a closed part, and
// the post shader integrates the volume over that length. Checked against the CPU path tracer on an L-shaped room and against the
// analytic length through two shells. Draws in headless Chrome (SwiftShader) from the Vite dev server; skips when Chrome cannot start.
import test from 'node:test';
import assert from 'node:assert/strict';
import { spawn } from 'node:child_process';
import { freePort } from './helpers/free-port.js';
import { launchChrome, skipOrThrow } from './helpers/dev-server-browser.js';
import { renderPathTracedImage } from '../src/path-tracer.js';
import { applyViewTransform } from '../src/view-transform.js';

async function startServer(port) {
  const server = spawn(process.execPath, ['node_modules/vite/bin/vite.js', '--host', '127.0.0.1', '--port', String(port), '--strictPort'], { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
  await new Promise((resolve, reject) => {
    const timeout = setTimeout(() => reject(new Error('The dev server did not start.')), 30000);
    server.stdout.on('data', (data) => {
      if (/ready in/.test(data.toString())) {
        clearTimeout(timeout);
        resolve();
      }
    });
    server.on('exit', (code) => {
      clearTimeout(timeout);
      reject(new Error(`The dev server exited ${code}.`));
    });
  });
  return server;
}

// An L-shaped prism: three unit squares in x and z (the notch is x in 1..2, z in 1..2), extruded in y from 0 to 1.
function lPrism(reverseWinding = false) {
  const outline = [[0, 0], [1, 0], [2, 0], [2, 1], [1, 1], [1, 2], [0, 2], [0, 1]];
  const positions = [];
  for (const height of [0, 1]) for (const [x, z] of outline) positions.push(x, height, z);
  const indices = [];
  const top = outline.length;
  for (const [firstCorner, secondCorner, thirdCorner, fourthCorner] of [[0, 1, 4, 7], [1, 2, 3, 4], [7, 4, 5, 6]]) {
    indices.push(firstCorner, secondCorner, thirdCorner, firstCorner, thirdCorner, fourthCorner, top + firstCorner, top + thirdCorner, top + secondCorner, top + firstCorner, top + fourthCorner, top + thirdCorner);
  }
  for (let corner = 0; corner < outline.length; corner++) {
    const next = (corner + 1) % outline.length;
    indices.push(corner, top + next, next, corner, top + corner, top + next);
  }
  if (reverseWinding) {
    for (let corner = 0; corner < indices.length; corner += 3) [indices[corner + 1], indices[corner + 2]] = [indices[corner + 2], indices[corner + 1]];
  }
  return { positions, indices };
}

// Two unit boxes two units apart along x, one closed mesh in two shells.
function dumbbell() {
  const corners = [0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1];
  const faces = [0, 2, 1, 0, 3, 2, 4, 5, 6, 4, 6, 7, 0, 1, 5, 0, 5, 4, 3, 7, 6, 3, 6, 2, 0, 4, 7, 0, 7, 3, 1, 2, 6, 1, 6, 5];
  const positions = [...corners];
  for (let index = 0; index < corners.length; index += 3) positions.push(corners[index] + 3, corners[index + 1], corners[index + 2]);
  return { positions, indices: [...faces, ...faces.map(index => index + 8)] };
}

const OPEN_BOX = { positions: dumbbell().positions.slice(0, 24), indices: dumbbell().indices.slice(0, 33) };
const LOOKING_DOWN = { eye: [0, 3, 0], target: [0, 0, 0], up: [0, 0, -1], fov: 40 };
// The L covers x and z from -0.4 to 0.4 with its notch where both are positive, from y 0.2 to 1.2.
const L_ROOM_MATRIX = [0.4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0.4, 0, -0.4, 0.2, -0.4, 1];
const MIXED_WINDING = { positions: dumbbell().positions.slice(0, 24), indices: [0, 1, 2, ...dumbbell().indices.slice(3, 36)] };
const IDENTITY = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
const SIZE = 64;

// Draws each scenario to a float target and returns the frame with the volume and the frame without it, as the display shows them.
// A white ground and a white wall at x = 8 give every ray something to dim.
async function drawInChrome(page, scenarios) {
  const task = async (list) => {
    const { box } = await import('/src/geometry.js');
    const { CrucibleRenderer } = await import('/src/renderer.js');
    const { createFloatOutput, readFloatPixels } = await import('/src/float-output.js');
    await import('/src/fog-shader.js');
    const canvas = document.createElement('canvas');
    canvas.width = list.size;
    canvas.height = list.size;
    canvas.style.width = list.size + 'px';
    canvas.style.height = list.size + 'px';
    document.body.append(canvas);
    const material = { tag: 'unlit', color: [1, 1, 1], metallic: 0, roughness: 1, emission: 0 };
    const ground = { name: 'ground', geometry: box(0, 1), matrix: new Float32Array([40, 0, 0, 0, 0, 0.2, 0, 0, 0, 0, 40, 0, 0, -0.1, 0, 1]), material };
    const wall = { name: 'wall', geometry: box(0, 1), matrix: new Float32Array([0.2, 0, 0, 0, 0, 40, 0, 0, 0, 0, 40, 0, 8, 0, 0, 1]), material };
    const results = [];
    for (const scenario of list.scenarios) {
      const geometry = { positions: new Float32Array(scenario.mesh.positions), normals: new Float32Array(scenario.mesh.positions.length), indices: new Uint32Array(scenario.mesh.indices) };
      const room = { name: 'room', visible: false, geometry, matrix: new Float32Array(scenario.matrix), material };
      const renderer = new CrucibleRenderer(canvas, [ground, wall, room], { hdr: true, depth: true, samples: 1, pixelRatio: 1, ground: false });
      const output = createFloatOutput(renderer.context, renderer.canvas.width, renderer.canvas.height);
      const draw = (localVolumes) => {
        const environment = { keyIntensity: 0, fillIntensity: 0, ambientIntensity: 0, localVolumes };
        renderer.render({ ...scenario.camera, time: 0, environment, outputFramebuffer: output.framebuffer });
        return readFloatPixels(renderer.context, output.framebuffer, renderer.canvas.width, renderer.canvas.height);
      };
      let message = null;
      let transmittance = null;
      try {
        const plain = draw([]);
        const filled = draw(scenario.localVolumes);
        transmittance = { plain: Array.from(plain), filled: Array.from(filled) };
      } catch (error) {
        message = error.message;
      }
      results.push({ message, transmittance, glError: renderer.context.getError() });
      output.dispose();
      renderer.dispose?.();
    }
    return results;
  };
  const frames = await page.evaluate(task, { scenarios, size: SIZE });
  return frames.map(frame => (frame.transmittance ? { ...frame, filled: frame.transmittance.filled, transmittance: transmittanceOf(frame.transmittance) } : frame));
}

// The display shows aces(light) ^ (1 / 2.2); this undoes both for a channel, so the ratio of two frames is the light that got through.
function lightOf(displayed) {
  let low = 0;
  let high = 64;
  for (let round = 0; round < 40; round++) {
    const middle = (low + high) / 2;
    if (applyViewTransform('aces', [middle, middle, middle])[0] ** (1 / 2.2) < displayed) low = middle;
    else high = middle;
  }
  return low;
}

function transmittanceOf({ plain, filled }) {
  return Array.from(filled, (value, index) => (index % 4 === 3 || plain[index] <= 0 ? 1 : lightOf(value) / lightOf(plain[index])));
}

async function openPage(context) {
  let chromium;
  try {
    ({ chromium } = await import('playwright'));
  } catch {
    skipOrThrow(context, 'playwright is not installed');
    return null;
  }
  const port = await freePort();
  const server = await startServer(port);
  try {
    const browser = await launchChrome(chromium);
    const page = await browser.newPage({ viewport: { width: SIZE, height: SIZE } });
    await page.goto(`http://127.0.0.1:${port}/`);
    return {
      page,
      async close() {
        await browser.close();
        server.kill();
      },
    };
  } catch (error) {
    server.kill();
    skipOrThrow(context, `Chrome did not start: ${error.message.split('\n')[0]}`);
    return null;
  }
}

function groundGeometry() {
  const half = 20;
  return {
    positions: new Float32Array([-half, 0, half, half, 0, half, half, 0, -half, -half, 0, -half]),
    normals: new Float32Array([0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0]),
    uvs: new Float32Array([0, 1, 1, 1, 1, 0, 0, 0]),
    indices: new Uint32Array([0, 1, 2, 0, 2, 3]),
  };
}

async function cpuTransmittance(fit) {
  const material = { tag: 'surface', metallic: 0, roughness: 1, color: [0, 0, 0], ior: 1, emission: 1, emissiveColor: [1, 1, 1] };
  const mesh = lPrism();
  const geometry = { positions: new Float32Array(mesh.positions), indices: new Uint32Array(mesh.indices), normals: new Float32Array(mesh.positions.length) };
  const room = { name: 'room', visible: false, material, matrix: L_ROOM_MATRIX, geometry };
  const environment = { localVolumes: [{ shape: 'mesh', part: 'room', fit, density: 1, falloff: 0, steps: 32 }] };
  const { linear } = await renderPathTracedImage({ parts: [{ name: 'ground', geometry: groundGeometry(), material }, room], environment, camera: LOOKING_DOWN }, { width: 16, height: 16, denoise: false, threads: 1, bounces: 1, samples: 1 });
  return (column, row) => linear[(row * 16 + column) * 3];
}

// WebGL stores rows bottom to top, so the row is counted from the other end. The mean of the inner 2 by 2 pixels of the 4 by 4 block of the 64 by 64 frame that one pixel of the 16 by 16 CPU image covers.
function blockMean(transmittance, column, row) {
  let total = 0;
  for (let y = row * 4 + 1; y < row * 4 + 3; y++) for (let x = column * 4 + 1; x < column * 4 + 3; x++) total += transmittance[((SIZE - 1 - y) * SIZE + x) * 4];
  return total / 4;
}

test('the rasterizer fills an L-shaped closed mesh like the CPU path tracer: arms dimmed, notch clear', async (context) => {
  const opened = await openPage(context);
  if (!opened) return;
  try {
    const smoke = { shape: 'mesh', part: 'room', density: 1, falloff: 0, steps: 16 };
    const [meshFit, boundsFit, reversed] = await drawInChrome(opened.page, [
      { mesh: lPrism(), matrix: L_ROOM_MATRIX, camera: LOOKING_DOWN, localVolumes: [{ ...smoke, fit: 'mesh' }] },
      { mesh: lPrism(), matrix: L_ROOM_MATRIX, camera: LOOKING_DOWN, localVolumes: [{ ...smoke, fit: 'bounds' }] },
      { mesh: lPrism(true), matrix: L_ROOM_MATRIX, camera: LOOKING_DOWN, localVolumes: [{ ...smoke, fit: 'mesh' }] },
    ]);
    for (const result of [meshFit, boundsFit, reversed]) {
      assert.equal(result.message, null);
      assert.equal(result.glError, 0);
    }
    const cpu = await cpuTransmittance('mesh');
    const regions = { arm: [5, 5], notch: [10, 10], outside: [1, 1] };
    for (const [name, [column, row]] of Object.entries(regions)) {
      const raster = blockMean(meshFit.transmittance, column, row);
      assert.ok(Math.abs(raster - cpu(column, row)) < 0.03, `${name}: raster ${raster} against CPU ${cpu(column, row)}`);
      assert.equal(blockMean(reversed.transmittance, column, row), raster, `${name}: a mesh wound the other way fills the same`);
    }
    assert.equal(blockMean(meshFit.transmittance, 10, 10), 1, 'the notch is clear');
    assert.ok(blockMean(boundsFit.transmittance, 10, 10) < 0.5, 'bounds fill the notch');
  } finally {
    await opened.close();
  }
});

test('the rasterizer sums every shell a ray crosses, from outside and from inside, and refuses an open or inconsistently wound mesh', async (context) => {
  const opened = await openPage(context);
  if (!opened) return;
  try {
    // A ray along +x at height 0.5 crosses 1 unit of each box of the dumbbell and a 2 unit gap.
    const side = { eye: [-6, 0.5, 0.5], target: [0, 0.5, 0.5], up: [0, 1, 0], fov: 20 };
    const smoke = { shape: 'mesh', part: 'room', fit: 'mesh', density: 0.7, falloff: 0, steps: 16 };
    const [shells, inside, open, mixed] = await drawInChrome(opened.page, [
      { mesh: dumbbell(), matrix: IDENTITY, camera: side, localVolumes: [smoke] },
      // The camera sits in the middle of the first box, so the ray leaves it after half a unit and crosses the second box.
      { mesh: dumbbell(), matrix: IDENTITY, camera: { eye: [0.5, 0.5, 0.5], target: [6, 0.5, 0.5], up: [0, 1, 0], fov: 20 }, localVolumes: [smoke] },
      { mesh: OPEN_BOX, matrix: IDENTITY, camera: side, localVolumes: [smoke] },
      { mesh: MIXED_WINDING, matrix: IDENTITY, camera: side, localVolumes: [smoke] },
    ]);
    const middle = (SIZE / 2) * SIZE + SIZE / 2;
    assert.equal(shells.message, null);
    assert.ok(Math.abs(shells.transmittance[middle * 4] - Math.exp(-0.7 * 2)) < 0.01, `two shells: ${shells.transmittance[middle * 4]}`);
    assert.equal(inside.message, null);
    assert.ok(Math.abs(inside.transmittance[middle * 4] - Math.exp(-0.7 * 1.5)) < 0.01, `from inside: ${inside.transmittance[middle * 4]}`);
    assert.match(open.message, /environment\.localVolumes\[0\]\.part "room" is not a closed mesh \(\d+ open edges\)/);
    assert.match(mixed.message, /environment\.localVolumes\[0\]\.part "room" has faces wound against each other/);
  } finally {
    await opened.close();
  }
});

test('a mesh that straddles the far plane keeps the stretch it has this side of it', async (context) => {
  const opened = await openPage(context);
  if (!opened) return;
  try {
    // The box spans 6 to 7 units from the eye and the far plane sits at 6.5, so its back face is clipped. A box half as long ends
    // at 6.5 inside a farther far plane and loses nothing, so both pixels must glow the same.
    const view = { eye: [-6, 0.5, 0.5], target: [0, 0.5, 0.5], up: [0, 1, 0], fov: 20 };
    const glowing = { shape: 'mesh', part: 'room', fit: 'mesh', density: 0.7, falloff: 0, steps: 16, emission: [1, 1, 1] };
    const singleBox = { positions: dumbbell().positions.slice(0, 24), indices: dumbbell().indices.slice(0, 36) };
    const halfLong = [0.5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
    const [clipped, whole] = await drawInChrome(opened.page, [
      { mesh: singleBox, matrix: IDENTITY, camera: { ...view, far: 6.5 }, localVolumes: [glowing] },
      { mesh: singleBox, matrix: halfLong, camera: { ...view, far: 7.5 }, localVolumes: [glowing] },
    ]);
    assert.equal(clipped.message, null);
    const middle = ((SIZE / 2) * SIZE + SIZE / 2) * 4;
    const expected = lightOf(whole.filled[middle]);
    assert.ok(lightOf(clipped.filled[middle]) > 0.4, 'the clipped volume still glows');
    assert.ok(Math.abs(lightOf(clipped.filled[middle]) - expected) < 0.02 * expected, `clipped ${lightOf(clipped.filled[middle])} against ${expected}`);
  } finally {
    await opened.close();
  }
});