Skip to content

3D engine that AI agents drive

12.3 KB
Raw Blame
import { compileProgram } from './gl-program.js';
import { assertClosedMesh } from './local-volume-parts.js';

// The rasterizer's half of a local volume with shape "mesh" and fit "mesh" (issue 807). Before the post pass, each such part is drawn
// without culling into a layer of one float target, and the post shader reads the result per pixel. A ray crosses a closed mesh an even
// number of times, and the length it spends inside is the sum over the back faces of (distance - the distance of each front face):
//   channel R  adds +distance at a back face and -distance at a front face, each clamped to [0, distance of the first surface];
//   channel G  keeps the nearest front face distance (the entry, used to order the volumes).
// That sum is exact for any number of shells and any shape, so there is no shell limit. The post shader then integrates the volume
// over that length in closed form, which is exact because a mesh volume has the same density everywhere inside (no falloff, no noise).
// Geometry far past the far plane is clipped, so a ray can meet a front face and lose its back face; the post shader repairs that.
// The cost is two draws of the part per frame, so a scene without mesh volumes pays nothing.
// Limits: the mesh must be closed and rigid; geometry nearer than the camera's near plane is clipped, so a ray loses the stretch
// between the eye and the near plane; the target is as wide as the canvas and as tall as the canvas times the number of mesh volumes.

const MESH_VERTEX = `#version 300 es
precision highp float;
layout(location = 0) in vec3 position;
uniform mat4 viewProjection;
uniform mat4 model;
out vec3 worldPosition;
void main() {
  vec4 world = model * vec4(position, 1.0);
  worldPosition = world.xyz;
  gl_Position = viewProjection * world;
}`;

const MESH_FRAGMENT = `#version 300 es
precision highp float;
precision highp sampler2D;
in vec3 worldPosition;
uniform sampler2D sceneDepth;
uniform mat4 inverseViewProjection;
uniform vec3 eye;
uniform float orthographic;
uniform vec2 resolution;
uniform float layerOffset;
uniform float entryPass;
out vec4 result;
void main() {
  vec2 pixel = vec2(gl_FragCoord.x, gl_FragCoord.y - layerOffset);
  vec2 clipXY = pixel / resolution * 2.0 - 1.0;
  vec4 farPoint = inverseViewProjection * vec4(clipXY, 1.0, 1.0);
  vec4 nearPoint = inverseViewProjection * vec4(clipXY, -1.0, 1.0);
  vec3 origin = orthographic > 0.5 ? nearPoint.xyz / nearPoint.w : eye;
  vec3 direction = normalize(farPoint.xyz - origin * farPoint.w);
  float depth = texelFetch(sceneDepth, ivec2(pixel), 0).r;
  vec4 surfacePoint = inverseViewProjection * vec4(clipXY, depth * 2.0 - 1.0, 1.0);
  float surfaceDistance = 1e30;
  if (surfacePoint.w > 0.0) surfaceDistance = max(0.0, dot(surfacePoint.xyz - origin * surfacePoint.w, direction) / surfacePoint.w);
  float distanceAlongRay = clamp(dot(worldPosition - origin, direction), 0.0, surfaceDistance);
  if (entryPass > 0.5) result = vec4(0.0, gl_FrontFacing ? distanceAlongRay : 1e30, 0.0, 0.0);
  else result = vec4(gl_FrontFacing ? -distanceAlongRay : distanceAlongRay, 0.0, 0.0, 0.0);
}`;

const UNIFORM_NAMES = ['viewProjection', 'model', 'sceneDepth', 'inverseViewProjection', 'eye', 'orthographic', 'resolution', 'layerOffset', 'entryPass'];
const NO_ENTRY = 1e30;

// Positive when the triangles wind counter-clockwise seen from outside, which is what the draw's front faces rely on.
function signedVolume(positions, indices) {
  let total = 0;
  for (let corner = 0; corner < indices.length; corner += 3) {
    const a = indices[corner] * 3;
    const b = indices[corner + 1] * 3;
    const thirdVertexOffset = indices[corner + 2] * 3;
    total += positions[a] * (positions[b + 1] * positions[thirdVertexOffset + 2] - positions[b + 2] * positions[thirdVertexOffset + 1])
      + positions[a + 1] * (positions[b + 2] * positions[thirdVertexOffset] - positions[b] * positions[thirdVertexOffset + 2])
      + positions[a + 2] * (positions[b] * positions[thirdVertexOffset + 1] - positions[b + 1] * positions[thirdVertexOffset]);
  }
  return total;
}

function matrixDeterminant(matrix) {
  return matrix[0] * (matrix[5] * matrix[10] - matrix[6] * matrix[9])
    - matrix[4] * (matrix[1] * matrix[10] - matrix[2] * matrix[9])
    + matrix[8] * (matrix[1] * matrix[6] - matrix[2] * matrix[5]);
}

function uploadGeometry(context, geometry) {
  const vertexArray = context.createVertexArray();
  context.bindVertexArray(vertexArray);
  const positionBuffer = context.createBuffer();
  context.bindBuffer(context.ARRAY_BUFFER, positionBuffer);
  context.bufferData(context.ARRAY_BUFFER, geometry.positions instanceof Float32Array ? geometry.positions : new Float32Array(geometry.positions), context.STATIC_DRAW);
  context.enableVertexAttribArray(0);
  context.vertexAttribPointer(0, 3, context.FLOAT, false, 0, 0);
  const shortIndices = geometry.indices instanceof Uint16Array;
  const indexBuffer = context.createBuffer();
  context.bindBuffer(context.ELEMENT_ARRAY_BUFFER, indexBuffer);
  context.bufferData(context.ELEMENT_ARRAY_BUFFER, shortIndices || geometry.indices instanceof Uint32Array ? geometry.indices : new Uint32Array(geometry.indices), context.STATIC_DRAW);
  context.bindVertexArray(null);
  return {
    vertexArray,
    buffers: [positionBuffer, indexBuffer],
    count: geometry.indices.length,
    indexType: shortIndices ? context.UNSIGNED_SHORT : context.UNSIGNED_INT,
    insideOut: signedVolume(geometry.positions, geometry.indices) < 0,
  };
}

function createState(renderer) {
  const context = renderer.context;
  if (!context.getExtension('EXT_float_blend')) throw new Error('Local volumes with fit "mesh" need the WebGL extension EXT_float_blend, which this browser does not offer; use fit "bounds"');
  const program = compileProgram(context, MESH_VERTEX, MESH_FRAGMENT);
  const uniforms = Object.fromEntries(UNIFORM_NAMES.map(name => [name, context.getUniformLocation(program, name)]));
  const state = { program, uniforms, geometries: new Map(), target: null };
  // The renderer's dispose gains this step here, so the core carries no call for a pass most scenes never create.
  const disposeRenderer = renderer.dispose.bind(renderer);
  renderer.dispose = () => {
    disposeMeshVolumePass(renderer, state);
    disposeRenderer();
  };
  return state;
}

function allocateTarget(renderer, state, layers) {
  const context = renderer.context;
  const width = renderer.canvas.width;
  const height = renderer.canvas.height * layers;
  if (state.target?.width === width && state.target.height === height) return;
  const limit = context.getParameter(context.MAX_TEXTURE_SIZE);
  if (width > limit || height > limit) {
    throw new RangeError(`${layers} mesh volumes at ${width}x${renderer.canvas.height} need a ${width}x${height} buffer, past this device's limit of ${limit}; use fewer volumes with fit "mesh", a smaller canvas or fit "bounds"`);
  }
  releaseTarget(renderer, state);
  const texture = context.createTexture();
  context.bindTexture(context.TEXTURE_2D, texture);
  context.texImage2D(context.TEXTURE_2D, 0, context.RG32F, width, height, 0, context.RG, context.FLOAT, null);
  context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.NEAREST);
  context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.NEAREST);
  context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_S, context.CLAMP_TO_EDGE);
  context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_T, context.CLAMP_TO_EDGE);
  const framebuffer = context.createFramebuffer();
  context.bindFramebuffer(context.FRAMEBUFFER, framebuffer);
  context.framebufferTexture2D(context.FRAMEBUFFER, context.COLOR_ATTACHMENT0, context.TEXTURE_2D, texture, 0);
  if (context.checkFramebufferStatus(context.FRAMEBUFFER) !== context.FRAMEBUFFER_COMPLETE) {
    context.deleteFramebuffer(framebuffer);
    context.deleteTexture(texture);
    throw new Error('The mesh volume buffer could not be allocated; use fit "bounds" or a smaller canvas');
  }
  state.target = { texture, framebuffer, width, height };
  renderer.renderTargets.record('meshVolumes', { width, height, format: 'RG32F' });
}

function releaseTarget(renderer, state) {
  if (!state.target) return;
  renderer.context.deleteTexture(state.target.texture);
  renderer.context.deleteFramebuffer(state.target.framebuffer);
  state.target = null;
  renderer.renderTargets.release('meshVolumes');
}

function releaseGeometry(context, gpu) {
  context.deleteVertexArray(gpu.vertexArray);
  gpu.buffers.forEach(buffer => context.deleteBuffer(buffer));
}

// A part whose geometry was replaced or removed no longer draws, so its buffers go.
function pruneGeometries(context, state, meshParts) {
  const inUse = new Set(meshParts.map(({ part }) => part.geometry));
  for (const [geometry, gpu] of state.geometries) {
    if (inUse.has(geometry)) continue;
    releaseGeometry(context, gpu);
    state.geometries.delete(geometry);
  }
}

function drawPart(context, state, part) {
  let gpu = state.geometries.get(part.geometry);
  if (!gpu) {
    gpu = uploadGeometry(context, part.geometry);
    state.geometries.set(part.geometry, gpu);
  }
  const matrix = part.matrix ?? [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
  context.uniformMatrix4fv(state.uniforms.model, false, matrix);
  context.frontFace(gpu.insideOut !== matrixDeterminant(matrix) < 0 ? context.CW : context.CCW);
  context.bindVertexArray(gpu.vertexArray);
  context.drawElements(context.TRIANGLES, gpu.count, gpu.indexType, 0);
}

/**
 * Draws every fit "mesh" volume of the frame into the mesh buffer: layer n holds the volume with slice n, R the length inside along
 * each pixel's ray and G the nearest entry. The caller rebinds the post framebuffer afterwards through the restore this makes.
 * Returns the texture, or null when the frame has no such volume.
 */
export function drawMeshVolumes(renderer, frame) {
  const meshParts = renderer.localVolumeBuffers?.meshParts ?? [];
  if (meshParts.length === 0) {
    if (renderer.meshVolumePass) pruneGeometries(renderer.context, renderer.meshVolumePass, meshParts);
    return null;
  }
  const context = renderer.context;
  const state = (renderer.meshVolumePass ??= createState(renderer));
  for (const { part, label } of meshParts) assertClosedMesh(part, label);
  pruneGeometries(context, state, meshParts);
  allocateTarget(renderer, state, meshParts.length);
  const { width, height } = renderer.canvas;
  const blendWasOn = context.isEnabled(context.BLEND);
  context.bindFramebuffer(context.FRAMEBUFFER, state.target.framebuffer);
  context.colorMask(true, true, true, true);
  context.clearBufferfv(context.COLOR, 0, [0, NO_ENTRY, 0, 0]);
  context.useProgram(state.program);
  context.disable(context.DEPTH_TEST);
  context.disable(context.CULL_FACE);
  context.enable(context.BLEND);
  context.blendFunc(context.ONE, context.ONE);
  context.activeTexture(context.TEXTURE1);
  context.bindTexture(context.TEXTURE_2D, renderer.depthTexture);
  context.uniform1i(state.uniforms.sceneDepth, 1);
  context.uniformMatrix4fv(state.uniforms.viewProjection, false, renderer.viewProjection);
  context.uniformMatrix4fv(state.uniforms.inverseViewProjection, false, renderer.inverseViewProjection);
  context.uniform3fv(state.uniforms.eye, frame.eye);
  context.uniform1f(state.uniforms.orthographic, frame.projection === 'orthographic' ? 1 : 0);
  context.uniform2f(state.uniforms.resolution, width, height);
  meshParts.forEach(({ part }, slice) => {
    context.viewport(0, slice * height, width, height);
    context.uniform1f(state.uniforms.layerOffset, slice * height);
    context.uniform1f(state.uniforms.entryPass, 0);
    context.blendEquation(context.FUNC_ADD);
    context.colorMask(true, false, false, false);
    drawPart(context, state, part);
    context.uniform1f(state.uniforms.entryPass, 1);
    context.blendEquation(context.MIN);
    context.colorMask(false, true, false, false);
    drawPart(context, state, part);
  });
  context.colorMask(true, true, true, true);
  context.frontFace(context.CCW);
  context.blendEquation(context.FUNC_ADD);
  if (!blendWasOn) context.disable(context.BLEND);
  context.bindVertexArray(null);
  context.bindFramebuffer(context.FRAMEBUFFER, frame.outputFramebuffer ?? null);
  context.viewport(0, 0, width, height);
  return state.target.texture;
}

function disposeMeshVolumePass(renderer, state) {
  const context = renderer.context;
  releaseTarget(renderer, state);
  state.geometries.forEach(gpu => releaseGeometry(context, gpu));
  context.deleteProgram(state.program);
  renderer.meshVolumePass = null;
}