Skip to content

3D engine that AI agents drive

13.8 KB
Raw Blame
// Reads a standard 3D Gaussian splat PLY: the file written by the original 3DGS training code and by most tools that
// export splats. It is binary little endian with one vertex per splat and the properties x, y, z, f_dc_0 to f_dc_2,
// opacity, scale_0 to scale_2 and rot_0 to rot_3. Higher-order spherical harmonics (f_rest_*) and normals are skipped,
// so a splat shows one colour from every direction. Other splat formats (.spz, .ksplat, .splat) are not read here.

import { runInChunks } from './gaussian-splat-chunks.js';

export const GAUSSIAN_SPLAT_LIMITS = Object.freeze({
  // A file this large cannot be read into memory safely in a browser tab.
  maximumBytes: 512 * 1024 * 1024,
  maximumCount: 2_000_000,
});
/** No option raises the splat count above this: the data texture then needs 192 MB of GPU memory. */
export const GAUSSIAN_SPLAT_HARD_COUNT_LIMIT = 4_000_000;

const REQUIRED_PROPERTIES = ['x', 'y', 'z', 'f_dc_0', 'f_dc_1', 'f_dc_2', 'opacity', 'scale_0', 'scale_1', 'scale_2', 'rot_0', 'rot_1', 'rot_2', 'rot_3'];
const SPHERICAL_HARMONIC_DC = 0.28209479177387814;
const HEADER_END_PATTERN = /end_header\r?\n/;
const MAXIMUM_HEADER_BYTES = 64 * 1024;

const PROPERTY_TYPES = {
  char: { bytes: 1, read: (view, offset) => view.getInt8(offset) },
  int8: { bytes: 1, read: (view, offset) => view.getInt8(offset) },
  uchar: { bytes: 1, read: (view, offset) => view.getUint8(offset) },
  uint8: { bytes: 1, read: (view, offset) => view.getUint8(offset) },
  short: { bytes: 2, read: (view, offset) => view.getInt16(offset, true) },
  int16: { bytes: 2, read: (view, offset) => view.getInt16(offset, true) },
  ushort: { bytes: 2, read: (view, offset) => view.getUint16(offset, true) },
  uint16: { bytes: 2, read: (view, offset) => view.getUint16(offset, true) },
  int: { bytes: 4, read: (view, offset) => view.getInt32(offset, true) },
  int32: { bytes: 4, read: (view, offset) => view.getInt32(offset, true) },
  uint: { bytes: 4, read: (view, offset) => view.getUint32(offset, true) },
  uint32: { bytes: 4, read: (view, offset) => view.getUint32(offset, true) },
  float: { bytes: 4, read: (view, offset) => view.getFloat32(offset, true) },
  float32: { bytes: 4, read: (view, offset) => view.getFloat32(offset, true) },
  double: { bytes: 8, read: (view, offset) => view.getFloat64(offset, true) },
  float64: { bytes: 8, read: (view, offset) => view.getFloat64(offset, true) },
};

function readHeaderText(bytes) {
  const searchLimit = Math.min(bytes.length, MAXIMUM_HEADER_BYTES);
  const text = new TextDecoder('latin1').decode(bytes.subarray(0, searchLimit));
  const endMatch = HEADER_END_PATTERN.exec(text);
  if (text.startsWith('ply') && !endMatch && bytes.length > MAXIMUM_HEADER_BYTES) {
    throw new Error(`Gaussian splat PLY header is longer than ${MAXIMUM_HEADER_BYTES} bytes (64 KiB) or has no "end_header" line. Remove comment lines or re-export the file.`);
  }
  if (!text.startsWith('ply') || !endMatch) {
    throw new Error('Not a PLY file: the header must start with "ply" and end with "end_header"');
  }
  return { text: text.slice(0, endMatch.index), bodyStart: endMatch.index + endMatch[0].length };
}

function parseHeader(headerText) {
  const lines = headerText.split(/\r?\n/).map(line => line.trim()).filter(Boolean);
  const formatLine = lines.find(line => line.startsWith('format '));
  if (formatLine?.split(/\s+/)[1] !== 'binary_little_endian') {
    throw new Error(`Gaussian splat PLY must be binary_little_endian, got "${formatLine ?? 'no format line'}"`);
  }
  let vertexCount = 0;
  let inVertexElement = false;
  let vertexElementSeen = false;
  let skippedElement = null;
  let skippedBytes = 0;
  const properties = [];
  const finishSkippedElement = () => {
    if (skippedElement) skippedBytes += skippedElement.count * skippedElement.rowBytes;
    skippedElement = null;
  };
  for (const line of lines) {
    const words = line.split(/\s+/);
    if (words[0] === 'element') {
      finishSkippedElement();
      inVertexElement = words[1] === 'vertex' && !vertexElementSeen;
      if (inVertexElement) {
        vertexElementSeen = true;
        vertexCount = Number(words[2]);
      } else if (!vertexElementSeen) {
        const count = Number(words[2]);
        if (!Number.isInteger(count) || count < 0) {
          throw new Error(`Gaussian splat PLY element "${words[1]}" before "vertex" has an invalid count "${words[2]}". Re-export the file.`);
        }
        skippedElement = { name: words[1], count, rowBytes: 0 };
      }
    } else if (words[0] === 'property' && skippedElement) {
      if (words[1] === 'list' || !PROPERTY_TYPES[words[1]]) {
        throw new Error(`Gaussian splat PLY element "${skippedElement.name}" before "vertex" has a variable-size or unsupported property, so its bytes cannot be skipped. Re-export the file with the vertex element first.`);
      }
      skippedElement.rowBytes += PROPERTY_TYPES[words[1]].bytes;
    } else if (words[0] === 'property' && inVertexElement) {
      if (words[1] === 'list' || !PROPERTY_TYPES[words[1]]) {
        throw new Error(`Gaussian splat PLY has an unsupported property type "${words[1]}"`);
      }
      properties.push({ name: words[2], type: words[1] });
    }
  }
  if (!Number.isInteger(vertexCount) || vertexCount <= 0) {
    throw new Error('Gaussian splat PLY has no vertex element with a positive count');
  }
  finishSkippedElement();
  return { vertexCount, properties, skippedBytes };
}

function propertyLayout(properties) {
  const layout = new Map();
  let offset = 0;
  for (const { name, type } of properties) {
    layout.set(name, { offset, read: PROPERTY_TYPES[type].read });
    offset += PROPERTY_TYPES[type].bytes;
  }
  const missing = REQUIRED_PROPERTIES.filter(name => !layout.has(name));
  if (missing.length > 0) {
    throw new Error(`Gaussian splat PLY lacks the properties ${missing.join(', ')}`);
  }
  return { layout, stride: offset };
}

export function resolveLimits(options) {
  const maximumBytes = options.maximumBytes ?? GAUSSIAN_SPLAT_LIMITS.maximumBytes;
  const maximumCount = options.maximumCount ?? GAUSSIAN_SPLAT_LIMITS.maximumCount;
  if (!(maximumBytes > 0) || !(maximumCount > 0)) {
    throw new RangeError('maximumBytes and maximumCount must be positive');
  }
  return { maximumBytes, maximumCount: Math.min(maximumCount, GAUSSIAN_SPLAT_HARD_COUNT_LIMIT) };
}

function toByte(unit) {
  return Math.max(0, Math.min(255, Math.round(unit * 255)));
}

/**
 * Parses a standard 3DGS PLY into flat typed arrays. Throws a RangeError, before reading any splat, when the file is
 * larger than `maximumBytes` (default 512 MiB) or holds more than `maximumCount` splats (default 2,000,000; never more
 * than 4,000,000). Returns `{ count, positions, colors, scales, rotations, bounds, skipped }`, where `skipped` is `{ count, fields }`: a splat with any non-finite field (a scale that overflows counts) is dropped, `count` is how many were dropped (0 for a clean file) and `fields` names the fields that were non-finite. It throws only when no splat is usable. In the arrays: positions are xyz, colors are
 * RGBA bytes (RGB from the DC spherical harmonic, A from the sigmoid of opacity), scales are the exponentiated
 * axis lengths in metres and rotations are unit quaternions in w, x, y, z order.
 * `upAxis` is `'y'` (default, no change) or `'z'` for a Z-up file such as one written by Blender: positions and rotations are
 * turned by -90 degrees about x, so the file's +z comes out on +y and each splat's orientation turns with it.
 */
export function parseGaussianSplatPly(buffer, options = {}) {
  const source = openSplatPly(buffer, options, 'parseGaussianSplatPly');
  const reader = createSplatReader(source);
  reader.readRange(0, source.vertexCount);
  return reader.result();
}

/**
 * The same result as `parseGaussianSplatPly`, byte for byte, read in slices of `chunkSize` splats (default 32,768) with a
 * wait of one macrotask between slices, so the page keeps drawing and answering input. `onProgress(fraction)` is called
 * after each slice with a rising fraction that ends at exactly 1. An aborted `signal` stops the read and rejects with an
 * error named `AbortError`. The header checks and size limits run first and reject before any splat is read.
 */
export async function parseGaussianSplatPlyAsync(buffer, options = {}) {
  const source = openSplatPly(buffer, options, 'parseGaussianSplatPlyAsync');
  const reader = createSplatReader(source);
  await runInChunks(source.vertexCount, options, 'parseGaussianSplatPlyAsync', reader.readRange);
  return reader.result();
}

function openSplatPly(buffer, options, caller) {
  const { maximumBytes, maximumCount } = resolveLimits(options);
  resolveUpAxis(options, caller);
  if (!(buffer instanceof ArrayBuffer) && !ArrayBuffer.isView(buffer)) {
    throw new TypeError(`${caller} expects an ArrayBuffer or a typed array`);
  }
  if (buffer.byteLength > maximumBytes) {
    throw new RangeError(`Gaussian splat file is ${buffer.byteLength} bytes, over the limit of ${maximumBytes}`);
  }
  const bytes = buffer instanceof ArrayBuffer ? new Uint8Array(buffer) : new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
  const { text, bodyStart } = readHeaderText(bytes);
  const { vertexCount, properties, skippedBytes } = parseHeader(text);
  if (!Number.isSafeInteger(skippedBytes)) {
    throw new Error('Gaussian splat PLY declares an element before "vertex" with an invalid count. Re-export the file.');
  }
  if (vertexCount > maximumCount) {
    throw new RangeError(`Gaussian splat file holds ${vertexCount} splats, over the limit of ${maximumCount}`);
  }
  const { layout, stride } = propertyLayout(properties);
  const vertexStart = bodyStart + skippedBytes;
  if (bytes.length - vertexStart < vertexCount * stride) {
    throw new Error(`Gaussian splat PLY is truncated: ${vertexCount} splats need ${vertexCount * stride} bytes, ${Math.max(0, bytes.length - vertexStart)} are present`);
  }
  const body = new DataView(bytes.buffer, bytes.byteOffset + vertexStart, vertexCount * stride);
  return { view: body, vertexCount, layout, stride, upAxis: resolveUpAxis(options, caller) };
}

const FIELD_NAMES = [...REQUIRED_PROPERTIES];
const UP_AXES = ['y', 'z'];
const HALF_SQUARE_ROOT = Math.SQRT1_2;

function resolveUpAxis(options, caller) {
  const upAxis = options.upAxis ?? 'y';
  if (!UP_AXES.includes(upAxis)) {
    throw new RangeError(`${caller} upAxis must be 'y' (the default, Crucible's up) or 'z' (Blender and CAD files), got ${String(upAxis)}`);
  }
  return upAxis;
}

// A turn of -90 degrees about x sends the file's +z (up) to +y: (x, y, z) becomes (x, z, -y).
// The same turn, as the quaternion (w, x, y, z) = (cos 45, -sin 45, 0, 0), is multiplied in front of each splat's own rotation.
function turnZUpToYUp(row) {
  const [x, y, z] = [row[0], row[1], row[2]];
  row[0] = x;
  row[1] = z;
  row[2] = -y;
  const [w, qx, qy, qz] = [row[10], row[11], row[12], row[13]];
  row[10] = HALF_SQUARE_ROOT * (w + qx);
  row[11] = HALF_SQUARE_ROOT * (qx - w);
  row[12] = HALF_SQUARE_ROOT * (qy + qz);
  row[13] = HALF_SQUARE_ROOT * (qz - qy);
}

function createSplatReader({ view, vertexCount, layout, stride, upAxis }) {
  const positions = new Float32Array(vertexCount * 3);
  const colors = new Uint8Array(vertexCount * 4);
  const scales = new Float32Array(vertexCount * 3);
  const rotations = new Float32Array(vertexCount * 4);
  const minimum = [Infinity, Infinity, Infinity];
  const maximum = [-Infinity, -Infinity, -Infinity];
  const fields = FIELD_NAMES.map(name => layout.get(name));
  const values = new Float64Array(fields.length);
  const badFieldNames = new Set();
  let kept = 0;
  function readRange(start, end) {
    for (let index = start; index < end; index++) readSplat(index);
  }
  // A splat with any non-finite field is dropped; the next good splat takes its slot.
  function readSplat(index) {
    const row = index * stride;
    let valid = true;
    for (let field = 0; field < fields.length; field++) {
      const value = fields[field].read(view, row + fields[field].offset);
      // Positions are stored as float32, so a value past that range counts as non-finite here, as an overflowing scale does.
      const isPosition = field < 3;
      values[field] = field >= 7 && field <= 9 ? Math.fround(Math.exp(value)) : isPosition ? Math.fround(value) : value;
      if (!Number.isFinite(values[field])) {
        badFieldNames.add(FIELD_NAMES[field]);
        valid = false;
      }
    }
    if (!valid) return;
    if (upAxis === 'z') turnZUpToYUp(values);
    writeSplat(kept++, values);
  }
  function writeSplat(slot, row) {
    for (let axis = 0; axis < 3; axis++) {
      positions[slot * 3 + axis] = row[axis];
      if (row[axis] < minimum[axis]) minimum[axis] = row[axis];
      if (row[axis] > maximum[axis]) maximum[axis] = row[axis];
      colors[slot * 4 + axis] = toByte(0.5 + SPHERICAL_HARMONIC_DC * row[3 + axis]);
      scales[slot * 3 + axis] = row[7 + axis];
    }
    colors[slot * 4 + 3] = toByte(1 / (1 + Math.exp(-row[6])));
    rotations.set(unitQuaternion(row[10], row[11], row[12], row[13]), slot * 4);
  }
  function result() {
    const skipped = { count: vertexCount - kept, fields: FIELD_NAMES.filter(name => badFieldNames.has(name)) };
    if (kept === 0) {
      throw new Error(`Gaussian splat PLY has no usable splat: every one of the ${vertexCount} has a non-finite ${skipped.fields.join(', ')}. Re-export the file.`);
    }
    if (skipped.count === 0) return { count: kept, positions, colors, scales, rotations, bounds: { minimum, maximum }, skipped };
    return {
      count: kept,
      positions: positions.slice(0, kept * 3),
      colors: colors.slice(0, kept * 4),
      scales: scales.slice(0, kept * 3),
      rotations: rotations.slice(0, kept * 4),
      bounds: { minimum, maximum },
      skipped,
    };
  }
  return { readRange, result };
}

function unitQuaternion(w, x, y, z) {
  const length = Math.hypot(w, x, y, z);
  if (length > 1e-12) return [w / length, x / length, y / length, z / length];
  return [1, 0, 0, 0];
}