Skip to content

3D engine that AI agents drive

11.2 KB
Raw Blame
// Reads a gzip SPZ splat file (versions 2 and 3), the compact format of Niantic and the .spz output of World Labs.
// Format source: https://github.com/nianticlabs/spz, src/cc/load-spz.cc (deserializePackedGaussians) and
// src/cc/splat-utils.h (unpackQuaternionFirstThree, unpackQuaternionSmallestThree, colorScale).
//
// The gzip stream holds a 16 byte little endian header, then one column per attribute, each holding all splats:
//   header   uint32 magic 0x5053474e ("NGSP"), uint32 version, uint32 count, uint8 shDegree, uint8 fractionalBits, uint8 flags, uint8 reserved
//   positions  count * 9 bytes: x, y, z as 24 bit two's complement fixed point, value = integer / 2^fractionalBits
//   alphas     count bytes: sigmoid(opacity) * 255
//   colors     count * 3 bytes: (byte / 255 - 0.5) / 0.15 is the DC spherical harmonic
//   scales     count * 3 bytes: log scale = byte / 16 - 10
//   rotations  version 2: count * 3 bytes, x y z as byte / 127.5 - 1 with w rebuilt (w >= 0);
//              version 3: count * 4 bytes, "smallest three": the top 2 bits name the largest component, then three
//              10 bit fields (9 bit magnitude, 1 sign bit) scaled by 1/sqrt(2), for components 3 down to 0 in x y z w order
//   sh         count * shDim * 3 bytes, skipped here (shDim is 0, 3, 8, 15 or 24 for degree 0 to 4)
// Version 4 (a zstd container) and the never released version 1 (float16 positions) are not read.

import { runInChunks } from './gaussian-splat-chunks.js';
import { resolveLimits } from './gaussian-splat-ply.js';

const SPZ_MAGIC = 0x5053474e;
const HEADER_BYTES = 16;
const SUPPORTED_VERSIONS = [2, 3];
const SPHERICAL_HARMONIC_DC = 0.28209479177387814;
const SPZ_COLOR_SCALE = 0.15;
const SH_DIMENSION_BY_DEGREE = [0, 3, 8, 15, 24];
const SMALLEST_THREE_MASK = (1 << 9) - 1;
const SQRT_HALF = Math.SQRT1_2;

function toByte(unit) {
  return Math.max(0, Math.min(255, Math.round(unit * 255)));
}

function toBytes(buffer, caller) {
  if (buffer instanceof ArrayBuffer) return new Uint8Array(buffer);
  if (ArrayBuffer.isView(buffer)) return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
  throw new TypeError(`${caller} expects an ArrayBuffer or a typed array`);
}

function readHeader(bytes, maximumCount) {
  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
  if (view.getUint32(0, true) !== SPZ_MAGIC) {
    throw new Error('Not an SPZ file: the decompressed data must start with the magic "NGSP"');
  }
  const version = view.getUint32(4, true);
  if (!SUPPORTED_VERSIONS.includes(version)) {
    throw new Error(`SPZ version ${version} is not supported, only versions ${SUPPORTED_VERSIONS.join(' and ')}: export the splats as SPZ version 3 or as PLY`);
  }
  const count = view.getUint32(8, true);
  const shDegree = view.getUint8(12);
  if (count === 0) throw new Error('SPZ file holds no splats');
  if (count > maximumCount) throw new RangeError(`Gaussian splat file holds ${count} splats, over the limit of ${maximumCount}`);
  if (shDegree >= SH_DIMENSION_BY_DEGREE.length) throw new Error(`SPZ file has an unsupported spherical harmonic degree ${shDegree}, the largest is 4`);
  const rotationBytes = version >= 3 ? 4 : 3;
  const shBytes = SH_DIMENSION_BY_DEGREE[shDegree] * 3;
  const alphasOffset = HEADER_BYTES + count * 9;
  const colorsOffset = alphasOffset + count;
  const scalesOffset = colorsOffset + count * 3;
  const rotationsOffset = scalesOffset + count * 3;
  return {
    version,
    count,
    fractionalScale: 2 ** -view.getUint8(13),
    rotationBytes,
    alphasOffset,
    colorsOffset,
    scalesOffset,
    rotationsOffset,
    totalBytes: rotationsOffset + count * (rotationBytes + shBytes),
  };
}

// Inflates the gzip stream and stops reading as soon as the splat columns are complete (an extension zone after them is
// ignored), so a small file cannot expand past the size its own header announces or past `maximumBytes`.
async function inflateSpz(compressed, limits, caller) {
  if (compressed[0] !== 0x1f || compressed[1] !== 0x8b) {
    const looksLikeVersion4 = compressed.length >= 4 && new DataView(compressed.buffer, compressed.byteOffset).getUint32(0, true) === SPZ_MAGIC;
    throw new Error(looksLikeVersion4
      ? 'SPZ version 4 (zstd) is not supported: export the splats as SPZ version 3 or as PLY'
      : 'Not an SPZ file: it must be a gzip stream');
  }
  if (typeof DecompressionStream !== 'function') {
    throw new Error(`${caller} needs DecompressionStream (browsers and Node 18 or later)`);
  }
  const reader = new Blob([compressed]).stream().pipeThrough(new DecompressionStream('gzip')).getReader();
  const chunks = [];
  let received = 0;
  let header = null;
  let wanted = Math.min(limits.maximumBytes, HEADER_BYTES);
  try {
    while (received < wanted || !header) {
      const { done, value } = await reader.read();
      if (done) break;
      chunks.push(value);
      received += value.length;
      if (!header && received >= HEADER_BYTES) {
        header = readHeader(concatenate(chunks, received), limits.maximumCount);
        if (header.totalBytes > limits.maximumBytes) {
          throw new RangeError(`Gaussian splat file inflates to ${header.totalBytes} bytes, over the limit of ${limits.maximumBytes}`);
        }
        wanted = header.totalBytes;
      }
    }
  } catch (error) {
    await reader.cancel().catch(() => {});
    if (error instanceof RangeError || error.message.includes('SPZ')) throw error;
    throw new Error(`SPZ file is not a valid gzip stream: ${error.message}`);
  }
  await reader.cancel().catch(() => {});
  if (!header) throw new Error('SPZ file is truncated: the decompressed data is shorter than the 16 byte header');
  if (received < header.totalBytes) {
    throw new Error(`SPZ file is truncated: ${header.count} splats need ${header.totalBytes} bytes, ${received} are present`);
  }
  return { bytes: concatenate(chunks, received).subarray(0, header.totalBytes), header };
}

function concatenate(chunks, totalLength) {
  const joined = new Uint8Array(totalLength);
  let offset = 0;
  for (const chunk of chunks) {
    joined.set(chunk, offset);
    offset += chunk.length;
  }
  return joined;
}

async function openSplatSpz(buffer, options, caller) {
  const limits = resolveLimits(options);
  const compressed = toBytes(buffer, caller);
  if (compressed.byteLength > limits.maximumBytes) {
    throw new RangeError(`Gaussian splat file is ${compressed.byteLength} bytes, over the limit of ${limits.maximumBytes}`);
  }
  return inflateSpz(compressed, limits, caller);
}

function readFixedPoint24(bytes, offset) {
  const unsigned = bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16);
  return unsigned & 0x800000 ? unsigned - 0x1000000 : unsigned;
}

// Quaternion in w, x, y, z order, unit length, for either rotation encoding.
function writeRotation(rotations, index, bytes, offset, version) {
  const base = index * 4;
  let x;
  let y;
  let z;
  let w;
  if (version >= 3) {
    const packed = (bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 24)) >>> 0;
    const components = [0, 0, 0, 0];
    const largest = packed >>> 30;
    let remaining = packed;
    let sumOfSquares = 0;
    for (let component = 3; component >= 0; component--) {
      if (component === largest) continue;
      const magnitude = remaining & SMALLEST_THREE_MASK;
      const negative = (remaining >>> 9) & 1;
      remaining >>>= 10;
      components[component] = (negative ? -1 : 1) * SQRT_HALF * magnitude / SMALLEST_THREE_MASK;
      sumOfSquares += components[component] ** 2;
    }
    components[largest] = Math.sqrt(Math.max(0, 1 - sumOfSquares));
    [x, y, z, w] = components;
  } else {
    x = bytes[offset] / 127.5 - 1;
    y = bytes[offset + 1] / 127.5 - 1;
    z = bytes[offset + 2] / 127.5 - 1;
    w = Math.sqrt(Math.max(0, 1 - x * x - y * y - z * z));
  }
  const length = Math.hypot(w, x, y, z);
  if (length > 1e-12) {
    rotations[base] = w / length;
    rotations[base + 1] = x / length;
    rotations[base + 2] = y / length;
    rotations[base + 3] = z / length;
  } else {
    rotations.set([1, 0, 0, 0], base);
  }
}

function createSpzReader(bytes, header) {
  const { count, version, fractionalScale, alphasOffset, colorsOffset, scalesOffset, rotationsOffset, rotationBytes } = header;
  const positions = new Float32Array(count * 3);
  const colors = new Uint8Array(count * 4);
  const scales = new Float32Array(count * 3);
  const rotations = new Float32Array(count * 4);
  const minimum = [Infinity, Infinity, Infinity];
  const maximum = [-Infinity, -Infinity, -Infinity];
  function readRange(start, end) {
    for (let index = start; index < end; index++) readSplat(index);
  }
  function readSplat(index) {
    for (let axis = 0; axis < 3; axis++) {
      const coordinate = readFixedPoint24(bytes, HEADER_BYTES + (index * 3 + axis) * 3) * fractionalScale;
      positions[index * 3 + axis] = coordinate;
      if (coordinate < minimum[axis]) minimum[axis] = coordinate;
      if (coordinate > maximum[axis]) maximum[axis] = coordinate;
      const colorByte = bytes[colorsOffset + index * 3 + axis];
      const harmonic = (colorByte / 255 - 0.5) / SPZ_COLOR_SCALE;
      colors[index * 4 + axis] = toByte(0.5 + SPHERICAL_HARMONIC_DC * harmonic);
      scales[index * 3 + axis] = Math.exp(bytes[scalesOffset + index * 3 + axis] / 16 - 10);
    }
    colors[index * 4 + 3] = bytes[alphasOffset + index];
    writeRotation(rotations, index, bytes, rotationsOffset + index * rotationBytes, version);
  }
  function result() {
    return { count, positions, colors, scales, rotations, bounds: { minimum, maximum }, skipped: { count: 0, fields: [] } };
  }
  return { readRange, result };
}

/**
 * Parses a gzip SPZ file (version 2 or 3, spherical harmonic degree 0 to 4 with the higher bands skipped) into the same
 * `{ count, positions, colors, scales, rotations, bounds }` as `parseGaussianSplatPly`, with the same limits:
 * `maximumBytes` (default 512 MiB, applied to the file and to its inflated size) and `maximumCount` (default 2,000,000,
 * never above 4,000,000) reject with a RangeError before any splat is read. Async because gzip is inflated with
 * `DecompressionStream`. Positions stay in the SPZ axes (x right, y up, z back); a PLY from a trainer is x right,
 * y down, z forward, so place one with `matrix` scale [1, -1, -1] when the two must agree.
 */
export async function parseGaussianSplatSpz(buffer, options = {}) {
  const { bytes, header } = await openSplatSpz(buffer, options, 'parseGaussianSplatSpz');
  const reader = createSpzReader(bytes, header);
  reader.readRange(0, header.count);
  return reader.result();
}

/**
 * The same result as `parseGaussianSplatSpz`, with the splats converted in slices of `chunkSize` (default 32,768) and a
 * wait of one macrotask between slices, as `parseGaussianSplatPlyAsync` does: `onProgress(fraction)` ends at exactly 1 and
 * an aborted `signal` rejects with an error named `AbortError`. Inflating the gzip stream does not block the thread.
 */
export async function parseGaussianSplatSpzAsync(buffer, options = {}) {
  const { bytes, header } = await openSplatSpz(buffer, options, 'parseGaussianSplatSpzAsync');
  const reader = createSpzReader(bytes, header);
  await runInChunks(header.count, options, 'parseGaussianSplatSpzAsync', reader.readRange);
  return reader.result();
}