Skip to content

3D engine that AI agents drive

13.0 KB
Raw Blame
import { deflateSync } from 'node:zlib';

import { PNG_SIGNATURE, pngChunk } from '../src/export-png.js';

// Node-side writers for the high-precision files of a render job: 16-bit PNG and OpenEXR. Both are small hand-written
// encoders (no dependency) and both take the float pixels the page reads back (RGBA float32, rows bottom to top).

const EXR_MAGIC = [0x76, 0x2f, 0x31, 0x01];
const EXR_VERSION = [2, 0, 0, 0];
const EXR_ZIP_COMPRESSION = 3;
const EXR_ZIP_ROWS_PER_BLOCK = 16;
const EXR_HALF = 1;
const EXR_FLOAT = 2;
const HALF_MAXIMUM = 65504;
const PNG_COLOR_TYPES = Object.freeze({ 1: 0, 3: 2, 4: 6 });
const PNG_SUB_FILTER = 1;
const MAXIMUM_PNG16_OBJECT_ID = 65535;

/** The 16 bit value of a colour channel in 0 to 1. */
function toSixteenBits(value) {
  return Math.round(Math.min(1, Math.max(0, value)) * 65535);
}

/**
 * A 16-bit PNG from interleaved samples (1 gray, 3 RGB or 4 RGBA channels), top row first, compressed with the Sub filter.
 * `samples` holds integers 0 to 65535.
 */
export function encodePng16(samples, width, height, channels) {
  if (!Object.hasOwn(PNG_COLOR_TYPES, channels)) throw new RangeError('A 16-bit PNG has 1, 3 or 4 channels.');
  if (samples.length !== width * height * channels) throw new RangeError('The sample count does not match the image size.');
  const bytesPerPixel = channels * 2;
  const rowBytes = width * bytesPerPixel;
  const filtered = Buffer.alloc((rowBytes + 1) * height);
  const rowSamples = width * channels;
  const row = Buffer.alloc(rowBytes);
  for (let rowIndex = 0; rowIndex < height; rowIndex++) {
    for (let index = 0; index < rowSamples; index++) row.writeUInt16BE(samples[rowIndex * rowSamples + index], index * 2);
    const target = rowIndex * (rowBytes + 1);
    filtered[target] = PNG_SUB_FILTER;
    for (let byteIndex = 0; byteIndex < rowBytes; byteIndex++) {
      const left = byteIndex >= bytesPerPixel ? row[byteIndex - bytesPerPixel] : 0;
      filtered[target + 1 + byteIndex] = (row[byteIndex] - left) & 0xff;
    }
  }
  const header = Buffer.alloc(13);
  header.writeUInt32BE(width, 0);
  header.writeUInt32BE(height, 4);
  header[8] = 16;
  header[9] = PNG_COLOR_TYPES[channels];
  return Buffer.concat([Buffer.from(PNG_SIGNATURE), pngChunk('IHDR', header), pngChunk('IDAT', deflateSync(filtered, { level: 6 })), pngChunk('IEND', new Uint8Array(0))]);
}

const float32Scratch = new Float32Array(1);
const uint32Scratch = new Uint32Array(float32Scratch.buffer);

/** IEEE half float bits of a number, rounded to nearest and clamped to the finite range. */
export function toHalfBits(value) {
  const clamped = Number.isNaN(value) ? 0 : Math.min(HALF_MAXIMUM, Math.max(-HALF_MAXIMUM, value));
  float32Scratch[0] = clamped;
  const bits = uint32Scratch[0];
  const sign = (bits >>> 16) & 0x8000;
  const exponent = ((bits >>> 23) & 0xff) - 127 + 15;
  const mantissa = bits & 0x7fffff;
  if (exponent <= 0) {
    if (exponent < -10) return sign;
    const shifted = (mantissa | 0x800000) >> (1 - exponent);
    return sign | ((shifted + 0x1000) >> 13);
  }
  return sign | (((exponent << 10) | (mantissa >> 13)) + ((mantissa >> 12) & 1));
}

function exrAttribute(name, type, value) {
  return Buffer.concat([Buffer.from(`${name}\0${type}\0`), uint32(value.length), value]);
}

function uint32(value) {
  const buffer = Buffer.alloc(4);
  buffer.writeInt32LE(value);
  return buffer;
}

function box(minimumX, minimumY, maximumX, maximumY) {
  const buffer = Buffer.alloc(16);
  [minimumX, minimumY, maximumX, maximumY].forEach((value, index) => buffer.writeInt32LE(value, index * 4));
  return buffer;
}

function channelList(channels) {
  const entries = channels.map(({ name, pixelType }) => {
    const entry = Buffer.alloc(name.length + 1 + 16);
    entry.write(name, 0, 'latin1');
    entry.writeInt32LE(pixelType, name.length + 1);
    entry.writeInt32LE(1, name.length + 9);
    entry.writeInt32LE(1, name.length + 13);
    return entry;
  });
  return Buffer.concat([...entries, Buffer.alloc(1)]);
}

/** The OpenEXR ZIP step: split even and odd bytes, delta code, deflate; a block that does not shrink is stored as it is. */
function zipBlock(raw) {
  const reordered = Buffer.alloc(raw.length);
  const half = (raw.length + 1) >> 1;
  for (let index = 0; index < raw.length; index++) {
    const target = index % 2 === 0 ? index / 2 : half + (index - 1) / 2;
    reordered[target] = raw[index];
  }
  let previous = reordered[0];
  for (let index = 1; index < reordered.length; index++) {
    const current = reordered[index];
    reordered[index] = (current - previous + 128 + 256) & 0xff;
    previous = current;
  }
  const compressed = deflateSync(reordered);
  return compressed.length < raw.length ? compressed : raw;
}

/**
 * An OpenEXR file (scanline, ZIP compression, one part). `channels` is [{ name, type: 'half' | 'float', values }] where
 * `values` is a Float32Array of width x height numbers, top row first. Channels are written in name order, as the format requires.
 */
export function encodeExr(width, height, channels) {
  const ordered = [...channels].sort((left, right) => (left.name < right.name ? -1 : 1))
    .map(channel => ({ ...channel, pixelType: channel.type === 'float' ? EXR_FLOAT : EXR_HALF }));
  const bytesPerPixel = ordered.reduce((total, channel) => total + (channel.pixelType === EXR_FLOAT ? 4 : 2), 0);
  const header = Buffer.concat([
    Buffer.from([...EXR_MAGIC, ...EXR_VERSION]),
    exrAttribute('channels', 'chlist', channelList(ordered)),
    exrAttribute('compression', 'compression', Buffer.from([EXR_ZIP_COMPRESSION])),
    exrAttribute('dataWindow', 'box2i', box(0, 0, width - 1, height - 1)),
    exrAttribute('displayWindow', 'box2i', box(0, 0, width - 1, height - 1)),
    exrAttribute('lineOrder', 'lineOrder', Buffer.from([0])),
    exrAttribute('pixelAspectRatio', 'float', Buffer.from(new Float32Array([1]).buffer)),
    exrAttribute('screenWindowCenter', 'v2f', Buffer.from(new Float32Array([0, 0]).buffer)),
    exrAttribute('screenWindowWidth', 'float', Buffer.from(new Float32Array([1]).buffer)),
    Buffer.alloc(1),
  ]);
  const blocks = [];
  for (let firstRow = 0; firstRow < height; firstRow += EXR_ZIP_ROWS_PER_BLOCK) {
    const rowCount = Math.min(EXR_ZIP_ROWS_PER_BLOCK, height - firstRow);
    const raw = Buffer.alloc(rowCount * width * bytesPerPixel);
    let offset = 0;
    for (let row = 0; row < rowCount; row++) {
      for (const channel of ordered) {
        for (let column = 0; column < width; column++) {
          const value = channel.values[(firstRow + row) * width + column];
          if (channel.pixelType === EXR_FLOAT) {
            raw.writeFloatLE(value, offset);
            offset += 4;
          } else {
            raw.writeUInt16LE(toHalfBits(value), offset);
            offset += 2;
          }
        }
      }
    }
    const data = zipBlock(raw);
    const blockHeader = Buffer.alloc(8);
    blockHeader.writeInt32LE(firstRow, 0);
    blockHeader.writeInt32LE(data.length, 4);
    blocks.push(Buffer.concat([blockHeader, data]));
  }
  const table = Buffer.alloc(blocks.length * 8);
  let position = header.length + table.length;
  blocks.forEach((block, index) => {
    table.writeBigUInt64LE(BigInt(position), index * 8);
    position += block.length;
  });
  return Buffer.concat([header, table, ...blocks]);
}

/** Interleaved bottom-up RGBA floats into one planar top-down Float32Array per requested component (0 to 3). */
function planes(pixels, width, height, components) {
  return components.map(component => {
    const plane = new Float32Array(width * height);
    for (let row = 0; row < height; row++) {
      const sourceRow = height - 1 - row;
      for (let column = 0; column < width; column++) plane[row * width + column] = pixels[(sourceRow * width + column) * 4 + component];
    }
    return plane;
  });
}

function byteOf(value) {
  return Math.round(Math.min(1, Math.max(0, value)) * 255);
}

/** The 16 bit depth step of a depth pass pixel: red is the high byte and green the low byte. */
function depthSteps(pixels, offset) {
  return byteOf(pixels[offset]) * 256 + byteOf(pixels[offset + 1]);
}

/** The 24 bit object ID of an object ID pass pixel, zero for no object. */
function objectIdentifier(pixels, offset) {
  return byteOf(pixels[offset]) + byteOf(pixels[offset + 1]) * 256 + byteOf(pixels[offset + 2]) * 65536;
}

function exrChannelsForPass({ pixels, width, height, pass, hasAlpha, depthRange, prefix = '' }) {
  if (pass === 'depth' || pass === 'objectId') {
    const values = new Float32Array(width * height);
    for (let row = 0; row < height; row++) {
      const sourceRow = height - 1 - row;
      for (let column = 0; column < width; column++) {
        const offset = (sourceRow * width + column) * 4;
        values[row * width + column] = pass === 'depth' ? depthSteps(pixels, offset) / 65535 * depthRange : objectIdentifier(pixels, offset);
      }
    }
    return [{ name: `${prefix}${pass === 'depth' ? 'Z' : 'id'}`, type: 'float', values }];
  }
  const [red, green, blue, alpha] = planes(pixels, width, height, [0, 1, 2, 3]);
  if (pass === 'uv') return [{ name: `${prefix}U`, type: 'half', values: red }, { name: `${prefix}V`, type: 'half', values: green }];
  if (pass === 'velocity') return [{ name: `${prefix}X`, type: 'half', values: red }, { name: `${prefix}Y`, type: 'half', values: green }];
  if (pass === 'position') return [{ name: `${prefix}X`, type: 'float', values: red }, { name: `${prefix}Y`, type: 'float', values: green }, { name: `${prefix}Z`, type: 'float', values: blue }];
  const channels = [{ name: `${prefix}R`, type: 'half', values: red }, { name: `${prefix}G`, type: 'half', values: green }, { name: `${prefix}B`, type: 'half', values: blue }];
  if (hasAlpha) channels.push({ name: `${prefix}A`, type: 'half', values: alpha });
  return channels;
}

function encodeExrPass(options) {
  return encodeExr(options.width, options.height, exrChannelsForPass(options));
}

export function encodeMultilayerExr({ width, height, beauty, passes = {}, hasAlpha = false, depthRange = 100 }) {
  if (beauty.length !== width * height * 4) throw new RangeError('The beauty pixel count does not match the image size.');
  const channels = [...exrChannelsForPass({ pixels: beauty, width, height, pass: undefined, hasAlpha, depthRange, prefix: 'Combined.' })];
  for (const [pass, pixels] of Object.entries(passes)) {
    if (pixels.length !== width * height * 4) throw new RangeError(`The ${pass} pixel count does not match the image size.`);
    channels.push(...exrChannelsForPass({ pixels, width, height, pass, hasAlpha: false, depthRange, prefix: `${pass}.` }));
  }
  return encodeExr(width, height, channels);
}

function encodePng16Pass({ pixels, width, height, pass, hasAlpha, depthRange = 100 }) {
  const pixelCount = width * height;
  if (pass === 'depth' || pass === 'objectId') {
    const samples = new Uint16Array(pixelCount);
    for (let row = 0; row < height; row++) {
      for (let column = 0; column < width; column++) {
        const offset = ((height - 1 - row) * width + column) * 4;
        const value = pass === 'depth' ? depthSteps(pixels, offset) : objectIdentifier(pixels, offset);
        if (value > MAXIMUM_PNG16_OBJECT_ID && pass === 'objectId') throw new RangeError('A 16-bit PNG holds object IDs up to 65535. Use format exr for this scene.');
        samples[row * width + column] = value;
      }
    }
    return encodePng16(samples, width, height, 1);
  }
  const channels = hasAlpha ? 4 : 3;
  const samples = new Uint16Array(pixelCount * channels);
  for (let row = 0; row < height; row++) {
    for (let column = 0; column < width; column++) {
      const offset = ((height - 1 - row) * width + column) * 4;
      const target = (row * width + column) * channels;
      const alpha = pixels[offset + 3];
      // The lit frame is premultiplied by alpha; a PNG stores straight colour.
      const scale = hasAlpha && alpha > 0 ? 1 / alpha : 1;
      for (let component = 0; component < 3; component++) {
        const raw = pixels[offset + component];
        const mapped = pass === 'position' ? raw / depthRange * 0.5 + 0.5
          : pass === 'velocity' ? (component < 2 ? raw * 0.5 + 0.5 : 0)
          : raw * scale;
        samples[target + component] = toSixteenBits(mapped);
      }
      if (hasAlpha) samples[target + 3] = toSixteenBits(alpha);
    }
  }
  return encodePng16(samples, width, height, channels);
}

/**
 * The file bytes of one rendered image in a high-precision format. `pixels` is the page's RGBA float readback
 * (rows bottom to top), `pass` names the render pass or is undefined for the lit image, and `hasAlpha` keeps the alpha
 * of a transparent lit image. Depth is written as metres over `depthRange` in EXR and as the raw 16-bit step in PNG.
 */
export function encodeRenderOutput({ format, pixels, width, height, pass, hasAlpha = false, depthRange = 100 }) {
  if (pixels.length !== width * height * 4) throw new RangeError('The pixel count does not match the image size.');
  const options = { pixels, width, height, pass, hasAlpha: hasAlpha && pass === undefined, depthRange };
  if (format === 'png16') return encodePng16Pass(options);
  if (format === 'exr') return encodeExrPass(options);
  throw new TypeError('format must be png16 or exr.');
}