Skip to content

3D engine that AI agents drive

22.6 KB
Raw Blame
import { mat4 } from 'gl-matrix';
import { createSlicer } from './cooperative.js';
import { COMPONENT_TYPES, attributeFloats, createAccessorReader } from './gltf-accessors.js';
import { createClips } from './gltf-animation.js';
import { layerTextureReferences } from './gltf-layers.js';
import { Matrix4, SceneNode } from './gltf-nodes.js';
import { rememberSourceEncoding } from './image-source-encoding.js';
import { createSkinPose, validateSkinGeometry } from './skinning.js';

const GEOMETRY_ATTRIBUTE_NAMES = {
  POSITION: 'position',
  NORMAL: 'normal',
  COLOR_0: 'color',
  TEXCOORD_0: 'uv',
  TEXCOORD_1: 'uv1',
  JOINTS_0: 'skinIndex',
  WEIGHTS_0: 'skinWeight',
};
const TEXTURE_SLOTS = [
  ['map', 'baseColorTexture', true],
  ['normalMap', 'normalTexture', false],
  ['metalnessMap', 'metallicRoughnessTexture', true],
  ['aoMap', 'occlusionTexture', false],
  ['emissiveMap', 'emissiveTexture', false],
];
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47];
// "RIFF", four size bytes that vary, then "WEBP".
const WEBP_SIGNATURE = [0x52, 0x49, 0x46, 0x46, null, null, null, null, 0x57, 0x45, 0x42, 0x50];
const LINEAR_SRGB = 'srgb-linear';
const COMPONENT_CODES = new Map(Object.entries(COMPONENT_TYPES).map(([code, ArrayType]) => [ArrayType, Number(code)]));

/**
 * Record a limit the file goes over: what was measured, the option, its value and the option value that fits.
 * `perUnit` is for ceilings that are a multiple of the option, such as twelve indices per allowed vertex.
 */
export function overLimit(overruns, what, measured, option, limit, perUnit = 1) {
  if (measured > limit * perUnit) {
    const ceiling = perUnit > 1 ? ` x ${perUnit}` : '';
    overruns.push({ text: `${what} ${measured}, over ${option} ${limit}${ceiling}`, option, needed: Math.ceil(measured / perUnit) });
  }
  return overruns;
}

/** Throw one error for every recorded limit, so a single failed call names each option to raise and the value that fits. */
export function throwOverruns(overruns) {
  if (!overruns.length) return;
  const needed = new Map();
  for (const overrun of overruns) needed.set(overrun.option, Math.max(needed.get(overrun.option) ?? 0, overrun.needed));
  const fix = [...needed].map(([option, value]) => `${option}: ${value}`).join(', ');
  throw new RangeError(`glTF over its limits: ${overruns.map(overrun => overrun.text).join('; ')}. Fix: pass { ${fix} }.`);
}

/** Primitives the loader found unusable (bad accessors) before decoding; the decoder leaves them out and the file still loads. */
export const unusablePrimitives = new WeakSet();

/** POINTS, LINES, LINE_LOOP and LINE_STRIP (modes 0 to 3) and primitives marked unusable are not drawn, so the decoder leaves them out. */
export function isLinePrimitive(primitive) {
  return (primitive.mode >= 0 && primitive.mode <= 3) || unusablePrimitives.has(primitive);
}

export function textureSourceIndex(texture) {
  return texture.extensions?.KHR_texture_basisu?.source ?? texture.extensions?.EXT_texture_webp?.source ?? texture.source;
}

function sanitizeNodeName(name) {
  return name.replace(/\s/g, '_').replace(/[[\].:/]/g, '');
}

function mimeTypeOf(imageDefinition, bytes) {
  if (imageDefinition.mimeType) return imageDefinition.mimeType;
  if (PNG_SIGNATURE.every((value, index) => bytes[index] === value)) return 'image/png';
  if (WEBP_SIGNATURE.every((value, index) => value === null || bytes[index] === value)) return 'image/webp';
  return 'image/jpeg';
}

async function decodeImage(bytes, mimeType, target) {
  const blob = new Blob([bytes], { type: mimeType });
  if (typeof createImageBitmap === 'function') {
    const options = { premultiplyAlpha: 'none', colorSpaceConversion: 'none', imageOrientation: 'none' };
    if (target) Object.assign(options, { resizeWidth: target.width, resizeHeight: target.height, resizeQuality: 'high' });
    return createImageBitmap(blob, options);
  }
  if (target) throw new Error('Scaling textures to fit maxTexturePixels requires createImageBitmap.');
  const url = URL.createObjectURL(blob);
  try {
    const image = new Image();
    image.src = url;
    await image.decode();
    return image;
  } finally {
    URL.revokeObjectURL(url);
  }
}

function decodedAttribute(attribute, normalized) {
  return {
    array: attribute.array,
    itemSize: attribute.itemSize,
    count: attribute.count,
    normalized: normalized ?? attribute.normalized === true,
    componentType: COMPONENT_CODES.get(attribute.array.constructor),
  };
}

/**
 * Scale each vertex's four weights to sum to one, as three.js does for skinned meshes. Negative weights are clamped to 0 first;
 * a vertex left with none is bound fully to joint 0. A vertex that was all zero stays zero for validateSkinGeometry to refuse.
 * `repairs` counts the clamped weights and the vertexes bound to joint 0, and `indices` is a Float32Array copy that may be edited.
 */
function normalizedWeights(weights, indices, repairs) {
  const values = attributeFloats(weights);
  for (let offset = 0; offset < values.length; offset += 4) {
    let total = 0;
    let clamped = 0;
    for (let influence = 0; influence < 4; influence++) {
      if (values[offset + influence] < 0) {
        values[offset + influence] = 0;
        clamped++;
      }
      total += values[offset + influence];
    }
    repairs.clampedWeights += clamped;
    if (total === 0 && clamped) {
      indices[offset] = 0;
      values[offset] = total = 1;
      repairs.boundToJointZero++;
    }
    if (total !== 0) for (let influence = 0; influence < 4; influence++) values[offset + influence] /= total;
  }
  return { array: values, itemSize: 4, count: weights.count, normalized: false, componentType: 5126 };
}

// Built outside decodeGltf so the returned function does not keep the scope's file bytes and accessor reader alive.
function releaserFor(releasers) {
  return () => releasers.forEach(releaseOne => releaseOne());
}

/**
 * Decode the selected glTF scene into plain scene nodes, geometry sources, materials, textures and animation clips.
 * The result has the shape the CrucibleRenderer conversion in gltf.js reads. Nothing here imports three.js: Draco, meshopt
 * and KTX2 decoders arrive as `decoders` objects that the caller supplies.
 */
export async function decodeGltf({ document, glbBinary, resourceBytes, decoders, limits, imageTargets = new Map(), sharedTextures }) {
  const releasers = [];
  const usedNames = new Map();
  const nodeCache = new Map();
  const meshNames = new Map();
  const sources = new Map();
  const materials = new Map();
  const textures = new Map();
  const images = new Map();
  const skeletons = new Map();
  const instanceMatrices = new Map();

  function bufferBytes(index) {
    const buffer = document.buffers?.[index];
    if (!buffer) return null;
    if (buffer.uri !== undefined) return resourceBytes.get(buffer.uri) ?? null;
    return index === 0 ? glbBinary : null;
  }

  const checkpoint = createSlicer();
  const reader = createAccessorReader(document, bufferBytes, decoders.meshopt);
  await reader.prepare();

  function uniqueName(originalName) {
    const sanitized = sanitizeNodeName(originalName);
    if (!usedNames.has(sanitized)) {
      usedNames.set(sanitized, 0);
      return sanitized;
    }
    // A later node may really be called "Bone_1", so skip suffixes that are taken: names stay unique and clip
    // tracks, which address nodes by name, never collide.
    let count = usedNames.get(sanitized);
    let candidate;
    do {
      count++;
      candidate = `${sanitized}_${count}`;
    } while (usedNames.has(candidate));
    usedNames.set(sanitized, count);
    usedNames.set(candidate, 0);
    return candidate;
  }

  function imageBytes(index) {
    const definition = document.images[index];
    if (definition.uri !== undefined) return resourceBytes.get(definition.uri);
    return reader.viewBytes(definition.bufferView);
  }

  function loadKtx2Image(bytes) {
    const loader = decoders.ktx2;
    const url = URL.createObjectURL(new Blob([bytes], { type: 'image/ktx2' }));
    return new Promise((resolve, reject) => loader.load(url, resolve, undefined, reject)).finally(() => URL.revokeObjectURL(url));
  }

  async function loadImage(sourceIndex, isKtx2) {
    const key = `${sourceIndex}:${isKtx2}`;
    if (images.has(key)) return images.get(key);
    const bytes = imageBytes(sourceIndex);
    let loaded;
    if (isKtx2) {
      const texture = await loadKtx2Image(bytes);
      loaded = { image: texture.image, isCompressedTexture: texture.isCompressedTexture === true, format: texture.format, mipmaps: texture.mipmaps };
      releasers.push(() => texture.dispose?.());
    } else {
      const mimeType = mimeTypeOf(document.images[sourceIndex], bytes);
      const scaleTarget = imageTargets.get(sourceIndex);
      const image = await decodeImage(bytes, mimeType, scaleTarget);
      // A scaled image no longer matches its file, so only an unscaled one lets export reuse the source bytes.
      if (!scaleTarget) rememberSourceEncoding(image, bytes.slice(), mimeType);
      loaded = { image, isCompressedTexture: false, format: undefined, mipmaps: undefined };
      releasers.push(() => image.close?.());
    }
    images.set(key, loaded);
    return loaded;
  }

  // A WebP source with a PNG or JPEG fallback falls back when this browser cannot decode WebP.
  async function loadTextureImage(definition) {
    const isKtx2 = Boolean(definition.extensions?.KHR_texture_basisu);
    const sourceIndex = textureSourceIndex(definition);
    const fallbackIndex = definition.source;
    const hasWebpFallback = !isKtx2 && definition.extensions?.EXT_texture_webp && Number.isInteger(fallbackIndex) && fallbackIndex !== sourceIndex;
    if (!hasWebpFallback) return loadImage(sourceIndex, isKtx2);
    try {
      return await loadImage(sourceIndex, false);
    } catch {
      return loadImage(fallbackIndex, false);
    }
  }

  async function loadTexture(textureIndex) {
    if (textures.has(textureIndex)) return textures.get(textureIndex);
    const definition = document.textures[textureIndex];
    // A texture that several files of one world carry is stored in the first file only; later files hold a stand-in image and
    // the same key, and take the texture the first file decoded.
    const shareKey = definition.extras?.sharedTexture;
    const shared = shareKey && sharedTextures?.get(shareKey);
    if (shared) return shared;
    const sampler = document.samplers?.[definition.sampler] ?? {};
    const { image, isCompressedTexture, format, mipmaps } = await loadTextureImage(definition);
    const texture = {
      image,
      isCompressedTexture,
      format,
      mipmaps,
      wrapS: sampler.wrapS ?? 10497,
      wrapT: sampler.wrapT ?? 10497,
      minFilter: sampler.minFilter ?? 9987,
      magFilter: sampler.magFilter ?? 9729,
    };
    textures.set(textureIndex, texture);
    if (shareKey) sharedTextures?.set(shareKey, texture);
    return texture;
  }

  async function loadMaterial(materialIndex) {
    if (materials.has(materialIndex)) return materials.get(materialIndex);
    const definition = materialIndex === undefined ? {} : document.materials[materialIndex];
    const pbr = definition.pbrMetallicRoughness ?? {};
    const material = {
      materialIndex,
      color: (pbr.baseColorFactor ?? [1, 1, 1]).slice(0, 3),
      metalness: pbr.metallicFactor ?? 1,
      roughness: pbr.roughnessFactor ?? 1,
      emissive: [...(definition.emissiveFactor ?? [0, 0, 0])],
      emissiveIntensity: definition.extensions?.KHR_materials_emissive_strength?.emissiveStrength ?? 1,
      aoMapIntensity: definition.occlusionTexture?.strength ?? 1,
    };
    for (const [field, sourceField, inPbrBlock] of TEXTURE_SLOTS) {
      const reference = inPbrBlock ? pbr[sourceField] : definition[sourceField];
      if (!reference) continue;
      try {
        material[field] = await loadTexture(reference.index);
      } catch (error) {
        throw new Error(`${sourceField} failed to decode. Use a valid PNG, JPEG or WebP image. ${error.message}`);
      }
    }
    const layerTextures = {};
    for (const [name, reference] of layerTextureReferences(definition)) {
      try {
        layerTextures[name] = await loadTexture(reference.index);
      } catch (error) {
        throw new Error(`${name} failed to decode. Use a valid PNG, JPEG or WebP image. ${error.message}`);
      }
    }
    if (Object.keys(layerTextures).length > 0) material.layerTextures = layerTextures;
    materials.set(materialIndex, material);
    return material;
  }

  function decodeDracoGeometry(primitive, extension) {
    const loader = decoders.draco;
    loader.preload?.();
    const attributeIds = {};
    const attributeTypes = {};
    for (const [semantic, id] of Object.entries(extension.attributes)) {
      const name = GEOMETRY_ATTRIBUTE_NAMES[semantic];
      if (!name) continue;
      attributeIds[name] = id;
      const accessor = document.accessors[primitive.attributes[semantic]];
      if (accessor) attributeTypes[name] = COMPONENT_TYPES[accessor.componentType].name;
    }
    const compressed = reader.viewBytes(extension.bufferView).slice().buffer;
    return new Promise((resolve, reject) => loader.decodeDracoFile(compressed, resolve, attributeIds, attributeTypes, LINEAR_SRGB, reject));
  }

  /** Decode one primitive's accessors into `source`. Draco decodes the whole primitive at once; every other attribute reads its accessor. */
  async function decodePrimitiveInto(source, primitive) {
    const dracoExtension = primitive.extensions?.KHR_draco_mesh_compression;
    const dracoGeometry = dracoExtension ? await decodeDracoGeometry(primitive, dracoExtension) : null;
    const attributes = {};
    for (const [semantic, accessorIndex] of Object.entries(primitive.attributes)) {
      const name = GEOMETRY_ATTRIBUTE_NAMES[semantic];
      if (!name) continue;
      if (dracoGeometry && dracoExtension.attributes[semantic] !== undefined) {
        const decoded = dracoGeometry.getAttribute(name);
        if (!decoded) throw new Error(`Draco geometry is missing ${semantic}.`);
        attributes[name] = decodedAttribute(decoded, document.accessors[accessorIndex]?.normalized === true);
        reader.applySparseToDecoded(accessorIndex, attributes[name]);
      } else {
        attributes[name] = await reader.readAccessorInSlices(accessorIndex, checkpoint);
      }
    }
    let index = null;
    if (primitive.indices !== undefined) {
      if (dracoGeometry) {
        index = decodedAttribute(dracoGeometry.getIndex());
        reader.applySparseToDecoded(primitive.indices, index);
      } else {
        index = await reader.readAccessorInSlices(primitive.indices, checkpoint);
      }
    }
    const targets = primitive.targets ?? [];
    source.attributes = attributes;
    source.index = index;
    source.morphPositions = targets.map(target => target.POSITION === undefined ? undefined : reader.readAccessor(target.POSITION));
    source.morphNormals = targets.map(target => target.NORMAL === undefined ? undefined : reader.readAccessor(target.NORMAL));
    if (source.skinJointCount !== undefined) prepareSkinnedSource(source, source.skinJointCount);
  }

  /**
   * The geometry source of one primitive. Nothing is decoded here: the conversion calls `load()` right before it reads the
   * attributes and drops them after the last node that uses the source, so only a few primitives are decoded at any time.
   */
  function loadPrimitiveSource(meshIndex, primitiveIndex) {
    const key = `${meshIndex}:${primitiveIndex}`;
    if (sources.has(key)) return sources.get(key);
    const primitive = document.meshes[meshIndex].primitives[primitiveIndex];
    let pending = null;
    const source = {
      uuid: key,
      attributes: null,
      index: null,
      morphPositions: null,
      morphNormals: null,
      load() {
        pending ??= decodePrimitiveInto(source, primitive);
        return pending;
      },
    };
    sources.set(key, source);
    return source;
  }

  /** Weights are normalized first, clamping negatives; joints and the result are then checked, so all-zero and non-finite weights are still reported. */
  function prepareSkinnedSource(source, jointCount) {
    if (source.skinPrepared) return;
    const { skinIndex, position } = source.attributes;
    const repairs = { clampedWeights: 0, boundToJointZero: 0 };
    const skinIndices = attributeFloats(skinIndex);
    const skinWeight = normalizedWeights(source.attributes.skinWeight, skinIndices, repairs);
    validateSkinGeometry({ positions: { length: position.count * 3 }, skinIndices, skinWeights: skinWeight.array }, createSkinPose(jointCount));
    source.skinWeightRepairs = repairs;
    if (repairs.boundToJointZero) source.attributes.skinIndex = { array: Uint16Array.from(skinIndices), itemSize: 4, count: skinIndex.count, normalized: false, componentType: 5123 };
    source.attributes.skinWeight = skinWeight;
    source.skinPrepared = true;
  }

  function instanceMatricesFor(nodeIndex, extension) {
    if (instanceMatrices.has(nodeIndex)) return instanceMatrices.get(nodeIndex);
    const { TRANSLATION, ROTATION, SCALE } = extension.attributes;
    const translations = TRANSLATION === undefined ? null : attributeFloats(reader.readAccessor(TRANSLATION));
    const rotations = ROTATION === undefined ? null : attributeFloats(reader.readAccessor(ROTATION));
    const scales = SCALE === undefined ? null : attributeFloats(reader.readAccessor(SCALE));
    const count = [TRANSLATION, ROTATION, SCALE].filter(index => index !== undefined).map(index => document.accessors[index].count)[0];
    throwOverruns(overLimit([], `node ${nodeIndex} instances`, count, 'maxInstances', limits.maxInstances));
    const matrices = new Float64Array(count * 16);
    for (let instance = 0; instance < count; instance++) {
      mat4.fromRotationTranslationScale(
        matrices.subarray(instance * 16, instance * 16 + 16),
        rotations ? rotations.subarray(instance * 4, instance * 4 + 4) : [0, 0, 0, 1],
        translations ? translations.subarray(instance * 3, instance * 3 + 3) : [0, 0, 0],
        scales ? scales.subarray(instance * 3, instance * 3 + 3) : [1, 1, 1],
      );
    }
    const result = { count, matrices };
    instanceMatrices.set(nodeIndex, result);
    return result;
  }

  function reserveMeshNames(meshIndex) {
    if (!meshNames.has(meshIndex)) {
      const mesh = document.meshes[meshIndex];
      const baseName = mesh.name || `mesh_${meshIndex}`;
      meshNames.set(meshIndex, mesh.primitives.map(() => uniqueName(baseName)));
    }
    return meshNames.get(meshIndex);
  }

  async function createPrimitiveObject(definition, nodeIndex, meshIndex, primitiveIndex) {
    const mesh = document.meshes[meshIndex];
    const primitive = mesh.primitives[primitiveIndex];
    const object = new SceneNode();
    object.isMesh = true;
    object.geometry = loadPrimitiveSource(meshIndex, primitiveIndex);
    object.material = await loadMaterial(primitive.material);
    object.name = reserveMeshNames(meshIndex)[primitiveIndex];
    object.association = { meshes: meshIndex, primitives: primitiveIndex };
    if (definition.skin !== undefined) {
      object.isSkinnedMesh = true;
      object.geometry.skinJointCount ??= document.skins[definition.skin].joints.length;
    }
    const targetCount = primitive.targets?.length ?? 0;
    if (targetCount) {
      object.morphTargetInfluences = Array.from({ length: targetCount }, (unused, target) => definition.weights?.[target] ?? mesh.weights?.[target] ?? 0);
    }
    const instancing = definition.extensions?.EXT_mesh_gpu_instancing;
    if (instancing) {
      const { count, matrices } = instanceMatricesFor(nodeIndex, instancing);
      object.isInstancedMesh = true;
      object.count = count;
      object.instanceMatrices = matrices;
    }
    return object;
  }

  async function loadSkeleton(skinIndex) {
    if (skeletons.has(skinIndex)) return skeletons.get(skinIndex);
    const skin = document.skins[skinIndex];
    const bones = [];
    for (const joint of skin.joints) bones.push(await buildNode(joint));
    const inverseValues = skin.inverseBindMatrices === undefined ? null : attributeFloats(reader.readAccessor(skin.inverseBindMatrices));
    const boneInverses = bones.map((bone, index) => inverseValues ? new Matrix4().fromArray(inverseValues, index * 16) : new Matrix4());
    const skeleton = { bones, boneInverses, dispose() {} };
    skeletons.set(skinIndex, skeleton);
    return skeleton;
  }

  async function buildNode(nodeIndex) {
    if (nodeCache.has(nodeIndex)) return nodeCache.get(nodeIndex);
    const definition = document.nodes[nodeIndex];
    const nodeName = definition.name ? uniqueName(definition.name) : '';
    const objects = [];
    if (definition.mesh !== undefined) {
      for (let primitiveIndex = 0; primitiveIndex < document.meshes[definition.mesh].primitives.length; primitiveIndex++) {
        if (isLinePrimitive(document.meshes[definition.mesh].primitives[primitiveIndex])) continue;
        objects.push(await createPrimitiveObject(definition, nodeIndex, definition.mesh, primitiveIndex));
      }
    }
    let node;
    if (objects.length === 1) {
      node = objects[0];
    } else {
      node = new SceneNode();
      if (objects.length) node.association = { meshes: definition.mesh };
      for (const object of objects) node.add(object);
    }
    if (definition.name) node.name = nodeName;
    node.association = { ...node.association, nodes: nodeIndex };
    if (definition.matrix !== undefined) new Matrix4().fromArray(definition.matrix).decompose(node.position, node.quaternion, node.scale);
    if (definition.translation !== undefined) node.position.fromArray(definition.translation);
    if (definition.rotation !== undefined) node.quaternion.fromArray(definition.rotation);
    if (definition.scale !== undefined) node.scale.fromArray(definition.scale);
    nodeCache.set(nodeIndex, node);
    if (definition.skin !== undefined) {
      const skeleton = await loadSkeleton(definition.skin);
      node.traverse(candidate => {
        if (!candidate.isSkinnedMesh) return;
        candidate.skeleton = skeleton;
        candidate.bindMatrix = new Matrix4();
      });
    }
    for (const childIndex of definition.children ?? []) node.add(await buildNode(childIndex));
    return node;
  }

  const release = releaserFor(releasers);
  try {
    const sceneDefinition = document.scenes?.[document.scene ?? 0];
    if (!sceneDefinition) throw new Error('glTF has no default scene.');
    const scene = new SceneNode();
    for (const rootIndex of sceneDefinition.nodes ?? []) scene.add(await buildNode(rootIndex));
    for (const animation of document.animations ?? []) {
      for (const channel of animation.channels ?? []) {
        if (channel.target.node !== undefined) await buildNode(channel.target.node);
      }
    }
    const animations = createClips(document, index => nodeCache.get(index), reader.readAccessor);
    scene.updateMatrixWorld();
    return { scene, animations, release };
  } catch (error) {
    release();
    throw error;
  }
}