Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

19.9 KB
Raw Blame
import { invalidField } from './field-error.js';
import { mat4 } from 'gl-matrix';
import { normalizeDrive } from './agent-drive-schema.js';

// Placing decoded glTF assets inside a procedural recipe scene. A node with an `asset` field is a group: the scene
// turns each mesh part of the asset into a generated child node, so terrain, lights, procedural meshes and any number of
// placed assets share one graph, one set of lights and one renderer. Every placement of the same asset uses the same
// geometry and the same material objects, so the renderer draws the copies as one instanced batch per material.
// A placement with a `clip` gets its own decoded copy (an "instance" above 0), because a pose is state of one decoded asset.

// The `assets` module also serves map sequences, so a recipe with one loads no extra module entry in the scene.
export * from './agent-texture-sequence.js';

export const ASSET_PLACEMENT_LIMITS = Object.freeze({ maxPlacements: 512, maxAssets: 16, maxAnimatedPlacements: 16, maxOverrides: 32 });

// The crowd code registers itself here when it loads, so this file (which the MCP host imports) never pulls it in.
let crowdModule = null;

export function registerCrowdModule(module) {
  crowdModule = module;
}

function crowdCode() {
  if (!crowdModule) throw new Error('This recipe places a crowd, which loads on demand: await loadAgentSceneModules(recipe) before createAgentScene or apply.');
  return crowdModule;
}

const ASSET_ID = /^[a-f0-9]{64}$/;
const PLACEMENT_FIELDS = ['id', 'materials', 'clip', 'drive', 'crowd'];
const CLIP_FIELDS = ['index', 'start', 'speed', 'loop'];
const COLOR_FIELDS = ['color', 'emissiveColor'];
const UNIT_FIELDS = ['metallic', 'roughness', 'opacity'];
const OVERRIDE_FIELDS = ['name', 'tag', ...COLOR_FIELDS, ...UNIT_FIELDS, 'emission'];
// A tag an override may set: foliage and grass make an imported tree sway in the wind and shade like a grown one.
const OVERRIDE_TAGS = ['surface', 'foliage', 'grass'];

function knownKeys(value, allowed, label) {
  if (!value || typeof value !== 'object' || Array.isArray(value)) {
    throw invalidField(label, value, 'an object, such as {}', TypeError);
  }
  for (const key of Object.keys(value)) {
    if (!allowed.includes(key)) {
      throw invalidField(`${label}.${key}`, value[key], `no such field; use one of ${allowed.join(', ')}`, TypeError);
    }
  }
}

function colorVector(value, label) {
  if (!Array.isArray(value) || value.length !== 3 || value.some(component => !Number.isFinite(component) || component < 0)) {
    throw invalidField(label, value, 'an array of 3 linear RGB numbers of 0 or more, such as [0.5, 0.2, 0.1]');
  }
  return [...value];
}

function normalizeMaterialOverride(override, index) {
  const path = `asset.materials[${index}]`;
  knownKeys(override, OVERRIDE_FIELDS, path);
  if (typeof override.name !== 'string' || !override.name || override.name.length > 256) {
    throw invalidField(`${path}.name`, override.name, 'the name of a source material, such as "paint"; scene_inspect lists the names under assets', TypeError);
  }
  const result = { name: override.name };
  if (override.tag !== undefined) {
    if (!OVERRIDE_TAGS.includes(override.tag)) throw invalidField(`${path}.tag`, override.tag, `one of ${OVERRIDE_TAGS.join(', ')}`);
    result.tag = override.tag;
  }
  for (const field of COLOR_FIELDS) {
    if (override[field] !== undefined) result[field] = colorVector(override[field], `${path}.${field}`);
  }
  for (const field of UNIT_FIELDS) {
    if (override[field] !== undefined) {
      if (!Number.isFinite(override[field]) || override[field] < 0 || override[field] > 1) {
        throw invalidField(`${path}.${field}`, override[field], 'a number from 0 to 1, such as 0.5');
      }
      result[field] = override[field];
    }
  }
  if (override.emission !== undefined) {
    if (!Number.isFinite(override.emission) || override.emission < 0) {
      throw invalidField(`${path}.emission`, override.emission, 'a number of 0 or more, such as 1');
    }
    result.emission = override.emission;
  }
  return result;
}

/** Validate the `asset` field of a node. Returns the canonical form kept in the recipe. */
export function normalizeAssetPlacement(value) {
  knownKeys(value, PLACEMENT_FIELDS, 'asset');
  if (typeof value.id !== 'string' || !ASSET_ID.test(value.id.toLowerCase())) {
    throw invalidField('asset.id', value.id, '64 hexadecimal characters, such as "ab12..." (a catalog asset id that asset_catalog lists)', TypeError);
  }
  const result = { id: value.id.toLowerCase() };
  if (value.materials !== undefined) {
    if (!Array.isArray(value.materials) || value.materials.length > ASSET_PLACEMENT_LIMITS.maxOverrides) {
      throw invalidField('asset.materials', Array.isArray(value.materials) ? `${value.materials.length} overrides` : value.materials, `a list of at most ${ASSET_PLACEMENT_LIMITS.maxOverrides} overrides, such as [{ "name": "paint", "color": [1, 0, 0] }]`);
    }
    const overrides = value.materials.map((override, index) => normalizeMaterialOverride(override, index));
    if (new Set(overrides.map(override => override.name)).size !== overrides.length) {
      throw invalidField('asset.materials', `${overrides.length} overrides`, 'overrides that each name a different source material, such as one override per name');
    }
    result.materials = overrides;
  }
  if (value.clip !== undefined) {
    knownKeys(value.clip, CLIP_FIELDS, 'asset.clip');
    const { index, start = 0, speed = 1, loop = true } = value.clip;
    if (!Number.isInteger(index) || index < 0) throw invalidField('asset.clip.index', index, 'a whole number of 0 or more, such as 0');
    if (!Number.isFinite(start) || start < 0 || start > 3600) throw invalidField('asset.clip.start', start, 'seconds from 0 to 3600, such as 0');
    if (!Number.isFinite(speed) || speed <= 0 || speed > 16) throw invalidField('asset.clip.speed', speed, 'a number above 0 and at most 16, such as 1');
    if (typeof loop !== 'boolean') throw invalidField('asset.clip.loop', loop, 'true or false', TypeError);
    result.clip = { index, start, speed, loop };
  }
  if (value.crowd !== undefined) {
    if (value.clip !== undefined || value.drive !== undefined) throw invalidField('asset.crowd', value.crowd, 'only one of clip, drive or crowd; drop the others, such as keeping just the crowd', TypeError);
    result.crowd = crowdCode().normalizeCrowd(value.crowd);
  }
  if (value.drive !== undefined) {
    if (value.clip !== undefined) throw invalidField('asset.drive', value.drive, 'a drive or a clip, not both; drop one, such as keeping just the drive', TypeError);
    result.drive = normalizeDrive(value.drive, 'asset.drive');
    if (result.drive.lights?.tailLights) {
      throw invalidField('asset.drive.lights.tailLights', value.drive.lights.tailLights, 'no tail lights on a placed asset, since placed assets share their materials; tail lights glow only in an asset recipe (motion.drive), such as motion.drive.lights.tailLights');
    }
  }
  return result;
}

/** The nodes a game recipe animates itself: a placed asset there is posed by the game's animation graph, frame by frame. */
export function gamePosedNodeIds(game) {
  const ids = new Set();
  if (game?.player?.animation && game.player.node) ids.add(game.player.node);
  for (const npc of game?.npcs ?? []) if (npc?.animation && npc.node) ids.add(npc.node);
  return ids;
}

function collectPlacements(value, found, depth = 0) {
  if (!value || typeof value !== 'object' || depth > 8) return;
  if (Array.isArray(value)) {
    for (const item of value) collectPlacements(item, found, depth + 1);
    return;
  }
  const placement = value.asset;
  if (placement && typeof placement === 'object' && typeof placement.id === 'string' && ASSET_ID.test(placement.id.toLowerCase())) {
    found.push(placement);
  }
  for (const [key, child] of Object.entries(value)) {
    if (key !== 'asset') collectPlacements(child, found, depth + 1);
  }
}

/**
 * The catalog assets a recipe (or a list of edit operations) places: `{ id, copies }` where `copies` is how many decoded
 * copies the host should load. Copy 0 serves every unanimated placement; each placement with a clip takes its own copy.
 */
export function recipeAssetNeeds(value) {
  const placements = [];
  collectPlacements(value, placements);
  const needs = new Map();
  const posedNodes = gamePosedNodeIds(value?.game);
  const posedPlacements = (value?.nodes ?? []).filter(node => node.asset && posedNodes.has(node.id)).map(node => node.asset);
  for (const placement of placements) {
    const id = placement.id.toLowerCase();
    const need = needs.get(id) ?? { id, copies: 1 };
    if (placement.clip || placement.drive || placement.crowd || posedPlacements.includes(placement)) need.copies++;
    needs.set(id, need);
  }
  return [...needs.values()];
}

export function assetCopyKey(id, copy) {
  return `${id}:${copy}`;
}

export function jsonMaterial(source) {
  const result = {};
  for (const [field, value] of Object.entries(source)) {
    if (typeof value === 'number' || typeof value === 'string' || typeof value === 'boolean') {
      result[field] = value;
    } else if (Array.isArray(value) && value.every(entry => typeof entry === 'number')) {
      result[field] = [...value];
    }
  }
  return result;
}

function overrideFields(override) {
  const fields = {};
  if (override?.tag !== undefined) fields.tag = override.tag;
  for (const field of [...COLOR_FIELDS, ...UNIT_FIELDS, 'emission']) {
    if (override?.[field] !== undefined) fields[field] = override[field];
  }
  return fields;
}

export function materialOutput(source, fields) {
  const output = { ...source };
  for (const [field, value] of Object.entries(fields)) {
    output[field] = Array.isArray(value) ? Object.freeze([...value]) : value;
  }
  if (fields.emission !== undefined && fields.emission > 0 && !output.emissiveColor?.some(component => component > 0)) {
    output.emissiveColor = Object.freeze([1, 1, 1]);
  }
  return Object.freeze(output);
}

/**
 * Generate the geometries, materials and child nodes of every placement into a normalized recipe, in place.
 * `assets` maps assetCopyKey(id, copy) to a decoded parseGltf asset; `createAssetDrive` is the loaded drive module's builder. Returns what the scene keeps outside the recipe JSON.
 */
export function expandAssetPlacements(recipe, assets, limits, outputCache = new Map(), createAssetDrive = null) {
  const placed = recipe.nodes.filter(node => node.asset);
  const runtime = { placements: [], geometryParts: new Map(), materialOutputs: new Map(), partExtras: new Map(), animators: [] };
  if (!placed.length) return runtime;
  if (!assets) {
    throw new Error('This recipe places catalog assets, which a host loads and decodes first. Pass them as the assets option of createAgentScene; MCP does this for scene_create.');
  }
  const distinct = new Set(placed.map(node => node.asset.id));
  const posedNodes = gamePosedNodeIds(recipe.game);
  const isAnimated = node => Boolean(node.asset.clip || node.asset.drive || node.asset.crowd || posedNodes.has(node.id));
  const animated = placed.filter(isAnimated);
  if (placed.length > ASSET_PLACEMENT_LIMITS.maxPlacements || distinct.size > ASSET_PLACEMENT_LIMITS.maxAssets ||
      animated.length > ASSET_PLACEMENT_LIMITS.maxAnimatedPlacements) {
    throw new RangeError(`A scene places at most ${ASSET_PLACEMENT_LIMITS.maxPlacements} assets, ${ASSET_PLACEMENT_LIMITS.maxAssets} different ones, ${ASSET_PLACEMENT_LIMITS.maxAnimatedPlacements} of them playing a clip; this recipe places ${placed.length}, ${distinct.size} and ${animated.length}.`);
  }
  const copyCounter = new Map();
  const geometryIds = new Set();
  const materialIds = new Map();
  const generatedNodes = [];
  for (const node of placed) {
    const { id: assetId, materials: overrides = [], clip, drive, crowd } = node.asset;
    const posed = posedNodes.has(node.id);
    let copy = 0;
    if (clip || drive || crowd || posed) {
      copy = (copyCounter.get(assetId) ?? 0) + 1;
      copyCounter.set(assetId, copy);
    }
    const asset = assets.get(assetCopyKey(assetId, copy));
    if (!asset) {
      throw new Error(`Asset ${assetId.slice(0, 12)} copy ${copy} is not loaded. The host loads it before createAgentScene or apply.`);
    }
    if (clip && !asset.animations[clip.index]) {
      throw invalidField(`nodes[${recipe.nodes.indexOf(node)}].asset.clip.index`, clip.index, `a clip index from 0 to ${asset.animations.length - 1}, since asset ${assetId.slice(0, 12)} has ${asset.animations.length} clips; scene_inspect lists them under assets`);
    }
    if (crowd) {
      const expanded = crowdCode().expandCrowd(recipe, node, asset, copy, outputCache, runtime, limits);
      generatedNodes.push(...expanded.nodes);
      runtime.placements.push(expanded.record);
      runtime.animators.push(expanded.record);
      continue;
    }
    const materialNames = new Set(asset.parts.map(part => part.material.name).filter(Boolean));
    for (const override of overrides) {
      if (!materialNames.has(override.name)) {
        throw invalidField(`nodes[${recipe.nodes.indexOf(node)}].asset.materials[${overrides.indexOf(override)}].name`, override.name, `one of ${[...materialNames].slice(0, 16).map(name => JSON.stringify(name)).join(', ') || 'none, since the asset materials are unnamed'}`);
      }
    }
    const overrideByName = new Map(overrides.map(override => [override.name, override]));
    const materialIndexes = new Map();
    const partNodeIds = [];
    asset.parts.forEach((part, index) => {
      const geometryId = `asset:${assetId.slice(0, 12)}:${copy}:${index}`;
      if (!geometryIds.has(geometryId)) {
        geometryIds.add(geometryId);
        recipe.geometries.push({ id: geometryId, type: 'asset', assetId, copy, part: index });
        runtime.geometryParts.set(geometryId, part.geometry);
      }
      if (!materialIndexes.has(part.material)) materialIndexes.set(part.material, materialIndexes.size);
      const fields = overrideFields(overrideByName.get(part.material.name));
      const materialKey = `asset:${assetId.slice(0, 12)}:${copy}:${materialIndexes.get(part.material)}:${JSON.stringify(fields)}`;
      if (!materialIds.has(materialKey)) {
        materialIds.set(materialKey, materialKey);
        recipe.materials.push({ id: materialKey, ...jsonMaterial(part.material), ...fields, assetSource: true });
        // The same output object across stages keeps the renderer's batches and the parts' identity stable.
        if (outputCache.get(materialKey)?.source !== part.material) {
          outputCache.set(materialKey, { source: part.material, output: materialOutput(part.material, fields) });
        }
        runtime.materialOutputs.set(materialKey, outputCache.get(materialKey).output);
      }
      const partNodeId = `${node.id}/${index}`;
      partNodeIds.push(partNodeId);
      generatedNodes.push({
        id: partNodeId, parent: node.id, label: String(part.name ?? partNodeId).slice(0, 256),
        translation: [0, 0, 0], rotation: [0, 0, 0, 1], scale: [1, 1, 1],
        geometry: geometryId, material: materialKey, assetPart: { placement: node.id, index, matrix: Array.from(part.matrix) },
      });
      runtime.partExtras.set(partNodeId, {
        ...(part.skin ? { skin: part.skin } : {}), ...(part.morph ? { morph: part.morph } : {}),
        ...(part.castShadow !== undefined ? { castShadow: part.castShadow } : {}),
      });
    });
    const record = { node: node.id, assetId, copy, asset, partNodeIds, clip: clip ?? null, pose: posed ? { nodeOverrides: [], morphWeights: undefined } : null, drive: drive ? createDrive(node.id, `nodes[${recipe.nodes.indexOf(node)}].asset.drive`, createAssetDrive, asset, drive) : null, drivePose: null };
    runtime.placements.push(record);
    if (clip || drive || posed) runtime.animators.push(record);
  }
  if (recipe.nodes.length + generatedNodes.length > limits.maxNodes) {
    throw new RangeError(`Placed assets expand to ${generatedNodes.length} part nodes, which with the ${recipe.nodes.length} authored nodes exceeds maxNodes ${limits.maxNodes}. Place fewer copies or simpler assets.`);
  }
  recipe.nodes.push(...generatedNodes);
  return runtime;
}

function createDrive(nodeId, fieldPath, createAssetDrive, asset, drive) {
  if (!createAssetDrive) throw new Error('This recipe drives a placed asset, which needs the drive code: await loadAgentSceneModules(recipe) before createAgentScene or apply.');
  try {
    return createAssetDrive(asset, drive, { fieldPath });
  } catch (error) {
    if (typeof error?.fieldPath !== 'string') error.message = `Node ${nodeId} drive: ${error.message}`;
    throw error;
  }
}

/** The time inside a clip for a scene time: held at the first frame before `start`, then looped or held at the end. */
export function clipTimeAt(clip, duration, time) {
  const elapsed = Math.max(0, ((time ?? 0) - clip.start) * clip.speed);
  if (duration <= 0) return 0;
  return clip.loop ? elapsed % duration : Math.min(elapsed, duration);
}

/**
 * Pose every animated placement at a scene time. Returns the world matrix of each part node and a commit/discard pair;
 * the pose is staged first so a failure leaves every asset where it was.
 */
export function stageAssetClips(runtime, graph, time) {
  const transactions = [];
  const matrices = new Map();
  try {
    for (const record of runtime.animators) {
      let transaction;
      if (record.crowd) {
        transactions.push(record.crowd.stage(time ?? 0));
        continue;
      }
      if (record.pose) {
        transaction = record.asset.stagePose({ animation: null, nodeOverrides: record.pose.nodeOverrides, morphWeights: record.pose.morphWeights });
      } else if (record.drive) {
        const driven = record.drive.sample(time ?? 0);
        record.drivePose = driven.pose;
        transaction = record.asset.stagePose({ nodeOverrides: driven.nodeOverrides });
      } else {
        const duration = record.asset.animations[record.clip.index].duration;
        transaction = record.asset.stagePose({ animation: { clip: record.clip.index, time: clipTimeAt(record.clip, duration, time) }, nodeOverrides: [] });
      }
      transactions.push(transaction);
      const placement = graph.getNode(record.node).worldMatrix;
      record.partNodeIds.forEach((id, index) => {
        matrices.set(id, mat4.multiply(new Float32Array(16), placement, transaction.parts[index].matrix));
      });
    }
  } catch (error) {
    for (const transaction of transactions) transaction.discard();
    throw error;
  }
  return {
    matrices,
    commit() {
      for (const transaction of transactions) transaction.commit();
    },
  };
}

/** What scene_inspect reports for each placement: the asset id, its world bounds, its clips and material names. */
export function describeAssetPlacements(runtime, graph, geometryBounds) {
  return runtime.placements.map(record => {
    if (record.crowd) return record.crowd.describe();
    const minimum = [Infinity, Infinity, Infinity];
    const maximum = [-Infinity, -Infinity, -Infinity];
    for (const id of record.partNodeIds) {
      const node = graph.getNode(id);
      const bounds = geometryBounds(node.geometry);
      for (let corner = 0; corner < 8; corner++) {
        const point = [0, 1, 2].map(axis => (corner & (1 << axis) ? bounds.max[axis] : bounds.min[axis]));
        for (let axis = 0; axis < 3; axis++) {
          const value = node.worldMatrix[axis] * point[0] + node.worldMatrix[4 + axis] * point[1] + node.worldMatrix[8 + axis] * point[2] + node.worldMatrix[12 + axis];
          minimum[axis] = Math.min(minimum[axis], value);
          maximum[axis] = Math.max(maximum[axis], value);
        }
      }
    }
    return {
      node: record.node, assetId: record.assetId, parts: record.partNodeIds.length,
      bounds: { min: minimum, max: maximum },
      clips: record.asset.animations.map((animation, index) => ({ index, name: animation.name ?? `clip ${index}`, duration: animation.duration })),
      materials: [...new Set(record.asset.parts.map(part => part.material.name).filter(Boolean))],
      ...(record.clip ? { playing: record.clip } : {}),
      ...(record.drive ? { drive: record.drive.describe(record.drivePose ?? undefined) } : {}),
    };
  });
}