Skip to content

3D engine that AI agents drive

9.5 KB
Raw Blame
import './support/load-agent-scene-modules.mjs';
import test from 'node:test';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import assert from 'node:assert/strict';
import { createAgentScene } from '../src/agent-scene.js';
import { assetCopyKey } from '../src/agent-scene-assets.js';
import { parseGltf } from '../src/gltf.js';
import { startGame } from '../src/game.js';
import { gameAssetIds, loadGameAssets } from '../src/game-assets.js';
import { writeWebBuild } from '../scripts/web-export.mjs';
import { MIXAMO_LIKE, UNREAL_LIKE, humanoidGlb, humanoidJoints, walkTracks } from './fixtures/humanoid-glb.js';

// A game recipe whose player is a placed catalog humanoid: the walk, run and idle clips come from another humanoid file and are
// retargeted to the player's skeleton, and the animation graph blends them by speed while the controller moves the node.

const SOURCE_ID = 'ab'.repeat(32);
const HERO_ID = 'cd'.repeat(32);
const WALK_SPEED = 1.2;
const RUN_SPEED = 2.4;
const FRAME_MILLISECONDS = 1000 / 60;
const FOOT_DRIFT_BOUND = 1e-4;
const CONTACT_HEIGHT = 1e-3;
const SETTLE_FRAMES = 30;

const SOURCE_JOINTS = humanoidJoints();
const HERO_JOINTS = humanoidJoints({ armTiltDegrees: 30 });
const parse = bytes => parseGltf(bytes, { maxVertices: 500000 });

function locomotionClips() {
  const clip = (name, duration, speed) => ({ name, duration, tracks: walkTracks(SOURCE_JOINTS, { speed, duration }) });
  return [clip('Idle', 1, 0), clip('Walk', 1, WALK_SPEED), clip('Run', 0.5, RUN_SPEED)];
}

async function loadAssets() {
  const assets = new Map();
  assets.set(assetCopyKey(SOURCE_ID, 0), await parse(humanoidGlb({ names: MIXAMO_LIKE, joints: SOURCE_JOINTS, clips: locomotionClips() })));
  assets.set(assetCopyKey(HERO_ID, 0), await parse(humanoidGlb({ names: UNREAL_LIKE, joints: HERO_JOINTS })));
  assets.set(assetCopyKey(HERO_ID, 1), await parse(humanoidGlb({ names: UNREAL_LIKE, joints: HERO_JOINTS })));
  return assets;
}

function heroAnimation(overrides = {}) {
  return {
    clips: {
      idle: { from: { asset: SOURCE_ID, clip: 0 } },
      walk: { from: { asset: SOURCE_ID, clip: 1 } },
      run: { from: { asset: SOURCE_ID, clip: 2 } },
    },
    parameters: { speed: { type: 'number' } },
    rootMotion: { bone: UNREAL_LIKE.hips },
    layers: [{
      name: 'base',
      states: [{
        name: 'move',
        blend: { parameter: 'speed', motions: [{ clip: 'idle', threshold: 0 }, { clip: 'walk', threshold: WALK_SPEED }, { clip: 'run', threshold: RUN_SPEED }] },
      }],
    }],
    ...overrides,
  };
}

function heroScene(assets) {
  return createAgentScene({
    format: 'crucible.scene', schemaVersion: 1,
    geometries: [], materials: [],
    nodes: [{ id: 'Hero', asset: { id: HERO_ID } }],
  }, {}, { assets });
}

function setGame(scene, game) {
  scene.apply([{ type: 'set-game', game }], { expectedRevision: scene.revision });
  return scene.exportRecipe();
}

const standInRenderer = () => ({ render() {}, resize() {}, updateParts() {}, replaceParts() {}, dispose() {} });

async function startRecipe(recipe, assets, replay) {
  const previous = { request: globalThis.requestAnimationFrame, cancel: globalThis.cancelAnimationFrame };
  globalThis.requestAnimationFrame = () => 1;
  globalThis.cancelAnimationFrame = () => {};
  const game = await startGame({}, recipe, { replay, assets, showStatus: false, createRenderer: standInRenderer });
  return {
    game,
    finish() {
      game.dispose();
      globalThis.requestAnimationFrame = previous.request;
      globalThis.cancelAnimationFrame = previous.cancel;
    },
  };
}

function soleParts(game, asset) {
  return ['sole_left', 'sole_right'].map(name => {
    const index = asset.parts.findIndex(part => part.name === name);
    assert.ok(index >= 0, `no part ${name}`);
    return game.scene.parts.find(part => part.name === `Hero/${index}`);
  });
}

test('a recipe validates an asset character animation at the boundary and names the fix', async () => {
  const assets = await loadAssets();
  const scene = heroScene(assets);
  const attempt = animation => () => scene.apply([{ type: 'set-game', game: { player: { node: 'Hero', animation } } }], { expectedRevision: scene.revision });
  assert.throws(attempt(heroAnimation({ clips: { walk: { from: { asset: 'nope', clip: 0 } } } })), /clips\.walk\.from\.asset is "nope"\. Pass a catalog asset id of 64 hexadecimal/);
  assert.throws(attempt(heroAnimation({ clips: { walk: { from: { clip: -1 } } } })), /clips\.walk\.from\.clip must be the index of a clip/);
  assert.throws(attempt(heroAnimation({ clips: { walk: { tracks: [] } } })), /clips\.walk\.tracks is not a field\. Use one of: from, loop, scale/);
  const recipe = setGame(scene, { player: { node: 'Hero', walkSpeed: WALK_SPEED, runSpeed: RUN_SPEED, animation: heroAnimation() } });
  assert.deepEqual(recipe.game.player.animation.drive, { speed: 'intentSpeed' });
  assert.deepEqual(recipe.game.player.animation.clips.walk, { from: { asset: SOURCE_ID, clip: 1 }, loop: true });
});

test('starting the game says which asset or clip is missing', async () => {
  const assets = await loadAssets();
  const recipe = setGame(heroScene(assets), { player: { node: 'Hero', animation: heroAnimation({ clips: { idle: { from: { asset: SOURCE_ID, clip: 7 } } }, layers: [{ name: 'base', states: [{ name: 'idle', clip: 'idle' }] }] }) } });
  await assert.rejects(startRecipe(recipe, assets, { version: 1, samples: 1, events: [] }), /clips\.idle\.from\.clip is 7, but asset abababababab has 3 clips\. Use an index from 0 to 2/);
  const withoutSource = new Map(assets);
  withoutSource.delete(assetCopyKey(SOURCE_ID, 0));
  const walkRecipe = setGame(heroScene(assets), { player: { node: 'Hero', animation: heroAnimation() } });
  await assert.rejects(startRecipe(walkRecipe, withoutSource, { version: 1, samples: 1, events: [] }), /needs catalog asset abababababab, which the host did not load/);
});

test('a retargeted character blends idle, walk and run by speed and a planted foot stays put', async () => {
  const assets = await loadAssets();
  const recipe = setGame(heroScene(assets), { player: { node: 'Hero', walkSpeed: WALK_SPEED, runSpeed: RUN_SPEED, animation: heroAnimation() } });
  const replay = { version: 1, samples: 600, events: [[1, 'keyboard/KeyW', 1]] };
  const { game, finish } = await startRecipe(recipe, assets, replay);
  try {
    const soles = soleParts(game, assets.get(assetCopyKey(HERO_ID, 0)));
    const contacts = soles.map(() => null);
    const drifts = [];
    for (let frame = 0; frame < 480; frame++) {
      game.runtime.step(FRAME_MILLISECONDS);
      if (frame < SETTLE_FRAMES) continue;
      soles.forEach((sole, index) => {
        const world = [sole.matrix[12], sole.matrix[13], sole.matrix[14]];
        if (world[1] > CONTACT_HEIGHT) {
          contacts[index] = null;
          return;
        }
        contacts[index] ??= { anchor: world, drift: 0 };
        contacts[index].drift = Math.max(contacts[index].drift, Math.hypot(world[0] - contacts[index].anchor[0], world[2] - contacts[index].anchor[2]));
        drifts.push(contacts[index].drift);
      });
    }
    assert.ok(game.controller.state.position[2] < -5, `the player walked forward, z ${game.controller.state.position[2]}`);
    assert.ok(drifts.length > 100, `feet were planted for ${drifts.length} samples`);
    assert.ok(Math.max(0, ...drifts) < FOOT_DRIFT_BOUND, `worst planted foot drift ${Math.max(0, ...drifts)} m`);
  } finally {
    finish();
  }
});

test('the web export carries the placed assets and the page loads them by URL', async () => {
  const sourceBytes = humanoidGlb({ names: MIXAMO_LIKE, joints: SOURCE_JOINTS, clips: locomotionClips() });
  const heroBytes = humanoidGlb({ names: UNREAL_LIKE, joints: HERO_JOINTS });
  const recipe = setGame(heroScene(await loadAssets()), { player: { node: 'Hero', walkSpeed: WALK_SPEED, runSpeed: RUN_SPEED, animation: heroAnimation() } });
  assert.deepEqual(gameAssetIds(recipe).sort(), [HERO_ID, SOURCE_ID].sort());

  const directory = await mkdtemp(join(tmpdir(), 'crucible-asset-export-'));
  const buildDirectory = join(directory, 'build');
  await mkdir(buildDirectory);
  await writeFile(join(buildDirectory, 'game.js'), 'export const startGame = () => {};');
  try {
    await assert.rejects(writeWebBuild({ recipe, buildDirectory, outputDirectory: join(directory, 'refused') }), /needs the bytes of the catalog assets placed by Hero/);
    const assetBytes = new Map([[HERO_ID, Buffer.from(heroBytes)], [SOURCE_ID, Buffer.from(sourceBytes)]]);
    const summary = await writeWebBuild({ recipe, buildDirectory, outputDirectory: join(directory, 'built'), assets: assetBytes });
    const page = await readFile(join(directory, 'built', 'index.html'), 'utf8');
    assert.ok(page.includes(`"${HERO_ID}":"./assets/${HERO_ID}.glb"`));
    assert.equal((await readFile(join(directory, 'built', 'assets', `${SOURCE_ID}.glb`))).byteLength, sourceBytes.byteLength);
    assert.ok(summary.bytes > sourceBytes.byteLength + heroBytes.byteLength);
  } finally {
    await rm(directory, { recursive: true, force: true });
  }

  const urls = { [HERO_ID]: 'hero.glb', [SOURCE_ID]: 'source.glb' };
  const files = { 'hero.glb': heroBytes, 'source.glb': sourceBytes };
  const loaded = await loadGameAssets(recipe, urls, async url => files[url]);
  assert.deepEqual([...loaded.keys()].sort(), [assetCopyKey(HERO_ID, 0), assetCopyKey(HERO_ID, 1), assetCopyKey(SOURCE_ID, 0)].sort());
  await assert.rejects(loadGameAssets(recipe, { [HERO_ID]: 'hero.glb' }, async () => files['hero.glb']), /assetUrls has no URL for it/);
});