Skip to content

3D engine that AI agents drive

19.0 KB
Raw Blame
// One showcase world for the hero loop, built in two steps so the next world can be fetched while the current one plays:
//   fetchWorld(descriptionUrl)  network only: the description, every GLB part and the sky, held as blobs.
//   buildWorld(download, canvas) parses the parts, uploads them to the GPU and returns a world that draws on that canvas.
// A world is whole before it is shown: the loop never draws the first part of a split world on its own. The one exception is a
// description with a first view (`glbFirstView`, `hdriPreview`): the whole world as a small, coarse copy that is drawn first and
// replaced by the full world through world.upgrade() once the full files have arrived and been prepared.
import { CrucibleRenderer, shaderFeatureRepresentatives } from '../../renderer.js';
import { createSharedTextures, loadGltf } from '../../gltf.js';
import { loadHdri } from '../../hdri.js';
import { createCascadedShadows } from '../../cascaded-shadows.js';
import { applyWaterReflection, attachWaterReflection } from '../../water-reflection.js';
import { readPartRanks } from '../../showcase-download.js';
import { prepareEffects } from './effects.js';
import { createSlicer, yieldToPage } from '../../cooperative.js';
import { loopedClipTime } from './clip-time.js';
import { withStandInTextures } from './construction-parts.js';

// The water mirror is off in the loop: once the camera moves between two draws the port world drew only its sky (the same happens
// on hero-b.html?scene=port when the camera is set by hand; with the mirror off every pose draws). Turn it on again when that is fixed.
const WATER_MIRROR_ENABLED = false;

// Lets the page draw and handle input between the phases of a build, so one phase never queues behind another.
const yieldToMain = yieldToPage;

// index.html starts the first world's files once the poster has loaded and leaves their promises in window.__heroPrefetch; each is
// taken once. Waiting for window.__heroPrefetchReady keeps this module from starting a second request for a file that is about to be
// prefetched, and from competing with the poster.
export async function takePrefetched(url) {
  await window.__heroPrefetchReady;
  const promise = window.__heroPrefetch?.[url];
  if (!promise) return null;
  delete window.__heroPrefetch[url];
  return promise.catch(() => null);
}

async function fetchBlob(url) {
  const prefetched = await takePrefetched(url);
  if (prefetched) return prefetched;
  const response = await fetch(url);
  if (!response.ok) throw new Error(`${url} is not available (${response.status})`);
  return response.blob();
}

function baseUrlOf(path) {
  return new URL('.', new URL(path, location.href)).href;
}

function inWholeWorldOrder(files) {
  if (files.some(file => !file.ranks || file.ranks.length !== file.parts.length)) return files.flatMap(file => file.parts);
  const ranked = files.flatMap(file => file.parts.map((part, index) => ({ part, rank: file.ranks[index] })));
  return ranked.sort((first, second) => first.rank - second.rank).map(entry => entry.part);
}

const TEXTURE_FIELDS = ['map', 'normalMap', 'metallicRoughnessMap', 'occlusionMap', 'emissiveMap'];
const SLICE_MILLISECONDS = 6;
const FIRST_REPLACE_SLICES = 8;
const MAXIMUM_REPLACE_SLICES = 32;

// Each texture upload is a synchronous GL call. The renderer caches uploads by descriptor, so uploading them here, a few
// milliseconds at a time with a yield between slices, leaves nothing for replaceParts to do in one block.
async function uploadTexturesInSlices(renderer, parts, isCancelled = () => false) {
  // A texture over 1 MiB would be one call of 30 to 80 ms; it goes up in bands first, and the loop below finds it cached.
  await renderer.preloadTextures(parts, { checkpoint: createSlicer(SLICE_MILLISECONDS) });
  if (isCancelled()) return;
  const seen = new Set();
  let sliceStart = performance.now();
  for (const part of parts) {
    for (const field of TEXTURE_FIELDS) {
      const descriptor = part.material?.[field];
      if (!descriptor || seen.has(descriptor)) continue;
      seen.add(descriptor);
      renderer.createTexture(descriptor);
      if (performance.now() - sliceStart > SLICE_MILLISECONDS) {
        await yieldToMain();
        if (isCancelled()) return;
        sliceStart = performance.now();
      }
    }
  }
}

// replaceParts keeps skinned, morphed and crowd batches only while they stay unchanged, so they start inside the renderer.
const isPinnedPart = part => Boolean(part.skin || part.morph || part.crowd || part.geometry.skinIndices || part.geometry.morphTargets);

// replaceParts builds every batch, buffer and culling table in one call. Feeding it the world in growing prefixes that
// end on a geometry boundary keeps each call small, because the batches it already built are kept as they are.
// The first slice is a guess (an eighth of the world); every slice after it is sized from how long the last one took, so a call
// lasts about `sliceMilliseconds` on a slow phone as well as on a fast desktop. Each call also pays for the parts before it
// (grouping checks, the part table), so a slice never shrinks below a thirty-second of the world.
export async function replacePartsInSlices(renderer, parts, { sliceMilliseconds = SLICE_MILLISECONDS, now = () => performance.now(), pause = yieldToMain } = {}) {
  const partsByGeometry = new Map();
  const pinned = parts.filter(isPinnedPart);
  for (const part of parts) {
    if (isPinnedPart(part)) continue;
    if (!partsByGeometry.has(part.geometry)) partsByGeometry.set(part.geometry, []);
    partsByGeometry.get(part.geometry).push(part);
  }
  const weightOf = (geometry, sharing) => geometry.indices.length + sharing.length * 300;
  const prefix = [...pinned];
  const totalWeight = [...partsByGeometry].reduce((total, [geometry, sharing]) => total + weightOf(geometry, sharing), 0);
  const smallestSliceWeight = totalWeight / MAXIMUM_REPLACE_SLICES;
  let targetWeight = totalWeight / FIRST_REPLACE_SLICES;
  let weight = 0;
  for (const [geometry, sharing] of partsByGeometry) {
    prefix.push(...sharing);
    weight += weightOf(geometry, sharing);
    if (weight < targetWeight) continue;
    const started = now();
    renderer.replaceParts([...prefix], { retainedUnchanged: true });
    const elapsed = now() - started;
    if (elapsed > 0) targetWeight = Math.max(smallestSliceWeight, weight * sliceMilliseconds / elapsed);
    weight = 0;
    await pause();
  }
  renderer.replaceParts(parts, { retainedUnchanged: true });
}

const bytesOf = (models, sky) => models.reduce((total, model) => total + model.size, sky.size);

async function readDescription(descriptionUrl) {
  return JSON.parse(await (await fetchBlob(descriptionUrl)).text());
}

// Every file of the full world.
export async function fetchDetail(description) {
  const paths = [description.glb, ...(description.glbRest ?? [])];
  const [models, sky] = await Promise.all([Promise.all(paths.map(fetchBlob)), fetchBlob(description.hdri)]);
  return { description, paths, models, sky, bytes: bytesOf(models, sky) };
}

export function hasFirstView(description) {
  return Boolean(description.glbFirstView && description.hdriPreview);
}

// Resolves with a download that is ready to build. With `firstView` and a description that has one, it holds the small files and a
// `detail` promise for the full world, and resolves as soon as the small files are in; the full files are requested only then, so
// the first view never shares the link with them. Otherwise it holds every file, as it always did.
// A link faster than this (megabits per second, measured on the first view's own download) gets the full world straight away: the
// full files arrive about as fast as the first view would have been built, so the extra step would only add work.
export const FAST_LINK_MEGABITS = 20;

// The speed of a finished download of `url`, from the page's resource timing: the body's bytes over the time between the first and the
// last byte. Answers Infinity for a response that came from the cache and null when the browser kept no timing for it.
export function measureLinkMegabits(url) {
  const absolute = new URL(url, globalThis.location?.href ?? 'http://localhost/').href;
  const entry = globalThis.performance?.getEntriesByName?.(absolute, 'resource')?.at(-1);
  if (!entry) return null;
  if (entry.transferSize === 0 && entry.encodedBodySize > 0) return Infinity;
  const seconds = (entry.responseEnd - entry.responseStart) / 1000;
  if (!(entry.encodedBodySize > 0) || !(seconds > 0)) return null;
  return (entry.encodedBodySize * 8) / seconds / 1e6;
}

async function fetchFirstViewFiles(description) {
  const paths = [description.glbFirstView];
  try {
    const [models, sky] = await Promise.all([Promise.all(paths.map(fetchBlob)), fetchBlob(description.hdriPreview)]);
    return { paths, models, sky };
  } catch {
    return null;
  }
}

// The build needs two chunks that load on demand (the meshopt decoder and replaceParts). Asking for them while the world's files are
// still downloading takes their round trips off the time between the last byte of the first view and its first frame.
function warmBuildChunks() {
  import('../../gltf-meshopt.js').catch(() => {});
  import('../../replace-textured-parts.js').catch(() => {});
}

export async function fetchWorld(descriptionUrl, { firstView = false, measureLink = measureLinkMegabits } = {}) {
  warmBuildChunks();
  const description = await readDescription(descriptionUrl);
  if (!firstView || !hasFirstView(description)) return fetchDetail(description);
  const firstViewFiles = await fetchFirstViewFiles(description);
  // A first view that is missing or corrupt must not cost the visitor the hero: the full world, which the page serves anyway, takes its place.
  if (!firstViewFiles) return fetchDetail(description);
  const { paths, models, sky } = firstViewFiles;
  if ((measureLink(description.glbFirstView) ?? 0) > FAST_LINK_MEGABITS) return fetchDetail(description);
  const detail = fetchDetail(description);
  detail.catch(() => {});
  return { description, paths, models, sky, bytes: bytesOf(models, sky), isFirstView: true, detail };
}

// The part of a swap that holds the draws: the sky prefilter, then the parts. Resolves false when the world was freed on the way.
// One replaceParts call, not slices: the draws are held either way, and measured on the port the slices made the hold twice as long.
export async function replaceRendererWorld(renderer, parts, environmentMap, isFreed) {
  const prefilterStart = performance.now();
  await renderer.prepareEnvironmentInSlices(environmentMap, createSlicer(SLICE_MILLISECONDS));
  recordSwapPhase('prefilter', prefilterStart);
  if (isFreed()) return false;
  // Large textures go to the driver in bands between slices, so replaceParts finds them uploaded.
  await renderer.preloadTextures(parts, { checkpoint: createSlicer(SLICE_MILLISECONDS) });
  if (isFreed()) return false;
  const replaceStart = performance.now();
  renderer.replaceParts(parts);
  recordSwapPhase('replaceParts', replaceStart);
  return !isFreed();
}

// Leaves the length of one phase of the swap in window.__phases, next to the laps of buildWorld, for the load measurement.
function recordSwapPhase(name, start) {
  const phases = globalThis.window?.__phases;
  if (phases) phases.push(['swap', name, Math.round(start), Math.round(performance.now())]);
}

// The first draw of a swapped world at the pose of its last frame, so the next frame does not link or allocate. A world that was
// never drawn (a page opened in a background tab) has no pose, and its first real frame does that work.
export async function prepareFrameAtLastPose(renderer, lastDraw, frameFor) {
  if (!lastDraw) return false;
  const prepareStart = performance.now();
  await renderer.prepareFrame(frameFor(lastDraw.camera, lastDraw.seconds));
  recordSwapPhase('prepareFrame', prepareStart);
  return true;
}

async function parseFiles(download, sharedTextures) {
  const { description, paths, models } = download;
  const files = [];
  let animated = null;
  for (const [index, model] of models.entries()) {
    const loaded = await loadGltf(model, { baseUrl: baseUrlOf(paths[index]), sharedTextures, ...description.gltfOptions });
    if (loaded.animations?.length) animated = loaded;
    files.push({ parts: loaded.parts, ranks: await readPartRanks(model) });
    await yieldToMain();
  }
  return { files, animated };
}

function readSky(description, sky) {
  return loadHdri(description.hdri, { ...description.hdriOptions, fetchImplementation: async () => new Response(sky) });
}

export async function buildWorld(download, canvas, overrides = {}) {
  const { description } = download;
  const timings = {};
  let lapStart = performance.now();
  const lap = name => {
    const now = performance.now();
    timings[name] = Math.round(now - lapStart);
    (window.__phases ??= []).push([description.id, name, Math.round(lapStart), Math.round(now)]);
    lapStart = now;
  };
  const sharedTextures = createSharedTextures();
  const parsedFirst = await parseFiles(download, sharedTextures);
  let animated = parsedFirst.animated;
  lap('parseMs');
  const options = description.renderer;
  const environmentMap = await readSky(description, download.sky);
  await yieldToMain();
  lap('hdriMs');
  let effects = prepareEffects(description.id, inWholeWorldOrder(parsedFirst.files));
  if (effects?.needsFogAddon) await import('../../fog-shader.js');
  const surfaceOf = parts => (description.waterReflection && WATER_MIRROR_ENABLED ? applyWaterReflection(parts) : { parts, surface: null });
  // Parts that exist only for the effects join the whole world.
  const wholeOf = (files, worldEffects) => surfaceOf([...inWholeWorldOrder(files), ...(worldEffects?.extraParts ?? [])]);
  // The renderer starts on a few parts that need the whole world's shader program, links it in the background, and takes the
  // rest through replaceParts in slices, the order the showcase page uses (a water mirror attached to a renderer built on the
  // whole world drew nothing after its first frame).
  const frame = { ...description.frame, exposure: description.exposure ?? 1 };
  const whole = wholeOf(parsedFirst.files, effects);
  const renderer = new CrucibleRenderer(canvas, withStandInTextures([...new Set([...shaderFeatureRepresentatives(whole.parts), ...whole.parts.filter(isPinnedPart)])], isPinnedPart), {
    ...options,
    ...overrides,
    pixelRatio: Math.min(devicePixelRatio, 1.25),
    deferProgramLink: true,
    deferEnvironmentPrefilter: true,
    ...effects?.rendererOptions,
    ...(options.shadowCascades ? { shadowCascades: createCascadedShadows(options.shadowCascades) } : {}),
    environment: { ...options.environment, map: environmentMap, ...(effects?.volumeFog ? { volumeFog: effects.volumeFog } : {}) },
  });
  lap('constructMs');
  await import('../../replace-textured-parts.js');
  // The programs compile on the driver's threads while the textures go up.
  await uploadTexturesInSlices(renderer, whole.parts);
  lap('uploadMs');
  await renderer.programsReady();
  lap('linkMs');
  // The sky prefilter runs a few milliseconds at a time on this world's own context, before any frame draws on it.
  await renderer.prepareEnvironmentInSlices(environmentMap, createSlicer(SLICE_MILLISECONDS));
  lap('environmentMs');
  await replacePartsInSlices(renderer, whole.parts);
  let detachReflection = attachWaterReflection(renderer, whole.surface);
  lap('rendererMs');
  let animationOrigin = null;
  let animationShadowStep = -1;
  let disposed = false;
  let drawsHeld = false;
  let lastDraw = null;
  let upgradedMap = null;
  let isFirstView = Boolean(download.isFirstView);
  const frameFor = (camera, seconds) => {
    const drawn = effects ? effects.apply(renderer, seconds, frame) : frame;
    const environment = upgradedMap ? { environment: { ...options.environment, map: upgradedMap } } : {};
    return { ...drawn, ...environment, eye: camera.eye, target: camera.target, fov: camera.fov, time: seconds };
  };
  return {
    id: description.id,
    title: description.title,
    description,
    canvas,
    renderer,
    timings,
    get disposed() { return disposed; },
    get isFirstView() { return isFirstView; },
    render(camera, seconds) {
      // While a swap replaces the parts the canvas keeps its last frame: a half-built world is never drawn.
      if (disposed || drawsHeld) return false;
      lastDraw = { camera: { eye: [...camera.eye], target: [...camera.target], fov: camera.fov }, seconds };
      // A GLB with an animation clip (vehicles on rails, people walking) plays from the world's first drawn frame and loops.
      if (animated) {
        animationOrigin ??= seconds;
        const clipSeconds = loopedClipTime(seconds - animationOrigin, animated.animations[0].duration);
        // The shadow map follows the clip 8 times a second, not on every frame.
        const shadowStep = Math.floor(clipSeconds * 8);
        renderer.updateParts(animated.applyAnimation(clipSeconds, 0), { shadows: shadowStep !== animationShadowStep });
        animationShadowStep = shadowStep;
      }
      renderer.render(frameFor(camera, seconds));
      return true;
    },
    // The first draw while the canvas is still hidden: programs link on the driver's threads and targets allocate now, not on the first frame of a fade.
    // Resolves false when the driver took longer than the timeout and the draw ran in one block.
    warm(camera) {
      return renderer.prepareFrame({ ...frame, eye: camera.eye, target: camera.target, fov: camera.fov, time: 0 });
    },
    // Replaces a first view with the full world, which `detail` holds. Everything slow (parsing, texture uploads) happens first
    // while the first view keeps drawing; only the swap holds the draws: the sky prefilter, one replaceParts and a prepared draw at
    // the pose of the last frame. Resolves true after the swap, false when the world was freed first.
    async upgrade(detail) {
      const parsed = await parseFiles(detail, createSharedTextures());
      if (disposed) return false;
      const fullEffects = prepareEffects(description.id, inWholeWorldOrder(parsed.files));
      const fullWhole = wholeOf(parsed.files, fullEffects);
      const fullMap = await readSky(description, detail.sky);
      await yieldToMain();
      await uploadTexturesInSlices(renderer, fullWhole.parts, () => disposed);
      if (disposed) return false;
      drawsHeld = true;
      try {
        const replaced = await replaceRendererWorld(renderer, fullWhole.parts, fullMap, () => disposed);
        if (!replaced) return false;
        detachReflection();
        detachReflection = attachWaterReflection(renderer, fullWhole.surface);
        effects = fullEffects;
        animated = parsed.animated;
        animationOrigin = null;
        upgradedMap = fullMap;
        isFirstView = false;
        await prepareFrameAtLastPose(renderer, lastDraw, frameFor);
        return !disposed;
      } catch (error) {
        // A world freed in the middle of its swap has no renderer left to finish on: that ends the swap, it does not fail it.
        if (disposed) return false;
        throw error;
      } finally {
        drawsHeld = false;
      }
    },
    resize: () => renderer.resize(),
    dispose() {
      if (disposed) return;
      disposed = true;
      detachReflection();
      renderer.dispose();
      canvas.remove();
    },
  };
}