3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
// One showcase world for the hero loop, built in two steps so the next world can be fetched while the current one plays:
// fetchWorld(descriptionUrl) network only: the description, every GLB part and the sky, held as blobs.
// buildWorld(download, canvas) parses the parts, uploads them to the GPU and returns a world that draws on that canvas.
// A world is whole before it is shown: the loop never draws the first part of a split world on its own. The one exception is a
// description with a first view (`glbFirstView`, `hdriPreview`): the whole world as a small, coarse copy that is drawn first and
// replaced by the full world through world.upgrade() once the full files have arrived and been prepared.
import { CrucibleRenderer, shaderFeatureRepresentatives } from '../../renderer.js';
import { createSharedTextures, loadGltf } from '../../gltf.js';
import { loadHdri } from '../../hdri.js';
import { createCascadedShadows } from '../../cascaded-shadows.js';
import { applyWaterReflection, attachWaterReflection } from '../../water-reflection.js';
import { readPartRanks } from '../../showcase-download.js';
import { prepareEffects } from './effects.js';
import { createSlicer, yieldToPage } from '../../cooperative.js';
import { loopedClipTime } from './clip-time.js';
import { withStandInTextures } from './construction-parts.js';
// The water mirror is off in the loop: once the camera moves between two draws the port world drew only its sky (the same happens
// on hero-b.html?scene=port when the camera is set by hand; with the mirror off every pose draws). Turn it on again when that is fixed.
const WATER_MIRROR_ENABLED = false;
// Lets the page draw and handle input between the phases of a build, so one phase never queues behind another.
const yieldToMain = yieldToPage;
// index.html starts the first world's files once the poster has loaded and leaves their promises in window.__heroPrefetch; each is
// taken once. Waiting for window.__heroPrefetchReady keeps this module from starting a second request for a file that is about to be
// prefetched, and from competing with the poster.
export async function takePrefetched(url) {
await window.__heroPrefetchReady;
const promise = window.__heroPrefetch?.[url];
if (!promise) return null;
delete window.__heroPrefetch[url];
return promise.catch(() => null);
}
async function fetchBlob(url) {
const prefetched = await takePrefetched(url);
if (prefetched) return prefetched;
const response = await fetch(url);
if (!response.ok) throw new Error(`${url} is not available (${response.status})`);
return response.blob();
}
function baseUrlOf(path) {
return new URL('.', new URL(path, location.href)).href;
}
function inWholeWorldOrder(files) {
if (files.some(file => !file.ranks || file.ranks.length !== file.parts.length)) return files.flatMap(file => file.parts);
const ranked = files.flatMap(file => file.parts.map((part, index) => ({ part, rank: file.ranks[index] })));
return ranked.sort((first, second) => first.rank - second.rank).map(entry => entry.part);
}
const TEXTURE_FIELDS = ['map', 'normalMap', 'metallicRoughnessMap', 'occlusionMap', 'emissiveMap'];
const SLICE_MILLISECONDS = 6;
const FIRST_REPLACE_SLICES = 8;
const MAXIMUM_REPLACE_SLICES = 32;
// Each texture upload is a synchronous GL call. The renderer caches uploads by descriptor, so uploading them here, a few
// milliseconds at a time with a yield between slices, leaves nothing for replaceParts to do in one block.
async function uploadTexturesInSlices(renderer, parts, isCancelled = () => false) {
// A texture over 1 MiB would be one call of 30 to 80 ms; it goes up in bands first, and the loop below finds it cached.
await renderer.preloadTextures(parts, { checkpoint: createSlicer(SLICE_MILLISECONDS) });
if (isCancelled()) return;
const seen = new Set();
let sliceStart = performance.now();
for (const part of parts) {
for (const field of TEXTURE_FIELDS) {
const descriptor = part.material?.[field];
if (!descriptor || seen.has(descriptor)) continue;
seen.add(descriptor);
renderer.createTexture(descriptor);
if (performance.now() - sliceStart > SLICE_MILLISECONDS) {
await yieldToMain();
if (isCancelled()) return;
sliceStart = performance.now();
}
}
}
}
// replaceParts keeps skinned, morphed and crowd batches only while they stay unchanged, so they start inside the renderer.
const isPinnedPart = part => Boolean(part.skin || part.morph || part.crowd || part.geometry.skinIndices || part.geometry.morphTargets);
// replaceParts builds every batch, buffer and culling table in one call. Feeding it the world in growing prefixes that
// end on a geometry boundary keeps each call small, because the batches it already built are kept as they are.
// The first slice is a guess (an eighth of the world); every slice after it is sized from how long the last one took, so a call
// lasts about `sliceMilliseconds` on a slow phone as well as on a fast desktop. Each call also pays for the parts before it
// (grouping checks, the part table), so a slice never shrinks below a thirty-second of the world.
export async function replacePartsInSlices(renderer, parts, { sliceMilliseconds = SLICE_MILLISECONDS, now = () => performance.now(), pause = yieldToMain } = {}) {
const partsByGeometry = new Map();
const pinned = parts.filter(isPinnedPart);
for (const part of parts) {
if (isPinnedPart(part)) continue;
if (!partsByGeometry.has(part.geometry)) partsByGeometry.set(part.geometry, []);
partsByGeometry.get(part.geometry).push(part);
}
const weightOf = (geometry, sharing) => geometry.indices.length + sharing.length * 300;
const prefix = [...pinned];
const totalWeight = [...partsByGeometry].reduce((total, [geometry, sharing]) => total + weightOf(geometry, sharing), 0);
const smallestSliceWeight = totalWeight / MAXIMUM_REPLACE_SLICES;
let targetWeight = totalWeight / FIRST_REPLACE_SLICES;
let weight = 0;
for (const [geometry, sharing] of partsByGeometry) {
prefix.push(...sharing);
weight += weightOf(geometry, sharing);
if (weight < targetWeight) continue;
const started = now();
renderer.replaceParts([...prefix], { retainedUnchanged: true });
const elapsed = now() - started;
if (elapsed > 0) targetWeight = Math.max(smallestSliceWeight, weight * sliceMilliseconds / elapsed);
weight = 0;
await pause();
}
renderer.replaceParts(parts, { retainedUnchanged: true });
}
const bytesOf = (models, sky) => models.reduce((total, model) => total + model.size, sky.size);
async function readDescription(descriptionUrl) {
return JSON.parse(await (await fetchBlob(descriptionUrl)).text());
}
// Every file of the full world.
export async function fetchDetail(description) {
const paths = [description.glb, ...(description.glbRest ?? [])];
const [models, sky] = await Promise.all([Promise.all(paths.map(fetchBlob)), fetchBlob(description.hdri)]);
return { description, paths, models, sky, bytes: bytesOf(models, sky) };
}
export function hasFirstView(description) {
return Boolean(description.glbFirstView && description.hdriPreview);
}
// Resolves with a download that is ready to build. With `firstView` and a description that has one, it holds the small files and a
// `detail` promise for the full world, and resolves as soon as the small files are in; the full files are requested only then, so
// the first view never shares the link with them. Otherwise it holds every file, as it always did.
// A link faster than this (megabits per second, measured on the first view's own download) gets the full world straight away: the
// full files arrive about as fast as the first view would have been built, so the extra step would only add work.
export const FAST_LINK_MEGABITS = 20;
// The speed of a finished download of `url`, from the page's resource timing: the body's bytes over the time between the first and the
// last byte. Answers Infinity for a response that came from the cache and null when the browser kept no timing for it.
export function measureLinkMegabits(url) {
const absolute = new URL(url, globalThis.location?.href ?? 'http://localhost/').href;
const entry = globalThis.performance?.getEntriesByName?.(absolute, 'resource')?.at(-1);
if (!entry) return null;
if (entry.transferSize === 0 && entry.encodedBodySize > 0) return Infinity;
const seconds = (entry.responseEnd - entry.responseStart) / 1000;
if (!(entry.encodedBodySize > 0) || !(seconds > 0)) return null;
return (entry.encodedBodySize * 8) / seconds / 1e6;
}
async function fetchFirstViewFiles(description) {
const paths = [description.glbFirstView];
try {
const [models, sky] = await Promise.all([Promise.all(paths.map(fetchBlob)), fetchBlob(description.hdriPreview)]);
return { paths, models, sky };
} catch {
return null;
}
}
// The build needs two chunks that load on demand (the meshopt decoder and replaceParts). Asking for them while the world's files are
// still downloading takes their round trips off the time between the last byte of the first view and its first frame.
function warmBuildChunks() {
import('../../gltf-meshopt.js').catch(() => {});
import('../../replace-textured-parts.js').catch(() => {});
}
export async function fetchWorld(descriptionUrl, { firstView = false, measureLink = measureLinkMegabits } = {}) {
warmBuildChunks();
const description = await readDescription(descriptionUrl);
if (!firstView || !hasFirstView(description)) return fetchDetail(description);
const firstViewFiles = await fetchFirstViewFiles(description);
// A first view that is missing or corrupt must not cost the visitor the hero: the full world, which the page serves anyway, takes its place.
if (!firstViewFiles) return fetchDetail(description);
const { paths, models, sky } = firstViewFiles;
if ((measureLink(description.glbFirstView) ?? 0) > FAST_LINK_MEGABITS) return fetchDetail(description);
const detail = fetchDetail(description);
detail.catch(() => {});
return { description, paths, models, sky, bytes: bytesOf(models, sky), isFirstView: true, detail };
}
// The part of a swap that holds the draws: the sky prefilter, then the parts. Resolves false when the world was freed on the way.
// One replaceParts call, not slices: the draws are held either way, and measured on the port the slices made the hold twice as long.
export async function replaceRendererWorld(renderer, parts, environmentMap, isFreed) {
const prefilterStart = performance.now();
await renderer.prepareEnvironmentInSlices(environmentMap, createSlicer(SLICE_MILLISECONDS));
recordSwapPhase('prefilter', prefilterStart);
if (isFreed()) return false;
// Large textures go to the driver in bands between slices, so replaceParts finds them uploaded.
await renderer.preloadTextures(parts, { checkpoint: createSlicer(SLICE_MILLISECONDS) });
if (isFreed()) return false;
const replaceStart = performance.now();
renderer.replaceParts(parts);
recordSwapPhase('replaceParts', replaceStart);
return !isFreed();
}
// Leaves the length of one phase of the swap in window.__phases, next to the laps of buildWorld, for the load measurement.
function recordSwapPhase(name, start) {
const phases = globalThis.window?.__phases;
if (phases) phases.push(['swap', name, Math.round(start), Math.round(performance.now())]);
}
// The first draw of a swapped world at the pose of its last frame, so the next frame does not link or allocate. A world that was
// never drawn (a page opened in a background tab) has no pose, and its first real frame does that work.
export async function prepareFrameAtLastPose(renderer, lastDraw, frameFor) {
if (!lastDraw) return false;
const prepareStart = performance.now();
await renderer.prepareFrame(frameFor(lastDraw.camera, lastDraw.seconds));
recordSwapPhase('prepareFrame', prepareStart);
return true;
}
async function parseFiles(download, sharedTextures) {
const { description, paths, models } = download;
const files = [];
let animated = null;
for (const [index, model] of models.entries()) {
const loaded = await loadGltf(model, { baseUrl: baseUrlOf(paths[index]), sharedTextures, ...description.gltfOptions });
if (loaded.animations?.length) animated = loaded;
files.push({ parts: loaded.parts, ranks: await readPartRanks(model) });
await yieldToMain();
}
return { files, animated };
}
function readSky(description, sky) {
return loadHdri(description.hdri, { ...description.hdriOptions, fetchImplementation: async () => new Response(sky) });
}
export async function buildWorld(download, canvas, overrides = {}) {
const { description } = download;
const timings = {};
let lapStart = performance.now();
const lap = name => {
const now = performance.now();
timings[name] = Math.round(now - lapStart);
(window.__phases ??= []).push([description.id, name, Math.round(lapStart), Math.round(now)]);
lapStart = now;
};
const sharedTextures = createSharedTextures();
const parsedFirst = await parseFiles(download, sharedTextures);
let animated = parsedFirst.animated;
lap('parseMs');
const options = description.renderer;
const environmentMap = await readSky(description, download.sky);
await yieldToMain();
lap('hdriMs');
let effects = prepareEffects(description.id, inWholeWorldOrder(parsedFirst.files));
if (effects?.needsFogAddon) await import('../../fog-shader.js');
const surfaceOf = parts => (description.waterReflection && WATER_MIRROR_ENABLED ? applyWaterReflection(parts) : { parts, surface: null });
// Parts that exist only for the effects join the whole world.
const wholeOf = (files, worldEffects) => surfaceOf([...inWholeWorldOrder(files), ...(worldEffects?.extraParts ?? [])]);
// The renderer starts on a few parts that need the whole world's shader program, links it in the background, and takes the
// rest through replaceParts in slices, the order the showcase page uses (a water mirror attached to a renderer built on the
// whole world drew nothing after its first frame).
const frame = { ...description.frame, exposure: description.exposure ?? 1 };
const whole = wholeOf(parsedFirst.files, effects);
const renderer = new CrucibleRenderer(canvas, withStandInTextures([...new Set([...shaderFeatureRepresentatives(whole.parts), ...whole.parts.filter(isPinnedPart)])], isPinnedPart), {
...options,
...overrides,
pixelRatio: Math.min(devicePixelRatio, 1.25),
deferProgramLink: true,
deferEnvironmentPrefilter: true,
...effects?.rendererOptions,
...(options.shadowCascades ? { shadowCascades: createCascadedShadows(options.shadowCascades) } : {}),
environment: { ...options.environment, map: environmentMap, ...(effects?.volumeFog ? { volumeFog: effects.volumeFog } : {}) },
});
lap('constructMs');
await import('../../replace-textured-parts.js');
// The programs compile on the driver's threads while the textures go up.
await uploadTexturesInSlices(renderer, whole.parts);
lap('uploadMs');
await renderer.programsReady();
lap('linkMs');
// The sky prefilter runs a few milliseconds at a time on this world's own context, before any frame draws on it.
await renderer.prepareEnvironmentInSlices(environmentMap, createSlicer(SLICE_MILLISECONDS));
lap('environmentMs');
await replacePartsInSlices(renderer, whole.parts);
let detachReflection = attachWaterReflection(renderer, whole.surface);
lap('rendererMs');
let animationOrigin = null;
let animationShadowStep = -1;
let disposed = false;
let drawsHeld = false;
let lastDraw = null;
let upgradedMap = null;
let isFirstView = Boolean(download.isFirstView);
const frameFor = (camera, seconds) => {
const drawn = effects ? effects.apply(renderer, seconds, frame) : frame;
const environment = upgradedMap ? { environment: { ...options.environment, map: upgradedMap } } : {};
return { ...drawn, ...environment, eye: camera.eye, target: camera.target, fov: camera.fov, time: seconds };
};
return {
id: description.id,
title: description.title,
description,
canvas,
renderer,
timings,
get disposed() { return disposed; },
get isFirstView() { return isFirstView; },
render(camera, seconds) {
// While a swap replaces the parts the canvas keeps its last frame: a half-built world is never drawn.
if (disposed || drawsHeld) return false;
lastDraw = { camera: { eye: [...camera.eye], target: [...camera.target], fov: camera.fov }, seconds };
// A GLB with an animation clip (vehicles on rails, people walking) plays from the world's first drawn frame and loops.
if (animated) {
animationOrigin ??= seconds;
const clipSeconds = loopedClipTime(seconds - animationOrigin, animated.animations[0].duration);
// The shadow map follows the clip 8 times a second, not on every frame.
const shadowStep = Math.floor(clipSeconds * 8);
renderer.updateParts(animated.applyAnimation(clipSeconds, 0), { shadows: shadowStep !== animationShadowStep });
animationShadowStep = shadowStep;
}
renderer.render(frameFor(camera, seconds));
return true;
},
// The first draw while the canvas is still hidden: programs link on the driver's threads and targets allocate now, not on the first frame of a fade.
// Resolves false when the driver took longer than the timeout and the draw ran in one block.
warm(camera) {
return renderer.prepareFrame({ ...frame, eye: camera.eye, target: camera.target, fov: camera.fov, time: 0 });
},
// Replaces a first view with the full world, which `detail` holds. Everything slow (parsing, texture uploads) happens first
// while the first view keeps drawing; only the swap holds the draws: the sky prefilter, one replaceParts and a prepared draw at
// the pose of the last frame. Resolves true after the swap, false when the world was freed first.
async upgrade(detail) {
const parsed = await parseFiles(detail, createSharedTextures());
if (disposed) return false;
const fullEffects = prepareEffects(description.id, inWholeWorldOrder(parsed.files));
const fullWhole = wholeOf(parsed.files, fullEffects);
const fullMap = await readSky(description, detail.sky);
await yieldToMain();
await uploadTexturesInSlices(renderer, fullWhole.parts, () => disposed);
if (disposed) return false;
drawsHeld = true;
try {
const replaced = await replaceRendererWorld(renderer, fullWhole.parts, fullMap, () => disposed);
if (!replaced) return false;
detachReflection();
detachReflection = attachWaterReflection(renderer, fullWhole.surface);
effects = fullEffects;
animated = parsed.animated;
animationOrigin = null;
upgradedMap = fullMap;
isFirstView = false;
await prepareFrameAtLastPose(renderer, lastDraw, frameFor);
return !disposed;
} catch (error) {
// A world freed in the middle of its swap has no renderer left to finish on: that ends the swap, it does not fail it.
if (disposed) return false;
throw error;
} finally {
drawsHeld = false;
}
},
resize: () => renderer.resize(),
dispose() {
if (disposed) return;
disposed = true;
detachReflection();
renderer.dispose();
canvas.remove();
},
};
}