3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Build the hero renderer on stand-in textures and upload large textures in bands (#779)
Nguyen Xuan Ancc41327eeb8f
diff --git a/changes/779-load-slices.md b/changes/779-load-slices.mdnew file mode 100644index 00000000..23f23ed7--- /dev/null+++ b/changes/779-load-slices.md@@ -0,0 +1 @@+- The home hero builds a world's renderer on one-pixel stand-in textures and uploads large textures in bands, which removes the 100 to 150 ms texture upload blocks of a world build (#779)diff --git a/docs/hero-loading.md b/docs/hero-loading.mdindex 56794bd6..30921b02 100644--- a/docs/hero-loading.md+++ b/docs/hero-loading.md@@ -8,7 +8,7 @@ The hero on the home page plays a flight over the Port showcase world. The world2. The inline script of `index.html` starts the first view as the head is parsed, without waiting for the poster: `scene.json`, `scene.first.glb` (about 2 MB) and `sky.preview.hdr` (256 by 128, 131 KB, a block average of the full `sky.hdr` at the same orientation; exposure, tone mapping, sun, fog and sky intensity come from the same `scene.json` for both, so only the sky's detail differs at the swap. Make it with `node scripts/downsample-hdr.mjs <sky.hdr> <sky.preview.hdr> 256 128`; the Fast 4G first frame budget keeps it under 150 KB).3. When those have settled, it starts the full world: `scene.glb`, `scene.rest.glb` and `sky.hdr`.4. `fetchWorld(url, { firstView: true })` in `src/hero-lab/loop/world.js` resolves as soon as the first view is in. It holds a `detail` promise for the full world.-5. The world is built from the first view and draws. `completeDetail` (`src/hero-lab/loop/detail.js`) then waits for `detail`, parses it and calls `world.upgrade(detail)`. The upgrade uploads the textures and the sky while the first view keeps drawing. Only the swap itself (the sky prefilter, one `replaceParts` and a prepared draw) holds the draws: about 140 ms on a desktop and a phone viewport (`swapHeldMs` of `scripts/measure-hero-load.mjs`, GPU shared with other jobs). Feeding `replaceParts` in slices was tried and doubled the hold (swap 0.8 s to 1.5 s) without shortening the longest gap between frames (about 110 ms either way).+5. The world is built from the first view and draws. `completeDetail` (`src/hero-lab/loop/detail.js`) then waits for `detail`, parses it and calls `world.upgrade(detail)`. The upgrade uploads the textures and the sky while the first view keeps drawing. A world's renderer is constructed on one-pixel stand-ins for the maps of its unpinned representative parts (`src/hero-lab/loop/construction-parts.js`), and every texture over 1 MiB then goes up in bands (`preloadTextures`), so no texture upload is one synchronous block. Only the swap itself (the sky prefilter, one `replaceParts` and a prepared draw) holds the draws: about 140 ms on a desktop and a phone viewport (`swapHeldMs` of `scripts/measure-hero-load.mjs`, GPU shared with other jobs). Feeding `replaceParts` in slices was tried and doubled the hold (swap 0.8 s to 1.5 s) without shortening the longest gap between frames (about 110 ms either way).A link faster than 20 Mbit/s (measured from the resource timing of the first view's own download) skips the first view and builds the full world straight away.diff --git a/src/hero-lab/loop/construction-parts.js b/src/hero-lab/loop/construction-parts.jsnew file mode 100644index 00000000..c724a39c--- /dev/null+++ b/src/hero-lab/loop/construction-parts.js@@ -0,0 +1,30 @@+// The parts a renderer is constructed on only decide which shader features the program has. The constructor uploads their textures in one+// block, and a world's representative parts carry several large ones. Constructing on stand-in textures of one pixel moves those uploads to+// uploadTexturesInSlices, which makes them a few milliseconds at a time. The stand-ins keep every field that decides a shader feature+// (which maps exist, color space, filtering), so the program is the same one.++const MAP_FIELDS = ['map', 'normalMap', 'metallicRoughnessMap', 'occlusionMap', 'emissiveMap'];++const createOnePixelImage = () => new ImageData(1, 1);++// A part keeps its real textures when something other than the constructor's batch decides its lifetime (pinned parts stay in the+// renderer through replaceParts) or when its textures are not one of the five material maps.+const keepsRealTextures = (part, isPinned) => isPinned(part) || Boolean(part.material?.layers || part.material?.splat || part.material?.lightMap);++/**+ * Returns `parts` with each unpinned part's material maps replaced by one-pixel stand-ins. A part without maps, and every pinned part, is+ * returned as it is. The stand-in parts are new objects, so a later replaceParts of the real parts never keeps a batch that holds a stand-in.+ */+export function withStandInTextures(parts, isPinned, createImage = createOnePixelImage) {+ const standIns = new Map();+ const standInOf = descriptor => {+ if (!standIns.has(descriptor)) standIns.set(descriptor, { ...descriptor, image: createImage() });+ return standIns.get(descriptor);+ };+ return parts.map(part => {+ if (keepsRealTextures(part, isPinned) || !MAP_FIELDS.some(field => part.material?.[field])) return part;+ const material = { ...part.material };+ for (const field of MAP_FIELDS) if (material[field]) material[field] = standInOf(material[field]);+ return { ...part, material };+ });+}diff --git a/src/hero-lab/loop/world.js b/src/hero-lab/loop/world.jsindex 3645e411..d87f5b50 100644--- a/src/hero-lab/loop/world.js+++ b/src/hero-lab/loop/world.js@@ -13,6 +13,7 @@ import { readPartRanks } from '../../showcase-download.js';import { prepareEffects } from './effects.js';import { createSlicer, yieldToPage } from '../../cooperative.js';import { loopedClipTime } from './clip-time.js';+import { withStandInTextures } from './construction-parts.js';// The water mirror is off in the loop: once the camera moves between two draws the port world drew only its sky (the same happens// on hero-b.html?scene=port when the camera is set by hand; with the mirror off every pose draws). Turn it on again when that is fixed.@@ -57,6 +58,9 @@ const REPLACE_SLICES = 8;// Each texture upload is a synchronous GL call. The renderer caches uploads by descriptor, so uploading them here, a few// milliseconds at a time with a yield between slices, leaves nothing for replaceParts to do in one block.async function uploadTexturesInSlices(renderer, parts, isCancelled = () => false) {+ // A texture over 1 MiB would be one call of 30 to 80 ms; it goes up in bands first, and the loop below finds it cached.+ await renderer.preloadTextures(parts, { checkpoint: createSlicer(SLICE_MILLISECONDS) });+ if (isCancelled()) return;const seen = new Set();let sliceStart = performance.now();for (const part of parts) {@@ -246,7 +250,7 @@ export async function buildWorld(download, canvas, overrides = {}) {// whole world drew nothing after its first frame).const frame = { ...description.frame, exposure: description.exposure ?? 1 };const whole = wholeOf(parsedFirst.files, effects);- const renderer = new CrucibleRenderer(canvas, [...new Set([...shaderFeatureRepresentatives(whole.parts), ...whole.parts.filter(isPinnedPart)])], {+ const renderer = new CrucibleRenderer(canvas, withStandInTextures([...new Set([...shaderFeatureRepresentatives(whole.parts), ...whole.parts.filter(isPinnedPart)])], isPinnedPart), {...options,...overrides,pixelRatio: Math.min(devicePixelRatio, 1.25),diff --git a/tests/hero-construction-parts.test.js b/tests/hero-construction-parts.test.jsnew file mode 100644index 00000000..b3874c3e--- /dev/null+++ b/tests/hero-construction-parts.test.js@@ -0,0 +1,85 @@+// The hero builds its renderer on stand-in textures, so the constructor uploads one-pixel images and the real textures go up in slices (#779).+// It runs in headless Chrome from the Vite dev server; it skips locally when Chrome cannot start and fails on CI.+import test from 'node:test';+import assert from 'node:assert/strict';+import { withStandInTextures } from '../src/hero-lab/loop/construction-parts.js';+import { withDevServerPage } from './helpers/dev-server-browser.js';++const isPinned = part => Boolean(part.skin);+const fakeImage = () => ({ width: 1, height: 1 });++test('withStandInTextures replaces the maps of unpinned parts and leaves pinned parts and the real descriptors alone', () => {+ const realMap = { image: { width: 2048, height: 2048 }, colorSpace: 'srgb', minFilter: 9987 };+ const realNormal = { image: { width: 1024, height: 1024 }, colorSpace: 'linear' };+ const textured = { name: 'wall', geometry: {}, material: { tag: 'surface', map: realMap, normalMap: realNormal } };+ const second = { name: 'floor', geometry: {}, material: { tag: 'surface', map: realMap } };+ const plain = { name: 'pipe', geometry: {}, material: { tag: 'surface' } };+ const pinned = { name: 'crane', skin: {}, geometry: {}, material: { tag: 'surface', map: realMap } };+ const [standInWall, standInFloor, samePlain, samePinned] = withStandInTextures([textured, second, plain, pinned], isPinned, fakeImage);+ assert.notEqual(standInWall, textured);+ assert.equal(samePlain, plain);+ assert.equal(samePinned, pinned);+ assert.equal(standInWall.material.map.image.width, 1);+ assert.equal(standInWall.material.normalMap.image.width, 1);+ assert.equal(standInWall.material.map.colorSpace, 'srgb');+ assert.equal(standInWall.material.normalMap.colorSpace, 'linear');+ assert.equal(standInFloor.material.map, standInWall.material.map, 'one stand-in per real descriptor');+ assert.equal(realMap.image.width, 2048, 'the real descriptor is untouched');+ assert.equal(textured.material.map, realMap);+});++test('a renderer built on stand-in textures uploads no real texture in its constructor and draws the same pixels after the sliced uploads (#779)', async (testContext) => {+ await withDevServerPage(testContext, async (page) => {+ const result = await page.evaluate(async () => {+ const { sphere } = await import('/src/geometry.js');+ const { CrucibleRenderer } = await import('/src/renderer.js');+ await import('/src/replace-textured-parts.js');+ const { withStandInTextures } = await import('/src/hero-lab/loop/construction-parts.js');+ const identity = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);+ const painter = document.createElement('canvas');+ painter.width = 256;+ painter.height = 256;+ const paint = painter.getContext('2d');+ for (let row = 0; row < 256; row++) {+ paint.fillStyle = `rgb(${row}, ${255 - row}, ${(row * 37) % 256})`;+ paint.fillRect(row % 256, row, 256, 1);+ }+ const descriptor = { image: painter, colorSpace: 'srgb', minFilter: 9987 };+ const geometry = sphere(24, 12);+ const parts = [{ name: 'ball', geometry, matrix: identity, material: { tag: 'surface', color: [1, 1, 1], metallic: 0, roughness: 0.6, emission: 0, map: descriptor } }];+ const frame = { eye: [0, 0, 3], target: [0, 0, 0], fov: 50, time: 0 };+ const build = async (constructionParts) => {+ const canvas = document.createElement('canvas');+ canvas.width = 64;+ canvas.height = 64;+ document.body.append(canvas);+ const context = canvas.getContext('webgl2', { preserveDrawingBuffer: true });+ const widths = [];+ const realTexImage2D = context.texImage2D.bind(context);+ context.texImage2D = (...args) => {+ if (args.length === 9 || args.length === 6) widths.push(args.length === 9 ? args[3] : args[5].width);+ return realTexImage2D(...args);+ };+ const renderer = new CrucibleRenderer(canvas, constructionParts, { samples: 1, pixelRatio: 1, deferProgramLink: true });+ const uploadedByConstructor = widths.filter(width => width >= 256).length;+ await renderer.programsReady();+ // What uploadTexturesInSlices does for the whole world, one descriptor at a time.+ renderer.createTexture(descriptor);+ renderer.replaceParts(parts, { retainedUnchanged: true });+ renderer.render(frame);+ const pixels = new Uint8Array(64 * 64 * 4);+ context.readPixels(0, 0, 64, 64, context.RGBA, context.UNSIGNED_BYTE, pixels);+ return { uploadedByConstructor, pixels: Array.from(pixels) };+ };+ const direct = await build(parts);+ const standIn = await build(withStandInTextures(parts, () => false));+ let differing = 0;+ for (let index = 0; index < direct.pixels.length; index++) if (direct.pixels[index] !== standIn.pixels[index]) differing++;+ return { directUploads: direct.uploadedByConstructor, standInUploads: standIn.uploadedByConstructor, differing, drawn: direct.pixels.some(value => value !== 0) };+ });+ assert.equal(result.drawn, true, 'the scene draws something');+ assert.equal(result.directUploads, 1, 'the control uploads the real texture in its constructor');+ assert.equal(result.standInUploads, 0, 'the stand-in build uploads no real texture in its constructor');+ assert.equal(result.differing, 0, 'the sliced path draws the same bytes');+ });+});