3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Slice the sky prefilter's map passes and size replaceParts slices by time (#779)
Nguyen Xuan Anb6a901faf20b
diff --git a/changes/779-world-load-slices-2.md b/changes/779-world-load-slices-2.mdnew file mode 100644index 00000000..b499e68d--- /dev/null+++ b/changes/779-world-load-slices-2.md@@ -0,0 +1 @@+- The home hero's world build hands the thread back more often: the sky prefilter's pass over the whole map runs in steps of a fixed texel count, `replaceParts` slices are sized from how long the last one took, and `replaceParts` computes each material's layer state once instead of once per value (#779)diff --git a/docs/hero-loading.md b/docs/hero-loading.mdindex 30921b02..43f35d2c 100644--- a/docs/hero-loading.md+++ b/docs/hero-loading.md@@ -8,7 +8,7 @@ The hero on the home page plays a flight over the Port showcase world. The world2. The inline script of `index.html` starts the first view as the head is parsed, without waiting for the poster: `scene.json`, `scene.first.glb` (about 2 MB) and `sky.preview.hdr` (256 by 128, 131 KB, a block average of the full `sky.hdr` at the same orientation; exposure, tone mapping, sun, fog and sky intensity come from the same `scene.json` for both, so only the sky's detail differs at the swap. Make it with `node scripts/downsample-hdr.mjs <sky.hdr> <sky.preview.hdr> 256 128`; the Fast 4G first frame budget keeps it under 150 KB).3. When those have settled, it starts the full world: `scene.glb`, `scene.rest.glb` and `sky.hdr`.4. `fetchWorld(url, { firstView: true })` in `src/hero-lab/loop/world.js` resolves as soon as the first view is in. It holds a `detail` promise for the full world.-5. The world is built from the first view and draws. `completeDetail` (`src/hero-lab/loop/detail.js`) then waits for `detail`, parses it and calls `world.upgrade(detail)`. The upgrade uploads the textures and the sky while the first view keeps drawing. A world's renderer is constructed on one-pixel stand-ins for the maps of its unpinned representative parts (`src/hero-lab/loop/construction-parts.js`), and every texture over 1 MiB then goes up in bands (`preloadTextures`), so no texture upload is one synchronous block. Only the swap itself (the sky prefilter, one `replaceParts` and a prepared draw) holds the draws: about 140 ms on a desktop and a phone viewport (`swapHeldMs` of `scripts/measure-hero-load.mjs`, GPU shared with other jobs). Feeding `replaceParts` in slices was tried and doubled the hold (swap 0.8 s to 1.5 s) without shortening the longest gap between frames (about 110 ms either way).+5. The world is built from the first view and draws. `completeDetail` (`src/hero-lab/loop/detail.js`) then waits for `detail`, parses it and calls `world.upgrade(detail)`. The upgrade uploads the textures and the sky while the first view keeps drawing. A world's renderer is constructed on one-pixel stand-ins for the maps of its unpinned representative parts (`src/hero-lab/loop/construction-parts.js`), and every texture over 1 MiB then goes up in bands (`preloadTextures`), so no texture upload is one synchronous block. Only the swap itself (the sky prefilter, one `replaceParts` and a prepared draw) holds the draws: about 140 ms on a desktop and a phone viewport (`swapHeldMs` of `scripts/measure-hero-load.mjs`, GPU shared with other jobs). During the build (not the swap) `replaceParts` is fed in growing prefixes sized from how long the last call took (about 8 ms each, at most 32 calls), and the sky prefilter checks the clock every 65,536 texels of its passes over the map. Feeding `replaceParts` in slices during the swap was tried and doubled the hold (swap 0.8 s to 1.5 s) without shortening the longest gap between frames (about 110 ms either way).A link faster than 20 Mbit/s (measured from the resource timing of the first view's own download) skips the first view and builds the full world straight away.diff --git a/src/environment.js b/src/environment.jsindex 7ee0c2e8..49900693 100644--- a/src/environment.js+++ b/src/environment.js@@ -97,7 +97,12 @@ function sphericalHarmonicBasis(x, y, z) {// Returns 27 floats (9 RGB triples) with basis constants and the Lambert kernel folded in,// so evaluateIrradiance(coefficients, normal) is the diffuse radiance for a white surface.-const IRRADIANCE_ROWS_PER_STEP = 32;+// Texels one step of a pass over the whole map handles, so a step costs about the same on a small map and a large one.+const PIXELS_PER_STEP = 65536;++function rowsPerStep(width) {+ return Math.max(1, Math.floor(PIXELS_PER_STEP / width));+}// One step per few rows of the map, so a caller can hand the thread back between steps.function* irradianceSteps(map) {@@ -112,6 +117,7 @@ function* irradianceSteps(map) {}const sums = new Float64Array(27);const pixelSolidAngleScale = (2 * Math.PI / width) * (Math.PI / height);+ const stepRows = rowsPerStep(width);for (let row = 0; row < height; row++) {const polar = (row + 0.5) / height * Math.PI;const sinePolar = Math.sin(polar);@@ -127,7 +133,7 @@ function* irradianceSteps(map) {sums[term * 3 + 2] += data[offset + 2] * termWeight;}}- if (row % IRRADIANCE_ROWS_PER_STEP === IRRADIANCE_ROWS_PER_STEP - 1) yield;+ if (row % stepRows === stepRows - 1) yield;}const coefficients = new Float32Array(27);for (let term = 0; term < 9; term++) {@@ -147,13 +153,14 @@ const IRRADIANCE_ROWS_PER_GRID_STEP = 4;// Averages the map down to at most 128 by 64 texels, weighting rows by their solid angle, so the cosine integral below// costs about four million multiply-adds however large the map is. Light is conserved, so a sun stays as bright overall.-function irradianceSourceTexels(map) {+function* irradianceSourceTexelSteps(map) {const { width, height, channels, data } = map;const scale = Math.min(1, MAXIMUM_IRRADIANCE_SOURCE_WIDTH / width);const sourceWidth = Math.max(4, Math.round(width * scale));const sourceHeight = Math.max(2, Math.round(height * scale));const sums = new Float64Array(sourceWidth * sourceHeight * 3);const weights = new Float64Array(sourceWidth * sourceHeight);+ const stepRows = rowsPerStep(width);for (let row = 0; row < height; row++) {const rowWeight = Math.sin((row + 0.5) / height * Math.PI);const targetRow = Math.min(sourceHeight - 1, Math.floor(row * sourceHeight / height));@@ -165,6 +172,7 @@ function irradianceSourceTexels(map) {sums[target * 3 + 2] += data[offset + 2] * rowWeight;weights[target] += rowWeight;}+ if (row % stepRows === stepRows - 1) yield;}const texelCount = sourceWidth * sourceHeight;const directions = new Float64Array(texelCount * 3);@@ -193,7 +201,7 @@ function irradianceSourceTexels(map) {// convolution, so a strong sun does not ring the way a nine-term spherical harmonic fit does.function* irradianceGridSteps(map) {assertEnvironmentMap(map);- const { texelCount, directions, radiance } = irradianceSourceTexels(map);+ const { texelCount, directions, radiance } = yield* irradianceSourceTexelSteps(map);const grid = new Float32Array(ENVIRONMENT_IRRADIANCE_WIDTH * ENVIRONMENT_IRRADIANCE_HEIGHT * 3);for (let row = 0; row < ENVIRONMENT_IRRADIANCE_HEIGHT; row++) {const polar = (row + 0.5) / ENVIRONMENT_IRRADIANCE_HEIGHT * Math.PI;@@ -445,8 +453,9 @@ function* environmentResourceSteps(context, map, { compileProgram, fullscreenVerconst layout = environmentSpecularLayout(map);const sourceLevelCount = Math.floor(Math.log2(Math.max(map.width, map.height))) + 1;const sourceTexture = createHalfFloatTexture(context, context.RGBA16F, sourceLevelCount, map.width, map.height, { wrapHorizontally: true });- for (let firstRow = 0; firstRow < map.height; firstRow += UPLOAD_ROWS_PER_STEP) {- const rowCount = Math.min(UPLOAD_ROWS_PER_STEP, map.height - firstRow);+ const uploadRows = Math.min(UPLOAD_ROWS_PER_STEP, rowsPerStep(map.width));+ for (let firstRow = 0; firstRow < map.height; firstRow += uploadRows) {+ const rowCount = Math.min(uploadRows, map.height - firstRow);context.texSubImage2D(context.TEXTURE_2D, 0, 0, firstRow, map.width, rowCount, context.RGBA, context.FLOAT, paddedHalfFloatRows(map, firstRow, rowCount));yield;}diff --git a/src/hero-lab/loop/world.js b/src/hero-lab/loop/world.jsindex d87f5b50..3067b978 100644--- a/src/hero-lab/loop/world.js+++ b/src/hero-lab/loop/world.js@@ -53,7 +53,8 @@ function inWholeWorldOrder(files) {const TEXTURE_FIELDS = ['map', 'normalMap', 'metallicRoughnessMap', 'occlusionMap', 'emissiveMap'];const SLICE_MILLISECONDS = 6;-const REPLACE_SLICES = 8;+const FIRST_REPLACE_SLICES = 8;+const MAXIMUM_REPLACE_SLICES = 32;// Each texture upload is a synchronous GL call. The renderer caches uploads by descriptor, so uploading them here, a few// milliseconds at a time with a yield between slices, leaves nothing for replaceParts to do in one block.@@ -83,7 +84,10 @@ const isPinnedPart = part => Boolean(part.skin || part.morph || part.crowd || pa// replaceParts builds every batch, buffer and culling table in one call. Feeding it the world in growing prefixes that// end on a geometry boundary keeps each call small, because the batches it already built are kept as they are.-async function replacePartsInSlices(renderer, parts) {+// The first slice is a guess (an eighth of the world); every slice after it is sized from how long the last one took, so a call+// lasts about `sliceMilliseconds` on a slow phone as well as on a fast desktop. Each call also pays for the parts before it+// (grouping checks, the part table), so a slice never shrinks below a thirty-second of the world.+export async function replacePartsInSlices(renderer, parts, { sliceMilliseconds = SLICE_MILLISECONDS, now = () => performance.now(), pause = yieldToMain } = {}) {const partsByGeometry = new Map();const pinned = parts.filter(isPinnedPart);for (const part of parts) {@@ -91,17 +95,22 @@ async function replacePartsInSlices(renderer, parts) {if (!partsByGeometry.has(part.geometry)) partsByGeometry.set(part.geometry, []);partsByGeometry.get(part.geometry).push(part);}+ const weightOf = (geometry, sharing) => geometry.indices.length + sharing.length * 300;const prefix = [...pinned];- const totalWeight = [...partsByGeometry].reduce((total, [geometry, sharing]) => total + geometry.indices.length + sharing.length * 300, 0);- const sliceWeight = totalWeight / REPLACE_SLICES;+ const totalWeight = [...partsByGeometry].reduce((total, [geometry, sharing]) => total + weightOf(geometry, sharing), 0);+ const smallestSliceWeight = totalWeight / MAXIMUM_REPLACE_SLICES;+ let targetWeight = totalWeight / FIRST_REPLACE_SLICES;let weight = 0;for (const [geometry, sharing] of partsByGeometry) {prefix.push(...sharing);- weight += geometry.indices.length + sharing.length * 300;- if (weight < sliceWeight) continue;- weight = 0;+ weight += weightOf(geometry, sharing);+ if (weight < targetWeight) continue;+ const started = now();renderer.replaceParts([...prefix], { retainedUnchanged: true });- await yieldToMain();+ const elapsed = now() - started;+ if (elapsed > 0) targetWeight = Math.max(smallestSliceWeight, weight * sliceMilliseconds / elapsed);+ weight = 0;+ await pause();}renderer.replaceParts(parts, { retainedUnchanged: true });}diff --git a/src/renderer.js b/src/renderer.jsindex 2a3cdec3..077344ac 100644--- a/src/renderer.js+++ b/src/renderer.js@@ -1446,6 +1446,12 @@ export class CrucibleRenderer {if (!this.geometryBuffers.has(geometry)) validateReplacementGeometry(geometry);}const retained = new Set();+ // Groups share a few materials, and the layer state is 28 validated values, so each material answers once per call.+ const layerStates = new Map();+ const layerStateOf = material => {+ if (!layerStates.has(material)) layerStates.set(material, materialLayerState(material));+ return layerStates.get(material);+ };const replacements = groups.map(group => {const first = group[0];const location = this.partLocations.get(first);@@ -1457,7 +1463,7 @@ export class CrucibleRenderer {batch.floor === (first.material.tag === 'floor') && batchPass(batch) === partPass(first) &&[batch.alphaMode, batch.alphaCutoff, batch.opacity, batch.doubleSided, batch.retro].every((value, index) => value === alphaState[index]) &&batch.mapState.every((value, index) => value === mapState[index]) &&- batch.layerState.every((value, index) => value === materialLayerState(first.material)[index]);+ batch.layerState.every((value, index) => value === layerStateOf(first.material)[index]);if (sameParts && sameState) {retained.add(batch);return { group, batch };diff --git a/tests/hero-world-slices.test.js b/tests/hero-world-slices.test.jsnew file mode 100644index 00000000..ac71343c--- /dev/null+++ b/tests/hero-world-slices.test.js@@ -0,0 +1,159 @@+// A hero world switch hands the main thread back every few milliseconds: the sky prefilter in steps of a fixed texel count and+// replaceParts in slices sized from how long the last one took (#779).+import test from 'node:test';+import assert from 'node:assert/strict';+import { createEnvironmentMap, createEnvironmentResources, createEnvironmentResourcesInSlices } from '../src/environment.js';+import { replacePartsInSlices } from '../src/hero-lab/loop/world.js';++function recordingContext() {+ const constants = { FRAMEBUFFER_COMPLETE: 36053, TEXTURE_2D: 3553 };+ return new Proxy({}, {+ get(_, name) {+ if (name in constants) return constants[name];+ if (name === 'getExtension') return () => ({});+ if (name === 'checkFramebufferStatus') return () => constants.FRAMEBUFFER_COMPLETE;+ return () => ({});+ },+ });+}++test('the sky prefilter checks the clock inside the pass over the whole map, not only after it', async () => {+ const width = 2048;+ const height = 1024;+ const data = new Float32Array(width * height * 3).map((_, index) => (index % 13) / 4);+ const map = createEnvironmentMap({ width, height, data });+ const programs = { compileProgram: () => ({}), fullscreenVertex: '' };+ let checkpoints = 0;+ const sliced = await createEnvironmentResourcesInSlices(recordingContext(), map, programs, async () => { checkpoints++; });+ const whole = createEnvironmentResources(recordingContext(), map, programs);+ assert.deepEqual(sliced.irradiance, whole.irradiance);+ assert.deepEqual(sliced.irradianceGrid, whole.irradianceGrid);+ // 65,536 texels a step: 32 rows of upload, 32 of the downsample, 32 of the spherical harmonics, then the grid, the levels and the mipmaps.+ assert.ok(checkpoints >= 100, `expected at least 100 steps for a 2048 by 1024 map, got ${checkpoints}`);+});++// Parts that share one geometry per group of three, so the slicer has many boundaries to choose from.+function makeWorld(geometryCount) {+ const parts = [];+ for (let geometryIndex = 0; geometryIndex < geometryCount; geometryIndex++) {+ const geometry = { indices: new Array(300).fill(0) };+ for (let copy = 0; copy < 3; copy++) parts.push({ name: `part-${geometryIndex}-${copy}`, geometry, material: {} });+ }+ return parts;+}++function makeClockedRenderer(millisecondsPerPart) {+ const clock = { time: 0 };+ const calls = [];+ const renderer = {+ replaceParts(parts, options) {+ const previous = calls.at(-1)?.parts.length ?? 0;+ // The cost of a call is the parts it adds, as with retainedUnchanged, plus a little for the ones before.+ clock.time += (parts.length - previous) * millisecondsPerPart + previous * millisecondsPerPart * 0.02;+ calls.push({ parts, options, cost: 0 });+ },+ };+ return { renderer, clock, calls };+}++test('replacePartsInSlices feeds the world in growing prefixes and ends on the whole list in the same order', async () => {+ const world = makeWorld(60);+ const { renderer, clock, calls } = makeClockedRenderer(0.05);+ await replacePartsInSlices(renderer, world, { now: () => clock.time, pause: async () => {} });+ assert.ok(calls.length > 2);+ assert.ok(calls.every(call => call.options.retainedUnchanged === true));+ for (let index = 1; index < calls.length; index++) {+ const earlier = calls[index - 1].parts;+ assert.ok(calls[index].parts.length >= earlier.length);+ assert.ok(earlier.every((part, position) => part === calls[index].parts[position]), 'each call starts with the parts of the one before');+ }+ assert.deepEqual(calls.at(-1).parts, world);+});++test('replacePartsInSlices cuts a slow world into more pieces than a fast one and keeps each piece near the budget', async () => {+ const world = makeWorld(300);+ const fast = makeClockedRenderer(0.001);+ await replacePartsInSlices(fast.renderer, world, { now: () => fast.clock.time, pause: async () => {} });+ const slow = makeClockedRenderer(0.2);+ const costs = [];+ let previousTime = 0;+ await replacePartsInSlices(slow.renderer, world, {+ sliceMilliseconds: 8,+ now: () => slow.clock.time,+ pause: async () => { costs.push(slow.clock.time - previousTime); previousTime = slow.clock.time; },+ });+ assert.ok(slow.calls.length > fast.calls.length, `a slow world needs more slices (${slow.calls.length}) than a fast one (${fast.calls.length})`);+ // The first piece is a guess; every piece after it is sized from the one before, so it stays close to the 8 ms budget.+ const afterFirst = costs.slice(1);+ assert.ok(afterFirst.length >= 3);+ assert.ok(Math.max(...afterFirst) < 8 * 2.5, `slices after the first should last about 8 ms, got ${afterFirst.map(cost => cost.toFixed(1)).join(', ')}`);+ assert.ok(slow.calls.length <= 32 + 1, 'the number of calls stays bounded, because each call pays for the parts before it');+});++test('replacePartsInSlices keeps pinned parts in every prefix and puts them first', async () => {+ const world = makeWorld(20);+ const pinned = { name: 'crane', skin: {}, geometry: { indices: [0, 1, 2] }, material: {} };+ const parts = [world[0], pinned, ...world.slice(1)];+ const { renderer, clock, calls } = makeClockedRenderer(0.5);+ await replacePartsInSlices(renderer, parts, { now: () => clock.time, pause: async () => {} });+ assert.ok(calls.slice(0, -1).every(call => call.parts[0] === pinned));+ assert.equal(calls.at(-1).parts, parts);+});++test('a world built in slices on a sliced sky draws the same pixels as the one-piece build (#779)', async (testContext) => {+ const { withDevServerPage } = await import('./helpers/dev-server-browser.js');+ await withDevServerPage(testContext, async (page) => {+ const result = await page.evaluate(async () => {+ const { sphere } = await import('/src/geometry.js');+ const { CrucibleRenderer } = await import('/src/renderer.js');+ const { createEnvironmentMap } = await import('/src/environment.js');+ const { createSlicer } = await import('/src/cooperative.js');+ const { replacePartsInSlices } = await import('/src/hero-lab/loop/world.js');+ const width = 256;+ const height = 128;+ const data = new Float32Array(width * height * 3).map((_, index) => 0.2 + ((index * 7) % 23) / 6);+ const map = createEnvironmentMap({ width, height, data, showBackground: true });+ const parts = [];+ for (let index = 0; index < 12; index++) {+ const x = (index % 4) * 1.1 - 1.65;+ const y = Math.floor(index / 4) * 1.1 - 1.1;+ const matrix = new Float32Array([0.5, 0, 0, 0, 0, 0.5, 0, 0, 0, 0, 0.5, 0, x, y, 0, 1]);+ parts.push({ name: `ball-${index}`, geometry: sphere(8 + index, 6 + index), matrix, material: { tag: 'surface', color: [0.9, 0.5 + index / 30, 0.3], metallic: index % 2, roughness: 0.2 + index / 20, emission: 0 } });+ }+ const frame = { eye: [0, 0, 5], target: [0, 0, 0], fov: 50, time: 0 };+ const build = async (sliced) => {+ const canvas = document.createElement('canvas');+ canvas.width = 96;+ canvas.height = 96;+ document.body.append(canvas);+ const context = canvas.getContext('webgl2', { preserveDrawingBuffer: true });+ const renderer = new CrucibleRenderer(canvas, [parts[0]], { samples: 1, pixelRatio: 1, deferProgramLink: true, deferEnvironmentPrefilter: sliced, environment: { map } });+ await renderer.programsReady();+ let replaceCalls = 0;+ const realReplace = renderer.replaceParts.bind(renderer);+ renderer.replaceParts = (...callArguments) => { replaceCalls++; return realReplace(...callArguments); };+ if (sliced) {+ await renderer.prepareEnvironmentInSlices(map, createSlicer(0));+ // A clock that runs 40 ms a call, so the slices shrink to the smallest the slicer allows.+ let time = 0;+ await replacePartsInSlices(renderer, parts, { now: () => (time += 40), pause: async () => {} });+ } else {+ renderer.replaceParts(parts, { retainedUnchanged: true });+ }+ renderer.render(frame);+ const pixels = new Uint8Array(96 * 96 * 4);+ context.readPixels(0, 0, 96, 96, context.RGBA, context.UNSIGNED_BYTE, pixels);+ return { replaceCalls, pixels: Array.from(pixels) };+ };+ const whole = await build(false);+ const sliced = await build(true);+ let differing = 0;+ for (let index = 0; index < whole.pixels.length; index++) if (whole.pixels[index] !== sliced.pixels[index]) differing++;+ return { differing, drawn: whole.pixels.some(value => value !== 0), wholeCalls: whole.replaceCalls, slicedCalls: sliced.replaceCalls };+ });+ assert.equal(result.drawn, true, 'the scene draws something');+ assert.equal(result.wholeCalls, 1);+ assert.ok(result.slicedCalls > 2, `the sliced build calls replaceParts several times, got ${result.slicedCalls}`);+ assert.equal(result.differing, 0, 'the sliced build draws the same bytes');+ });+});