Skip to content

3D engine that AI agents drive

Set the dense-lod floor to 4000 triangles and measure shimmer and pops (#2157)

Nguyen Xuan Ana296c0e781e1
diff --git a/bench/glaze-passes/harness.js b/bench/glaze-passes/harness.js
index a777cedd..b0ee4b9f 100644
--- a/bench/glaze-passes/harness.js
+++ b/bench/glaze-passes/harness.js
@@ -90,7 +90,7 @@ function replaceColourShaders(replaceVertex) {
};
}
-window.measureScene = async function measureScene({ description, glbUrl, hdriUrl, frames = 200, capture = false, alternate = false, segmented = true, width = 1280, height = 720, rendererOverrides = {}, denseLods = false, denseLodsMinimumTriangles = undefined, experiment = null }) {
+window.measureScene = async function measureScene({ description, glbUrl, hdriUrl, frames = 200, capture = false, alternate = false, segmented = true, width = 1280, height = 720, rendererOverrides = {}, denseLods = false, denseLodsMinimumTriangles = undefined, experiment = null, sequence = null }) {
const canvas = document.querySelector('#viewport');
const experiments = new Set(String(experiment ?? '').split(','));
if (experiments.has('no-invariant')) {
@@ -215,6 +215,8 @@ window.measureScene = async function measureScene({ description, glbUrl, hdriUrl
for (let offset = 0; offset < pixels.length; offset += 0x8000) binary += String.fromCharCode(...pixels.subarray(offset, offset + 0x8000));
capturedFrame = btoa(binary);
}
+ let sequenceTriangles = null;
+ if (sequence) sequenceTriangles = await captureSequence({ renderer, context, canvas, frame, eye: description.eye, target: description.target, ...sequence });
if (experiment === "probe") return probeAlongView(asset.parts, description.eye, description.target);
const geometries = new Set(asset.parts.map(part => part.geometry));
const geometryStats = {
@@ -243,6 +245,7 @@ window.measureScene = async function measureScene({ description, glbUrl, hdriUrl
}])) : undefined,
pairedOnOverOffMedian: alternate ? Number(median(alternating.on.slice(0, alternating.off.length).map((value, index) => value / alternating.off[index])).toFixed(3)) : undefined,
capturedFrame,
+ sequenceTriangles,
centerPixel: Array.from(ONE_PIXEL),
glError: context.getError(),
};
@@ -285,3 +288,30 @@ function probeAlongView(parts, eye, target) {
hits.sort((first, second) => first.near - second.near);
return { hits: hits.slice(0, 12) };
}
+
+// A dolly along the view direction, from `backMetres` behind the camera to `forwardMetres` in front of it. Each frame
+// is read back whole and handed to window.deliverFrame(index, base64) (run.mjs writes it); returns the triangles drawn per frame.
+async function captureSequence({ renderer, context, canvas, frame, eye, target, frames = 60, backMetres = 6, forwardMetres = 5 }) {
+ const length = Math.hypot(...[0, 1, 2].map(axis => target[axis] - eye[axis]));
+ const unit = [0, 1, 2].map(axis => (target[axis] - eye[axis]) / length);
+ const pixels = new Uint8Array(canvas.width * canvas.height * 4);
+ const originalDrawInstanced = context.drawElementsInstanced.bind(context);
+ const triangleCounts = [];
+ for (let index = 0; index < frames; index++) {
+ const metres = -backMetres + (backMetres + forwardMetres) * index / (frames - 1);
+ const moved = { ...frame, eye: eye.map((value, axis) => value + unit[axis] * metres), target: target.map((value, axis) => value + unit[axis] * metres) };
+ let triangles = 0;
+ context.drawElementsInstanced = (mode, count, type, offset, instances) => {
+ triangles += (count / 3) * instances;
+ return originalDrawInstanced(mode, count, type, offset, instances);
+ };
+ renderer.render(moved);
+ context.drawElementsInstanced = originalDrawInstanced;
+ triangleCounts.push(triangles);
+ context.readPixels(0, 0, canvas.width, canvas.height, context.RGBA, context.UNSIGNED_BYTE, pixels);
+ let binary = '';
+ for (let offset = 0; offset < pixels.length; offset += 0x8000) binary += String.fromCharCode(...pixels.subarray(offset, offset + 0x8000));
+ await window.deliverFrame(index, btoa(binary));
+ }
+ return triangleCounts;
+}
diff --git a/bench/glaze-passes/run.mjs b/bench/glaze-passes/run.mjs
index 1f34abb6..9b7297a4 100644
--- a/bench/glaze-passes/run.mjs
+++ b/bench/glaze-passes/run.mjs
@@ -1,7 +1,7 @@
// Times each Glaze pass on the GPU for scene files: node bench/glaze-passes/run.mjs <out.json> <scene-directory>[,<scene-directory>...] [whole]
// Add '"capture":true' to the overrides to write one full frame per scene to <out.json>.<scene>.rgba (compare two with diff-frames.mjs).
// A scene directory holds scene.json, scene.glb and sky.hdr as written for the Blender demo scenes (see src/file-scene.js).
-import { readFile, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { basename, resolve } from 'node:path';
import { chromium } from 'playwright';
import { createServer } from 'vite';
@@ -26,6 +26,10 @@ try {
const description = JSON.parse(await readFile(`${directory}/scene.json`, 'utf8'));
const page = await browser.newPage({ viewport: { width: 1280, height: 720 } });
page.on('pageerror', error => console.error('page error:', error.message));
+ if (overrides.sequence) {
+ await mkdir(overrides.sequence.directory, { recursive: true });
+ await page.exposeFunction('deliverFrame', (index, base64) => writeFile(`${overrides.sequence.directory}/frame-${String(index).padStart(3, '0')}.rgba`, Buffer.from(base64, 'base64')));
+ }
await page.route('**/__scene/**', async (route, request) => {
const name = new URL(request.url()).pathname.split('/').pop();
await route.fulfill({ path: `${directory}/${name}` });
diff --git a/bench/glaze-passes/sequence-metrics.mjs b/bench/glaze-passes/sequence-metrics.mjs
new file mode 100644
index 00000000..b52086c8
--- /dev/null
+++ b/bench/glaze-passes/sequence-metrics.mjs
@@ -0,0 +1,127 @@
+// Between-frame shimmer and level popping of camera moves written by run.mjs with '"sequence":{"directory":...}':
+// node bench/glaze-passes/sequence-metrics.mjs <reference-directory> <directory>[,<directory>...] [width] [height]
+// The reference is the same move drawn without levels. For each directory it prints:
+// shimmer: mean absolute high-pass (value minus its 5 x 5 box blur) of the frame-to-frame luma difference, over the
+// detailed pixels (the top tenth by spatial high-pass energy in the reference), on the 0 to 255 scale;
+// the reference's own value is the floor of what the move itself causes.
+// addedShimmer: the same measure on the error against the reference (the frame minus the reference frame), so
+// the motion's own edges cancel and only what the levels add is left.
+// pop: RMSE over the frame of the change, from one frame to the next, of the error against the reference
+// (popMedian is the steady drift of the error); popSpike is the largest amount by which one frame's
+// change exceeds the median change of the six frames around it, which is a level switch.
+import { readdir, readFile } from 'node:fs/promises';
+
+const [referenceDirectory, directoryList, widthText = '1280', heightText = '720'] = process.argv.slice(2);
+if (!referenceDirectory || !directoryList) throw new Error('Pass the reference frame directory and a comma separated list of frame directories.');
+const width = Number(widthText);
+const height = Number(heightText);
+const HIGH_PASS_RADIUS = 2;
+const DETAIL_FRACTION = 0.1;
+
+async function readLumaFrames(directory) {
+ const names = (await readdir(directory)).filter(name => name.endsWith('.rgba')).sort();
+ const frames = [];
+ for (const name of names) {
+ const rgba = await readFile(`${directory}/${name}`);
+ if (rgba.length !== width * height * 4) throw new Error(`${directory}/${name} must be ${width} x ${height} RGBA.`);
+ const luma = new Float32Array(width * height);
+ for (let pixel = 0; pixel < luma.length; pixel++) luma[pixel] = 0.2126 * rgba[pixel * 4] + 0.7152 * rgba[pixel * 4 + 1] + 0.0722 * rgba[pixel * 4 + 2];
+ frames.push(luma);
+ }
+ return frames;
+}
+
+function boxBlur(image) {
+ const horizontal = new Float32Array(image.length);
+ const size = HIGH_PASS_RADIUS * 2 + 1;
+ for (let y = 0; y < height; y++) {
+ for (let x = 0; x < width; x++) {
+ let sum = 0;
+ for (let offset = -HIGH_PASS_RADIUS; offset <= HIGH_PASS_RADIUS; offset++) sum += image[y * width + Math.min(width - 1, Math.max(0, x + offset))];
+ horizontal[y * width + x] = sum / size;
+ }
+ }
+ const blurred = new Float32Array(image.length);
+ for (let y = 0; y < height; y++) {
+ for (let x = 0; x < width; x++) {
+ let sum = 0;
+ for (let offset = -HIGH_PASS_RADIUS; offset <= HIGH_PASS_RADIUS; offset++) sum += horizontal[Math.min(height - 1, Math.max(0, y + offset)) * width + x];
+ blurred[y * width + x] = sum / size;
+ }
+ }
+ return blurred;
+}
+
+function highPass(image) {
+ const blurred = boxBlur(image);
+ return image.map((value, index) => value - blurred[index]);
+}
+
+function difference(later, earlier) {
+ return later.map((value, index) => value - earlier[index]);
+}
+
+function detailMask(referenceFrames) {
+ const energy = new Float32Array(width * height);
+ for (const frame of referenceFrames) highPass(frame).forEach((value, index) => { energy[index] += Math.abs(value) / referenceFrames.length; });
+ const threshold = [...energy].sort((first, second) => first - second)[Math.floor(energy.length * (1 - DETAIL_FRACTION))];
+ return energy.map(value => (value >= threshold ? 1 : 0));
+}
+
+function shimmer(frames, mask) {
+ let total = 0;
+ const maskedPixels = mask.reduce((sum, value) => sum + value, 0);
+ for (let index = 1; index < frames.length; index++) {
+ const filtered = highPass(difference(frames[index], frames[index - 1]));
+ let sum = 0;
+ for (let pixel = 0; pixel < filtered.length; pixel++) if (mask[pixel]) sum += Math.abs(filtered[pixel]);
+ total += sum / maskedPixels;
+ }
+ return total / (frames.length - 1);
+}
+
+function errorFrames(frames, referenceFrames) {
+ return frames.map((frame, index) => difference(frame, referenceFrames[index]));
+}
+
+function popSpike(jumps) {
+ let largest = { excess: -Infinity, frame: 0 };
+ jumps.forEach((jump, index) => {
+ if (index < 3 || index > jumps.length - 4) return;
+ const around = jumps.filter((_, other) => other !== index && Math.abs(other - index) <= 3).sort((first, second) => first - second);
+ const excess = jump - around[Math.floor(around.length / 2)];
+ if (excess > largest.excess) largest = { excess, frame: index + 1 };
+ });
+ return largest;
+}
+
+function popJumps(frames, referenceFrames) {
+ const errors = frames.map((frame, index) => difference(frame, referenceFrames[index]));
+ const jumps = [];
+ for (let index = 1; index < errors.length; index++) {
+ const change = difference(errors[index], errors[index - 1]);
+ jumps.push(Math.sqrt(change.reduce((sum, value) => sum + value * value, 0) / change.length));
+ }
+ return jumps;
+}
+
+const referenceFrames = await readLumaFrames(referenceDirectory);
+const mask = detailMask(referenceFrames);
+console.log(JSON.stringify({ directory: referenceDirectory, reference: true, shimmer: Number(shimmer(referenceFrames, mask).toFixed(4)) }));
+for (const directory of directoryList.split(',')) {
+ const frames = await readLumaFrames(directory);
+ if (frames.length !== referenceFrames.length) throw new Error(`${directory} has ${frames.length} frames, the reference ${referenceFrames.length}.`);
+ const jumps = popJumps(frames, referenceFrames);
+ const sorted = [...jumps].sort((first, second) => first - second);
+ const largest = Math.max(...jumps);
+ console.log(JSON.stringify({
+ directory,
+ shimmer: Number(shimmer(frames, mask).toFixed(4)),
+ popMax: Number(largest.toFixed(4)),
+ popMedian: Number(sorted[Math.floor(sorted.length / 2)].toFixed(4)),
+ popMaxFrame: jumps.indexOf(largest) + 1,
+ addedShimmer: Number(shimmer(errorFrames(frames, referenceFrames), mask).toFixed(4)),
+ popSpike: Number(popSpike(jumps).excess.toFixed(4)),
+ popSpikeFrame: popSpike(jumps).frame,
+ }));
+}
diff --git a/changes/2144.md b/changes/2144.md
index c14aebf6..caef251a 100644
--- a/changes/2144.md
+++ b/changes/2144.md
@@ -1 +1 @@
-- A scene of many parts with a few thousand triangles each, such as Lone Monk, now gets levels of detail by default: `addLodsToDenseParts` starts at 1500 triangles instead of 20000, which drew 1.06 instead of 1.80 million triangles and took a courtyard view from 4.36 to 3.51 ms on the GPU (a slight change in far detail; #2144).
+- A scene of many parts with a few thousand triangles each, such as Lone Monk, now gets levels of detail by default: `addLodsToDenseParts` starts at 4000 triangles instead of 20000, which drew 1.06 instead of 1.80 million triangles and took a courtyard view from 4.36 to 3.38 ms on the GPU (a slight change in far detail; #2144). `bench/glaze-passes` can write a camera move and measure the shimmer and level pops of it (#2157).
diff --git a/docs/culling-lod.md b/docs/culling-lod.md
index e0f3d9e3..a7e548c8 100644
--- a/docs/culling-lod.md
+++ b/docs/culling-lod.md
@@ -58,8 +58,10 @@ A scan with about one triangle per pixel or more is shaded once per sample at ne
`generateGeometryLods` keeps the uvs, light uvs, colours and normals of a geometry that has them: each level is the geometry with some edges collapsed onto a vertex that stays, so no attribute is blended. Vertices on a UV seam, a hard edge or an open border never move, and a level can keep more triangles than its ratio when those vertices leave no safe collapse. `error` is the square root of the largest quadric cost of any collapse so far, an estimate of how far the surface moved, so it is a distance in the geometry's units (each plane counts once, so scaling the mesh by k scales it by k) and sits on the cautious side for small meshes with many seams. A geometry may have at most `MAXIMUM_SIMPLIFY_TRIANGLES` triangles and `MAXIMUM_SIMPLIFY_VERTICES` vertices (2 million each); a bigger one throws a RangeError, so split the part or skip levels for it (`addLodsToDenseParts` leaves such parts as they are).
-`addLodsToDenseParts(parts, { pixels })` from `src/dense-lods.js` gives levels to the parts of a scene that has more than one triangle per pixel (`pixels` is the canvas width times its height times the pixel ratio squared). A part gets levels when it has at least `minimumTriangles` (default 1500, so a scene of many meshes with a few thousand triangles each gets them too, #2144) triangles and can take them; parts with the same geometry share one set. It returns `parts` itself for a scene under the density. It is not applied by itself, and it takes seconds on a million triangles, so call it once while loading. On the Monster scene (1.6 million triangles, 1280 by 720, 4 samples) called with the canvas size, the frame went from 5.8 to 1.9 ms, and Lone Monk (5.6 million) from 15.4 to 9.5 ms (measured before the error metric was changed to a distance); the picture changes slightly (root mean square 2.0 of 255), because far parts draw from coarser levels (#1163).
+`addLodsToDenseParts(parts, { pixels })` from `src/dense-lods.js` gives levels to the parts of a scene that has more than one triangle per pixel (`pixels` is the canvas width times its height times the pixel ratio squared). A part gets levels when it has at least `minimumTriangles` (default 4000, so a scene of many meshes with a few thousand triangles each gets them too, #2144, #2157) triangles and can take them; parts with the same geometry share one set. It returns `parts` itself for a scene under the density. It is not applied by itself, and it takes seconds on a million triangles, so call it once while loading. On the Monster scene (1.6 million triangles, 1280 by 720, 4 samples) called with the canvas size, the frame went from 5.8 to 1.9 ms, and Lone Monk (5.6 million) from 15.4 to 9.5 ms (measured before the error metric was changed to a distance); the picture changes slightly (root mean square 2.0 of 255), because far parts draw from coarser levels (#1163).
A scene file (`scene.json`, rendered by `scripts/render-media.mjs --scene file`) gets the same levels by default; `"renderer": { "denseLods": false }` keeps every triangle of a dense scan. With the option on, `partsForRendererOptions` in `src/dense-lods.js` calls `addLodsToDenseParts` at the canvas size and pixel ratio before the renderer is built (#1163).
-On Lone Monk seen from the courtyard (a camera that shows the scene, 1280 by 720, 4 samples), drawing from the levels of parts of 1500 triangles and up, not 20000 and up, sent 1.06 million triangles instead of 1.80 million and cut the GPU frame from 4.36 to 3.51 ms (no levels at all: 3.26 million triangles, 7.12 ms). The picture differs from the 20000 floor by a root mean square of 1.74 of 255 (2.51 from no levels). `node bench/glaze-passes/run.mjs <out.json> <scene directories> whole '{"denseLods":true,"denseLodsMinimumTriangles":1500}'` measures it; `'{"engine":"three"}'` draws the same file with three.js, and `bench/glaze-passes/still-pair.mjs` writes the picture pair and the RMSE (#2144).
+On Lone Monk seen from the courtyard (a camera that shows the scene, 1280 by 720, 4 samples), drawing from the levels of parts of 4000 triangles and up, not 20000 and up, sent 1.06 million triangles instead of 1.80 million and cut the GPU frame from 4.36 to 3.38 ms (1500 and up: 3.51 to 3.62 ms, 8000 and up: 3.30 ms) (no levels at all: 3.26 million triangles, 7.12 ms). The picture differs from the 20000 floor by a root mean square of 1.74 of 255 (2.51 from no levels). `node bench/glaze-passes/run.mjs <out.json> <scene directories> whole '{"denseLods":true,"denseLodsMinimumTriangles":4000}'` measures it; `'{"engine":"three"}'` draws the same file with three.js, and `bench/glaze-passes/still-pair.mjs` writes the picture pair and the RMSE (#2144).
+
+A camera move shows what the levels add between frames (#2157). Add `"sequence":{"directory":"<frames>","frames":60}` to the options to write 60 frames of a dolly from 6 m behind the camera to 5 m in front of it, and run `node bench/glaze-passes/sequence-metrics.mjs <frames drawn with denseLods false> <frames>[,<frames>]`: it prints shimmer (the high-passed frame-to-frame difference over the detailed tenth of the picture), addedShimmer (the same on the error against the move without levels) and popSpike (the biggest jump of that error beyond its neighbours). On Lone Monk the floors 4000, 8000 and 1500 gave the same addedShimmer, 1.16 against 0.76 at 20000 (the move's own shimmer is 12.6 of 255, with or without levels), and a popSpike of 0.42 against 0.63 at 20000. The extra shimmer comes from the meshes of 8000 to 12000 triangles, which is also where the saved triangles are, so no floor between 1500 and 8000 separates speed from shimmer; `lodPixelError: 0.5` brought it to 0.87 for 1.28 million triangles and 4.04 ms.
diff --git a/src/dense-lods.js b/src/dense-lods.js
index 6b441517..4bf978ed 100644
--- a/src/dense-lods.js
+++ b/src/dense-lods.js
@@ -6,8 +6,8 @@ import { DENSE_TRIANGLES_PER_PIXEL, partTakesLods } from './renderer.js';
// fewest triangles whose error stays under `lodPixelError` pixels (#1163). On such a scene every pixel is shaded about
// once per triangle that touches it, so a level with a quarter of the triangles shades far fewer quads.
-// Meshes of a few thousand triangles, many of them, are as dense as a few huge scans, so the floor sits low (#2144).
-export const DEFAULT_MINIMUM_TRIANGLES = 1500;
+// Meshes of a few thousand triangles, many of them, are as dense as a few huge scans, so the floor sits low (#2144). Lone Monk draws as fast from 4000 as from 1500 and shimmers no more (#2157).
+export const DEFAULT_MINIMUM_TRIANGLES = 4000;
/** Triangles over pixels for these parts on a canvas of `pixels` pixels (the canvas size times the pixel ratio squared). */
export function trianglesPerPixel(parts, pixels) {
diff --git a/tests/geometry-simplify.test.js b/tests/geometry-simplify.test.js
index 343d0248..6c8d6bd5 100644
--- a/tests/geometry-simplify.test.js
+++ b/tests/geometry-simplify.test.js
@@ -226,12 +226,15 @@ test('simplifyGeometry still takes an index equal to the last vertex (#1900)', (
});
test('addLodsToDenseParts gives levels to mid-size meshes by default, so a scene of many of them is not left at full detail (#2144)', () => {
- const midSize = sphere(40, 30);
+ const midSize = sphere(60, 40);
+ const small = sphere(50, 30);
const tiny = sphere(8, 8);
assert.ok(midSize.indices.length / 3 >= DEFAULT_MINIMUM_TRIANGLES, 'the sphere is a mid-size mesh');
assert.ok(midSize.indices.length / 3 < 20000, 'it is under the old 20000 floor');
- const parts = [surfacePart('mid', midSize), surfacePart('tiny', tiny)];
+ assert.ok(small.indices.length / 3 < DEFAULT_MINIMUM_TRIANGLES, 'a smaller mesh sits under the floor');
+ const parts = [surfacePart('mid', midSize), surfacePart('small', small), surfacePart('tiny', tiny)];
const result = addLodsToDenseParts(parts, { pixels: 100 });
assert.ok(result[0].lods.length > 1, 'a mesh over the default floor gets levels');
- assert.equal(result[1].lods, undefined, 'a mesh under the default floor does not');
+ assert.equal(result[1].lods, undefined, 'a mesh under the default floor does not (4000 triangles, not 1500: the lower floor shimmered more and drew no faster, #2157)');
+ assert.equal(result[2].lods, undefined, 'a tiny mesh does not');
});