3D engine that AI agents drive
Trace swapped pixels to their face in the flicker probe (#751)
Nguyen Xuan Anfaaa359a4329
diff --git a/bench/flicker/ground-truth.js b/bench/flicker/ground-truth.jsindex dcf76a42..d4850c8c 100644--- a/bench/flicker/ground-truth.js+++ b/bench/flicker/ground-truth.js@@ -118,7 +118,7 @@ function hasUniformSurroundings(keyAt, width, height, x, y) {* - `tied`: contested pixels where the scene itself does not say which face is in front (the flush pass off, or a seam); left out of the other counts;* - `unjudged`: contested pixels whose face has too few clean pixels to know its colour;* - `edge`: pixels left out for having another colour next to them.- * `mismatchMask` marks the wrong pixels for a picture.+ * `mismatchMask` marks the wrong pixels for a picture: 1 for a wrong pixel, 2 for a swapped one.*/export function compareWithGroundTruth({ pixels, world, cast, width, height, threshold = DEFAULT_COLOUR_THRESHOLD }) {const keyAt = new Int32Array(width * height);@@ -169,6 +169,7 @@ export function compareWithGroundTruth({ pixels, world, cast, width, height, thrconst rival = references[cast.rivals[pixel * RIVALS_PER_PIXEL + slot]];if (rival && channelDistance(pixels, pixel * 4, rival) <= threshold) {result.swapped++;+ mismatchMask[pixel] = 2;break;}}diff --git a/bench/flicker/pixel-cast.js b/bench/flicker/pixel-cast.jsindex dcb9382f..173c3f09 100644--- a/bench/flicker/pixel-cast.js+++ b/bench/flicker/pixel-cast.js@@ -129,7 +129,7 @@ function isTie(world, distances, triangles, base) {/*** Casts one ray through every pixel centre of a `width` x `height` image. `world` is the result of `collectWorldTriangles`,- * `camera` the `{ eye, target, up, fov }` of the frame. Returns `{ owner, rivals, tied }`: `owner` is the visible triangle of+ * `camera` the `{ eye, target, up, fov }` of the frame. Returns `{ owner, rivals, tied, distance }`: `distance` is the depth of the visible face in front of the camera (0 for the background); `owner` is the visible triangle of* each pixel (`NO_TRIANGLE` for the background), `rivals` holds up to two colour keys of other triangles on the same ray, or* -1, and `tied` marks pixels where two faces of different colour lie at one distance with one rank. Only the six nearest* faces of a ray are kept, which is enough to name its visible face and the faces right behind it.@@ -150,10 +150,12 @@ export function castPixelOwners(world, camera, width, height) {const owner = new Int32Array(width * height).fill(NO_TRIANGLE);const rivals = new Int32Array(width * height * RIVALS_PER_PIXEL).fill(-1);const tied = new Uint8Array(width * height);+ const distance = new Float64Array(width * height);for (let pixel = 0; pixel < width * height; pixel++) {const base = pixel * HITS_PER_PIXEL;if (hitTriangles[base] === NO_TRIANGLE) continue;owner[pixel] = hitTriangles[base];+ distance[pixel] = hitDistances[base];tied[pixel] = isTie(world, hitDistances, hitTriangles, base) ? 1 : 0;const ownKey = world.keyOf[hitTriangles[base]];let stored = 0;@@ -163,5 +165,5 @@ export function castPixelOwners(world, camera, width, height) {if (!known) rivals[pixel * RIVALS_PER_PIXEL + stored++] = key;}}- return { owner, rivals, tied };+ return { owner, rivals, tied, distance };}diff --git a/bench/flicker/probe.mjs b/bench/flicker/probe.mjsindex ded6900e..a6f2c4de 100644--- a/bench/flicker/probe.mjs+++ b/bench/flicker/probe.mjs@@ -74,6 +74,9 @@ try {const spots = JSON.parse(readFileSync(spotsFile, 'utf8'));for (const configuration of configurations) {await open(configuration);+ // The flush pass of a large asset takes seconds; wait for the scene to be built, then for the pass when it is on.+ await page.waitForFunction(() => window.flicker.info.bounds, null, { polling: 500 });+ if (configuration.offset !== '0') await page.waitForFunction(() => window.flicker.info.flush, null, { polling: 500 });await page.waitForTimeout(1500);console.log(JSON.stringify({ flush: await page.evaluate(() => window.flicker.info.flush) }));const label = Object.entries(configuration).map(([key, value]) => `${key}${value}`).join('-');diff --git a/bench/flicker/viewer.js b/bench/flicker/viewer.jsindex 1298321c..74100779 100644--- a/bench/flicker/viewer.js+++ b/bench/flicker/viewer.js@@ -258,6 +258,75 @@ async function truth() {return { ...counts, triangles: world.count, castMilliseconds: Math.round(performance.now() - started), width, height };}+// The triangles of another colour key that lie under the centre of triangle `target` within a decimetre of its plane, as+// `{ triangle, offset, rank, key, facing }` with `offset` how far the target stands in front of it along the target's normal.+function trianglesBehind(world, target) {+ const corners = world.corners;+ const point = index => [corners[index * 9], corners[index * 9 + 1], corners[index * 9 + 2]];+ const centre = [0, 1, 2].map(axis => (corners[target * 9 + axis] + corners[target * 9 + 3 + axis] + corners[target * 9 + 6 + axis]) / 3);+ const edgeOne = [0, 1, 2].map(axis => corners[target * 9 + 3 + axis] - point(target)[axis]);+ const edgeTwo = [0, 1, 2].map(axis => corners[target * 9 + 6 + axis] - point(target)[axis]);+ const normal = [edgeOne[1] * edgeTwo[2] - edgeOne[2] * edgeTwo[1], edgeOne[2] * edgeTwo[0] - edgeOne[0] * edgeTwo[2], edgeOne[0] * edgeTwo[1] - edgeOne[1] * edgeTwo[0]];+ const length = Math.hypot(...normal);+ const unit = normal.map(value => value / length);+ const found = [];+ for (let other = 0; other < world.count; other++) {+ if (other === target || world.keyOf[other] === world.keyOf[target]) continue;+ const first = point(other);+ const second = [0, 1, 2].map(axis => corners[other * 9 + 3 + axis]);+ const third = [0, 1, 2].map(axis => corners[other * 9 + 6 + axis]);+ const side = [0, 1, 2].map(axis => second[axis] - first[axis]);+ const back = [0, 1, 2].map(axis => third[axis] - first[axis]);+ const otherNormal = [side[1] * back[2] - side[2] * back[1], side[2] * back[0] - side[0] * back[2], side[0] * back[1] - side[1] * back[0]];+ const otherLength = Math.hypot(...otherNormal);+ const alignment = (otherNormal[0] * unit[0] + otherNormal[1] * unit[1] + otherNormal[2] * unit[2]) / otherLength;+ if (Math.abs(alignment) < 0.99) continue;+ const offset = unit.reduce((sum, value, axis) => sum + value * (first[axis] - centre[axis]), 0);+ if (Math.abs(offset) > 0.1) continue;+ const projected = centre.map((value, axis) => value + unit[axis] * offset);+ const signs = [[first, second], [second, third], [third, first]].map(([from, to]) => {+ const edge = to.map((value, axis) => value - from[axis]);+ const toPoint = projected.map((value, axis) => value - from[axis]);+ const turn = [edge[1] * toPoint[2] - edge[2] * toPoint[1], edge[2] * toPoint[0] - edge[0] * toPoint[2], edge[0] * toPoint[1] - edge[1] * toPoint[0]];+ return turn[0] * otherNormal[0] + turn[1] * otherNormal[1] + turn[2] * otherNormal[2];+ });+ if (signs.every(sign => sign >= 0)) found.push({ triangle: other, offset: -offset, rank: world.ranks[other], key: world.keyOf[other], facing: alignment > 0 ? 'same' : 'opposite' });+ }+ return found.slice(0, 6);+}++// The swapped pixels of the current frame grouped by the triangle that should be visible there: its part, rank and corners in+// world space, the pixel count and the median distance. Lists the 'limit' largest groups; this is how a swap is traced to a face.+function swapDetails(limit = 12) {+ const { width, height } = canvas;+ const scratch = document.createElement('canvas');+ scratch.width = width;+ scratch.height = height;+ const context = scratch.getContext('2d', { willReadFrequently: true });+ renderer.render(frameFor(0));+ const pixels = new Uint8ClampedArray(readFrame(context));+ const world = collectWorldTriangles(window.flickerParts, flushOrderOf);+ const cast = castPixelOwners(world, frameFor(0), width, height);+ const { mismatchMask } = compareWithGroundTruth({ pixels, world, cast, width, height });+ const groups = new Map();+ for (let pixel = 0; pixel < mismatchMask.length; pixel++) {+ if (mismatchMask[pixel] !== 2) continue;+ const group = groups.get(cast.owner[pixel]) ?? { triangle: cast.owner[pixel], pixels: 0, distances: [] };+ group.pixels++;+ const neighbours = [-1, 1, -width, width, -width - 1, -width + 1, width - 1, width + 1];+ if (neighbours.some(step => cast.owner[pixel + step] !== cast.owner[pixel])) group.onTriangleEdge = (group.onTriangleEdge ?? 0) + 1;+ group.distances.push(cast.distance[pixel]);+ groups.set(cast.owner[pixel], group);+ }+ renderer.render(frameFor(0));+ return [...groups.values()].sort((first, second) => second.pixels - first.pixels).slice(0, limit).map(group => ({+ triangle: group.triangle, pixels: group.pixels, onTriangleEdge: group.onTriangleEdge ?? 0, rank: world.ranks[group.triangle], key: world.keyOf[group.triangle],+ behind: trianglesBehind(world, group.triangle),+ distance: group.distances.sort((first, second) => first - second)[group.distances.length >> 1],+ corners: Array.from(world.corners.subarray(group.triangle * 9, group.triangle * 9 + 9)),+ }));+}+async function probe() {if (!renderer) return null;if (!options.ids) {@@ -472,7 +541,7 @@ bindControls();requestAnimationFrame(tick);document.querySelector('#speedValue').textContent = '';window.flicker = {- setCamera, probe, truth, build, options, frameWholeScene,+ setCamera, probe, truth, swapDetails, build, options, frameWholeScene,get info() { return { depthMode: depthModeLabel(), ...(renderer ? nearAndFar() : {}), extent, bounds, canvas: [canvas.width, canvas.height], memory: renderer?.memoryReport?.(), flush: flushReport }; },render: () => { markDirty(); renderer?.render(frameFor(0)); },traps: Object.keys(TRAPS),diff --git a/tests/flicker-ground-truth.test.js b/tests/flicker-ground-truth.test.jsindex ddcb8cdc..dde8abd8 100644--- a/tests/flicker-ground-truth.test.js+++ b/tests/flicker-ground-truth.test.js@@ -82,6 +82,26 @@ test('a pixel with another colour next to it is an edge, not a judged pixel', ()assert.ok(outcome.edge > 0);});+test('the ray cast reports the depth of the visible face in front of the camera, and 0 for the background', () => {+ const { cast } = castWallWithPlaque(1);+ assert.ok(Math.abs(cast.distance[CENTRE] - 9) < 0.01, `plaque at ${cast.distance[CENTRE]}`);+ assert.ok(Math.abs(cast.distance[0] - 10) < 0.01, `wall at ${cast.distance[0]}`);+ const empty = castPixelOwners(collectWorldTriangles([], () => null), CAMERA, WIDTH, HEIGHT);+ assert.ok(empty.distance.every(distance => distance === 0));+});++test('a swapped pixel is marked 2 in the mismatch mask and a merely wrong one 1', () => {+ const { world, cast } = castWallWithPlaque(0.01);+ const swappedPixel = CENTRE;+ const wrongPixel = CENTRE + 2;+ const pixels = paintFrame(cast, pixel => (pixel === swappedPixel ? WALL_PIXEL : pixel === wrongPixel ? [20, 200, 20] : null));+ const outcome = compareWithGroundTruth({ pixels, world, cast, width: WIDTH, height: HEIGHT });+ assert.equal(outcome.mismatchMask[swappedPixel], 2);+ assert.equal(outcome.mismatchMask[wrongPixel], 1);+ assert.equal(outcome.swapped, 1);+ assert.equal(outcome.wrong, 2);+});+test('a plaque exactly on the wall leaves the plaque alone at its place once the flush pass ran', () => {const parts = wallWithPlaque(0);resolveFlushParts(parts);