Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Trace swapped pixels to their face in the flicker probe (#751)

Nguyen Xuan Anfaaa359a4329
diff --git a/bench/flicker/ground-truth.js b/bench/flicker/ground-truth.js
index dcf76a42..d4850c8c 100644
--- a/bench/flicker/ground-truth.js
+++ b/bench/flicker/ground-truth.js
@@ -118,7 +118,7 @@ function hasUniformSurroundings(keyAt, width, height, x, y) {
* - `tied`: contested pixels where the scene itself does not say which face is in front (the flush pass off, or a seam); left out of the other counts;
* - `unjudged`: contested pixels whose face has too few clean pixels to know its colour;
* - `edge`: pixels left out for having another colour next to them.
- * `mismatchMask` marks the wrong pixels for a picture.
+ * `mismatchMask` marks the wrong pixels for a picture: 1 for a wrong pixel, 2 for a swapped one.
*/
export function compareWithGroundTruth({ pixels, world, cast, width, height, threshold = DEFAULT_COLOUR_THRESHOLD }) {
const keyAt = new Int32Array(width * height);
@@ -169,6 +169,7 @@ export function compareWithGroundTruth({ pixels, world, cast, width, height, thr
const rival = references[cast.rivals[pixel * RIVALS_PER_PIXEL + slot]];
if (rival && channelDistance(pixels, pixel * 4, rival) <= threshold) {
result.swapped++;
+ mismatchMask[pixel] = 2;
break;
}
}
diff --git a/bench/flicker/pixel-cast.js b/bench/flicker/pixel-cast.js
index dcb9382f..173c3f09 100644
--- a/bench/flicker/pixel-cast.js
+++ b/bench/flicker/pixel-cast.js
@@ -129,7 +129,7 @@ function isTie(world, distances, triangles, base) {
/**
* Casts one ray through every pixel centre of a `width` x `height` image. `world` is the result of `collectWorldTriangles`,
- * `camera` the `{ eye, target, up, fov }` of the frame. Returns `{ owner, rivals, tied }`: `owner` is the visible triangle of
+ * `camera` the `{ eye, target, up, fov }` of the frame. Returns `{ owner, rivals, tied, distance }`: `distance` is the depth of the visible face in front of the camera (0 for the background); `owner` is the visible triangle of
* each pixel (`NO_TRIANGLE` for the background), `rivals` holds up to two colour keys of other triangles on the same ray, or
* -1, and `tied` marks pixels where two faces of different colour lie at one distance with one rank. Only the six nearest
* faces of a ray are kept, which is enough to name its visible face and the faces right behind it.
@@ -150,10 +150,12 @@ export function castPixelOwners(world, camera, width, height) {
const owner = new Int32Array(width * height).fill(NO_TRIANGLE);
const rivals = new Int32Array(width * height * RIVALS_PER_PIXEL).fill(-1);
const tied = new Uint8Array(width * height);
+ const distance = new Float64Array(width * height);
for (let pixel = 0; pixel < width * height; pixel++) {
const base = pixel * HITS_PER_PIXEL;
if (hitTriangles[base] === NO_TRIANGLE) continue;
owner[pixel] = hitTriangles[base];
+ distance[pixel] = hitDistances[base];
tied[pixel] = isTie(world, hitDistances, hitTriangles, base) ? 1 : 0;
const ownKey = world.keyOf[hitTriangles[base]];
let stored = 0;
@@ -163,5 +165,5 @@ export function castPixelOwners(world, camera, width, height) {
if (!known) rivals[pixel * RIVALS_PER_PIXEL + stored++] = key;
}
}
- return { owner, rivals, tied };
+ return { owner, rivals, tied, distance };
}
diff --git a/bench/flicker/probe.mjs b/bench/flicker/probe.mjs
index ded6900e..a6f2c4de 100644
--- a/bench/flicker/probe.mjs
+++ b/bench/flicker/probe.mjs
@@ -74,6 +74,9 @@ try {
const spots = JSON.parse(readFileSync(spotsFile, 'utf8'));
for (const configuration of configurations) {
await open(configuration);
+ // The flush pass of a large asset takes seconds; wait for the scene to be built, then for the pass when it is on.
+ await page.waitForFunction(() => window.flicker.info.bounds, null, { polling: 500 });
+ if (configuration.offset !== '0') await page.waitForFunction(() => window.flicker.info.flush, null, { polling: 500 });
await page.waitForTimeout(1500);
console.log(JSON.stringify({ flush: await page.evaluate(() => window.flicker.info.flush) }));
const label = Object.entries(configuration).map(([key, value]) => `${key}${value}`).join('-');
diff --git a/bench/flicker/viewer.js b/bench/flicker/viewer.js
index 1298321c..74100779 100644
--- a/bench/flicker/viewer.js
+++ b/bench/flicker/viewer.js
@@ -258,6 +258,75 @@ async function truth() {
return { ...counts, triangles: world.count, castMilliseconds: Math.round(performance.now() - started), width, height };
}
+// The triangles of another colour key that lie under the centre of triangle `target` within a decimetre of its plane, as
+// `{ triangle, offset, rank, key, facing }` with `offset` how far the target stands in front of it along the target's normal.
+function trianglesBehind(world, target) {
+ const corners = world.corners;
+ const point = index => [corners[index * 9], corners[index * 9 + 1], corners[index * 9 + 2]];
+ const centre = [0, 1, 2].map(axis => (corners[target * 9 + axis] + corners[target * 9 + 3 + axis] + corners[target * 9 + 6 + axis]) / 3);
+ const edgeOne = [0, 1, 2].map(axis => corners[target * 9 + 3 + axis] - point(target)[axis]);
+ const edgeTwo = [0, 1, 2].map(axis => corners[target * 9 + 6 + axis] - point(target)[axis]);
+ const normal = [edgeOne[1] * edgeTwo[2] - edgeOne[2] * edgeTwo[1], edgeOne[2] * edgeTwo[0] - edgeOne[0] * edgeTwo[2], edgeOne[0] * edgeTwo[1] - edgeOne[1] * edgeTwo[0]];
+ const length = Math.hypot(...normal);
+ const unit = normal.map(value => value / length);
+ const found = [];
+ for (let other = 0; other < world.count; other++) {
+ if (other === target || world.keyOf[other] === world.keyOf[target]) continue;
+ const first = point(other);
+ const second = [0, 1, 2].map(axis => corners[other * 9 + 3 + axis]);
+ const third = [0, 1, 2].map(axis => corners[other * 9 + 6 + axis]);
+ const side = [0, 1, 2].map(axis => second[axis] - first[axis]);
+ const back = [0, 1, 2].map(axis => third[axis] - first[axis]);
+ const otherNormal = [side[1] * back[2] - side[2] * back[1], side[2] * back[0] - side[0] * back[2], side[0] * back[1] - side[1] * back[0]];
+ const otherLength = Math.hypot(...otherNormal);
+ const alignment = (otherNormal[0] * unit[0] + otherNormal[1] * unit[1] + otherNormal[2] * unit[2]) / otherLength;
+ if (Math.abs(alignment) < 0.99) continue;
+ const offset = unit.reduce((sum, value, axis) => sum + value * (first[axis] - centre[axis]), 0);
+ if (Math.abs(offset) > 0.1) continue;
+ const projected = centre.map((value, axis) => value + unit[axis] * offset);
+ const signs = [[first, second], [second, third], [third, first]].map(([from, to]) => {
+ const edge = to.map((value, axis) => value - from[axis]);
+ const toPoint = projected.map((value, axis) => value - from[axis]);
+ const turn = [edge[1] * toPoint[2] - edge[2] * toPoint[1], edge[2] * toPoint[0] - edge[0] * toPoint[2], edge[0] * toPoint[1] - edge[1] * toPoint[0]];
+ return turn[0] * otherNormal[0] + turn[1] * otherNormal[1] + turn[2] * otherNormal[2];
+ });
+ if (signs.every(sign => sign >= 0)) found.push({ triangle: other, offset: -offset, rank: world.ranks[other], key: world.keyOf[other], facing: alignment > 0 ? 'same' : 'opposite' });
+ }
+ return found.slice(0, 6);
+}
+
+// The swapped pixels of the current frame grouped by the triangle that should be visible there: its part, rank and corners in
+// world space, the pixel count and the median distance. Lists the 'limit' largest groups; this is how a swap is traced to a face.
+function swapDetails(limit = 12) {
+ const { width, height } = canvas;
+ const scratch = document.createElement('canvas');
+ scratch.width = width;
+ scratch.height = height;
+ const context = scratch.getContext('2d', { willReadFrequently: true });
+ renderer.render(frameFor(0));
+ const pixels = new Uint8ClampedArray(readFrame(context));
+ const world = collectWorldTriangles(window.flickerParts, flushOrderOf);
+ const cast = castPixelOwners(world, frameFor(0), width, height);
+ const { mismatchMask } = compareWithGroundTruth({ pixels, world, cast, width, height });
+ const groups = new Map();
+ for (let pixel = 0; pixel < mismatchMask.length; pixel++) {
+ if (mismatchMask[pixel] !== 2) continue;
+ const group = groups.get(cast.owner[pixel]) ?? { triangle: cast.owner[pixel], pixels: 0, distances: [] };
+ group.pixels++;
+ const neighbours = [-1, 1, -width, width, -width - 1, -width + 1, width - 1, width + 1];
+ if (neighbours.some(step => cast.owner[pixel + step] !== cast.owner[pixel])) group.onTriangleEdge = (group.onTriangleEdge ?? 0) + 1;
+ group.distances.push(cast.distance[pixel]);
+ groups.set(cast.owner[pixel], group);
+ }
+ renderer.render(frameFor(0));
+ return [...groups.values()].sort((first, second) => second.pixels - first.pixels).slice(0, limit).map(group => ({
+ triangle: group.triangle, pixels: group.pixels, onTriangleEdge: group.onTriangleEdge ?? 0, rank: world.ranks[group.triangle], key: world.keyOf[group.triangle],
+ behind: trianglesBehind(world, group.triangle),
+ distance: group.distances.sort((first, second) => first - second)[group.distances.length >> 1],
+ corners: Array.from(world.corners.subarray(group.triangle * 9, group.triangle * 9 + 9)),
+ }));
+}
+
async function probe() {
if (!renderer) return null;
if (!options.ids) {
@@ -472,7 +541,7 @@ bindControls();
requestAnimationFrame(tick);
document.querySelector('#speedValue').textContent = '';
window.flicker = {
- setCamera, probe, truth, build, options, frameWholeScene,
+ setCamera, probe, truth, swapDetails, build, options, frameWholeScene,
get info() { return { depthMode: depthModeLabel(), ...(renderer ? nearAndFar() : {}), extent, bounds, canvas: [canvas.width, canvas.height], memory: renderer?.memoryReport?.(), flush: flushReport }; },
render: () => { markDirty(); renderer?.render(frameFor(0)); },
traps: Object.keys(TRAPS),
diff --git a/tests/flicker-ground-truth.test.js b/tests/flicker-ground-truth.test.js
index ddcb8cdc..dde8abd8 100644
--- a/tests/flicker-ground-truth.test.js
+++ b/tests/flicker-ground-truth.test.js
@@ -82,6 +82,26 @@ test('a pixel with another colour next to it is an edge, not a judged pixel', ()
assert.ok(outcome.edge > 0);
});
+test('the ray cast reports the depth of the visible face in front of the camera, and 0 for the background', () => {
+ const { cast } = castWallWithPlaque(1);
+ assert.ok(Math.abs(cast.distance[CENTRE] - 9) < 0.01, `plaque at ${cast.distance[CENTRE]}`);
+ assert.ok(Math.abs(cast.distance[0] - 10) < 0.01, `wall at ${cast.distance[0]}`);
+ const empty = castPixelOwners(collectWorldTriangles([], () => null), CAMERA, WIDTH, HEIGHT);
+ assert.ok(empty.distance.every(distance => distance === 0));
+});
+
+test('a swapped pixel is marked 2 in the mismatch mask and a merely wrong one 1', () => {
+ const { world, cast } = castWallWithPlaque(0.01);
+ const swappedPixel = CENTRE;
+ const wrongPixel = CENTRE + 2;
+ const pixels = paintFrame(cast, pixel => (pixel === swappedPixel ? WALL_PIXEL : pixel === wrongPixel ? [20, 200, 20] : null));
+ const outcome = compareWithGroundTruth({ pixels, world, cast, width: WIDTH, height: HEIGHT });
+ assert.equal(outcome.mismatchMask[swappedPixel], 2);
+ assert.equal(outcome.mismatchMask[wrongPixel], 1);
+ assert.equal(outcome.swapped, 1);
+ assert.equal(outcome.wrong, 2);
+});
+
test('a plaque exactly on the wall leaves the plaque alone at its place once the flush pass ran', () => {
const parts = wallWithPlaque(0);
resolveFlushParts(parts);