3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Harden splat packing and sorting for hand-built data (#2435)
Nguyen Xuan Anff75fe1d1c82
diff --git a/changes/2435.md b/changes/2435.mdnew file mode 100644index 00000000..c5ada5e9--- /dev/null+++ b/changes/2435.md@@ -0,0 +1 @@+- Gaussian splats: a rotation of any length packs its unit covariance, `createGaussianSplats` checks `bounds` at the call, and the sorter re-sorts tiny scaled scenes (#2435, #2436, #2437).diff --git a/docs/gaussian-splats.md b/docs/gaussian-splats.mdindex ed55af55..82dfcfd1 100644--- a/docs/gaussian-splats.md+++ b/docs/gaussian-splats.md@@ -60,7 +60,7 @@ const splats = await createGaussianSplatsAsync(data, options);## Hand-built data-Data written by code, not parsed, is checked at the call. `count` must be a whole number, `positions` and `scales` need `count * 3` values (Float32Array or Float64Array), `rotations` `count * 4` (w, x, y, z) and `colors` `count * 4` bytes (Uint8Array, starting on a 4-byte boundary of its buffer). A longer array is fine, a shorter or missing one throws an error that names the field, the length it needs and the length it got. `matrix` and the options object must be left out or be a 16-number array and an object; `null` throws. The same checks run in `createGaussianSplatsAsync` before any splat is packed.+Data written by code, not parsed, is checked at the call. `count` must be a whole number, `positions` and `scales` need `count * 3` values (Float32Array or Float64Array), `rotations` `count * 4` (w, x, y, z) and `colors` `count * 4` bytes (Uint8Array, starting on a 4-byte boundary of its buffer). A longer array is fine, a shorter or missing one throws an error that names the field, the length it needs and the length it got. `bounds` must be `{ minimum, maximum }`, each three finite numbers (the parse result carries it), or a `TypeError` says so. A rotation of any length counts as its unit version, and all zero as no rotation. `matrix` and the options object must be left out or be a 16-number array and an object; `null` throws. The same checks run in `createGaussianSplatsAsync` before any splat is packed.## What it drawsdiff --git a/src/gaussian-splat-sort.js b/src/gaussian-splat-sort.jsindex 72f32c9a..3074af74 100644--- a/src/gaussian-splat-sort.js+++ b/src/gaussian-splat-sort.js@@ -4,6 +4,8 @@// out of the order. Splats closer than one step of depth (the span divided by 65536) may draw in either order.const DEPTH_STEPS = 65536;+// Relative to the length of the depth axis, which carries any uniform scale of the splat matrix: a fixed tolerance+// would never re-sort a scene scaled down to 1e-6.const UNCHANGED_TOLERANCE = 1e-6;function viewDepthRange(modelView, { minimum, maximum }) {@@ -35,7 +37,8 @@ export function createSplatSorter(positions, count, bounds) {function sort(modelView) {const axis = [modelView[2], modelView[6], modelView[10], modelView[14]];- const unchanged = axis.every((value, index) => Math.abs(value - lastAxis[index]) <= UNCHANGED_TOLERANCE);+ const tolerance = UNCHANGED_TOLERANCE * Math.hypot(axis[0], axis[1], axis[2]);+ const unchanged = axis.every((value, index) => Math.abs(value - lastAxis[index]) <= tolerance);if (unchanged) return { order, visibleCount, changed: false };lastAxis.set(axis);const { nearest, farthest } = viewDepthRange(modelView, bounds);diff --git a/src/gaussian-splats.js b/src/gaussian-splats.jsindex 083e7dde..5b36350a 100644--- a/src/gaussian-splats.js+++ b/src/gaussian-splats.js@@ -122,8 +122,16 @@ void main() {const UNIFORM_NAMES = ['splatData', 'modelView', 'projection', 'focal', 'viewport', 'hdrEnabled', 'exposure', 'alphaCut'];export const GAUSSIAN_SPLAT_SHADER_SOURCES = Object.freeze({ vertex: VERTEX_SOURCE, fragment: FRAGMENT_SOURCE, opaqueFragment: OPAQUE_FRAGMENT_SOURCE });-function covarianceOf(scales, rotations, index) {+// A quaternion of any length stands for the rotation of its unit version; all zero stands for no rotation.+function unitQuaternion(rotations, index) {const [w, x, y, z] = rotations.subarray(index * 4, index * 4 + 4);+ const length = Math.hypot(w, x, y, z);+ if (!(length > 0) || !Number.isFinite(length)) return length === 0 ? [1, 0, 0, 0] : [w, x, y, z];+ return [w / length, x / length, y / length, z / length];+}++function covarianceOf(scales, rotations, index) {+ const [w, x, y, z] = unitQuaternion(rotations, index);const rotation = [1 - 2 * (y * y + z * z), 2 * (x * y + w * z), 2 * (x * z - w * y),2 * (x * y - w * z), 1 - 2 * (x * x + z * z), 2 * (y * z + w * x),@@ -180,6 +188,13 @@ function checkSplatArray(data, { field, stride, types, meaning }, caller) {}}+function checkSplatBounds(bounds, caller) {+ const isCorner = corner => (Array.isArray(corner) || ArrayBuffer.isView(corner)) && corner.length === 3 && Array.from(corner).every(Number.isFinite);+ if (!isCorner(bounds?.minimum) || !isCorner(bounds?.maximum)) {+ throw new TypeError(`${caller} needs data.bounds as { minimum, maximum }, each three finite numbers (the box that holds every splat centre; parseGaussianSplatPly returns it). Pass the bounds from the parse result`);+ }+}+function resolveSplatOptions(options, caller) {if (options === undefined) return {};if (options === null || typeof options !== 'object') {@@ -205,6 +220,7 @@ function checkSplatInput(data, maximumCount, caller) {if (!Number.isInteger(data.count)) throw new RangeError(`${caller} needs data.count to be a whole number of splats, got ${data.count}`);if (data.count > limit) throw new RangeError(`Gaussian splat scene holds ${data.count} splats, over the limit of ${limit}`);for (const description of SPLAT_ARRAYS) checkSplatArray(data, description, caller);+ checkSplatBounds(data.bounds, caller);}function resolveAlphaCut(alphaCut, caller) {diff --git a/tests/gaussian-splat-robustness.test.js b/tests/gaussian-splat-robustness.test.jsnew file mode 100644index 00000000..6af8f537--- /dev/null+++ b/tests/gaussian-splat-robustness.test.js@@ -0,0 +1,51 @@+import assert from 'node:assert/strict';+import test from 'node:test';+import { createGaussianSplats, createGaussianSplatsAsync } from '../src/gaussian-splats.js';+import { createSplatSorter } from '../src/gaussian-splat-sort.js';++function buildSplatData(rotation = [1, 0, 0, 0]) {+ return {+ count: 1,+ positions: new Float32Array(3),+ colors: new Uint8Array(4),+ scales: Float32Array.of(1, 2, 3),+ rotations: Float32Array.from(rotation),+ bounds: { minimum: [0, 0, 0], maximum: [1, 1, 1] },+ };+}++function packedCovariance(data) {+ return Array.from(new Float32Array(createGaussianSplats(data).packed.words.buffer).subarray(4, 10));+}++test('a rotation that is not unit length packs the same covariance as its unit version', () => {+ const unit = packedCovariance(buildSplatData([Math.SQRT1_2, 0, 0, Math.SQRT1_2]));+ const doubled = packedCovariance(buildSplatData([2 * Math.SQRT1_2, 0, 0, 2 * Math.SQRT1_2]));+ doubled.forEach((value, index) => assert.ok(Math.abs(value - unit[index]) < 1e-5, `entry ${index}: ${value} against ${unit[index]}`));+});++test('an all zero rotation packs the covariance of no rotation', () => {+ const none = packedCovariance(buildSplatData());+ assert.deepEqual(packedCovariance(buildSplatData([0, 0, 0, 0])), none);+});++for (const [name, build] of [['createGaussianSplats', createGaussianSplats], ['createGaussianSplatsAsync', createGaussianSplatsAsync]]) {+ test(`${name} refuses bounds that are missing or not finite before drawing`, async () => {+ const badBounds = [undefined, { minimum: [0, 0, 0] }, { minimum: [0, 0], maximum: [1, 1, 1] }, { minimum: [0, 0, NaN], maximum: [1, 1, 1] }];+ for (const bounds of badBounds) {+ await assert.rejects(async () => build({ ...buildSplatData(), bounds }), error => error instanceof TypeError && /data\.bounds/.test(error.message) && /minimum, maximum/.test(error.message));+ }+ });+}++test('the sorter re-sorts when the view turns round under a tiny uniform scale', () => {+ const positions = Float32Array.of(0, 0, -1e6, 0, 0, -3e6);+ const sorter = createSplatSorter(positions, 2, { minimum: [0, 0, -3e6], maximum: [0, 0, -1e6] });+ const scale = 1e-8;+ const forward = Float32Array.of(scale, 0, 0, 0, 0, scale, 0, 0, 0, 0, scale, 0, 0, 0, 0, 1);+ const turned = Float32Array.of(scale, 0, 0, 0, 0, scale, 0, 0, 0, 0, -scale, 0, 0, 0, 0, 1);+ assert.deepEqual(Array.from(sorter.sort(forward).order.subarray(0, 2)), [1, 0]);+ const result = sorter.sort(turned);+ assert.equal(result.changed, true, 'a turned view must be reported as changed');+ assert.equal(result.visibleCount, 0, 'both splats are now behind the camera');+});