Skip to content

3D engine that AI agents drive

Harden splat packing and sorting for hand-built data (#2435)

Nguyen Xuan Anff75fe1d1c82
diff --git a/changes/2435.md b/changes/2435.md
new file mode 100644
index 00000000..c5ada5e9
--- /dev/null
+++ b/changes/2435.md
@@ -0,0 +1 @@
+- Gaussian splats: a rotation of any length packs its unit covariance, `createGaussianSplats` checks `bounds` at the call, and the sorter re-sorts tiny scaled scenes (#2435, #2436, #2437).
diff --git a/docs/gaussian-splats.md b/docs/gaussian-splats.md
index ed55af55..82dfcfd1 100644
--- a/docs/gaussian-splats.md
+++ b/docs/gaussian-splats.md
@@ -60,7 +60,7 @@ const splats = await createGaussianSplatsAsync(data, options);
## Hand-built data
-Data written by code, not parsed, is checked at the call. `count` must be a whole number, `positions` and `scales` need `count * 3` values (Float32Array or Float64Array), `rotations` `count * 4` (w, x, y, z) and `colors` `count * 4` bytes (Uint8Array, starting on a 4-byte boundary of its buffer). A longer array is fine, a shorter or missing one throws an error that names the field, the length it needs and the length it got. `matrix` and the options object must be left out or be a 16-number array and an object; `null` throws. The same checks run in `createGaussianSplatsAsync` before any splat is packed.
+Data written by code, not parsed, is checked at the call. `count` must be a whole number, `positions` and `scales` need `count * 3` values (Float32Array or Float64Array), `rotations` `count * 4` (w, x, y, z) and `colors` `count * 4` bytes (Uint8Array, starting on a 4-byte boundary of its buffer). A longer array is fine, a shorter or missing one throws an error that names the field, the length it needs and the length it got. `bounds` must be `{ minimum, maximum }`, each three finite numbers (the parse result carries it), or a `TypeError` says so. A rotation of any length counts as its unit version, and all zero as no rotation. `matrix` and the options object must be left out or be a 16-number array and an object; `null` throws. The same checks run in `createGaussianSplatsAsync` before any splat is packed.
## What it draws
diff --git a/src/gaussian-splat-sort.js b/src/gaussian-splat-sort.js
index 72f32c9a..3074af74 100644
--- a/src/gaussian-splat-sort.js
+++ b/src/gaussian-splat-sort.js
@@ -4,6 +4,8 @@
// out of the order. Splats closer than one step of depth (the span divided by 65536) may draw in either order.
const DEPTH_STEPS = 65536;
+// Relative to the length of the depth axis, which carries any uniform scale of the splat matrix: a fixed tolerance
+// would never re-sort a scene scaled down to 1e-6.
const UNCHANGED_TOLERANCE = 1e-6;
function viewDepthRange(modelView, { minimum, maximum }) {
@@ -35,7 +37,8 @@ export function createSplatSorter(positions, count, bounds) {
function sort(modelView) {
const axis = [modelView[2], modelView[6], modelView[10], modelView[14]];
- const unchanged = axis.every((value, index) => Math.abs(value - lastAxis[index]) <= UNCHANGED_TOLERANCE);
+ const tolerance = UNCHANGED_TOLERANCE * Math.hypot(axis[0], axis[1], axis[2]);
+ const unchanged = axis.every((value, index) => Math.abs(value - lastAxis[index]) <= tolerance);
if (unchanged) return { order, visibleCount, changed: false };
lastAxis.set(axis);
const { nearest, farthest } = viewDepthRange(modelView, bounds);
diff --git a/src/gaussian-splats.js b/src/gaussian-splats.js
index 083e7dde..5b36350a 100644
--- a/src/gaussian-splats.js
+++ b/src/gaussian-splats.js
@@ -122,8 +122,16 @@ void main() {
const UNIFORM_NAMES = ['splatData', 'modelView', 'projection', 'focal', 'viewport', 'hdrEnabled', 'exposure', 'alphaCut'];
export const GAUSSIAN_SPLAT_SHADER_SOURCES = Object.freeze({ vertex: VERTEX_SOURCE, fragment: FRAGMENT_SOURCE, opaqueFragment: OPAQUE_FRAGMENT_SOURCE });
-function covarianceOf(scales, rotations, index) {
+// A quaternion of any length stands for the rotation of its unit version; all zero stands for no rotation.
+function unitQuaternion(rotations, index) {
const [w, x, y, z] = rotations.subarray(index * 4, index * 4 + 4);
+ const length = Math.hypot(w, x, y, z);
+ if (!(length > 0) || !Number.isFinite(length)) return length === 0 ? [1, 0, 0, 0] : [w, x, y, z];
+ return [w / length, x / length, y / length, z / length];
+}
+
+function covarianceOf(scales, rotations, index) {
+ const [w, x, y, z] = unitQuaternion(rotations, index);
const rotation = [
1 - 2 * (y * y + z * z), 2 * (x * y + w * z), 2 * (x * z - w * y),
2 * (x * y - w * z), 1 - 2 * (x * x + z * z), 2 * (y * z + w * x),
@@ -180,6 +188,13 @@ function checkSplatArray(data, { field, stride, types, meaning }, caller) {
}
}
+function checkSplatBounds(bounds, caller) {
+ const isCorner = corner => (Array.isArray(corner) || ArrayBuffer.isView(corner)) && corner.length === 3 && Array.from(corner).every(Number.isFinite);
+ if (!isCorner(bounds?.minimum) || !isCorner(bounds?.maximum)) {
+ throw new TypeError(`${caller} needs data.bounds as { minimum, maximum }, each three finite numbers (the box that holds every splat centre; parseGaussianSplatPly returns it). Pass the bounds from the parse result`);
+ }
+}
+
function resolveSplatOptions(options, caller) {
if (options === undefined) return {};
if (options === null || typeof options !== 'object') {
@@ -205,6 +220,7 @@ function checkSplatInput(data, maximumCount, caller) {
if (!Number.isInteger(data.count)) throw new RangeError(`${caller} needs data.count to be a whole number of splats, got ${data.count}`);
if (data.count > limit) throw new RangeError(`Gaussian splat scene holds ${data.count} splats, over the limit of ${limit}`);
for (const description of SPLAT_ARRAYS) checkSplatArray(data, description, caller);
+ checkSplatBounds(data.bounds, caller);
}
function resolveAlphaCut(alphaCut, caller) {
diff --git a/tests/gaussian-splat-robustness.test.js b/tests/gaussian-splat-robustness.test.js
new file mode 100644
index 00000000..6af8f537
--- /dev/null
+++ b/tests/gaussian-splat-robustness.test.js
@@ -0,0 +1,51 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createGaussianSplats, createGaussianSplatsAsync } from '../src/gaussian-splats.js';
+import { createSplatSorter } from '../src/gaussian-splat-sort.js';
+
+function buildSplatData(rotation = [1, 0, 0, 0]) {
+ return {
+ count: 1,
+ positions: new Float32Array(3),
+ colors: new Uint8Array(4),
+ scales: Float32Array.of(1, 2, 3),
+ rotations: Float32Array.from(rotation),
+ bounds: { minimum: [0, 0, 0], maximum: [1, 1, 1] },
+ };
+}
+
+function packedCovariance(data) {
+ return Array.from(new Float32Array(createGaussianSplats(data).packed.words.buffer).subarray(4, 10));
+}
+
+test('a rotation that is not unit length packs the same covariance as its unit version', () => {
+ const unit = packedCovariance(buildSplatData([Math.SQRT1_2, 0, 0, Math.SQRT1_2]));
+ const doubled = packedCovariance(buildSplatData([2 * Math.SQRT1_2, 0, 0, 2 * Math.SQRT1_2]));
+ doubled.forEach((value, index) => assert.ok(Math.abs(value - unit[index]) < 1e-5, `entry ${index}: ${value} against ${unit[index]}`));
+});
+
+test('an all zero rotation packs the covariance of no rotation', () => {
+ const none = packedCovariance(buildSplatData());
+ assert.deepEqual(packedCovariance(buildSplatData([0, 0, 0, 0])), none);
+});
+
+for (const [name, build] of [['createGaussianSplats', createGaussianSplats], ['createGaussianSplatsAsync', createGaussianSplatsAsync]]) {
+ test(`${name} refuses bounds that are missing or not finite before drawing`, async () => {
+ const badBounds = [undefined, { minimum: [0, 0, 0] }, { minimum: [0, 0], maximum: [1, 1, 1] }, { minimum: [0, 0, NaN], maximum: [1, 1, 1] }];
+ for (const bounds of badBounds) {
+ await assert.rejects(async () => build({ ...buildSplatData(), bounds }), error => error instanceof TypeError && /data\.bounds/.test(error.message) && /minimum, maximum/.test(error.message));
+ }
+ });
+}
+
+test('the sorter re-sorts when the view turns round under a tiny uniform scale', () => {
+ const positions = Float32Array.of(0, 0, -1e6, 0, 0, -3e6);
+ const sorter = createSplatSorter(positions, 2, { minimum: [0, 0, -3e6], maximum: [0, 0, -1e6] });
+ const scale = 1e-8;
+ const forward = Float32Array.of(scale, 0, 0, 0, 0, scale, 0, 0, 0, 0, scale, 0, 0, 0, 0, 1);
+ const turned = Float32Array.of(scale, 0, 0, 0, 0, scale, 0, 0, 0, 0, -scale, 0, 0, 0, 0, 1);
+ assert.deepEqual(Array.from(sorter.sort(forward).order.subarray(0, 2)), [1, 0]);
+ const result = sorter.sort(turned);
+ assert.equal(result.changed, true, 'a turned view must be reported as changed');
+ assert.equal(result.visibleCount, 0, 'both splats are now behind the camera');
+});