Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Refuse a native export whose tables together exceed one message, and reject --trace with --no-encode (#1055)

Nguyen Xuan An9ac70494c0ea
diff --git a/bench/cycles-parity/native-export-size-check.mjs b/bench/cycles-parity/native-export-size-check.mjs
new file mode 100644
index 00000000..2e46bec4
--- /dev/null
+++ b/bench/cycles-parity/native-export-size-check.mjs
@@ -0,0 +1,25 @@
+// Job-level check of the export size rule (#1055): about 1,500,000 unique triangles with 150,000 instances stay inside both job bounds but
+// not inside one message, so the export must refuse with the new RangeError; the same triangles with 20,000 instances must export.
+// NODE_OPTIONS=--max-old-space-size=16384 node bench/cycles-parity/native-export-size-check.mjs
+import * as pack from '../../src/path-tracer-gpu-pack.js';
+import { buildNativeInput } from '../../packages/render-native/pack-input.mjs';
+import { buildFlattenedJobNativeInput, buildJobNativeInput } from '../../packages/render-native/job-input.mjs';
+import { buildTraceScene, refuseLocalVolumes, resolveTexture } from '../../src/path-tracer-scene.js';
+import { NATIVE_EXPORT_MAXIMUM_CHARACTERS, exportNativeSceneTables } from '../../scripts/native-bake-page.js';
+import { NATIVE_CAP_ENVIRONMENT, NATIVE_CAP_PACK_OPTIONS, buildNativeCapParts, nativeCapCamera } from './native-cap-scene.mjs';
+import { buildCardField } from './native-instance-scene.mjs';
+
+const modules = { buildTraceScene, refuseLocalVolumes, resolveTexture, buildNativeInput, buildJobNativeInput, buildFlattenedJobNativeInput, pack };
+
+function exportWithCards(cardCount) {
+ const parts = [...buildNativeCapParts(1_500_000, true).parts, ...buildCardField(cardCount, cardCount)];
+ try {
+ const exported = exportNativeSceneTables({ parts, environment: NATIVE_CAP_ENVIRONMENT, camera: nativeCapCamera(), options: NATIVE_CAP_PACK_OPTIONS, kernel: 'cycles', instancing: true, modules });
+ const characters = Object.values(exported.tables).reduce((total, text) => total + text.length, 0);
+ return { cardCount, outcome: 'exported', instances: exported.instances, triangles: exported.triangles, characters, limit: NATIVE_EXPORT_MAXIMUM_CHARACTERS, instancing: exported.instancing ?? 'kept' };
+ } catch (error) {
+ return { cardCount, outcome: 'refused', exportTooLarge: error.exportTooLarge === true, message: error.message };
+ }
+}
+
+for (const cardCount of [20_000, 150_000]) console.log(`NATIVE_EXPORT_SIZE ${JSON.stringify(exportWithCards(cardCount))}`);
diff --git a/bench/cycles-parity/native-instance-measure.mjs b/bench/cycles-parity/native-instance-measure.mjs
index 9e292e88..28d26faf 100644
--- a/bench/cycles-parity/native-instance-measure.mjs
+++ b/bench/cycles-parity/native-instance-measure.mjs
@@ -74,6 +74,10 @@ const count = Number(readFlag(argumentList, '--count', 250000));
const variant = readFlag(argumentList, '--variant', 'instanced');
const samples = Number(readFlag(argumentList, '--samples', 16));
const encode = !argumentList.includes('--no-encode');
+if (argumentList.includes('--trace') && !encode) {
+ console.error('--trace needs the encoded tables: drop --no-encode, or drop --trace.');
+ process.exit(2);
+}
const redFrom = Math.max(0, count - Math.floor(count / 4));
const camera = cardFieldCamera(count);
const result = { variant, count };
diff --git a/docs/cycles-parity.md b/docs/cycles-parity.md
index 64fd6eaf..8c5bc760 100644
--- a/docs/cycles-parity.md
+++ b/docs/cycles-parity.md
@@ -1460,7 +1460,7 @@ Measured on an RTX 4090 box, one process per row, a field of two-triangle cards
| 500,001 | 3.24 | 7.13 | 128.0 | 277.3 M | 2,363 | 0.73 | 0.100 | 796 |
| 1,000,000 | 7.17 | 14.34 | 256.0 | 554.7 M | 4,511 | 1.59 | 0.193 | 1,557 |
-Each instance costs 416 bytes packed (160 for the instance and 256 for its material) and 555 base64 characters. What bounds it `[measured, inferred for the reason]`: the table bytes and V8 strings, not the card or the acceleration build. At 1,000,000 instances the largest table is 341 M characters, which encodes, but the tables together are 554.7 M characters, above the 536,870,888 of one V8 string, so a page that returns the export as one string (a JSON message) would fail; 500,000 instances are 277 M characters, 52% of that limit. The job stops at 500,000 for this reason, and the library ceiling is 1,000,000, the size measured on the card. The card was never the limit: the acceleration build took 0.19 s at 1,000,000 instances and the traces ran in 1.6 s. The hardware allows 16,777,216 instances, and nothing above 1,000,000 instances was measured.
+Each instance costs 416 bytes packed (160 for the instance and 256 for its material) and 555 base64 characters. What bounds it `[measured, inferred for the reason]`: the table bytes and V8 strings, not the card or the acceleration build. At 1,000,000 instances the largest table is 341 M characters, which encodes, but the tables together are 554.7 M characters, above the 536,870,888 of one V8 string, so a page that returns the export as one string (a JSON message) would fail; 500,000 instances alone are 277 M characters, 52% of that limit, but the tables of one export share the message, so the bounds are not independent. The export sums the real base64 length of every table (`4 * ceil(bytes / 3)` each) before encoding, and refuses above 95% of the V8 limit (510,027,343 characters, `NATIVE_EXPORT_MAXIMUM_CHARACTERS`) with a `RangeError` that names the total and the largest tables and says to pass fewer parts or fewer unique triangles. The job rethrows that refusal (as it does the texture pool one), so the job report says why the native export failed and no other packing is tried. Worked example `[measured sizes]`: 1,500,000 unique triangles are 448 M characters (299 each), which leaves 62 M for instances at 555 each: about 110,000 instances (the 65,536 of before, 36 M, still fit; 150,000 instances, 83 M, give 531 M and are refused). Without triangles, 500,000 instances (277 M) fit alone. The job stops at 500,000 instances, and the library ceiling is 1,000,000, the size measured on the card. The card was never the limit: the acceleration build took 0.19 s at 1,000,000 instances and the traces ran in 1.6 s. The hardware allows 16,777,216 instances, and nothing above 1,000,000 instances was measured.
Pictures: scenes under the old bounds are byte for byte the same. The PNG SHA-256 of three scenes (the 20,000 triangle cap scene with copies, a field of 3,000 shared cards and the same cards merged) on the job export of `cycles` instanced, `cycles` flattened and `legacy` are equal on `main` and on this change, and the refusal of the scene the flattened and legacy routes cannot take has the same words (`bench/cycles-parity/native-identity-pngs.mjs`, `[measured]`). Over the old bound, `tests/native-instancing-limit.test.js` traces 85,537 parts (the last 20,000 red, all past index 65,536) and the same cards merged into two meshes: 3,381 red pixels in both, mean ratio within 2% per channel, tiles within 5%, and a relative mean squared error of 1.5e-7, against 1.4e-3 between two seeds of the merged picture.
diff --git a/docs/path-tracer.md b/docs/path-tracer.md
index 7c6eee5a..0809257d 100644
--- a/docs/path-tracer.md
+++ b/docs/path-tracer.md
@@ -61,7 +61,7 @@ Also checked before any ray is traced: samples times pixels times bounces at mos
`buildTraceScene(parts, environment, { instancing: true })` keeps parts that share one geometry object as instances of it, and `buildNativeInputV2` packs them as one bottom-level structure per geometry plus one instance record per part, for `traceNative` with `kernel: "cycles"`. Without the option the scene is flattened as before, and a scene where no two parts share a geometry is flattened too, with output identical to the default.
- Sharing is decided after the trace geometry is resolved. A skinned, flush-lifted or box-mapped part gets a geometry of its own, and a geometry with per-triangle materials is not shared either: those parts are moved into world space once and traced as instances with an identity matrix.
-- Limits: 400,000 triangles counted once per shared geometry, 1,024 distinct geometries (the part limit, counted on geometries) and 65,536 instances. `buildTraceScene(parts, environment, { instancing: true, maximumTriangles, maximumInstances })` replaces the 400,000 for a scene exported to the native tracer: an integer from 1 to 4,000,000 (`MAX_NATIVE_TRACE_TRIANGLES`) bounds the unique triangles (all triangles of a flattened scene), and four times that bounds the triangles with every copy counted. A value outside that range is refused, and the CPU and WebGL tracers never pass it, so they keep 400,000 and their error text. `maximumInstances` does the same for the 65,536 instances: an integer from 1 to 1,000,000 (`MAX_NATIVE_TRACE_INSTANCES`), counted over the parts of an instanced scene (each part is one instance, and the unique-geometry bound stays 1,024). A value outside that range is refused, and a flattened scene ignores it. A native render job passes 1,500,000 triangles and 500,000 instances (see "Triangle bound of a native scene" and "Instance bound of a native scene" in [cycles-parity.md](./cycles-parity.md)). An instanced scene has no world-space triangle arrays or CPU BVH, so only `buildNativeInputV2` reads it.
+- Limits: 400,000 triangles counted once per shared geometry, 1,024 distinct geometries (the part limit, counted on geometries) and 65,536 instances. `buildTraceScene(parts, environment, { instancing: true, maximumTriangles, maximumInstances })` replaces the 400,000 for a scene exported to the native tracer: an integer from 1 to 4,000,000 (`MAX_NATIVE_TRACE_TRIANGLES`) bounds the unique triangles (all triangles of a flattened scene), and four times that bounds the triangles with every copy counted. A value outside that range is refused, and the CPU and WebGL tracers never pass it, so they keep 400,000 and their error text. `maximumInstances` does the same for the 65,536 instances: an integer from 1 to 1,000,000 (`MAX_NATIVE_TRACE_INSTANCES`), counted over the parts of an instanced scene (each part is one instance, and the unique-geometry bound stays 1,024). A value outside that range is refused, and a flattened scene ignores it. A native render job passes 1,500,000 triangles and 500,000 instances, and its export also refuses a scene whose tables together exceed 95% of one V8 string (about 299 characters per unique triangle and 555 per instance: 1,500,000 triangles leave room for about 110,000 instances; see "Triangle bound of a native scene" and "Instance bound of a native scene" in [cycles-parity.md](./cycles-parity.md)). An instanced scene has no world-space triangle arrays or CPU BVH, so only `buildNativeInputV2` reads it.
- A part's material is its own entry in `materials`, so copies may differ in material; a multi-material mesh stays one geometry with the materials the part names.
- `buildNativeInput` (the `legacy` kernel) refuses an instanced scene with an error; build the scene without `instancing` for that kernel. `traceNative` with `kernel: "legacy"` refuses a FORMAT-v2 input the same way.
- Render jobs with `renderer.backend: "native"` keep repeated geometry as instances when the job traces with the `cycles` kernel (the default): the job page exports FORMAT-v2 tables of the unique triangles and one record per part, the job runner rebuilds the input for `traceNative`, and the report names the `instances` and counts `triangles` once per shared geometry. With `JS67_NATIVE_KERNEL=legacy`, `JS67_NATIVE_INSTANCING=off` (flattened triangles on the `cycles` kernel, to compare the two), or a scene where no geometry is shared, the job exports flattened geometry as before; on the `cycles` kernel the materials are still sent whole (next point), and only the legacy kernel reads the v1 tables.
diff --git a/scripts/native-bake-page.js b/scripts/native-bake-page.js
index 15aeeb01..8fcca180 100644
--- a/scripts/native-bake-page.js
+++ b/scripts/native-bake-page.js
@@ -49,6 +49,35 @@ export const NATIVE_JOB_MAXIMUM_TRIANGLES = 1_500_000;
// The instance bound of the same export (see buildTraceScene's maximumInstances).
export const NATIVE_JOB_MAXIMUM_INSTANCES = 500_000;
+// Every table of the export travels as base64 inside one evaluate result, one JSON string, and V8 refuses a string above
+// 536,870,888 characters. The bounds above are independent (about 299 characters per unique triangle, 555 per instance), so the
+// export checks the sum of its real tables against this share of the limit; the rest is the JSON around the tables.
+export const V8_MAXIMUM_STRING_LENGTH = 536_870_888;
+export const NATIVE_EXPORT_MAXIMUM_CHARACTERS = Math.floor(V8_MAXIMUM_STRING_LENGTH * 0.95);
+
+/** The base64 length of tables of these byte lengths: `4 * ceil(bytes / 3)` each, summed. */
+export function encodedExportLength(byteLengths) {
+ return byteLengths.reduce((total, bytes) => total + 4 * Math.ceil(bytes / 3), 0);
+}
+
+/**
+ * Refuses tables, given as `[name, byteLength]` pairs, that would not fit one message. The error carries `exportTooLarge`, so the
+ * export rethrows it (like `texturePoolFull`) and the job reports it instead of trying another packing of the same scene.
+ */
+export function requireTableSizesFit(tables) {
+ const total = encodedExportLength(tables.map(([, bytes]) => bytes));
+ if (total <= NATIVE_EXPORT_MAXIMUM_CHARACTERS) return;
+ const largest = [...tables].sort((first, second) => second[1] - first[1]).slice(0, 3).map(([name, bytes]) => `${name} ${encodedExportLength([bytes])}`).join(', ');
+ const error = new RangeError(`The native scene export is ${total} base64 characters in total (largest tables: ${largest}), over the ${NATIVE_EXPORT_MAXIMUM_CHARACTERS} one message can carry. Pass fewer parts (each instance takes about 555 characters), fewer unique triangles (about 299 each), or smaller textures.`);
+ error.exportTooLarge = true;
+ throw error;
+}
+
+/** `requireTableSizesFit` over the typed-array tables of a packed input. */
+export function requireExportFits(input) {
+ requireTableSizesFit(Object.entries(input).filter(([, values]) => ArrayBuffer.isView(values)).map(([name, values]) => [name, values.byteLength]));
+}
+
/**
* The scene of a native render job as the tables the job runner traces. With the "cycles" kernel (`kernel`) every scene is
* exported as FORMAT-v2, so its materials reach the kernel whole; parts that share a geometry stay instances when
@@ -68,10 +97,11 @@ export function exportNativeSceneTables({ parts, environment, camera, options, k
if (traceScene.instances) {
try {
const input = buildJobNativeInput(traceScene, camera, options, pack, resolveTexture);
+ requireExportFits(input);
return { ...encodeNativeInputV2(input), triangleCount: input.triangleCount, triangles: input.triangleCount, instances: input.instanceCount, wholeMaterials };
} catch (error) {
// A scene over the texture pool stays over it flattened: the caller gets that refusal, not a v1 atlas one.
- if (!(error instanceof RangeError) || error.texturePoolFull) throw error;
+ if (!(error instanceof RangeError) || error.texturePoolFull || error.exportTooLarge) throw error;
flattenedBecause = error.message;
traceScene = flattenRefusedScene(buildTraceScene, parts, environment, nativeLimits, flattenedBecause);
}
@@ -81,9 +111,10 @@ export function exportNativeSceneTables({ parts, environment, camera, options, k
if (wholeMaterials) {
try {
const input = buildFlattenedJobNativeInput(traceScene, camera, options, pack, resolveTexture);
+ requireExportFits(input);
return { ...encodeNativeInputV2(input), triangleCount: input.triangleCount, triangles: traceScene.indices.length / 3, wholeMaterials, ...instancingNote };
} catch (error) {
- if (error.texturePoolFull) throw error;
+ if (error.texturePoolFull || error.exportTooLarge) throw error;
// The scene stays traceable: the v1 tables carry what the kernel read before, and the report says why they were used.
materialsReason = error.message;
}
@@ -94,6 +125,7 @@ export function exportNativeSceneTables({ parts, environment, camera, options, k
// The v1 tables of the "cycles" job fallback cannot carry visibility: the scene is drawn without it, and the report says so.
const visibilityNote = traceScene.partRayVisibility ? { rayVisibility: 'ignored', rayVisibilityReason: materialsReason } : {};
const input = buildNativeInputOrExplain(buildNativeInput, { ...traceScene, partRayVisibility: null }, camera, options, pack, materialsReason);
+ requireExportFits(input);
return { tables: encodeNativeTables(input), triangleCount: input.triangleCount, triangles: traceScene.indices.length / 3, wholeMaterials: false,
...(materialsReason === undefined ? {} : { materialsReason }), ...visibilityNote, ...instancingNote };
}
diff --git a/tests/path-tracer-native-instance-limit.test.js b/tests/path-tracer-native-instance-limit.test.js
index 8d676a32..3a050840 100644
--- a/tests/path-tracer-native-instance-limit.test.js
+++ b/tests/path-tracer-native-instance-limit.test.js
@@ -2,7 +2,15 @@ import assert from 'node:assert/strict';
import test from 'node:test';
import { buildNativeInput } from '../packages/render-native/pack-input.mjs';
import { buildFlattenedJobNativeInput, buildJobNativeInput } from '../packages/render-native/job-input.mjs';
-import { exportNativeSceneTables, NATIVE_JOB_MAXIMUM_INSTANCES } from '../scripts/native-bake-page.js';
+import {
+ encodedExportLength,
+ exportNativeSceneTables,
+ NATIVE_EXPORT_MAXIMUM_CHARACTERS,
+ NATIVE_JOB_MAXIMUM_INSTANCES,
+ NATIVE_JOB_MAXIMUM_TRIANGLES,
+ requireTableSizesFit,
+ V8_MAXIMUM_STRING_LENGTH,
+} from '../scripts/native-bake-page.js';
import { createCamera } from '../src/path-tracer.js';
import * as pack from '../src/path-tracer-gpu-pack.js';
import {
@@ -95,3 +103,52 @@ test('a native render job with the legacy kernel keeps its flattened bounds, not
assert.throws(() => exportNativeSceneTables({ parts: copiesOfTriangle(MAX_TRACE_INSTANCES + 1), environment: ENVIRONMENT, camera, options: packOptions, kernel: 'legacy', instancing: true, modules }),
{ name: 'RangeError', message: /1 to 1024 parts/ });
});
+
+const TRIANGLE_RECORD_BYTES = 224;
+const INSTANCE_AND_MATERIAL_BYTES = 160 + 256;
+
+test('the base64 length of tables is 4 * ceil(bytes / 3) each, as Buffer measures it', () => {
+ for (const bytes of [0, 1, 2, 3, 4, 5, 299, 1000]) {
+ assert.equal(encodedExportLength([bytes]), Buffer.alloc(bytes).toString('base64').length, `${bytes} bytes`);
+ }
+ assert.equal(encodedExportLength([1, 2, 3]), 4 + 4 + 4);
+ assert.equal(encodedExportLength([]), 0);
+});
+
+test('the export share is under the V8 string limit', () => {
+ assert.equal(V8_MAXIMUM_STRING_LENGTH, 536_870_888);
+ assert.ok(NATIVE_EXPORT_MAXIMUM_CHARACTERS < V8_MAXIMUM_STRING_LENGTH);
+ assert.ok(NATIVE_EXPORT_MAXIMUM_CHARACTERS > 0.9 * V8_MAXIMUM_STRING_LENGTH);
+});
+
+test('the job triangle bound with the 65,536 instances of before still fits one message', () => {
+ const tables = [['triangles', NATIVE_JOB_MAXIMUM_TRIANGLES * TRIANGLE_RECORD_BYTES], ['instances and materials', 65_536 * INSTANCE_AND_MATERIAL_BYTES]];
+ requireTableSizesFit(tables);
+});
+
+test('both job bounds at once are refused before encoding, with the totals and what to pass', () => {
+ const tables = [['triangles', NATIVE_JOB_MAXIMUM_TRIANGLES * TRIANGLE_RECORD_BYTES], ['materials', 150_000 * 256], ['instances', 150_000 * 160]];
+ assert.throws(() => requireTableSizesFit(tables), error => {
+ assert.equal(error.name, 'RangeError');
+ assert.equal(error.exportTooLarge, true);
+ assert.match(error.message, new RegExp(`${encodedExportLength(tables.map(([, bytes]) => bytes))} base64 characters in total`));
+ assert.match(error.message, /largest tables: triangles 448000000, materials 51200000/);
+ assert.match(error.message, /Pass fewer parts.*fewer unique triangles/);
+ return true;
+ });
+});
+
+test('an export over one message is rethrown, not flattened or packed another way', () => {
+ const huge = new Uint8Array(1);
+ Object.defineProperty(huge, 'byteLength', { value: 400_000_000 });
+ let flattenedCalls = 0;
+ const countingFlatten = () => {
+ flattenedCalls++;
+ throw new Error('the flattened packer must not run');
+ };
+ const oversized = () => ({ triangles: huge, triangleCount: 1, instanceCount: 2 });
+ assert.throws(() => exportNativeSceneTables({ parts: copiesOfTriangle(2), environment: ENVIRONMENT, camera, options: packOptions, kernel: 'cycles', instancing: true,
+ modules: { ...modules, buildJobNativeInput: oversized, buildFlattenedJobNativeInput: countingFlatten } }),
+ error => error.exportTooLarge === true && /base64 characters in total/.test(error.message));
+ assert.equal(flattenedCalls, 0);
+});