3D engine that AI agents drive
Let a scene built for the native path tracer take more than 400,000 triangles (#1041)
Nguyen Xuan An5c9c02068c13
diff --git a/bench/cycles-parity/instanced-scene-in-batches.mjs b/bench/cycles-parity/instanced-scene-in-batches.mjsdeleted file mode 100644index ef653944..00000000--- a/bench/cycles-parity/instanced-scene-in-batches.mjs+++ /dev/null@@ -1,106 +0,0 @@-// buildTraceScene({ instancing: true }) refuses more than MAX_TRACE_TRIANGLES (400,000) unique triangles, counted once per shared-// geometry. The native cycles kernel has no such limit (bench/cycles-parity/large-scene.mjs packs bigger scenes directly), and the-// Classroom scene holds 427,769 unique triangles. This builds the instanced scene in two batches that each fit under the cap and joins-// their instance and material lists, so the bench needs no change to src/.-import { MAX_TRACE_TRIANGLES } from '../../src/path-tracer-scene.js';-import { buildTraceSceneWithManyLights } from './trace-scene-many-lights.mjs';--function triangleCountOf(geometry) {- return geometry.indices.length / 3;-}--function groupPartsByGeometry(parts) {- const groups = new Map();- for (const part of parts) {- if (!groups.has(part.geometry)) {- groups.set(part.geometry, []);- }- groups.get(part.geometry).push(part);- }- return groups;-}--function uniqueTriangleTotal(geometries) {- let total = 0;- for (const geometry of geometries) {- total += triangleCountOf(geometry);- }- return total;-}--/**- * Splits the geometries into one batch that fits the cap and, when the scene is over it, a second batch seeded with the biggest- * shared geometries, so each batch still has shared geometry (a batch without any is flattened by buildTraceScene).- */-export function splitGeometriesIntoBatches(groups) {- const allGeometries = [...groups.keys()];- if (uniqueTriangleTotal(allGeometries) <= MAX_TRACE_TRIANGLES) {- return [allGeometries];- }- const sharedBySize = allGeometries- .filter((geometry) => groups.get(geometry).length > 1)- .sort((first, second) => triangleCountOf(second) - triangleCountOf(first));- const secondBatch = new Set();- let firstBatch = new Set(allGeometries);- for (const geometry of sharedBySize) {- if (uniqueTriangleTotal(firstBatch) <= MAX_TRACE_TRIANGLES && secondBatch.size > 0) {- break;- }- secondBatch.add(geometry);- firstBatch.delete(geometry);- }- if (uniqueTriangleTotal(firstBatch) > MAX_TRACE_TRIANGLES || uniqueTriangleTotal(secondBatch) > MAX_TRACE_TRIANGLES) {- throw new RangeError('The scene needs more than two batches of instanced geometry; split it further in instanced-scene-in-batches.mjs.');- }- return [[...firstBatch], [...secondBatch]];-}--// FORMAT-v2 instance visibility bits: 1 camera, 2 diffuse, 4 glossy, 8 transmission, 16 shadow, 32 volume scatter.-const RAY_VISIBILITY_BITS = { camera: 1, diffuse: 2, glossy: 4, transmission: 8, shadow: 16, volumeScatter: 32 };-const ALL_RAY_KINDS = 63;--function visibilityFlagsFor(part) {- let flags = ALL_RAY_KINDS;- for (const [kind, isVisible] of Object.entries(part.rayVisibility ?? {})) {- if (isVisible === false) {- flags &= ~RAY_VISIBILITY_BITS[kind];- }- }- return flags;-}--// Each instance of a batch scene is its part, in order; a part that hides from some ray kinds gets those bits cleared.-function applyRayVisibility(scene, batchParts) {- scene.instances.forEach((instance, index) => {- if (batchParts[index].rayVisibility) {- instance.visibilityFlags = visibilityFlagsFor(batchParts[index]);- }- });-}--export function buildInstancedTraceSceneInBatches(parts, environment) {- const groups = groupPartsByGeometry(parts);- const batches = splitGeometriesIntoBatches(groups);- const scenes = batches.map((geometries) => {- const batchParts = geometries.flatMap((geometry) => groups.get(geometry));- const scene = buildTraceSceneWithManyLights(batchParts, environment, { instancing: true });- if (!scene.instances) {- throw new RangeError('A batch of the instanced scene has no shared geometry and was flattened.');- }- applyRayVisibility(scene, batchParts);- return scene;- });- const [firstScene, ...otherScenes] = scenes;- const joined = { ...firstScene, instances: [...firstScene.instances], materials: [...firstScene.materials], traceMaterials: [...firstScene.traceMaterials] };- for (const scene of otherScenes) {- const partOffset = joined.materials.length;- for (const instance of scene.instances) {- joined.instances.push({ ...instance, materialIndex: instance.materialIndex + partOffset, objectId: instance.objectId + partOffset });- }- joined.materials.push(...scene.materials);- joined.traceMaterials.push(...scene.traceMaterials);- joined.surfaceOffset = Math.max(joined.surfaceOffset, scene.surfaceOffset);- joined.farDistance = Math.max(joined.farDistance, scene.farDistance);- }- return { scene: joined, batchCount: batches.length, uniqueTrianglesPerBatch: batches.map(uniqueTriangleTotal) };-}diff --git a/bench/cycles-parity/instanced-scene.mjs b/bench/cycles-parity/instanced-scene.mjsnew file mode 100644index 00000000..5aac3882--- /dev/null+++ b/bench/cycles-parity/instanced-scene.mjs@@ -0,0 +1,37 @@+// The instanced trace scene of a bench scene for the native cycles kernel: copies of one geometry stay instances, and a part that hides+// from some ray kinds gets those visibility bits cleared (buildTraceScene does not read `rayVisibility`). The scene is built with the+// native triangle bound (#1041), so a scene above the 400,000 unique triangles of the CPU and WebGL paths, such as the Classroom, packs directly.+import { MAX_NATIVE_TRACE_TRIANGLES } from '../../src/path-tracer-scene.js';+import { buildTraceSceneWithManyLights } from './trace-scene-many-lights.mjs';++// FORMAT-v2 instance visibility bits: 1 camera, 2 diffuse, 4 glossy, 8 transmission, 16 shadow, 32 volume scatter.+const RAY_VISIBILITY_BITS = { camera: 1, diffuse: 2, glossy: 4, transmission: 8, shadow: 16, volumeScatter: 32 };+const ALL_RAY_KINDS = 63;++function visibilityFlagsFor(part) {+ let flags = ALL_RAY_KINDS;+ for (const [kind, isVisible] of Object.entries(part.rayVisibility ?? {})) {+ if (isVisible === false) {+ flags &= ~RAY_VISIBILITY_BITS[kind];+ }+ }+ return flags;+}++// Each instance of the scene is its part, in order.+function applyRayVisibility(scene, parts) {+ scene.instances.forEach((instance, index) => {+ if (parts[index].rayVisibility) {+ instance.visibilityFlags = visibilityFlagsFor(parts[index]);+ }+ });+}++export function buildInstancedTraceScene(parts, environment) {+ const scene = buildTraceSceneWithManyLights(parts, environment, { instancing: true, maximumTriangles: MAX_NATIVE_TRACE_TRIANGLES });+ if (!scene.instances) {+ throw new RangeError('The scene has no shared geometry and was flattened.');+ }+ applyRayVisibility(scene, parts);+ return scene;+}diff --git a/bench/cycles-parity/native-cap-measure.mjs b/bench/cycles-parity/native-cap-measure.mjsnew file mode 100644index 00000000..58a666a9--- /dev/null+++ b/bench/cycles-parity/native-cap-measure.mjs@@ -0,0 +1,158 @@+// Measures the page-side pack of a scene above the CPU and WebGL triangle bound (#1041), and optionally traces it on the native tracer.+// One child process per case, so the peak memory of a case is its own. Each child prints one NATIVE_CAP_CASE line of JSON.+// node bench/cycles-parity/native-cap-measure.mjs --sizes 400000,1000000 [--variants instanced,flattened] [--trace] [--out <json file>]+// [--png-directory <dir>] [--samples 16]+// Run with NODE_OPTIONS=--max-old-space-size=16384 for sizes above 2,000,000 triangles.+// instanced: buildTraceScene({ instancing: true, maximumTriangles }) + buildJobNativeInput + encodeNativeInputV2, the job export path.+// flattened: the four meshes alone, buildTraceScene({ maximumTriangles }) (world-space triangles and a CPU BVH) + buildFlattenedJobNativeInput + encodeNativeInputV2.+// expanded: the same parts as instanced (copies included) but flattened, the picture to compare the instanced one with.+import { execFile, spawn } from 'node:child_process';+import { createHash } from 'node:crypto';+import { mkdirSync, writeFileSync } from 'node:fs';+import { dirname, join } from 'node:path';+import { fileURLToPath } from 'node:url';+import { promisify } from 'node:util';+import { getHeapStatistics } from 'node:v8';+import * as pack from '../../src/path-tracer-gpu-pack.js';+import { buildNativeInput } from '../../packages/render-native/pack-input.mjs';+import { buildFlattenedJobNativeInput, buildJobNativeInput } from '../../packages/render-native/job-input.mjs';+import { buildTraceScene, resolveTexture } from '../../src/path-tracer-scene.js';+import { encodeNativeInputV2 } from '../../scripts/native-bake-page.js';+import { readFlag } from './sequence-bench-core.mjs';+import { NATIVE_CAP_ENVIRONMENT, NATIVE_CAP_PACK_OPTIONS, buildNativeCapParts, nativeCapCamera } from './native-cap-scene.mjs';++const runCommand = promisify(execFile);+const THIS_FILE = fileURLToPath(import.meta.url);+const V8_STRING_LIMIT = 536_870_888;+const BYTES_PER_MEGABYTE = 1024 * 1024;++function secondsSince(startedAt) {+ return Number(((performance.now() - startedAt) / 1000).toFixed(3));+}++function megabytes(bytes) {+ return Number((bytes / BYTES_PER_MEGABYTE).toFixed(1));+}++function describeTables(exported) {+ const base64Lengths = Object.values(exported.tables).map(text => text.length);+ const base64Total = base64Lengths.reduce((total, length) => total + length, 0);+ return { base64Megabytes: megabytes(base64Total), largestTableCharacters: Math.max(...base64Lengths), jsonCharacters: base64Total };+}++function packedBytesOf(input) {+ return Object.values(input).reduce((total, value) => total + (ArrayBuffer.isView(value) ? value.byteLength : 0), 0);+}++function packScene(variant, uniqueTriangles, maximumTriangles) {+ const instanced = variant === 'instanced';+ const { parts, uniqueTriangles: actualUnique } = buildNativeCapParts(uniqueTriangles, variant !== 'flattened');+ const camera = nativeCapCamera();+ const phases = {};+ let startedAt = performance.now();+ const traceScene = buildTraceScene(parts, NATIVE_CAP_ENVIRONMENT, { instancing: instanced, maximumTriangles });+ phases.buildSceneSeconds = secondsSince(startedAt);+ startedAt = performance.now();+ const input = instanced+ ? buildJobNativeInput(traceScene, camera, NATIVE_CAP_PACK_OPTIONS, pack, resolveTexture)+ : buildFlattenedJobNativeInput(traceScene, camera, NATIVE_CAP_PACK_OPTIONS, pack, resolveTexture);+ phases.packSeconds = secondsSince(startedAt);+ startedAt = performance.now();+ const encoded = encodeNativeInputV2(input);+ phases.encodeSeconds = secondsSince(startedAt);+ const exported = { ...encoded, triangleCount: input.triangleCount, triangles: instanced ? input.triangleCount : traceScene.indices.length / 3, instances: input.instanceCount, wholeMaterials: true };+ return { exported, phases, actualUnique, packedBytes: packedBytesOf(input), instances: input.instanceCount ?? 0 };+}++async function traceExported(exported, samples, pngPath) {+ const tracer = await import('../../packages/render-native/index.mjs');+ const { traceFrameNative } = await import('../../scripts/agent-native-trace.mjs');+ const { width, height } = NATIVE_CAP_PACK_OPTIONS;+ const frame = await traceFrameNative({ tracer, exported: { ...exported, ignored: [] }, width, height, samples, bounces: NATIVE_CAP_PACK_OPTIONS.bounces, seed: 1, denoise: 'none', kernel: 'cycles' });+ const png = Buffer.from(frame.data, 'base64');+ if (pngPath) {+ mkdirSync(dirname(pngPath), { recursive: true });+ writeFileSync(pngPath, png);+ }+ const { report } = frame;+ return { tracedSeconds: Number(report.tracedSeconds.toFixed(3)), accelerationBuildSeconds: report.accelerationBuildSeconds, uploadSeconds: report.uploadSeconds, adapter: report.gpu, inputBytes: report.inputBytes, pngSha256: createHash('sha256').update(png).digest('hex') };+}++async function runChild(argumentList) {+ const variant = readFlag(argumentList, '--variant', 'instanced');+ const size = Number(readFlag(argumentList, '--size', 400000));+ const samples = Number(readFlag(argumentList, '--samples', 16));+ const pngDirectory = readFlag(argumentList, '--png-directory', null);+ const maximumTriangles = Number(readFlag(argumentList, '--maximum', 4_000_000));+ const result = { variant, requestedUniqueTriangles: size };+ const startedAt = performance.now();+ const packed = packScene(variant, size, maximumTriangles);+ Object.assign(result, { uniqueTriangles: packed.actualUnique, instances: packed.instances, ...packed.phases, packTotalSeconds: secondsSince(startedAt),+ packedMegabytes: megabytes(packed.packedBytes), ...describeTables(packed.exported) });+ result.overStringLimit = result.largestTableCharacters > V8_STRING_LIMIT || result.jsonCharacters > V8_STRING_LIMIT;+ result.heapUsedMegabytes = megabytes(process.memoryUsage().heapUsed);+ result.peakResidentMegabytes = megabytes(process.resourceUsage().maxRSS * 1024);+ if (argumentList.includes('--trace')) {+ const pngPath = pngDirectory ? join(pngDirectory, `${variant}-${size}.png`) : null;+ Object.assign(result, { trace: await traceExported(packed.exported, samples, pngPath) });+ }+ console.log(`NATIVE_CAP_CASE ${JSON.stringify(result)}`);+}++async function readCardMegabytes() {+ try {+ const { stdout } = await runCommand('nvidia-smi', ['--query-gpu=memory.used', '--format=csv,noheader,nounits']);+ return Number(stdout.trim().split('\n')[0]);+ } catch (error) {+ return Number.NaN;+ }+}++async function runCaseInChild(variant, size, extraArguments) {+ const childArguments = [THIS_FILE, '--child', '--variant', variant, '--size', String(size), ...extraArguments];+ const idleCardMegabytes = await readCardMegabytes();+ const child = spawn(process.execPath, childArguments, { stdio: ['ignore', 'pipe', 'inherit'], env: process.env });+ let output = '';+ child.stdout.on('data', chunk => { output += chunk; });+ let finished = false;+ const exited = new Promise(resolve => child.once('exit', code => { finished = true; resolve(code); }));+ let peakCardMegabytes = idleCardMegabytes;+ while (!finished) {+ const card = await readCardMegabytes();+ if (Number.isFinite(card)) peakCardMegabytes = Math.max(peakCardMegabytes, card);+ await new Promise(resolve => setTimeout(resolve, 150));+ }+ const code = await exited;+ const line = output.split('\n').find(text => text.startsWith('NATIVE_CAP_CASE '));+ if (!line) return { variant, requestedUniqueTriangles: size, failed: `child exited with code ${code} and no result line` };+ const result = JSON.parse(line.slice('NATIVE_CAP_CASE '.length));+ if (Number.isFinite(idleCardMegabytes) && result.trace) {+ result.trace.cardIdleMegabytes = idleCardMegabytes;+ result.trace.cardPeakMegabytes = peakCardMegabytes;+ result.trace.cardDeltaMegabytes = peakCardMegabytes - idleCardMegabytes;+ }+ return result;+}++async function runParent(argumentList) {+ const sizes = readFlag(argumentList, '--sizes', '400000').split(',').map(Number);+ const variants = readFlag(argumentList, '--variants', 'instanced,flattened').split(',');+ const passThrough = ['--samples', '--png-directory', '--maximum'].flatMap(flag => (argumentList.includes(flag) ? [flag, readFlag(argumentList, flag, '')] : []));+ if (argumentList.includes('--trace')) passThrough.push('--trace');+ const results = [];+ for (const size of sizes) {+ for (const variant of variants) {+ const result = await runCaseInChild(variant, size, passThrough);+ console.log(JSON.stringify(result));+ results.push(result);+ }+ }+ const outPath = readFlag(argumentList, '--out', null);+ if (outPath) {+ mkdirSync(dirname(outPath), { recursive: true });+ writeFileSync(outPath, `${JSON.stringify(results, null, 2)}\n`);+ }+}++const argumentList = process.argv.slice(2);+await (argumentList.includes('--child') ? runChild(argumentList) : runParent(argumentList));diff --git a/bench/cycles-parity/native-cap-scene.mjs b/bench/cycles-parity/native-cap-scene.mjsnew file mode 100644index 00000000..af5df360--- /dev/null+++ b/bench/cycles-parity/native-cap-scene.mjs@@ -0,0 +1,53 @@+// A deterministic scene of four large meshes for the native triangle bound (#1041): the meshes are rippled grids side by side, two of+// them appear a second time (sharing their geometry object, so an instanced build keeps them as instances), and one emissive+// quad lights them. `uniqueTriangles` is about the triangles of the four meshes together.+import { createCamera } from '../../src/path-tracer.js';+import { buildRippledGrid } from './large-scene.mjs';++const MESH_COUNT = 4;+const COPIED_MESH_COUNT = 2;+const MESH_SPACING = 7;+const MESH_HALF_SIZE = 3;++export const NATIVE_CAP_ENVIRONMENT = { keyColor: [3, 3, 3], lightDirection: [0.4, 0.8, 0.5] };+export const NATIVE_CAP_PACK_OPTIONS = { width: 480, height: 270, bounces: 4, seed: 1, fireflyClamp: 20, noiseThreshold: 0 };++const SURFACE_MATERIAL = { color: [0.6, 0.55, 0.5], roughness: 0.7 };+const LAMP_MATERIAL = { color: [1, 0.95, 0.9], emission: 6, roughness: 1 };++const LAMP_QUAD = {+ positions: new Float32Array([-1, 0, -1, 1, 0, -1, 1, 0, 1, -1, 0, 1]),+ normals: new Float32Array([0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0]),+ uvs: new Float32Array([0, 0, 1, 0, 1, 1, 0, 1]),+ colors: new Float32Array(12).fill(1),+ indices: new Uint32Array([0, 1, 2, 0, 2, 3]),+};++function translation(x, y, z, scale = 1) {+ return new Float32Array([scale, 0, 0, 0, 0, scale, 0, 0, 0, 0, scale, 0, x, y, z, 1]);+}++function buildMesh(meshIndex, cells) {+ const phase = 1 + meshIndex * 0.35;+ return buildRippledGrid(cells, MESH_HALF_SIZE, (x, z) => 0.35 * Math.sin(phase * x) * Math.cos(phase * z));+}++/** The parts of the scene: `uniqueTriangles` is the target for the four meshes together; `copies` adds the two shared copies. */+export function buildNativeCapParts(uniqueTriangles, copies) {+ const cells = Math.max(1, Math.round(Math.sqrt(uniqueTriangles / MESH_COUNT / 2)));+ const meshes = Array.from({ length: MESH_COUNT }, (_, meshIndex) => buildMesh(meshIndex, cells));+ const leftEdge = -MESH_SPACING * (MESH_COUNT - 1) / 2;+ const parts = meshes.map((geometry, meshIndex) => ({ name: `mesh-${meshIndex}`, geometry, matrix: translation(leftEdge + meshIndex * MESH_SPACING, 0, 0), material: SURFACE_MATERIAL }));+ if (copies) {+ for (let meshIndex = 0; meshIndex < COPIED_MESH_COUNT; meshIndex++) {+ parts.push({ name: `copy-${meshIndex}`, geometry: meshes[meshIndex], matrix: translation(leftEdge + meshIndex * MESH_SPACING, 0, MESH_SPACING), material: SURFACE_MATERIAL });+ }+ }+ parts.push({ name: 'lamp', geometry: LAMP_QUAD, matrix: translation(0, 6, 0, 9), material: LAMP_MATERIAL });+ return { parts, uniqueTriangles: meshes.reduce((total, mesh) => total + mesh.indices.length / 3, 0) };+}++export function nativeCapCamera() {+ const { width, height } = NATIVE_CAP_PACK_OPTIONS;+ return createCamera({ eye: [0, 11, 24], target: [0, 0, 2], fov: 50, near: 0.05, far: 200 }, width, height);+}diff --git a/bench/cycles-parity/packed-table-hashes.mjs b/bench/cycles-parity/packed-table-hashes.mjsnew file mode 100644index 00000000..aa02c4c6--- /dev/null+++ b/bench/cycles-parity/packed-table-hashes.mjs@@ -0,0 +1,44 @@+// SHA-256 of every packed table of two scenes under the old 400,000 triangle bound, to show a change to buildTraceScene's triangle bound+// leaves the bytes alone (#1041): the lit-interior reference scene (flattened, as v1 tables and as FORMAT-v2 job tables) and an+// instanced scene (FORMAT-v2 job tables). Run it before and after a change and compare the output.+// node bench/cycles-parity/packed-table-hashes.mjs [--native-bound]+// --native-bound builds with maximumTriangles, which a build before #1041 ignores.+import { createHash } from 'node:crypto';+import { createCamera } from '../../src/path-tracer.js';+import * as pack from '../../src/path-tracer-gpu-pack.js';+import { buildNativeInput } from '../../packages/render-native/pack-input.mjs';+import { buildFlattenedJobNativeInput, buildJobNativeInput } from '../../packages/render-native/job-input.mjs';+import { buildTraceScene, resolveTexture } from '../../src/path-tracer-scene.js';+import { buildTraceSceneWithManyLights } from './trace-scene-many-lights.mjs';+import { loadReferenceScene } from './scene-bridge.mjs';+import { NATIVE_CAP_ENVIRONMENT, NATIVE_CAP_PACK_OPTIONS, buildNativeCapParts, nativeCapCamera } from './native-cap-scene.mjs';++const LIT_INTERIOR_OPTIONS = { width: 960, height: 540, bounces: 8, seed: 1, fireflyClamp: 1e9, noiseThreshold: 0 };++function hashTables(input) {+ const hashes = {};+ for (const [name, value] of Object.entries(input)) {+ if (ArrayBuffer.isView(value)) hashes[name] = createHash('sha256').update(Buffer.from(value.buffer, value.byteOffset, value.byteLength)).digest('hex');+ else if (typeof value === 'number') hashes[name] = value;+ }+ return hashes;+}++async function litInteriorHashes(buildOptions) {+ const { traceSource } = await loadReferenceScene('lit-interior');+ const camera = createCamera(traceSource.camera, LIT_INTERIOR_OPTIONS.width, LIT_INTERIOR_OPTIONS.height);+ const scene = buildTraceSceneWithManyLights(traceSource.parts, traceSource.environment, buildOptions);+ return {+ flattenedV1: hashTables(buildNativeInput(scene, camera, LIT_INTERIOR_OPTIONS, pack)),+ flattenedJobV2: hashTables(buildFlattenedJobNativeInput(scene, camera, LIT_INTERIOR_OPTIONS, pack, resolveTexture)),+ };+}++function instancedSceneHashes(buildOptions) {+ const { parts } = buildNativeCapParts(20_000, true);+ const scene = buildTraceScene(parts, NATIVE_CAP_ENVIRONMENT, { instancing: true, ...buildOptions });+ return { instancedJobV2: hashTables(buildJobNativeInput(scene, nativeCapCamera(), NATIVE_CAP_PACK_OPTIONS, pack, resolveTexture)) };+}++const buildOptions = process.argv.includes('--native-bound') ? { maximumTriangles: 4_000_000 } : {};+console.log(JSON.stringify({ litInterior: await litInteriorHashes(buildOptions), instanced: instancedSceneHashes(buildOptions) }, null, 1));diff --git a/bench/cycles-parity/trace-native.mjs b/bench/cycles-parity/trace-native.mjsindex 5ddb09b8..3d85956d 100644--- a/bench/cycles-parity/trace-native.mjs+++ b/bench/cycles-parity/trace-native.mjs@@ -6,7 +6,7 @@ import { dirname, resolve } from 'node:path';import { createCamera } from '../../src/path-tracer.js';import * as packModule from '../../src/path-tracer-gpu-pack.js';import { buildTraceSceneWithManyLights } from './trace-scene-many-lights.mjs';-import { buildInstancedTraceSceneInBatches } from './instanced-scene-in-batches.mjs';+import { buildInstancedTraceScene } from './instanced-scene.mjs';import * as renderNative from '../../packages/render-native/index.mjs';import { writeFloat32Npy } from './npy.mjs';import { reconstructTraceSceneSource } from './scene-bridge.mjs';@@ -321,8 +321,7 @@ async function runCli() {if (options.instancing && options.kernel !== 'cycles') {throw new RangeError('Option --instancing needs --kernel cycles: the legacy kernel packs flattened scenes only.');}- const instancedBatches = options.instancing ? buildInstancedTraceSceneInBatches(traceSource.parts, traceSource.environment) : null;- const traceScene = instancedBatches ? instancedBatches.scene : buildTraceSceneWithManyLights(traceSource.parts, traceSource.environment);+ const traceScene = options.instancing ? buildInstancedTraceScene(traceSource.parts, traceSource.environment) : buildTraceSceneWithManyLights(traceSource.parts, traceSource.environment);// render-cycles.py takes the lens from the flags only, so a lens set in the scene file is dropped here to keep the two sides comparable.const { fStop: ignoredSceneFStop, ...sceneCameraWithoutLens } = traceSource.camera;const camera = createCamera({ ...sceneCameraWithoutLens, ...options.lens }, options.width, options.height);@@ -416,7 +415,6 @@ async function runCli() {triangleCount: nativeInput.triangleCount,instanceCount: nativeInput.instanceCount ?? null,instancing: options.instancing,- instanceBatches: instancedBatches ? instancedBatches.uniqueTrianglesPerBatch : null,adapter: traced.adapter,backend: traced.backend,features: traced.features,diff --git a/scripts/native-bake-page.js b/scripts/native-bake-page.jsindex d53550d7..acac1ab6 100644--- a/scripts/native-bake-page.js+++ b/scripts/native-bake-page.js@@ -43,6 +43,10 @@ export function encodeNativeInputV2(input) {return { schemaVersion: input.schemaVersion, tables: encodeNativeTables(input), types, counts };}+// The triangle bound of a scene exported to the "cycles" kernel (see buildTraceScene's maximumTriangles); the legacy kernel and every+// other tracer keep the 400,000 of the CPU and WebGL paths.+export const NATIVE_JOB_MAXIMUM_TRIANGLES = 1_000_000;+/*** The scene of a native render job as the tables the job runner traces. With the "cycles" kernel (`kernel`) every scene is* exported as FORMAT-v2, so its materials reach the kernel whole; parts that share a geometry stay instances when@@ -55,7 +59,8 @@ export function encodeNativeInputV2(input) {export function exportNativeSceneTables({ parts, environment, camera, options, kernel, instancing, modules }) {const { buildTraceScene, refuseLocalVolumes, resolveTexture, buildNativeInput, buildJobNativeInput, buildFlattenedJobNativeInput, pack } = modules;const wholeMaterials = kernel === 'cycles';- let traceScene = buildTraceScene(parts, environment, { instancing: wholeMaterials && instancing === true });+ const triangleLimit = wholeMaterials ? { maximumTriangles: NATIVE_JOB_MAXIMUM_TRIANGLES } : {};+ let traceScene = buildTraceScene(parts, environment, { instancing: wholeMaterials && instancing === true, ...triangleLimit });refuseLocalVolumes(traceScene, 'the native path tracer');let flattenedBecause;if (traceScene.instances) {@@ -65,7 +70,7 @@ export function exportNativeSceneTables({ parts, environment, camera, options, k} catch (error) {if (!(error instanceof RangeError)) throw error;flattenedBecause = error.message;- traceScene = buildTraceScene(parts, environment);+ traceScene = buildTraceScene(parts, environment, triangleLimit);}}const instancingNote = flattenedBecause === undefined ? {} : { instancing: 'flattened', instancingReason: flattenedBecause };diff --git a/src/path-tracer-scene.js b/src/path-tracer-scene.jsindex a51b9611..060cae40 100644--- a/src/path-tracer-scene.js+++ b/src/path-tracer-scene.js@@ -14,6 +14,11 @@ import { liftRankedTriangles } from './flush-lift.js';// and colours, one resolved material per part, the lights in plain arrays, and the environment map as a sampling table.export const MAX_TRACE_TRIANGLES = 400_000;+// The bound a scene built for the native "cycles" tracer may ask for with `options.maximumTriangles` (unique triangles of an+// instanced scene, all triangles of a flattened one). The CPU and WebGL tracers never pass it and keep MAX_TRACE_TRIANGLES.+export const MAX_NATIVE_TRACE_TRIANGLES = 4_000_000;+// With that option an instanced scene also counts every copy: at most this many times `maximumTriangles` triangles in total.+export const NATIVE_INSTANCED_TOTAL_FACTOR = 4;export const MAX_TRACE_PARTS = 1024;// An instanced trace counts triangles once per shared geometry and bounds the copies on their own.export const MAX_TRACE_INSTANCES = 65_536;@@ -258,7 +263,20 @@ function writeWorldPart(world, part, geometry, partIndex, vertexOffset, triangleworld.triangleMaterial.fill(partIndex, triangleOffset, triangleOffset + geometry.indices.length / 3);}-function flattenParts(parts, geometries = parts.map(traceGeometry)) {+/**+ * The triangle bounds of one build: the CPU and WebGL default, or the native bound an exporter for the "cycles" kernel+ * asks for with `maximumTriangles`. A bad value is refused here so a typo cannot lift the bound.+ */+function resolveTriangleLimits(options) {+ const requested = options?.maximumTriangles;+ if (requested === undefined) return { native: false, unique: MAX_TRACE_TRIANGLES, total: Infinity };+ if (!Number.isInteger(requested) || requested < 1 || requested > MAX_NATIVE_TRACE_TRIANGLES) {+ throw new RangeError(`buildTraceScene option maximumTriangles is an integer from 1 to ${MAX_NATIVE_TRACE_TRIANGLES} for the native path tracer, got ${requested}. Leave it out for the CPU and WebGL path tracers, which take at most ${MAX_TRACE_TRIANGLES} triangles.`);+ }+ return { native: true, unique: requested, total: requested * NATIVE_INSTANCED_TOTAL_FACTOR };+}++function flattenParts(parts, geometries = parts.map(traceGeometry), limits = resolveTriangleLimits()) {let vertexTotal = 0;let indexTotal = 0;parts.forEach((part, index) => validatePart(part, geometries[index]));@@ -266,7 +284,11 @@ function flattenParts(parts, geometries = parts.map(traceGeometry)) {vertexTotal += geometry.positions.length / 3;indexTotal += geometry.indices.length;}- if (indexTotal / 3 > MAX_TRACE_TRIANGLES) throw new RangeError(`A path trace takes at most ${MAX_TRACE_TRIANGLES} triangles in total.`);+ if (indexTotal / 3 > limits.unique) {+ throw new RangeError(limits.native+ ? `A native path trace takes at most ${limits.unique} triangles in total, got ${indexTotal / 3}. Share geometry between copies (they stay instances) or simplify the meshes.`+ : `A path trace takes at most ${MAX_TRACE_TRIANGLES} triangles in total.`);+ }const world = allocateWorld(vertexTotal, indexTotal);let vertexOffset = 0;let triangleOffset = 0;@@ -385,9 +407,11 @@ function measureInstancedParts(parts, geometries) {const seen = new Set();let uniqueGeometries = 0;let uniqueTriangles = 0;+ let totalTriangles = 0;parts.forEach((part, index) => {const geometry = geometries[index];validateMatrix(part);+ totalTriangles += geometry.indices.length / 3;if (sharesAuthoredGeometry(part, geometry)) {if (seen.has(geometry)) return;seen.add(geometry);@@ -396,18 +420,21 @@ function measureInstancedParts(parts, geometries) {uniqueGeometries++;uniqueTriangles += geometry.indices.length / 3;});- return { uniqueGeometries, uniqueTriangles };+ return { uniqueGeometries, uniqueTriangles, totalTriangles };}-function requireInstancedCaps(parts, { uniqueGeometries, uniqueTriangles }) {+function requireInstancedCaps(parts, { uniqueGeometries, uniqueTriangles, totalTriangles }, limits) {if (parts.length > MAX_TRACE_INSTANCES) {throw new RangeError(`An instanced path trace takes at most ${MAX_TRACE_INSTANCES} instances, got ${parts.length}. Pass fewer parts, or merge small copies into one mesh.`);}if (uniqueGeometries > MAX_TRACE_PARTS) {throw new RangeError(`An instanced path trace takes at most ${MAX_TRACE_PARTS} distinct geometries, got ${uniqueGeometries}. Share one geometry object between copies instead of building a new one per part.`);}- if (uniqueTriangles > MAX_TRACE_TRIANGLES) {- throw new RangeError(`An instanced path trace takes at most ${MAX_TRACE_TRIANGLES} unique triangles, counted once per shared geometry, got ${uniqueTriangles}. Share geometry between copies or simplify the meshes.`);+ if (uniqueTriangles > limits.unique) {+ throw new RangeError(`An instanced path trace takes at most ${limits.unique} unique triangles, counted once per shared geometry, got ${uniqueTriangles}. Share geometry between copies or simplify the meshes.`);+ }+ if (totalTriangles > limits.total) {+ throw new RangeError(`An instanced native path trace takes at most ${limits.total} triangles with every copy counted, got ${totalTriangles}. Pass fewer copies of the large meshes or simplify them.`);}}@@ -471,8 +498,8 @@ function boundingBoxOf(positions) {* list a flattened scene calls `materials`), with no world-space triangle arrays and no CPU BVH. Only* `buildNativeInputV2` reads this shape, for the native "cycles" kernel.*/-function buildInstancedTraceScene(parts, geometries, environment, localVolumes) {- requireInstancedCaps(parts, measureInstancedParts(parts, geometries));+function buildInstancedTraceScene(parts, geometries, environment, localVolumes, limits) {+ requireInstancedCaps(parts, measureInstancedParts(parts, geometries), limits);const traceMaterials = parts.map(part => resolveMaterial(part.material ?? {}));const materials = parts.map((part, index) => ({ ...(part.material ?? {}), ...traceMaterials[index] }));const instances = parts.map((part, index) => instanceFor(part, geometries[index], index, materials[index]));@@ -493,16 +520,18 @@ function buildInstancedTraceScene(parts, geometries, environment, localVolumes)/*** Builds the scene a path trace reads. With `options.instancing` and at least one geometry shared by two parts, the result* holds `instances` instead of flattened triangles (see `buildInstancedTraceScene`); with no shared geometry, or without the- * option, it is the flattened scene, unchanged.+ * option, it is the flattened scene, unchanged. `options.maximumTriangles` (an integer up to MAX_NATIVE_TRACE_TRIANGLES) is for+ * a scene exported to the native "cycles" tracer only: it replaces the 400,000 triangle bound of the CPU and WebGL tracers.*/export function buildTraceScene(allParts, environment = {}, options = {}) {+ const limits = resolveTriangleLimits(options);const { volumes: localVolumes, boundsOnlyParts } = resolveTraceLocalVolumes(environment.localVolumes, allParts);const parts = boundsOnlyParts.length ? allParts.filter(part => !boundsOnlyParts.includes(part)) : allParts;const tryInstancing = options.instancing === true && Array.isArray(parts) && parts.length > 0;const geometries = tryInstancing ? parts.map(traceGeometry) : undefined;- if (tryInstancing && countSharedGeometries(parts, geometries) > 0) return buildInstancedTraceScene(parts, geometries, environment, localVolumes);+ if (tryInstancing && countSharedGeometries(parts, geometries) > 0) return buildInstancedTraceScene(parts, geometries, environment, localVolumes, limits);if (!Array.isArray(parts) || parts.length < 1 || parts.length > MAX_TRACE_PARTS) throw new RangeError(`A path trace takes 1 to ${MAX_TRACE_PARTS} parts.`);- const geometry = flattenParts(parts, geometries);+ const geometry = flattenParts(parts, geometries, limits);const bvh = buildTriangleBvh(geometry.positions, geometry.indices);let extent = 1e-3;for (let index = 0; index < geometry.positions.length; index++) extent = Math.max(extent, Math.abs(geometry.positions[index]));diff --git a/tests/cycles-parity-instanced-batches.test.js b/tests/cycles-parity-instanced-batches.test.jsdeleted file mode 100644index 32fa4736..00000000--- a/tests/cycles-parity-instanced-batches.test.js+++ /dev/null@@ -1,32 +0,0 @@-import assert from 'node:assert/strict';-import test from 'node:test';-import { MAX_TRACE_TRIANGLES } from '../src/path-tracer-scene.js';-import { splitGeometriesIntoBatches } from '../bench/cycles-parity/instanced-scene-in-batches.mjs';--function geometryWithTriangles(triangleCount) {- return { indices: new Uint32Array(triangleCount * 3) };-}--function groupsFor(entries) {- return new Map(entries.map(([geometry, useCount]) => [geometry, Array.from({ length: useCount }, () => ({ geometry }))]));-}--test('a scene under the cap stays in one batch', () => {- const groups = groupsFor([[geometryWithTriangles(1000), 3], [geometryWithTriangles(500), 1]]);- assert.equal(splitGeometriesIntoBatches(groups).length, 1);-});--test('a scene over the cap splits into two batches that each fit and each hold a shared geometry', () => {- const big = geometryWithTriangles(300000);- const shared = geometryWithTriangles(60000);- const single = geometryWithTriangles(67769);- const groups = groupsFor([[big, 2], [shared, 4], [single, 1]]);- const batches = splitGeometriesIntoBatches(groups);- assert.equal(batches.length, 2);- for (const batch of batches) {- const total = batch.reduce((sum, geometry) => sum + geometry.indices.length / 3, 0);- assert.ok(total <= MAX_TRACE_TRIANGLES);- assert.ok(batch.some((geometry) => groups.get(geometry).length > 1));- }- assert.equal(batches.flat().length, 3);-});diff --git a/tests/cycles-parity-instanced-scene.test.js b/tests/cycles-parity-instanced-scene.test.jsnew file mode 100644index 00000000..d6b5e826--- /dev/null+++ b/tests/cycles-parity-instanced-scene.test.js@@ -0,0 +1,33 @@+import assert from 'node:assert/strict';+import test from 'node:test';+import { buildInstancedTraceScene } from '../bench/cycles-parity/instanced-scene.mjs';+import { MAX_TRACE_TRIANGLES } from '../src/path-tracer-scene.js';++const SHARED_GEOMETRY = {+ positions: new Float32Array([0, 0, 0, 1, 0, 0, 0, 1, 0]),+ normals: new Float32Array([0, 0, 1, 0, 0, 1, 0, 0, 1]),+ indices: new Uint32Array([0, 1, 2]),+};+const ENVIRONMENT = { keyColor: [1, 1, 1], lightDirection: [0, 1, 0] };++function copy(name, x, extra = {}) {+ return { name, geometry: SHARED_GEOMETRY, matrix: new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, 0, 0, 1]), material: { color: [0.8, 0.8, 0.8] }, ...extra };+}++test('a part that hides from camera and shadow rays loses those visibility bits, the others keep every kind', () => {+ const scene = buildInstancedTraceScene([copy('a', 0), copy('b', 2, { rayVisibility: { camera: false, shadow: false } })], ENVIRONMENT);+ assert.equal(scene.instances.length, 2);+ assert.equal(scene.instances[0].visibilityFlags, undefined);+ assert.equal(scene.instances[1].visibilityFlags, 63 & ~1 & ~16);+});++test('a scene above 400,000 unique triangles builds directly, in the order of its parts', () => {+ const big = { ...SHARED_GEOMETRY, indices: new Uint32Array((MAX_TRACE_TRIANGLES + 1) * 3).map((_, slot) => slot % 3) };+ const parts = [copy('a', 0), copy('b', 2)].map(part => ({ ...part, geometry: big }));+ const scene = buildInstancedTraceScene(parts, ENVIRONMENT);+ assert.deepEqual(scene.instances.map(instance => instance.materialIndex), [0, 1]);+});++test('a scene without shared geometry is refused', () => {+ assert.throws(() => buildInstancedTraceScene([copy('a', 0)], ENVIRONMENT), { name: 'RangeError', message: /no shared geometry/ });+});diff --git a/tests/path-tracer-native-triangle-limit.test.js b/tests/path-tracer-native-triangle-limit.test.jsnew file mode 100644index 00000000..b17105d6--- /dev/null+++ b/tests/path-tracer-native-triangle-limit.test.js@@ -0,0 +1,110 @@+import assert from 'node:assert/strict';+import test from 'node:test';+import { buildNativeInput } from '../packages/render-native/pack-input.mjs';+import { buildFlattenedJobNativeInput, buildJobNativeInput } from '../packages/render-native/job-input.mjs';+import { exportNativeSceneTables, NATIVE_JOB_MAXIMUM_TRIANGLES } from '../scripts/native-bake-page.js';+import { createCamera } from '../src/path-tracer.js';+import * as pack from '../src/path-tracer-gpu-pack.js';+import {+ buildTraceScene,+ MAX_NATIVE_TRACE_TRIANGLES,+ MAX_TRACE_TRIANGLES,+ NATIVE_INSTANCED_TOTAL_FACTOR,+ refuseLocalVolumes,+ resolveTexture,+} from '../src/path-tracer-scene.js';++// Every triangle reuses the same three vertices: valid for the builder and cheap to allocate.+function triangleStack(triangleCount) {+ return {+ positions: new Float32Array([0, 0, 0, 1, 0, 0, 0, 1, 0]),+ normals: new Float32Array([0, 0, 1, 0, 0, 1, 0, 0, 1]),+ indices: new Uint32Array(triangleCount * 3).map((_, slot) => slot % 3),+ };+}++function identityShifted(x) {+ return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, 0, 0, 1]);+}++function copiesOf(geometry, copyCount) {+ return Array.from({ length: copyCount }, (_, index) => ({ name: `copy-${index}`, geometry, matrix: identityShifted(index * 2), material: { color: [0.8, 0.8, 0.8] } }));+}++const ENVIRONMENT = { keyColor: [1, 1, 1], lightDirection: [0, 1, 0] };+const OLD_FLATTENED_MESSAGE = 'A path trace takes at most 400000 triangles in total.';++test('the CPU and WebGL bound is still 400,000 and a build without the option keeps the old messages', () => {+ assert.equal(MAX_TRACE_TRIANGLES, 400_000);+ const flattened = copiesOf(triangleStack(MAX_TRACE_TRIANGLES + 1), 1);+ assert.throws(() => buildTraceScene(flattened, ENVIRONMENT), { name: 'RangeError', message: OLD_FLATTENED_MESSAGE });+ const instanced = copiesOf(triangleStack(MAX_TRACE_TRIANGLES + 1), 2);+ assert.throws(() => buildTraceScene(instanced, ENVIRONMENT, { instancing: true }), {+ name: 'RangeError',+ message: 'An instanced path trace takes at most 400000 unique triangles, counted once per shared geometry, got 400001. Share geometry between copies or simplify the meshes.',+ });+});++test('an instanced scene above 400,000 unique triangles builds with the native option and keeps its instances', () => {+ const parts = copiesOf(triangleStack(MAX_TRACE_TRIANGLES + 1), 3);+ const scene = buildTraceScene(parts, ENVIRONMENT, { instancing: true, maximumTriangles: 500_000 });+ assert.equal(scene.instances.length, 3);+ assert.equal(new Set(scene.instances.map(instance => instance.geometry)).size, 1);+});++test('an instanced scene is refused above the unique bound that was asked for, with the bound in the message', () => {+ const parts = copiesOf(triangleStack(11), 2);+ assert.throws(() => buildTraceScene(parts, ENVIRONMENT, { instancing: true, maximumTriangles: 10 }),+ { name: 'RangeError', message: /at most 10 unique triangles, counted once per shared geometry, got 11\. Share geometry between copies or simplify the meshes\./ });+});++test('an instanced scene counts every copy against four times the bound', () => {+ assert.equal(NATIVE_INSTANCED_TOTAL_FACTOR, 4);+ assert.equal(buildTraceScene(copiesOf(triangleStack(10), 4), ENVIRONMENT, { instancing: true, maximumTriangles: 10 }).instances.length, 4);+ assert.throws(() => buildTraceScene(copiesOf(triangleStack(10), 5), ENVIRONMENT, { instancing: true, maximumTriangles: 10 }),+ { name: 'RangeError', message: /at most 40 triangles with every copy counted, got 50\. Pass fewer copies of the large meshes or simplify them\./ });+});++test('without the option copies are not counted in total, as before', () => {+ assert.equal(buildTraceScene(copiesOf(triangleStack(10), 50), ENVIRONMENT, { instancing: true }).instances.length, 50);+});++test('a flattened scene takes the native bound too, and says it is the native one', () => {+ assert.equal(buildTraceScene(copiesOf(triangleStack(12), 1), ENVIRONMENT, { maximumTriangles: 12 }).indices.length, 36);+ assert.throws(() => buildTraceScene(copiesOf(triangleStack(13), 1), ENVIRONMENT, { maximumTriangles: 12 }),+ { name: 'RangeError', message: /A native path trace takes at most 12 triangles in total, got 13\. Share geometry between copies/ });+});++test('the native bound has a hard ceiling, and a scene above it is refused', () => {+ assert.equal(MAX_NATIVE_TRACE_TRIANGLES, 4_000_000);+ const parts = copiesOf(triangleStack(MAX_NATIVE_TRACE_TRIANGLES + 1), 2);+ assert.throws(() => buildTraceScene(parts, ENVIRONMENT, { instancing: true, maximumTriangles: MAX_NATIVE_TRACE_TRIANGLES }),+ { name: 'RangeError', message: /at most 4000000 unique triangles, counted once per shared geometry, got 4000001/ });+});++test('an option value that is not an integer from 1 to the ceiling is refused with the range', () => {+ const parts = copiesOf(triangleStack(3), 2);+ for (const value of [0, -5, 1.5, Number.NaN, Infinity, '1000', null, MAX_NATIVE_TRACE_TRIANGLES + 1]) {+ assert.throws(() => buildTraceScene(parts, ENVIRONMENT, { instancing: true, maximumTriangles: value }),+ { name: 'RangeError', message: /maximumTriangles is an integer from 1 to 4000000 for the native path tracer.*400000/ }, `value ${String(value)}`);+ }+});++const camera = createCamera({ eye: [0, 0, 8], target: [0, 0, 0], fov: 50 }, 8, 8);+const packOptions = { width: 8, height: 8, bounces: 2, seed: 1, fireflyClamp: 20, noiseThreshold: 0 };+const modules = { buildTraceScene, refuseLocalVolumes, resolveTexture, buildNativeInput, buildJobNativeInput, buildFlattenedJobNativeInput, pack };++test('a native render job with the cycles kernel exports an instanced scene above 400,000 unique triangles', () => {+ assert.ok(NATIVE_JOB_MAXIMUM_TRIANGLES > MAX_TRACE_TRIANGLES && NATIVE_JOB_MAXIMUM_TRIANGLES <= MAX_NATIVE_TRACE_TRIANGLES);+ const parts = copiesOf(triangleStack(MAX_TRACE_TRIANGLES + 1), 2);+ const exported = exportNativeSceneTables({ parts, environment: ENVIRONMENT, camera, options: packOptions, kernel: 'cycles', instancing: true, modules });+ assert.equal(exported.instances, 2);+ assert.equal(exported.instancing, undefined);+ assert.equal(exported.triangleCount, MAX_TRACE_TRIANGLES + 1);+});++test('a native render job with the legacy kernel keeps the 400,000 bound', () => {+ const parts = copiesOf(triangleStack(MAX_TRACE_TRIANGLES + 1), 1);+ assert.throws(() => exportNativeSceneTables({ parts, environment: ENVIRONMENT, camera, options: packOptions, kernel: 'legacy', instancing: false, modules }),+ { name: 'RangeError', message: OLD_FLATTENED_MESSAGE });+});