Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Add renderer.backend native to render jobs (issue #626)

Nguyen Xuan An74d1892c07ce
diff --git a/bench/native-tracer/job.mjs b/bench/native-tracer/job.mjs
new file mode 100644
index 00000000..df87c707
--- /dev/null
+++ b/bench/native-tracer/job.mjs
@@ -0,0 +1,68 @@
+// One MCP image render job of a recipe with settings.renderer.backend set, timed from submit to completed. Run it through gpu-lock.
+// node bench/native-tracer/job.mjs <output directory> --backend native|gpu|cpu [--recipe tests/fixtures/demo-room.recipe.json]
+// [--samples 64] [--width 1920] [--height 1080] [--label run1] [--denoise true]
+// Writes job-<label>.png and job-<label>.json (seconds, the metrics.pathTracer report and the job state).
+import { copyFile, mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises';
+import { constants, setPriority, tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { Client } from '@modelcontextprotocol/client';
+import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';
+
+const argumentsGiven = process.argv.slice(2);
+const outputDirectory = resolve(argumentsGiven[0]);
+
+function valueAfter(flag, fallback) {
+ const index = argumentsGiven.indexOf(flag);
+ return index >= 0 ? argumentsGiven[index + 1] : fallback;
+}
+
+const backend = valueAfter('--backend', 'native');
+const label = valueAfter('--label', 'run1');
+const width = Number(valueAfter('--width', 1920));
+const height = Number(valueAfter('--height', 1080));
+const samples = Number(valueAfter('--samples', 64));
+const denoise = JSON.parse(valueAfter('--denoise', 'true'));
+const recipePath = valueAfter('--recipe', 'tests/fixtures/demo-room.recipe.json');
+
+setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);
+await mkdir(outputDirectory, { recursive: true });
+const workDirectory = await mkdtemp(join(tmpdir(), '67js-native-job-'));
+const transport = new StdioClientTransport({ command: process.execPath,
+ args: [resolve('bin/67js-mcp.mjs'), '--backend', 'gpu', '--memory', '384', '--jobs', join(workDirectory, 'jobs')], cwd: workDirectory, stderr: 'pipe' });
+let diagnostics = '';
+transport.stderr.on('data', bytes => { diagnostics = (diagnostics + bytes).slice(-4096); });
+const client = new Client({ name: '67js-native-job-bench', version: '1' });
+
+async function call(name, toolArguments = {}, timeout = 600000) {
+ return client.callTool({ name, arguments: toolArguments }, { timeout });
+}
+
+try {
+ await client.connect(transport);
+ const recipe = JSON.parse(await readFile(recipePath, 'utf8'));
+ const created = (await call('scene_create', { recipe }, 600000)).structuredContent;
+ const settings = { kind: 'image', width, height, renderer: { type: 'pathTracer', backend, samples, bounces: 8, denoise, maxSeconds: 600 } };
+ const started = performance.now();
+ const submitted = await call('scene_render_submit', { sessionId: created.sessionId, expectedRevision: 0, settings });
+ if (submitted.isError) throw new Error(submitted.content?.[0]?.text);
+ const { jobId } = submitted.structuredContent;
+ let record;
+ for (;;) {
+ record = (await call('scene_render_status', { jobId })).structuredContent;
+ if (['completed', 'cancelled', 'failed'].includes(record.state)) break;
+ await delay(100);
+ }
+ const jobSeconds = (performance.now() - started) / 1000;
+ const result = (await call('scene_render_result', { jobId })).structuredContent;
+ if (record.state === 'completed') await copyFile(result.artifacts[0].path, join(outputDirectory, `job-${label}.png`));
+ const summary = { backend, label, width, height, samples, denoise, state: record.state, jobSeconds, pathTracer: (result.metrics ?? record.metrics)?.pathTracer, error: record.error };
+ await writeFile(join(outputDirectory, `job-${label}.json`), JSON.stringify(summary, null, 1));
+ console.log(JSON.stringify(summary));
+ await call('scene_render_release', { jobId });
+} catch (error) {
+ console.error(diagnostics);
+ throw error;
+} finally {
+ await client.close();
+}
diff --git a/changes/626-jobs.md b/changes/626-jobs.md
new file mode 100644
index 00000000..97c9822a
--- /dev/null
+++ b/changes/626-jobs.md
@@ -0,0 +1 @@
+- renderer.backend "native" on render jobs (image, stills, video): hardware ray queries through the optional @codepawlhq/67js-render-native package, then the usual wavelet or OIDN denoise and PNG. Without the package, its binary or a Vulkan ray query adapter the job runs on gpu (cpu on a cpu runtime) and metrics.pathTracer.backendNote says why (#626)
diff --git a/docs/agent-render-jobs.md b/docs/agent-render-jobs.md
index f70a6505..92ce1894 100644
--- a/docs/agent-render-jobs.md
+++ b/docs/agent-render-jobs.md
@@ -134,7 +134,7 @@ An `image`, `stills` or `video` job can ask for the offline [path tracer](https:
"renderer": { "type": "pathTracer", "backend": "gpu", "samples": 64, "bounces": 8, "denoise": true, "seed": 0, "maxSeconds": 120 } }
```
-Every field after `type` is optional. `backend` is `cpu` (the default, byte-identical on SwiftShader smokes) or `gpu`, which traces in a WebGL2 shader and needs a runtime started on the GPU backend (a `--backend cpu` runtime refuses it). The report in `metrics.pathTracer` names the `backend` that ran, the `gpu`, `gpuCompileSeconds` and `gpuKernelCompiles`. A `stills` or `video` job traces every frame with the same options: the kernel compiles once per job page and each frame reuses it, so a film shows `gpuKernelCompiles: 1`. A video of traced frames goes through the ffmpeg encoder (the browser H.264 encoder has no drawn frame to read), `maxSeconds` limits each frame, and the job deadline is `maxSeconds` times the frame count plus 60 seconds, up to the job ceiling. Limits: `samples` 1 to 1024, `bounces` 1 to 16, `seed` 0 to 2147483647, `maxSeconds` 1 to 600 (the job's deadline is `maxSeconds` plus 60 seconds; a 1920 by 1080 still of a 180,000 triangle room at 64 samples took 246 s, so give a large still `maxSeconds` 600). `passes` are refused with it. The tracer loads in the job page on demand, uses eight threads at below-normal priority, stops pixels that have converged (`report.averageSamples`), and writes `render.png`; `scene_render_result` metrics carry `pathTracer` with the samples, threads, seconds and what the scene asked for that was not drawn. The same recipe and seed give the same PNG. A job that passes `maxSeconds` fails and writes no image.
+Every field after `type` is optional. `backend` is `cpu` (the default, byte-identical on SwiftShader smokes), `gpu`, which traces in a WebGL2 shader and needs a runtime started on the GPU backend (a `--backend cpu` runtime refuses it), or `native`, which traces on hardware ray queries (Vulkan) through the optional `@codepawlhq/67js-render-native` package. A native job exports the scene from the job page, traces it in the job runner, then denoises (wavelet or OIDN) and writes the PNG as the other backends do. It never fails for lack of the package: without it, without its built binary or without a Vulkan ray query adapter the job runs on `gpu` (`cpu` on a cpu runtime) and `metrics.pathTracer.backendNote` says why; `metrics.pathTracer.backend` always names what ran. The look up costs one import and is cached per process. Native draws no volume fog or subsurface scattering (listed in `ignored`), and `noiseThreshold` and `maxSeconds` do not apply to it. The report in `metrics.pathTracer` names the `backend` that ran, the `gpu`, `gpuCompileSeconds` and `gpuKernelCompiles`. A `stills` or `video` job traces every frame with the same options: the kernel compiles once per job page and each frame reuses it, so a film shows `gpuKernelCompiles: 1`. A video of traced frames goes through the ffmpeg encoder (the browser H.264 encoder has no drawn frame to read), `maxSeconds` limits each frame, and the job deadline is `maxSeconds` times the frame count plus 60 seconds, up to the job ceiling. Limits: `samples` 1 to 1024, `bounces` 1 to 16, `seed` 0 to 2147483647, `maxSeconds` 1 to 600 (the job's deadline is `maxSeconds` plus 60 seconds; a 1920 by 1080 still of a 180,000 triangle room at 64 samples took 246 s, so give a large still `maxSeconds` 600). `passes` are refused with it. The tracer loads in the job page on demand, uses eight threads at below-normal priority, stops pixels that have converged (`report.averageSamples`), and writes `render.png`; `scene_render_result` metrics carry `pathTracer` with the samples, threads, seconds and what the scene asked for that was not drawn. The same recipe and seed give the same PNG. A job that passes `maxSeconds` fails and writes no image.
## Controller API
diff --git a/docs/path-tracer.md b/docs/path-tracer.md
index 1e32fc6c..03784207 100644
--- a/docs/path-tracer.md
+++ b/docs/path-tracer.md
@@ -58,6 +58,8 @@ The same scene and options give the same bytes. Random numbers come from a hash
## Choosing a backend
+`backend: "native"` is an agent render job setting, not a `renderPathTracedImage` option: [`packages/render-native`](../packages/render-native/README.md) runs the same kernel on Vulkan ray queries. On the demo room at 1920 by 1080 and 64 samples an MCP image job took 25 to 35 s with native against 58 s with gpu (trace alone 2.7 to 3.1 s against 38.8 s; the rest is the page export, the wavelet denoise and the PNG), FLIP 0.0047 between them. A missing package, binary or adapter falls back to gpu with a note in the job report. See [agent-render-jobs.md](agent-render-jobs.md#path-tracer).
+
`backend: 'cpu'` (the default) is the reference: it reuses `src/triangle-bvh.js` and `src/ray-query.js`, runs on workers in the job page next to the scene, needs no GPU, and gives the same bytes for the same scene and options. It measured 25 to 238 seconds for a 960 by 540 still at 32 samples on a Ryzen 5 5600X with four threads, while another session also loaded the machine (table in `bench/reference/README.md`).
`backend: 'gpu'` (issue #302) runs the same algorithm in a WebGL2 fragment shader (`src/path-tracer-gpu.js`, loaded only for this backend, so a CPU render and the core bundle do not carry it). The BVH, triangles, materials, maps, lights and the environment table are packed into float textures (`src/path-tracer-gpu-pack.js`), the random numbers are the same integer hash and PCG stream as the CPU tracer and are drawn in the same order, and each sample adds into four float targets (colour and depth, albedo, normal, luminance and its square) that feed the same denoiser. Float32 maths and a 24-bit random value make the pixels differ a little from the CPU tracer: FLIP (LDR) of GPU against CPU at the same seed was 0.0001 to 0.0018 on the 16 path tracer reference scenes at 480 by 270 and 32 samples. It needs a page with WebGL2, `EXT_color_buffer_float` and `EXT_float_blend`, and throws without them, so choose it only where those exist. It traces everything the CPU tracer does (lights with linking, all materials and maps including emissive maps, glass, volume fog, the sky); it does not use `noiseThreshold` (every pixel draws every sample) or `threads`, and a sample that produced a NaN or an infinity counts as black. The first still of a page compiles the kernel (13 to 30 s), later ones reuse it.
diff --git a/packages/render-native/README.md b/packages/render-native/README.md
index 6f2146e6..f0deb9bf 100644
--- a/packages/render-native/README.md
+++ b/packages/render-native/README.md
@@ -1,6 +1,6 @@
# @codepawlhq/67js-render-native
-Optional. Traces a 67js scene on the GPU with hardware ray queries (wgpu, Vulkan) instead of the browser's software BVH walk. It uses the same WGSL path tracer as the browser kernel, so one seed gives the same picture. It is not a root dependency and no render job calls it yet.
+Optional. Traces a 67js scene on the GPU with hardware ray queries (wgpu, Vulkan) instead of the browser's software BVH walk. It uses the same WGSL path tracer as the browser kernel, so one seed gives the same picture. It is not a root dependency. Render jobs with `settings.renderer.backend: "native"` use it when it is installed and built, and fall back to the `gpu` backend with a note when it is not (see docs/agent-render-jobs.md).
Needs Rust (stable) and a Vulkan GPU with ray queries. On Windows build for MSVC; the default GNU toolchain crashes any wgpu 30 program. The DX12 backend does not report ray queries, so the addon uses Vulkan only.
@@ -17,4 +17,4 @@ const input = buildNativeInput(traceScene, camera, options, pack);
const { rgba, adapter, tracedSeconds } = traceNative(input, options);
```
-`rgba` is linear float RGBA, rows from the top, averaged over the samples. `bench/native-tracer` compares it with the browser kernel.
+`rgba` is linear float RGBA, rows from the top, averaged over the samples. `albedo` and `normal` (three floats per pixel, normalise the normal), `depth` and `variance` (one float) are the first-hit features the wavelet and OIDN denoisers read. `pack-input.mjs` holds `buildNativeInput` without Node imports, so the job page packs the scene itself. `bench/native-tracer` compares it with the browser kernel.
diff --git a/packages/render-native/index.mjs b/packages/render-native/index.mjs
index a1ed6a25..b2edb5a5 100644
--- a/packages/render-native/index.mjs
+++ b/packages/render-native/index.mjs
@@ -5,63 +5,16 @@ import { createRequire } from 'node:module';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
+export { buildNativeInput } from './pack-input.mjs';
+
const PACKAGE_DIRECTORY = dirname(fileURLToPath(import.meta.url));
const ADDON_PATH = join(PACKAGE_DIRECTORY, 'build', 'render-native.node');
-const PARAMETER_BYTES = 12 * 16;
-const SUN_CONE_COSINE = Math.cos(Math.PI / 360);
/** The path of the built addon, or null when `npm run build` has not run. */
export function findNativeAddon() {
return existsSync(ADDON_PATH) ? ADDON_PATH : null;
}
-function writeParameters(scene, camera, options, lightCount, atlasWidth) {
- const buffer = new ArrayBuffer(PARAMETER_BYTES);
- const floats = new Float32Array(buffer);
- const integers = new Int32Array(buffer);
- const words = new Uint32Array(buffer);
- const { sun } = scene.lights;
- const { environment } = scene;
- floats.set([...camera.eye, 0], 0);
- floats.set([...camera.forward, 0], 4);
- floats.set([...camera.right, 0], 8);
- floats.set([...camera.above, 0], 12);
- floats.set([...(sun?.direction ?? [0, 1, 0]), 0], 16);
- floats.set([...(sun?.color ?? [0, 0, 0]), SUN_CONE_COSINE], 20);
- floats.set([environment?.width ?? 1, environment?.height ?? 1, environment?.intensity ?? 1, environment?.total ?? 1], 24);
- floats.set([environment?.cosine ?? 1, environment?.sine ?? 0, scene.surfaceOffset, scene.farDistance], 28);
- floats.set([camera.halfWidth, camera.halfHeight, options.fireflyClamp ?? 20, 0], 32);
- integers.set([options.width, options.height, options.bounces, lightCount], 36);
- integers.set([(sun ? 1 : 0) + lightCount + (environment ? 1 : 0), sun ? 1 : 0, environment ? 1 : 0, 0], 40);
- words.set([options.seed >>> 0, atlasWidth, 0, 0], 44);
- return new Uint8Array(buffer);
-}
-
-/**
- * Packs a trace scene (from `buildTraceScene`) and its camera into the tables the kernel reads.
- * `pack` is the module `src/path-tracer-gpu-pack.js`, passed in so this package never imports the engine's source.
- */
-export function buildNativeInput(scene, camera, options, pack) {
- const atlas = pack.packAtlas(scene.materials);
- const lights = pack.packLights(scene.lights);
- const environment = pack.packEnvironment(scene.environment);
- const environmentTables = new Float32Array(environment ? environment.height + environment.width * environment.height : 4);
- if (environment) {
- environmentTables.set(environment.rows.data.subarray(0, environment.height), 0);
- environmentTables.set(environment.pixels.data.subarray(0, environment.width * environment.height), environment.height);
- }
- return {
- triangles: pack.packTriangles(scene).data,
- triangleCount: scene.bvh.triangleOrder.length,
- materials: pack.packMaterials(scene.materials, atlas).data,
- lights: lights.table.data,
- atlas: new Uint32Array(atlas.data.buffer, atlas.data.byteOffset, Math.floor(atlas.data.byteLength / 4)),
- environmentColors: environment?.colors ?? new Float32Array(4),
- environmentTables,
- parameters: writeParameters(scene, camera, options, lights.count, atlas.width),
- };
-}
-
/**
* Traces the input on the GPU and resolves linear RGBA float data (`rgba`, width * height * 4, rows from the top, the
* average of `samples` paths) with the adapter name and timings in seconds.
diff --git a/packages/render-native/pack-input.mjs b/packages/render-native/pack-input.mjs
new file mode 100644
index 00000000..89dbb115
--- /dev/null
+++ b/packages/render-native/pack-input.mjs
@@ -0,0 +1,50 @@
+// Packs a trace scene into the tables the native kernel reads. It imports nothing from Node, so the job page can run it too.
+const PARAMETER_BYTES = 12 * 16;
+const SUN_CONE_COSINE = Math.cos(Math.PI / 360);
+
+function writeParameters(scene, camera, options, lightCount, atlasWidth) {
+ const buffer = new ArrayBuffer(PARAMETER_BYTES);
+ const floats = new Float32Array(buffer);
+ const integers = new Int32Array(buffer);
+ const words = new Uint32Array(buffer);
+ const { sun } = scene.lights;
+ const { environment } = scene;
+ floats.set([...camera.eye, 0], 0);
+ floats.set([...camera.forward, 0], 4);
+ floats.set([...camera.right, 0], 8);
+ floats.set([...camera.above, 0], 12);
+ floats.set([...(sun?.direction ?? [0, 1, 0]), 0], 16);
+ floats.set([...(sun?.color ?? [0, 0, 0]), SUN_CONE_COSINE], 20);
+ floats.set([environment?.width ?? 1, environment?.height ?? 1, environment?.intensity ?? 1, environment?.total ?? 1], 24);
+ floats.set([environment?.cosine ?? 1, environment?.sine ?? 0, scene.surfaceOffset, scene.farDistance], 28);
+ floats.set([camera.halfWidth, camera.halfHeight, options.fireflyClamp ?? 20, 0], 32);
+ integers.set([options.width, options.height, options.bounces, lightCount], 36);
+ integers.set([(sun ? 1 : 0) + lightCount + (environment ? 1 : 0), sun ? 1 : 0, environment ? 1 : 0, 0], 40);
+ words.set([options.seed >>> 0, atlasWidth, 0, 0], 44);
+ return new Uint8Array(buffer);
+}
+
+/**
+ * Packs a trace scene (from `buildTraceScene`) and its camera into the tables the kernel reads.
+ * `pack` is the module `src/path-tracer-gpu-pack.js`, passed in so this package never imports the engine's source.
+ */
+export function buildNativeInput(scene, camera, options, pack) {
+ const atlas = pack.packAtlas(scene.materials);
+ const lights = pack.packLights(scene.lights);
+ const environment = pack.packEnvironment(scene.environment);
+ const environmentTables = new Float32Array(environment ? environment.height + environment.width * environment.height : 4);
+ if (environment) {
+ environmentTables.set(environment.rows.data.subarray(0, environment.height), 0);
+ environmentTables.set(environment.pixels.data.subarray(0, environment.width * environment.height), environment.height);
+ }
+ return {
+ triangles: pack.packTriangles(scene).data,
+ triangleCount: scene.bvh.triangleOrder.length,
+ materials: pack.packMaterials(scene.materials, atlas).data,
+ lights: lights.table.data,
+ atlas: new Uint32Array(atlas.data.buffer, atlas.data.byteOffset, Math.floor(atlas.data.byteLength / 4)),
+ environmentColors: environment?.colors ?? new Float32Array(4),
+ environmentTables,
+ parameters: writeParameters(scene, camera, options, lights.count, atlas.width),
+ };
+}
diff --git a/packages/render-native/package.json b/packages/render-native/package.json
index 1e7d3e6d..53381cff 100644
--- a/packages/render-native/package.json
+++ b/packages/render-native/package.json
@@ -9,6 +9,7 @@
},
"files": [
"index.mjs",
+ "pack-input.mjs",
"build.mjs",
"build.rs",
"Cargo.toml",
diff --git a/packages/render-native/src/lib.rs b/packages/render-native/src/lib.rs
index 81efb19b..9ebd27ce 100644
--- a/packages/render-native/src/lib.rs
+++ b/packages/render-native/src/lib.rs
@@ -44,6 +44,12 @@ pub struct NativeTraceOptions {
pub struct NativeTraceResult {
/// Linear RGBA, averaged over the samples, row by row from the top.
pub rgba: Float32Array,
+ /// First-hit albedo and normal (three floats per pixel, averaged over the samples; normalise the normal), the first-hit
+ /// depth and the variance of each pixel's mean luminance: the features the wavelet and OIDN denoisers read.
+ pub albedo: Float32Array,
+ pub normal: Float32Array,
+ pub depth: Float32Array,
+ pub variance: Float32Array,
pub adapter: String,
pub backend: String,
pub setup_seconds: f64,
@@ -285,7 +291,7 @@ pub fn trace_scene(input: NativeTraceInput, options: NativeTraceOptions) -> Resu
let _ = device.poll(wgpu::PollType::wait_indefinitely());
}
}
- let readback_size = pixel_count * 16;
+ let readback_size = pixel_count * 16 * 4;
let readback = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("readback"),
size: readback_size,
@@ -299,16 +305,35 @@ pub fn trace_scene(input: NativeTraceInput, options: NativeTraceOptions) -> Resu
let _ = device.poll(wgpu::PollType::wait_indefinitely());
let mapped = readback.slice(..).get_mapped_range().map_err(|error| failure(format!("Could not read the image back: {error}")))?;
let sums: &[f32] = bytemuck::cast_slice(&mapped);
+ let plane_length = pixel_count as usize * 4;
let sample_scale = 1.0 / options.samples.max(1) as f32;
+ let variance_divisor = options.samples.saturating_sub(1).max(1) as f32;
let mut rgba = Vec::with_capacity(pixel_count as usize * 4);
- for pixel in sums.chunks_exact(4) {
- rgba.extend_from_slice(&[pixel[0] * sample_scale, pixel[1] * sample_scale, pixel[2] * sample_scale, 1.0]);
+ let mut albedo = Vec::with_capacity(pixel_count as usize * 3);
+ let mut normal = Vec::with_capacity(pixel_count as usize * 3);
+ let mut depth = Vec::with_capacity(pixel_count as usize);
+ let mut variance = Vec::with_capacity(pixel_count as usize);
+ for pixel in 0..pixel_count as usize {
+ let radiance = &sums[pixel * 4..pixel * 4 + 4];
+ let albedo_sum = &sums[plane_length + pixel * 4..plane_length + pixel * 4 + 3];
+ let normal_sum = &sums[2 * plane_length + pixel * 4..2 * plane_length + pixel * 4 + 3];
+ let statistics = &sums[3 * plane_length + pixel * 4..3 * plane_length + pixel * 4 + 2];
+ rgba.extend_from_slice(&[radiance[0] * sample_scale, radiance[1] * sample_scale, radiance[2] * sample_scale, 1.0]);
+ depth.push(radiance[3] * sample_scale);
+ albedo.extend(albedo_sum.iter().map(|value| value * sample_scale));
+ normal.extend(normal_sum.iter().map(|value| value * sample_scale));
+ let mean_luminance = statistics[0] * sample_scale;
+ variance.push((statistics[1] * sample_scale - mean_luminance * mean_luminance).max(0.0) / variance_divisor);
}
drop(mapped);
let traced_seconds = trace_started.elapsed().as_secs_f64();
Ok(NativeTraceResult {
rgba: Float32Array::new(rgba),
+ albedo: Float32Array::new(albedo),
+ normal: Float32Array::new(normal),
+ depth: Float32Array::new(depth),
+ variance: Float32Array::new(variance),
adapter: adapter_info.name,
backend: format!("{:?}", adapter_info.backend),
setup_seconds,
diff --git a/scripts/agent-native-trace.mjs b/scripts/agent-native-trace.mjs
new file mode 100644
index 00000000..d83ce503
--- /dev/null
+++ b/scripts/agent-native-trace.mjs
@@ -0,0 +1,136 @@
+// renderer.backend "native" on render jobs: the optional hardware ray query tracer (packages/render-native) traces the scene
+// the job page exported, then the job runner denoises and writes the PNG. The package and its binary are optional: when they,
+// or a Vulkan ray query adapter, are missing, the job runs on the gpu or cpu backend and the report says why.
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+// The names sit in variables so the bundler leaves the imports alone; the package resolves from the user's node_modules.
+const NATIVE_PACKAGE = '@codepawlhq/67js-render-native';
+// A source checkout keeps the package in packages/, beside scripts/ or, from the bundled server, two folders above build/mcp.
+const CHECKOUT_PACKAGE_URLS = ['../packages/render-native/index.mjs', '../../packages/render-native/index.mjs']
+ .map(relative => new URL(/* @vite-ignore */ relative, import.meta.url));
+const TABLE_NAMES = ['triangles', 'materials', 'lights', 'atlas', 'environmentColors', 'environmentTables', 'parameters'];
+const TABLE_TYPES = { triangles: Float32Array, materials: Float32Array, lights: Float32Array, atlas: Uint32Array,
+ environmentColors: Float32Array, environmentTables: Float32Array, parameters: Uint8Array };
+
+async function importNativePackage(importPackage) {
+ try {
+ return await importPackage(NATIVE_PACKAGE);
+ } catch {
+ for (const packageUrl of CHECKOUT_PACKAGE_URLS) {
+ if (!existsSync(fileURLToPath(packageUrl))) continue;
+ try {
+ return await importPackage(packageUrl.href);
+ } catch {
+ return null;
+ }
+ }
+ return null;
+ }
+}
+
+/**
+ * A cached look for the native tracer. Looking costs one import and one file check, done once per process. `markUnusable`
+ * records a failure from a real trace (usually no ray query adapter), so later jobs skip the attempt and keep the reason.
+ */
+export function createNativeTracerLocator({ importPackage = specifier => import(specifier) } = {}) {
+ let lookup;
+ let unusableReason = null;
+ return {
+ async find() {
+ lookup ??= importNativePackage(importPackage).then(tracer => {
+ if (!tracer) return { reason: `renderer.backend "native" needs ${NATIVE_PACKAGE}.` };
+ if (!tracer.findNativeAddon()) return { reason: `renderer.backend "native" needs the built binary of ${NATIVE_PACKAGE} (npm run build in that package).` };
+ return { tracer };
+ });
+ const found = await lookup;
+ if (unusableReason) return { reason: unusableReason };
+ return found;
+ },
+ markUnusable(reason) {
+ unusableReason = `renderer.backend "native" could not trace (${reason}), which usually means no Vulkan ray query adapter.`;
+ },
+ };
+}
+
+/**
+ * Which backend a job's frames run on. "native" falls back to gpu, or to cpu on a runtime without a GPU, and `note` says why.
+ * Every other request runs as asked, and the caller still rejects "gpu" on a cpu runtime.
+ */
+export async function chooseTraceBackend({ requested, runtimeBackend, locator }) {
+ if (requested !== 'native') return { backend: requested };
+ const found = await locator.find();
+ if (found.tracer) return { backend: 'native', tracer: found.tracer };
+ const fallback = runtimeBackend === 'cpu' ? 'cpu' : 'gpu';
+ return { backend: fallback, note: `${found.reason} The ${fallback} backend ran instead.` };
+}
+
+function decodeTables(exported) {
+ const input = { triangleCount: exported.triangleCount };
+ for (const name of TABLE_NAMES) {
+ const bytes = Buffer.from(exported.tables[name], 'base64');
+ const View = TABLE_TYPES[name];
+ const copy = new Uint8Array(bytes.length);
+ copy.set(bytes);
+ input[name] = new View(copy.buffer, 0, copy.length / View.BYTES_PER_ELEMENT);
+ }
+ return input;
+}
+
+function normalizeNormals(normal) {
+ for (let offset = 0; offset < normal.length; offset += 3) {
+ const length = Math.hypot(normal[offset], normal[offset + 1], normal[offset + 2]);
+ if (length > 1e-6) {
+ normal[offset] /= length;
+ normal[offset + 1] /= length;
+ normal[offset + 2] /= length;
+ }
+ }
+}
+
+function withoutAlpha(rgba, pixelCount) {
+ const color = new Float32Array(pixelCount * 3);
+ for (let pixel = 0; pixel < pixelCount; pixel++) {
+ color.set(rgba.subarray(pixel * 4, pixel * 4 + 3), pixel * 3);
+ }
+ return color;
+}
+
+/**
+ * Traces one frame the page exported and returns the PNG (base64) with the job report. `denoise` is "none", "wavelet" or
+ * "oidn"; `denoiser` is the loaded OIDN package for "oidn". Throws when the tracer cannot run, so the caller can fall back.
+ */
+export async function traceFrameNative({ tracer, exported, width, height, samples, bounces, seed, denoise, denoiser }) {
+ const started = performance.now();
+ const traced = tracer.traceNative(decodeTables(exported), { width, height, samples });
+ const tracedSeconds = (performance.now() - started) / 1000;
+ const pixelCount = width * height;
+ const features = { color: withoutAlpha(traced.rgba, pixelCount), albedo: traced.albedo, normal: traced.normal,
+ depth: traced.depth, variance: traced.variance };
+ normalizeNormals(features.normal);
+ const report = { width, height, samples, bounces, seed, denoised: false, denoiser: 'none', threads: 1, backend: 'native',
+ gpu: traced.adapter, nativeBackend: traced.backend, setupSeconds: traced.setupSeconds, compileSeconds: traced.compileSeconds,
+ uploadSeconds: traced.uploadSeconds, accelerationBuildSeconds: traced.accelerationBuildSeconds,
+ averageSamples: samples, triangles: exported.triangles, tracedSeconds, ignored: exported.ignored };
+ let linear = features.color;
+ const denoiseStarted = performance.now();
+ if (denoise === 'oidn') {
+ try {
+ linear = await denoiser.denoiseWithOidn({ width, height, color: features.color, albedo: features.albedo, normal: features.normal });
+ Object.assign(report, { denoised: true, denoiser: 'oidn', oidnVersion: denoiser.OIDN_VERSION });
+ } catch (error) {
+ report.denoiserNote = `OIDN failed (${error.message}): the wavelet filter ran instead.`;
+ denoise = 'wavelet';
+ }
+ }
+ if (denoise === 'wavelet') {
+ const { denoiseRadiance } = await import('../src/path-tracer-denoise.js');
+ linear = denoiseRadiance({ width, height, ...features });
+ Object.assign(report, { denoised: true, denoiser: 'wavelet' });
+ }
+ if (report.denoised) report.denoiseSeconds = (performance.now() - denoiseStarted) / 1000;
+ const [{ toneMapToRgba8 }, { encodePng }] = await Promise.all([import('../src/path-tracer.js'), import('../src/export-png.js')]);
+ const png = encodePng(toneMapToRgba8(linear, width, height), width, height);
+ report.seconds = (performance.now() - started) / 1000;
+ return { data: Buffer.from(png).toString('base64'), report };
+}
diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.js
index f602bd73..d6ede53c 100644
--- a/scripts/agent-preview-browser.js
+++ b/scripts/agent-preview-browser.js
@@ -788,6 +788,35 @@ window.__agentPreview = {
}
return { data: encodeTracedPng(result.rgba, result.width, result.height), report: result.report };
},
+ // The scene at one instant packed into the tables the native ray query tracer reads (renderer.backend "native"). The job
+ // runner traces them outside the page; each table comes back as base64 bytes.
+ async exportNativeScene({ sessionId, animation, motionTime, options }) {
+ const record = requireSession(sessionId);
+ if (record.asset && (animation !== undefined || motionTime !== undefined)) {
+ record.scene.sampleFrame({ ...(animation === undefined ? {} : { animation }),
+ ...(motionTime === undefined ? {} : { motionTime }) });
+ } else if (motionTime !== undefined) {
+ record.scene.sampleMotion(motionTime);
+ }
+ const [{ createCamera, describeIgnoredFeatures }, { buildTraceScene }, pack, { buildNativeInput }] = await Promise.all([
+ import('../src/path-tracer.js'), import('../src/path-tracer-scene.js'), import('../src/path-tracer-gpu-pack.js'),
+ import('../packages/render-native/pack-input.mjs'),
+ ]);
+ const environment = { ...record.scene.environment, map: agentEnvironmentMap(record.scene.environment) };
+ const { sceneUsesSubsurface } = pack;
+ const traceScene = buildTraceScene(record.scene.parts, environment);
+ const camera = createCamera(record.scene.camera, options.width, options.height);
+ const input = buildNativeInput(traceScene, camera, options, pack);
+ const tables = {};
+ for (const [name, values] of Object.entries(input)) {
+ if (ArrayBuffer.isView(values)) tables[name] = bytesToBase64(new Uint8Array(values.buffer, values.byteOffset, values.byteLength));
+ }
+ // The native kernel has no participating media yet, so say so instead of claiming a fog or a subsurface look it did not trace.
+ const ignored = describeIgnoredFeatures(record.scene.parts, environment);
+ if (environment.volumeFog?.density > 0) ignored.push('volumeFog is not traced by the native backend.');
+ if (sceneUsesSubsurface(traceScene.materials)) ignored.push('subsurface is not traced by the native backend.');
+ return { tables, triangleCount: input.triangleCount, triangles: traceScene.indices.length / 3, ignored };
+ },
// The PNG of linear pixels (three floats each, base64) that a denoiser outside the page returned, or of the wavelet
// filter over the kept features when none came back.
async finishPathTrace({ width, height, linear }) {
diff --git a/scripts/agent-render-jobs.mjs b/scripts/agent-render-jobs.mjs
index b4edb394..d8e65c8c 100644
--- a/scripts/agent-render-jobs.mjs
+++ b/scripts/agent-render-jobs.mjs
@@ -146,6 +146,7 @@ function settingsFrom(input, clips, recipe) {
return result;
}
+const PATH_TRACER_BACKENDS = ['cpu', 'gpu', 'native'];
const FRACTIONAL_RENDERER_FIELDS = new Set(['maxSeconds', 'noiseThreshold']);
function rendererFrom(renderer, passes) {
@@ -155,7 +156,7 @@ function rendererFrom(renderer, passes) {
const denoise = renderer.denoise ?? true;
if (typeof denoise !== 'boolean' && !PATH_TRACER_DENOISERS.includes(denoise)) throw new TypeError(`renderer.denoise must be a boolean or one of ${PATH_TRACER_DENOISERS.join(', ')}.`);
const backend = renderer.backend ?? 'cpu';
- if (backend !== 'cpu' && backend !== 'gpu') throw new TypeError('renderer.backend must be cpu or gpu.');
+ if (!PATH_TRACER_BACKENDS.includes(backend)) throw new TypeError('renderer.backend must be cpu, gpu or native.');
const result = { type: 'pathTracer', backend, denoise };
for (const name of Object.keys(PATH_TRACER_JOB_LIMITS)) {
const { minimum, maximum, default: fallback } = PATH_TRACER_JOB_LIMITS[name];
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index 61908b6b..b9d5cd75 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -14,6 +14,7 @@ import { MATERIAL_SNIPPET_CAPABILITIES } from '../src/material-snippet.js';
import { AGENT_SCENE_CAPABILITIES, AGENT_SCENE_DEFAULT_LIMITS, RENDER_PASS_NAMES, RENDER_PASS_DEPTH_RANGE, VIDEO_ENCODERS, MAXIMUM_VIDEO_SECONDS, MAXIMUM_JOB_SIDE, MAXIMUM_JOB_PIXELS, MAXIMUM_MEMORY_MIB, BAKE_WORKER_MAXIMUM_SECONDS } from '../src/agent-scene-capabilities.js';
import { PATH_TRACER_CAPABILITIES } from '../src/path-tracer-capabilities.js';
import { chooseDenoiser, finishWithOidn } from './agent-path-trace-denoise.mjs';
+import { chooseTraceBackend, createNativeTracerLocator, traceFrameNative } from './agent-native-trace.mjs';
import { createAgentAssetCatalog } from './agent-asset-catalog.mjs';
import { geometryOnlyGlb } from './agent-query-asset.mjs';
import { AGENT_ASSET_LIMITS, assertCombinedAssetBytes } from '../src/agent-asset-limits.js';
@@ -77,6 +78,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
if (!['gpu', 'cpu'].includes(backend)) {
throw new RangeError('backend must be gpu or cpu.');
}
+ const nativeTracerLocator = createNativeTracerLocator();
if (!Number.isInteger(memory) || memory < 32 || memory > MAXIMUM_MEMORY_MIB) {
throw new RangeError(`memory must be an integer from 32 to ${MAXIMUM_MEMORY_MIB} MiB.`);
}
@@ -459,10 +461,29 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
const { type, ...tracerOptions } = settings.renderer;
if (settings.renderer.backend === 'gpu' && backend === 'cpu') throw new Error('renderer.backend "gpu" needs a runtime on the GPU backend, and this one runs on SwiftShader.');
const choice = await chooseDenoiser(settings.renderer.denoise);
- let traced = await jobPage.evaluate(input => window.__agentPreview.pathTrace(input), { ...frameInput, options: { width, height, ...tracerOptions, denoise: choice.denoise, threads: PATH_TRACER_THREADS } });
- if (traced.features) {
- traced = await finishWithOidn({ traced, denoiser: choice.denoiser, drawLinear: input => jobPage.evaluate(drawInput => window.__agentPreview.finishPathTrace(drawInput), input) });
+ const trace = await chooseTraceBackend({ requested: settings.renderer.backend, runtimeBackend: backend, locator: nativeTracerLocator });
+ const traceOptions = { width, height, ...tracerOptions, backend: trace.backend === 'native' ? 'gpu' : trace.backend, denoise: choice.denoise, threads: PATH_TRACER_THREADS };
+ let traced;
+ let backendNote = trace.note;
+ if (trace.backend === 'native') {
+ try {
+ const exported = await jobPage.evaluate(input => window.__agentPreview.exportNativeScene(input), { ...frameInput, options: { width, height, bounces: tracerOptions.bounces, seed: tracerOptions.seed, fireflyClamp: 20 } });
+ traced = await traceFrameNative({ tracer: trace.tracer, exported, width, height, samples: tracerOptions.samples, bounces: tracerOptions.bounces,
+ seed: tracerOptions.seed, denoise: choice.denoise === true ? 'wavelet' : choice.denoise === false ? 'none' : choice.denoise, denoiser: choice.denoiser });
+ } catch (error) {
+ nativeTracerLocator.markUnusable(error.message);
+ backendNote = (await nativeTracerLocator.find()).reason;
+ traceOptions.backend = backend === 'cpu' ? 'cpu' : 'gpu';
+ backendNote += ' The ' + traceOptions.backend + ' backend ran instead.';
+ }
+ }
+ if (!traced) {
+ traced = await jobPage.evaluate(input => window.__agentPreview.pathTrace(input), { ...frameInput, options: traceOptions });
+ if (traced.features) {
+ traced = await finishWithOidn({ traced, denoiser: choice.denoiser, drawLinear: input => jobPage.evaluate(drawInput => window.__agentPreview.finishPathTrace(drawInput), input) });
+ }
}
+ if (backendNote) traced.report.backendNote = backendNote;
if (choice.note) traced.report.denoiserNote = choice.note;
metrics.renderSeconds += (performance.now() - renderStarted) / 1000;
metrics.width = width;
diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjs
index 68a1b530..aaea8696 100644
--- a/scripts/mcp-server.mjs
+++ b/scripts/mcp-server.mjs
@@ -700,7 +700,7 @@ const renderSettings = z.strictObject({
animation: z.strictObject({ clip: z.number().int().min(0), start: z.number().min(0).max(3600).optional() }).optional(),
motion: z.strictObject({ start: z.number().min(0).max(3600).optional() }).optional(),
rendererTime: z.strictObject({ start: z.number().min(0).max(3600).optional(), rate: z.number().min(-16).max(16).optional() }).optional(),
- renderer: z.strictObject({ type: z.literal('pathTracer'), backend: z.enum(['cpu', 'gpu']).optional(), samples: z.number().optional(), bounces: z.number().optional(), denoise: z.union([z.boolean(), z.enum(['wavelet', 'oidn', 'none'])]).optional(), seed: z.number().optional(), maxSeconds: z.number().optional(), noiseThreshold: z.number().optional() }).optional().describe('path tracer, backend gpu is faster; see scene_capabilities pathTracer.'),
+ renderer: z.strictObject({ type: z.literal('pathTracer'), backend: z.enum(['cpu', 'gpu', 'native']).optional(), samples: z.number().optional(), bounces: z.number().optional(), denoise: z.union([z.boolean(), z.enum(['wavelet', 'oidn', 'none'])]).optional(), seed: z.number().optional(), maxSeconds: z.number().optional(), noiseThreshold: z.number().optional() }).optional().describe('path tracer, backend gpu is faster, native fastest if installed; see scene_capabilities pathTracer.'),
timeline: z.strictObject({ start: z.number().min(0).max(60).optional() }).optional().describe('Play the recipe timeline from start (default 0); duration defaults to the rest of it. It owns the motion clock, so leave motion and animation out.'),
audio: z.strictObject({ file: z.string().max(68), offset: z.number().min(0).max(3600).optional(), track: z.enum(['aac', 'pcm']).optional() }).optional().describe('Video only: a file name in the --audio folder (.wav, .mp3, .ogg, .flac) mixed in from offset seconds.'),
encoder: z.enum(VIDEO_ENCODERS).optional().describe('Video only: cpu (default) or nvenc.'),
diff --git a/src/path-tracer-capabilities.js b/src/path-tracer-capabilities.js
index d2886ec7..d1ac573e 100644
--- a/src/path-tracer-capabilities.js
+++ b/src/path-tracer-capabilities.js
@@ -16,7 +16,7 @@ export const PATH_TRACER_DENOISERS = Object.freeze(['wavelet', 'oidn', 'none']);
export const OIDN_DEFAULT_SAMPLES = 16;
export const PATH_TRACER_CAPABILITIES = Object.freeze({
- setting: 'settings.renderer { type: "pathTracer", backend ("cpu" default | "gpu"), samples, bounces, denoise (true | false | "wavelet" | "oidn" | "none"), seed, maxSeconds, noiseThreshold }; image, stills and video jobs, no passes. backend gpu needs a runtime on the GPU backend; the report names backend, gpuCompileSeconds and gpuKernelCompiles (1 for a whole film). Past maxSeconds on one frame the job fails.',
+ setting: 'settings.renderer { type: "pathTracer", backend ("cpu" default | "gpu" | "native"), samples, bounces, denoise (true | false | "wavelet" | "oidn" | "none"), seed, maxSeconds, noiseThreshold }; image, stills and video jobs, no passes. backend gpu needs a runtime on the GPU backend; backend native uses hardware ray queries through the optional @codepawlhq/67js-render-native package and falls back to gpu (cpu on a cpu runtime) with metrics.pathTracer.backendNote when the package, its binary or a Vulkan ray query adapter is missing (it draws no volume fog or subsurface, named in report.ignored); the report names backend, gpuCompileSeconds and gpuKernelCompiles (1 for a whole film). Past maxSeconds on one frame the job fails.',
noiseThreshold: 'Default 0 draws every sample. Above 0 a pixel stops once its standard error relative to its mean is under the value, checked from 16 samples on: faster and noisier, and it cost quality on spot-area (score 0.0658 at 0, 0.0900 at 0.03). docs/path-tracer.md has the table.',
traces: 'emissive maps and strength; volume fog as a uniform medium (density, colour, anisotropy, box; noise, drift and ambient are not used); the material subsurface block as a bounded random walk on the cpu and gpu backends (at most 32 scattering events, mean free path radius times scale in metres), and a card with nothing behind it as thin translucency.',
skips: 'occlusion maps, liquids, anisotropy, iridescence, sheen, caustics, depth of field, bloom; metrics.pathTracer.ignored in the job result lists what the scene asked for.',
diff --git a/tests/agent-native-trace.test.js b/tests/agent-native-trace.test.js
new file mode 100644
index 00000000..3f25634d
--- /dev/null
+++ b/tests/agent-native-trace.test.js
@@ -0,0 +1,52 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { chooseTraceBackend, createNativeTracerLocator } from '../scripts/agent-native-trace.mjs';
+
+const builtTracer = { findNativeAddon: () => '/build/render-native.node' };
+
+function locatorFor(importPackage) {
+ return createNativeTracerLocator({ importPackage });
+}
+
+test('a native request runs native when the package and its binary are present', async () => {
+ const locator = locatorFor(async () => builtTracer);
+ const choice = await chooseTraceBackend({ requested: 'native', runtimeBackend: 'gpu', locator });
+ assert.equal(choice.backend, 'native');
+ assert.equal(choice.tracer, builtTracer);
+ assert.equal(choice.note, undefined);
+});
+
+test('without the package a native job falls back to gpu, or to cpu on a cpu runtime, and says why', async () => {
+ const locator = locatorFor(async () => { throw new Error('Cannot find package'); });
+ const onGpu = await chooseTraceBackend({ requested: 'native', runtimeBackend: 'gpu', locator });
+ assert.equal(onGpu.backend, 'gpu');
+ assert.match(onGpu.note, /needs @codepawlhq\/67js-render-native/);
+ assert.match(onGpu.note, /gpu backend ran instead/);
+ const onCpu = await chooseTraceBackend({ requested: 'native', runtimeBackend: 'cpu', locator });
+ assert.equal(onCpu.backend, 'cpu');
+ assert.match(onCpu.note, /cpu backend ran instead/);
+});
+
+test('a package without its built binary falls back and names the build step', async () => {
+ const locator = locatorFor(async () => ({ findNativeAddon: () => null }));
+ const choice = await chooseTraceBackend({ requested: 'native', runtimeBackend: 'gpu', locator });
+ assert.equal(choice.backend, 'gpu');
+ assert.match(choice.note, /built binary/);
+});
+
+test('a trace that failed once makes later native requests fall back with that reason, without importing again', async () => {
+ let imports = 0;
+ const locator = locatorFor(async () => { imports += 1; return builtTracer; });
+ assert.equal((await chooseTraceBackend({ requested: 'native', runtimeBackend: 'gpu', locator })).backend, 'native');
+ locator.markUnusable('no adapter supports ray queries');
+ const choice = await chooseTraceBackend({ requested: 'native', runtimeBackend: 'gpu', locator });
+ assert.equal(choice.backend, 'gpu');
+ assert.match(choice.note, /no adapter supports ray queries/);
+ assert.equal(imports, 1, 'the package is looked up once per process');
+});
+
+test('cpu and gpu requests never touch the native package', async () => {
+ const locator = locatorFor(async () => { throw new Error('must not be imported'); });
+ assert.deepEqual(await chooseTraceBackend({ requested: 'gpu', runtimeBackend: 'gpu', locator }), { backend: 'gpu' });
+ assert.deepEqual(await chooseTraceBackend({ requested: 'cpu', runtimeBackend: 'cpu', locator }), { backend: 'cpu' });
+});
diff --git a/tests/agent-render-jobs.test.js b/tests/agent-render-jobs.test.js
index 9b369bca..da72dad7 100644
--- a/tests/agent-render-jobs.test.js
+++ b/tests/agent-render-jobs.test.js
@@ -435,7 +435,10 @@ test('a path tracer job runs on the cpu unless it asks for the gpu, and film job
const onGpu = await submitWith({ backend: 'gpu' });
await terminal(controller, onGpu.jobId);
assert.equal(controller.status(onGpu.jobId).settings.renderer.backend, 'gpu');
- await assert.rejects(submitWith({ backend: 'metal' }), /backend must be cpu or gpu/);
+ const onNative = await submitWith({ backend: 'native' });
+ await terminal(controller, onNative.jobId);
+ assert.equal(controller.status(onNative.jobId).settings.renderer.backend, 'native');
+ await assert.rejects(submitWith({ backend: 'metal' }), /backend must be cpu, gpu or native/);
const film = await submitWith({ backend: 'gpu', samples: 16 }, 'video');
assert.equal(controller.status(film.jobId).settings.renderer.samples, 16);
await controller.close();