Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Make a recipe's viewTransform and look reach the renderer (part of #282)

Nguyen Xuan And2e838bf9bc4
diff --git a/bench/size/budgets.json b/bench/size/budgets.json
index b27cdec0..3383d5d2 100644
--- a/bench/size/budgets.json
+++ b/bench/size/budgets.json
@@ -6,7 +6,7 @@
"agent-scene": 100000
},
"budgets": {
- "entry:core": 89150,
+ "entry:core": 89500,
"entry:model": 14400,
"entry:nature": 13200,
"entry:motion": 15260,
@@ -35,9 +35,9 @@
"entry:path-tracer": 17110,
"entry:texture-set": 900,
"entry:agent-scene": 81220,
- "entry:agent-scene+lazy": 265860,
+ "entry:agent-scene+lazy": 266190,
"entry:agent-asset": 112550,
- "entry:agent-asset+lazy": 277170,
+ "entry:agent-asset+lazy": 277500,
"entry:hdri": 3300,
"entry:camera-effects": 8600,
"entry:water-reflection": 26580,
@@ -46,12 +46,12 @@
"entry:material-layer-shaders": 21400,
"entry:light-linking": 19720,
"entry:cascaded-shadows": 28530,
- "entry:view-transform-shader": 20650,
+ "entry:view-transform-shader": 20980,
"entry:view-transform": 2120,
- "scene:lit-box": 66140,
- "scene:gltf-box": 90410,
- "scene:gltf-box+lazy": 93740,
- "scene:agent-scene": 110320,
- "scene:agent-scene+lazy": 294970
+ "scene:lit-box": 66490,
+ "scene:gltf-box": 90750,
+ "scene:gltf-box+lazy": 94080,
+ "scene:agent-scene": 110670,
+ "scene:agent-scene+lazy": 295640
}
}
diff --git a/package.json b/package.json
index 7de43e2d..bae1bf4a 100644
--- a/package.json
+++ b/package.json
@@ -115,6 +115,7 @@
"smoke:kernel-workers": "node scripts/smoke-kernel-workers.mjs",
"smoke:agent-topology": "node scripts/smoke-agent-topology.mjs",
"smoke:agent-look": "node scripts/smoke-agent-look.mjs",
+ "smoke:view-transform": "node scripts/smoke-view-transform.mjs",
"smoke:agent-assets": "node scripts/smoke-agent-assets.mjs",
"smoke:agent-render-jobs": "node scripts/smoke-agent-render-jobs.mjs",
"smoke:render-passes": "node scripts/smoke-render-passes.mjs",
diff --git a/scripts/smoke-view-transform.mjs b/scripts/smoke-view-transform.mjs
new file mode 100644
index 00000000..11440327
--- /dev/null
+++ b/scripts/smoke-view-transform.mjs
@@ -0,0 +1,104 @@
+// Draws the agent preview fixture through one renderer while a recipe switches environment.viewTransform and environment.look,
+// in HDR (the curve runs in the post pass) and in 8-bit (it runs in the main program). The three curves must give different
+// pixels, the look must change them again, going back to aces must restore the first picture exactly, and the shaders must
+// compile without a console error or warning: npm run smoke:view-transform [cpu|gpu]
+import assert from 'node:assert/strict';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { constants, setPriority } from 'node:os';
+import { createServer } from 'vite';
+import { chromium } from 'playwright';
+import { createAgentPreviewFixture } from './agent-preview-fixture.mjs';
+
+setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);
+const backend = process.argv[2] ?? 'cpu';
+assert.ok(['cpu', 'gpu'].includes(backend));
+await mkdir('artifacts', { recursive: true });
+const recipe = createAgentPreviewFixture();
+recipe.environment = { world: true, lightDirection: [-3, 6, 4], keyColor: [4, 3.4, 2.6], fillIntensity: 0.25, ambientIntensity: 0.35, skyTop: [0.2, 0.35, 0.6], skyBottom: [0.7, 0.75, 0.8], fogDensity: 0 };
+const VARIANTS = [
+ { name: 'aces', environment: { viewTransform: 'aces' } },
+ { name: 'agx', environment: { viewTransform: 'agx' } },
+ { name: 'neutral', environment: { viewTransform: 'neutral' } },
+ { name: 'agx+warm', environment: { viewTransform: 'agx', look: 'warm' } },
+ { name: 'aces again', environment: { viewTransform: 'aces', look: null } },
+];
+
+function harness(hdr) {
+ return `<!doctype html><html><body style="margin:0"><canvas style="width:384px;height:256px;display:block"></canvas><script type="module">
+import {createAgentScene,loadAllAgentSceneModules} from '/src/agent-scene.js';
+import {Renderer67} from '/src/renderer.js';
+await loadAllAgentSceneModules();
+const session=createAgentScene(${JSON.stringify(recipe)});
+const renderer=new Renderer67(document.querySelector('canvas'),session.parts,{ground:false,hdr:${hdr},samples:1,pixelRatio:1});
+renderer.resize();
+const context=renderer.context;
+function draw(){
+ renderer.render({...session.camera,environment:session.environment,effects:session.effects,exposure:session.effects.exposure,time:0});
+ const pixels=new Uint8Array(context.drawingBufferWidth*context.drawingBufferHeight*4);
+ context.readPixels(0,0,context.drawingBufferWidth,context.drawingBufferHeight,context.RGBA,context.UNSIGNED_BYTE,pixels);
+ return pixels;
+}
+window.__viewTransformProof={session,renderer,draw};
+</script></body></html>`;
+}
+
+function measure(pixels) {
+ const sums = [0, 0, 0];
+ for (let offset = 0; offset < pixels.length; offset += 4) for (let channel = 0; channel < 3; channel++) sums[channel] += pixels[offset + channel];
+ return sums.map(sum => Math.round(sum / (pixels.length / 4) * 10) / 10);
+}
+
+const server = await createServer({ server: { host: '127.0.0.1', port: 0 }, logLevel: 'silent' });
+let browser;
+const report = {};
+try {
+ await server.listen();
+ const args = ['--enable-webgl', '--ignore-gpu-blocklist', '--disable-renderer-backgrounding'];
+ if (backend === 'cpu') args.push('--use-angle=swiftshader', '--enable-unsafe-swiftshader');
+ browser = await chromium.launch({ executablePath: 'C:/Program Files/Google/Chrome/Application/chrome.exe', headless: true, args });
+ for (const hdr of [true, false]) {
+ const label = hdr ? 'hdr' : 'sdr';
+ await writeFile(`artifacts/view-transform-${label}.html`, harness(hdr));
+ const page = await browser.newPage({ viewport: { width: 384, height: 256 } });
+ const problems = [];
+ page.on('pageerror', error => problems.push(error.message));
+ // The driver's ReadPixels stall note and the browser's favicon request are not shader problems.
+ const noise = /GL Driver Message \(OpenGL, Performance|status of 404/;
+ page.on('console', message => { if (['error', 'warning'].includes(message.type()) && !noise.test(message.text())) problems.push(message.text()); });
+ await page.goto(`http://127.0.0.1:${server.httpServer.address().port}/artifacts/view-transform-${label}.html`);
+ await page.waitForFunction(() => window.__viewTransformProof, null, { timeout: 120000 });
+ const result = await page.evaluate(variants => {
+ const { session, renderer, draw } = window.__viewTransformProof;
+ const frames = [];
+ for (const variant of variants) {
+ session.apply([{ type: 'set-environment', environment: variant.environment }], { expectedRevision: session.revision });
+ frames.push({ name: variant.name, pixels: Array.from(draw()) });
+ }
+ const glError = renderer.context.getError();
+ renderer.dispose();
+ return { frames, glError };
+ }, VARIANTS);
+ const frames = result.frames.map(frame => ({ name: frame.name, pixels: Uint8Array.from(frame.pixels) }));
+ const means = Object.fromEntries(frames.map(frame => [frame.name, measure(frame.pixels)]));
+ report[label] = { means, glError: result.glError, problems };
+ // Mean absolute difference per colour channel, in 0 to 255 steps: two curves can share a channel mean and still differ.
+ const differs = (first, second) => {
+ let total = 0;
+ for (let offset = 0; offset < first.length; offset += 4) for (let channel = 0; channel < 3; channel++) total += Math.abs(first[offset + channel] - second[offset + channel]);
+ return total / (first.length / 4 * 3) >= 1;
+ };
+ const byName = Object.fromEntries(frames.map(frame => [frame.name, frame.pixels]));
+ assert.deepEqual(problems, [], `${label}: console errors or warnings`);
+ assert.equal(result.glError, 0, `${label}: WebGL error`);
+ assert.ok(differs(byName.aces, byName.agx), `${label}: agx looks like aces`);
+ assert.ok(differs(byName.aces, byName.neutral), `${label}: neutral looks like aces`);
+ assert.ok(differs(byName.agx, byName.neutral), `${label}: neutral looks like agx`);
+ assert.ok(differs(byName.agx, byName['agx+warm']), `${label}: the warm look changes nothing`);
+ assert.ok(byName['aces again'].every((value, index) => value === byName.aces[index]), `${label}: going back to aces does not restore the picture`);
+ await page.close();
+ }
+ console.log(JSON.stringify(report, null, 2));
+} finally {
+ await browser?.close();
+ await server.close();
+}
diff --git a/src/renderer.js b/src/renderer.js
index 5eac886d..76cc937f 100644
--- a/src/renderer.js
+++ b/src/renderer.js
@@ -477,9 +477,13 @@ export class Renderer67 {
this.diagnostics = [];
this.activeLightDirection = environment.lightDirection ?? [-3, 6, 4];
this.shadowCascades = shadowCascades;
- this.viewTransform = viewTransform;
- this.look = look;
- if ((viewTransform !== 'aces' || look) && !viewTransformShader.apply) throw new Error("viewTransform and look need their shader code: add import '@codepawlhq/67js/addons/view-transform-shader' (a recipe: await loadAgentSceneModules(recipe)).");
+ // The constructor options are the default; a frame's environment (a recipe's viewTransform and look) overrides them.
+ this.defaultViewTransform = viewTransform;
+ this.defaultLook = look;
+ this.viewTransform = 'aces';
+ this.look = null;
+ this.lookSource = null;
+ this.selectViewTransform(viewTransform, look);
this.memoryBudgetMiB = memoryBudgetMiB;
this.hdr = Boolean(hdr && !benchmark && this.context.getExtension('EXT_color_buffer_float'));
if (volume.density > 0 && !this.hdr) throw new Error('Volume fog requires EXT_color_buffer_float for HDR rendering');
@@ -637,6 +641,41 @@ export class Renderer67 {
if (!this.benchmark && !this.multisampleFramebuffer) this.allocateFrameTargets(this.canvas.width, this.canvas.height);
}
+ selectViewTransform(viewTransform, lookSource) {
+ if ((viewTransform !== 'aces' || lookSource) && !viewTransformShader.apply) throw new Error("viewTransform and look need their shader code: add import '@codepawlhq/67js/addons/view-transform-shader' (a recipe: await loadAgentSceneModules(recipe)).");
+ this.viewTransform = viewTransform;
+ this.lookSource = lookSource;
+ this.look = lookSource ? viewTransformShader.resolveLook(lookSource) : null;
+ }
+
+ // A recipe's environment.viewTransform and environment.look reach the renderer in each frame; a change recompiles the
+ // programs that carry the display curve (main, liquid, environment background, post) before the frame is drawn.
+ syncViewTransform(environment) {
+ const viewTransform = environment.viewTransform ?? this.defaultViewTransform;
+ const lookSource = environment.look ?? this.defaultLook;
+ if (viewTransform === this.viewTransform && lookSource === this.lookSource) return;
+ viewTransformShader.dispose?.(this);
+ this.selectViewTransform(viewTransform, lookSource);
+ if (this.benchmark) return;
+ const context = this.context;
+ this.rebuildMainProgram();
+ for (const variant of this.postVariants.values()) context.deleteProgram(variant.pending ? finishProgram(context, variant.pending) : variant.program);
+ this.postVariants.clear();
+ context.deleteProgram(this.postProgram);
+ this.postProgram = finishProgram(context, startProgram(context, fullscreenVertex, this.withViewTransform(postFragment, 'post')));
+ this.postUniforms = postUniformLocations(context, this.postProgram);
+ const liquidNames = Object.keys(this.liquidUniforms);
+ context.deleteProgram(this.liquidProgram);
+ this.liquidProgram = finishProgram(context, startProgram(context, vertexShader, this.withViewTransform(liquidFragmentShader, 'liquid')));
+ this.liquidUniforms = Object.fromEntries(liquidNames.map(name => [name, context.getUniformLocation(this.liquidProgram, name)]));
+ context.useProgram(this.liquidProgram);
+ context.uniform1i(this.liquidUniforms.skinInfluences, 6);
+ context.uniform1i(this.liquidUniforms.jointPalette, 7);
+ context.uniform1i(this.liquidUniforms.morphTargets, 8);
+ if (this.environmentBackgroundProgram) context.deleteProgram(this.environmentBackgroundProgram);
+ this.environmentBackgroundProgram = null;
+ }
+
// The display curve and look of this renderer, spliced into one of its programs (view-transform-shader.js); the default curve changes nothing.
withViewTransform(source, kind) {
if (this.viewTransform === 'aces' && !this.look) return source;
@@ -2080,6 +2119,7 @@ export class Renderer67 {
if (!Number.isFinite(lightLength) || lightLength < 1e-6 || !Array.from(direction).every(component => typeof component === 'number' && Math.abs(component) <= 1e6)) throw new RangeError('Volume light direction must have a finite nonzero length and bounded components');
}
updateLightingState(this.lighting, environment, { world: this.world });
+ this.syncViewTransform(environment);
// The shadow view follows the sun, so a moved sun redraws the map.
const lightChanged = String(this.activeLightDirection) !== String(environment.lightDirection);
this.activeLightDirection = environment.lightDirection;
diff --git a/src/view-transform-shader.js b/src/view-transform-shader.js
index 521987b7..8c199927 100644
--- a/src/view-transform-shader.js
+++ b/src/view-transform-shader.js
@@ -1,5 +1,5 @@
import { viewTransformShader } from './shaders.js';
-import { AGX_CONSTANTS, NEUTRAL_CONSTANTS } from './view-transform.js';
+import { AGX_CONSTANTS, NEUTRAL_CONSTANTS, namedLook } from './view-transform.js';
import { VIEW_TRANSFORM_NAMES } from './view-transform-names.js';
// The GLSL of the AgX and Khronos PBR Neutral view transforms and of the look LUT. It lives outside shaders.js so a
@@ -76,6 +76,14 @@ function apply(source, kind, viewTransform, hasLook) {
}
const lookTextures = new WeakMap();
+const namedLooks = new Map();
+
+/** A look as the renderer draws it: a name from LOOK_NAMES is built once, a parsed .cube look is used as is. */
+function resolveLook(look) {
+ if (typeof look !== 'string') return look;
+ if (!namedLooks.has(look)) namedLooks.set(look, namedLook(look));
+ return namedLooks.get(look);
+}
function createLookTexture(context, look) {
const { size, data } = look;
@@ -107,6 +115,6 @@ function dispose(renderer) {
lookTextures.delete(renderer);
}
-Object.assign(viewTransformShader, { apply, bindLook, dispose });
+Object.assign(viewTransformShader, { apply, bindLook, dispose, resolveLook });
export { apply as applyViewTransformToShader };