3D engine that AI agents drive
Make a recipe's viewTransform and look reach the renderer (part of #282)
Nguyen Xuan And2e838bf9bc4
diff --git a/bench/size/budgets.json b/bench/size/budgets.jsonindex b27cdec0..3383d5d2 100644--- a/bench/size/budgets.json+++ b/bench/size/budgets.json@@ -6,7 +6,7 @@"agent-scene": 100000},"budgets": {- "entry:core": 89150,+ "entry:core": 89500,"entry:model": 14400,"entry:nature": 13200,"entry:motion": 15260,@@ -35,9 +35,9 @@"entry:path-tracer": 17110,"entry:texture-set": 900,"entry:agent-scene": 81220,- "entry:agent-scene+lazy": 265860,+ "entry:agent-scene+lazy": 266190,"entry:agent-asset": 112550,- "entry:agent-asset+lazy": 277170,+ "entry:agent-asset+lazy": 277500,"entry:hdri": 3300,"entry:camera-effects": 8600,"entry:water-reflection": 26580,@@ -46,12 +46,12 @@"entry:material-layer-shaders": 21400,"entry:light-linking": 19720,"entry:cascaded-shadows": 28530,- "entry:view-transform-shader": 20650,+ "entry:view-transform-shader": 20980,"entry:view-transform": 2120,- "scene:lit-box": 66140,- "scene:gltf-box": 90410,- "scene:gltf-box+lazy": 93740,- "scene:agent-scene": 110320,- "scene:agent-scene+lazy": 294970+ "scene:lit-box": 66490,+ "scene:gltf-box": 90750,+ "scene:gltf-box+lazy": 94080,+ "scene:agent-scene": 110670,+ "scene:agent-scene+lazy": 295640}}diff --git a/package.json b/package.jsonindex 7de43e2d..bae1bf4a 100644--- a/package.json+++ b/package.json@@ -115,6 +115,7 @@"smoke:kernel-workers": "node scripts/smoke-kernel-workers.mjs","smoke:agent-topology": "node scripts/smoke-agent-topology.mjs","smoke:agent-look": "node scripts/smoke-agent-look.mjs",+ "smoke:view-transform": "node scripts/smoke-view-transform.mjs","smoke:agent-assets": "node scripts/smoke-agent-assets.mjs","smoke:agent-render-jobs": "node scripts/smoke-agent-render-jobs.mjs","smoke:render-passes": "node scripts/smoke-render-passes.mjs",diff --git a/scripts/smoke-view-transform.mjs b/scripts/smoke-view-transform.mjsnew file mode 100644index 00000000..11440327--- /dev/null+++ b/scripts/smoke-view-transform.mjs@@ -0,0 +1,104 @@+// Draws the agent preview fixture through one renderer while a recipe switches environment.viewTransform and environment.look,+// in HDR (the curve runs in the post pass) and in 8-bit (it runs in the main program). The three curves must give different+// pixels, the look must change them again, going back to aces must restore the first picture exactly, and the shaders must+// compile without a console error or warning: npm run smoke:view-transform [cpu|gpu]+import assert from 'node:assert/strict';+import { mkdir, writeFile } from 'node:fs/promises';+import { constants, setPriority } from 'node:os';+import { createServer } from 'vite';+import { chromium } from 'playwright';+import { createAgentPreviewFixture } from './agent-preview-fixture.mjs';++setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);+const backend = process.argv[2] ?? 'cpu';+assert.ok(['cpu', 'gpu'].includes(backend));+await mkdir('artifacts', { recursive: true });+const recipe = createAgentPreviewFixture();+recipe.environment = { world: true, lightDirection: [-3, 6, 4], keyColor: [4, 3.4, 2.6], fillIntensity: 0.25, ambientIntensity: 0.35, skyTop: [0.2, 0.35, 0.6], skyBottom: [0.7, 0.75, 0.8], fogDensity: 0 };+const VARIANTS = [+ { name: 'aces', environment: { viewTransform: 'aces' } },+ { name: 'agx', environment: { viewTransform: 'agx' } },+ { name: 'neutral', environment: { viewTransform: 'neutral' } },+ { name: 'agx+warm', environment: { viewTransform: 'agx', look: 'warm' } },+ { name: 'aces again', environment: { viewTransform: 'aces', look: null } },+];++function harness(hdr) {+ return `<!doctype html><html><body style="margin:0"><canvas style="width:384px;height:256px;display:block"></canvas><script type="module">+import {createAgentScene,loadAllAgentSceneModules} from '/src/agent-scene.js';+import {Renderer67} from '/src/renderer.js';+await loadAllAgentSceneModules();+const session=createAgentScene(${JSON.stringify(recipe)});+const renderer=new Renderer67(document.querySelector('canvas'),session.parts,{ground:false,hdr:${hdr},samples:1,pixelRatio:1});+renderer.resize();+const context=renderer.context;+function draw(){+ renderer.render({...session.camera,environment:session.environment,effects:session.effects,exposure:session.effects.exposure,time:0});+ const pixels=new Uint8Array(context.drawingBufferWidth*context.drawingBufferHeight*4);+ context.readPixels(0,0,context.drawingBufferWidth,context.drawingBufferHeight,context.RGBA,context.UNSIGNED_BYTE,pixels);+ return pixels;+}+window.__viewTransformProof={session,renderer,draw};+</script></body></html>`;+}++function measure(pixels) {+ const sums = [0, 0, 0];+ for (let offset = 0; offset < pixels.length; offset += 4) for (let channel = 0; channel < 3; channel++) sums[channel] += pixels[offset + channel];+ return sums.map(sum => Math.round(sum / (pixels.length / 4) * 10) / 10);+}++const server = await createServer({ server: { host: '127.0.0.1', port: 0 }, logLevel: 'silent' });+let browser;+const report = {};+try {+ await server.listen();+ const args = ['--enable-webgl', '--ignore-gpu-blocklist', '--disable-renderer-backgrounding'];+ if (backend === 'cpu') args.push('--use-angle=swiftshader', '--enable-unsafe-swiftshader');+ browser = await chromium.launch({ executablePath: 'C:/Program Files/Google/Chrome/Application/chrome.exe', headless: true, args });+ for (const hdr of [true, false]) {+ const label = hdr ? 'hdr' : 'sdr';+ await writeFile(`artifacts/view-transform-${label}.html`, harness(hdr));+ const page = await browser.newPage({ viewport: { width: 384, height: 256 } });+ const problems = [];+ page.on('pageerror', error => problems.push(error.message));+ // The driver's ReadPixels stall note and the browser's favicon request are not shader problems.+ const noise = /GL Driver Message \(OpenGL, Performance|status of 404/;+ page.on('console', message => { if (['error', 'warning'].includes(message.type()) && !noise.test(message.text())) problems.push(message.text()); });+ await page.goto(`http://127.0.0.1:${server.httpServer.address().port}/artifacts/view-transform-${label}.html`);+ await page.waitForFunction(() => window.__viewTransformProof, null, { timeout: 120000 });+ const result = await page.evaluate(variants => {+ const { session, renderer, draw } = window.__viewTransformProof;+ const frames = [];+ for (const variant of variants) {+ session.apply([{ type: 'set-environment', environment: variant.environment }], { expectedRevision: session.revision });+ frames.push({ name: variant.name, pixels: Array.from(draw()) });+ }+ const glError = renderer.context.getError();+ renderer.dispose();+ return { frames, glError };+ }, VARIANTS);+ const frames = result.frames.map(frame => ({ name: frame.name, pixels: Uint8Array.from(frame.pixels) }));+ const means = Object.fromEntries(frames.map(frame => [frame.name, measure(frame.pixels)]));+ report[label] = { means, glError: result.glError, problems };+ // Mean absolute difference per colour channel, in 0 to 255 steps: two curves can share a channel mean and still differ.+ const differs = (first, second) => {+ let total = 0;+ for (let offset = 0; offset < first.length; offset += 4) for (let channel = 0; channel < 3; channel++) total += Math.abs(first[offset + channel] - second[offset + channel]);+ return total / (first.length / 4 * 3) >= 1;+ };+ const byName = Object.fromEntries(frames.map(frame => [frame.name, frame.pixels]));+ assert.deepEqual(problems, [], `${label}: console errors or warnings`);+ assert.equal(result.glError, 0, `${label}: WebGL error`);+ assert.ok(differs(byName.aces, byName.agx), `${label}: agx looks like aces`);+ assert.ok(differs(byName.aces, byName.neutral), `${label}: neutral looks like aces`);+ assert.ok(differs(byName.agx, byName.neutral), `${label}: neutral looks like agx`);+ assert.ok(differs(byName.agx, byName['agx+warm']), `${label}: the warm look changes nothing`);+ assert.ok(byName['aces again'].every((value, index) => value === byName.aces[index]), `${label}: going back to aces does not restore the picture`);+ await page.close();+ }+ console.log(JSON.stringify(report, null, 2));+} finally {+ await browser?.close();+ await server.close();+}diff --git a/src/renderer.js b/src/renderer.jsindex 5eac886d..76cc937f 100644--- a/src/renderer.js+++ b/src/renderer.js@@ -477,9 +477,13 @@ export class Renderer67 {this.diagnostics = [];this.activeLightDirection = environment.lightDirection ?? [-3, 6, 4];this.shadowCascades = shadowCascades;- this.viewTransform = viewTransform;- this.look = look;- if ((viewTransform !== 'aces' || look) && !viewTransformShader.apply) throw new Error("viewTransform and look need their shader code: add import '@codepawlhq/67js/addons/view-transform-shader' (a recipe: await loadAgentSceneModules(recipe)).");+ // The constructor options are the default; a frame's environment (a recipe's viewTransform and look) overrides them.+ this.defaultViewTransform = viewTransform;+ this.defaultLook = look;+ this.viewTransform = 'aces';+ this.look = null;+ this.lookSource = null;+ this.selectViewTransform(viewTransform, look);this.memoryBudgetMiB = memoryBudgetMiB;this.hdr = Boolean(hdr && !benchmark && this.context.getExtension('EXT_color_buffer_float'));if (volume.density > 0 && !this.hdr) throw new Error('Volume fog requires EXT_color_buffer_float for HDR rendering');@@ -637,6 +641,41 @@ export class Renderer67 {if (!this.benchmark && !this.multisampleFramebuffer) this.allocateFrameTargets(this.canvas.width, this.canvas.height);}+ selectViewTransform(viewTransform, lookSource) {+ if ((viewTransform !== 'aces' || lookSource) && !viewTransformShader.apply) throw new Error("viewTransform and look need their shader code: add import '@codepawlhq/67js/addons/view-transform-shader' (a recipe: await loadAgentSceneModules(recipe)).");+ this.viewTransform = viewTransform;+ this.lookSource = lookSource;+ this.look = lookSource ? viewTransformShader.resolveLook(lookSource) : null;+ }++ // A recipe's environment.viewTransform and environment.look reach the renderer in each frame; a change recompiles the+ // programs that carry the display curve (main, liquid, environment background, post) before the frame is drawn.+ syncViewTransform(environment) {+ const viewTransform = environment.viewTransform ?? this.defaultViewTransform;+ const lookSource = environment.look ?? this.defaultLook;+ if (viewTransform === this.viewTransform && lookSource === this.lookSource) return;+ viewTransformShader.dispose?.(this);+ this.selectViewTransform(viewTransform, lookSource);+ if (this.benchmark) return;+ const context = this.context;+ this.rebuildMainProgram();+ for (const variant of this.postVariants.values()) context.deleteProgram(variant.pending ? finishProgram(context, variant.pending) : variant.program);+ this.postVariants.clear();+ context.deleteProgram(this.postProgram);+ this.postProgram = finishProgram(context, startProgram(context, fullscreenVertex, this.withViewTransform(postFragment, 'post')));+ this.postUniforms = postUniformLocations(context, this.postProgram);+ const liquidNames = Object.keys(this.liquidUniforms);+ context.deleteProgram(this.liquidProgram);+ this.liquidProgram = finishProgram(context, startProgram(context, vertexShader, this.withViewTransform(liquidFragmentShader, 'liquid')));+ this.liquidUniforms = Object.fromEntries(liquidNames.map(name => [name, context.getUniformLocation(this.liquidProgram, name)]));+ context.useProgram(this.liquidProgram);+ context.uniform1i(this.liquidUniforms.skinInfluences, 6);+ context.uniform1i(this.liquidUniforms.jointPalette, 7);+ context.uniform1i(this.liquidUniforms.morphTargets, 8);+ if (this.environmentBackgroundProgram) context.deleteProgram(this.environmentBackgroundProgram);+ this.environmentBackgroundProgram = null;+ }+// The display curve and look of this renderer, spliced into one of its programs (view-transform-shader.js); the default curve changes nothing.withViewTransform(source, kind) {if (this.viewTransform === 'aces' && !this.look) return source;@@ -2080,6 +2119,7 @@ export class Renderer67 {if (!Number.isFinite(lightLength) || lightLength < 1e-6 || !Array.from(direction).every(component => typeof component === 'number' && Math.abs(component) <= 1e6)) throw new RangeError('Volume light direction must have a finite nonzero length and bounded components');}updateLightingState(this.lighting, environment, { world: this.world });+ this.syncViewTransform(environment);// The shadow view follows the sun, so a moved sun redraws the map.const lightChanged = String(this.activeLightDirection) !== String(environment.lightDirection);this.activeLightDirection = environment.lightDirection;diff --git a/src/view-transform-shader.js b/src/view-transform-shader.jsindex 521987b7..8c199927 100644--- a/src/view-transform-shader.js+++ b/src/view-transform-shader.js@@ -1,5 +1,5 @@import { viewTransformShader } from './shaders.js';-import { AGX_CONSTANTS, NEUTRAL_CONSTANTS } from './view-transform.js';+import { AGX_CONSTANTS, NEUTRAL_CONSTANTS, namedLook } from './view-transform.js';import { VIEW_TRANSFORM_NAMES } from './view-transform-names.js';// The GLSL of the AgX and Khronos PBR Neutral view transforms and of the look LUT. It lives outside shaders.js so a@@ -76,6 +76,14 @@ function apply(source, kind, viewTransform, hasLook) {}const lookTextures = new WeakMap();+const namedLooks = new Map();++/** A look as the renderer draws it: a name from LOOK_NAMES is built once, a parsed .cube look is used as is. */+function resolveLook(look) {+ if (typeof look !== 'string') return look;+ if (!namedLooks.has(look)) namedLooks.set(look, namedLook(look));+ return namedLooks.get(look);+}function createLookTexture(context, look) {const { size, data } = look;@@ -107,6 +115,6 @@ function dispose(renderer) {lookTextures.delete(renderer);}-Object.assign(viewTransformShader, { apply, bindLook, dispose });+Object.assign(viewTransformShader, { apply, bindLook, dispose, resolveLook });export { apply as applyViewTransformToShader };