3D engine that AI agents drive
Let post passes read the object ID pass (part of #707)
Nguyen Xuan Ana68f4c7c0e72
diff --git a/bench/post/post-steps.js b/bench/post/post-steps.jsindex 1571dccc..b046f2e9 100644--- a/bench/post/post-steps.js+++ b/bench/post/post-steps.js@@ -3,6 +3,8 @@ import { Renderer67 } from '/src/renderer.js';import { box } from '/src/geometry.js';import { installCameraEffects } from '/src/camera-effects.js';import { installRenderPasses } from '/src/render-passes.js';+const userPassPath = '/src/user-pass.js';+const userPassModule = await import(/* @vite-ignore */ userPassPath).catch(() => null);// The named steps are an optional module: this harness loads it only when the source tree has it, so the same page also// draws the default picture of an older tree.@@ -54,6 +56,19 @@ const CASES = {'key-chroma-masked': { composite: { background: { color: [0.9, 0.35, 0.1] }, key: { chroma: { color: GREEN_SCREEN, tolerance: 0.25 } }, mask: { depth: { near: 0, far: 3 } } } },'key-luma': { composite: { background: { color: [0.1, 0.2, 0.6] }, key: { luma: { low: 0.05, high: 0.3 } } } },};+// A user pass that reads the object ID pass: it tints only the part it lists, as a snippet would to grade one object.+const USER_PASS_CASES = {+ 'user-pass-hero': [{+ name: 'tint-hero',+ objects: ['hero'],+ fragment: [+ 'vec4 userPass(vec2 uv) {',+ ' vec4 color = texture(sceneColor, uv);',+ ' return objectSlotAt(uv) == 0 ? vec4(color.r * 1.4, color.g * 0.6, color.b * 0.4, color.a) : color;',+ '}',+ ].join('\n'),+ }],+};const GREEN_CASES = ['green-default', 'key-chroma', 'key-chroma-masked'];const camera = { eye: [3.4, 1.8, 4.6], target: [0, 0.8, -1], fov: 45 };@@ -71,9 +86,13 @@ async function drawCase(name, { warmupFrames = 4, measuredFrames = 20 } = {}) {const canvas = document.querySelector('canvas');const renderer = installRenderPasses(installCameraEffects(new Renderer67(canvas, sceneParts(GREEN_CASES.includes(name)), { ground: false, hdr: true, depth: true, samples: 4, pixelRatio: 1, memoryBudgetMiB: 1024, environment })));if (postSteps) postSteps.installPostSteps(renderer);+ if (USER_PASS_CASES[name] && userPassModule) {+ userPassModule.installUserPasses(renderer);+ renderer.userPasses.set(userPassModule.normalizeUserPasses(USER_PASS_CASES[name]));+ }renderer.resize();const frame = { ...camera, environment, time: 0, exposure: 1, effects: { ambientOcclusion: 0, ...CASES[name] } };- const usesObjects = postSteps?.usesObjectMask(postSteps.resolvePostSteps(frame.effects)) ?? false;+ const usesObjects = Boolean(USER_PASS_CASES[name]) || (postSteps?.usesObjectMask(postSteps.resolvePostSteps(frame.effects)) ?? false);const drawFrame = () => {// A mask by parts reads the object ID pass, drawn first.if (usesObjects) renderer.render({ ...frame, pass: 'objectId' });@@ -102,5 +121,5 @@ async function drawCase(name, { warmupFrames = 4, measuredFrames = 20 } = {}) {}window.hasPostSteps = Boolean(postSteps);-window.caseNames = Object.keys(CASES);+window.caseNames = [...Object.keys(CASES), ...Object.keys(USER_PASS_CASES)];window.drawCase = drawCase;diff --git a/bench/post/post-steps.mjs b/bench/post/post-steps.mjsindex 0cb105e7..69238fc7 100644--- a/bench/post/post-steps.mjs+++ b/bench/post/post-steps.mjs@@ -27,7 +27,7 @@ try {await page.goto(`${serverUrl}/bench/post/post-steps.html`, { timeout: PAGE_LOAD_TIMEOUT });await page.waitForFunction(() => window.drawCase, null, { timeout: PAGE_LOAD_TIMEOUT });const hasPostSteps = await page.evaluate(() => window.hasPostSteps);- const names = (await page.evaluate(() => window.caseNames)).filter(name => hasPostSteps || !/color-balance|glare|lens|grade|sharpen|kuwahara|posterize|pixelate/.test(name));+ const names = (await page.evaluate(() => window.caseNames)).filter(name => hasPostSteps || !/color-balance|user-pass|glare|lens|grade|sharpen|kuwahara|posterize|pixelate/.test(name));await mkdir(outputFolder, { recursive: true });const medians = new Map(names.map(name => [name, []]));const hashes = {};diff --git a/changes/707-pass-objects.md b/changes/707-pass-objects.mdnew file mode 100644index 00000000..bf17ae4d--- /dev/null+++ b/changes/707-pass-objects.md@@ -0,0 +1 @@+- A post pass can list parts in `objects` and call `objectSlotAt(uv)` to read the object ID pass, so a snippet grades or blurs one object (#707)diff --git a/changes/707-pass-objects.size.md b/changes/707-pass-objects.size.mdnew file mode 100644index 00000000..f8c6da2a--- /dev/null+++ b/changes/707-pass-objects.size.md@@ -0,0 +1,2 @@+- entry:user-pass: a pass that lists parts reads the object ID pass through objectSlotAt, which needs the part-name check, the id lookup and the texture binding (751 bytes gzip); no other bundle runs that code+- scene:agent-scene+lazy: the same code, once, in the lazy chunk the scene loads only for a recipe with postPasses (913 bytes gzip)diff --git a/docs/user-passes.md b/docs/user-passes.mdindex 84003eba..3e55cbb2 100644--- a/docs/user-passes.md+++ b/docs/user-passes.md@@ -20,6 +20,7 @@ Passes run in order on the finished frame: after depth of field, motion blur, li| `fragment` | GLSL ES 3.00 text that defines `vec4 userPass(vec2 uv)`. Helper functions and `const` values may come before it. 4096 bytes at most. || `uniforms` | Named values. A number is a `float`; `[x, y]`, `[x, y, z]` and `[x, y, z, w]` are `vec2`, `vec3` and `vec4`; an int is `{ "type": "int", "value": 4 }`. || `scale` | 0.05 to 1 (default 1). The pass renders at this fraction of the width and height, and whatever reads it enlarges it with nearest filtering. A `scale` of 0.25 on the last pass is a pixel-art downscale. |+| `objects` | Optional list of 1 to 16 part names (node ids, as the object ID pass lists them). `objectSlotAt(uv)` tells which of them is under a pixel. The preview and render jobs draw the object ID pass first when a pass lists parts. |You do not declare uniforms or the version line; the engine writes them. A snippet sees these names:@@ -31,6 +32,7 @@ You do not declare uniforms or the version line; the engine writes them. A snipp| `time` | `float` | Frame time in seconds. || `depthAt(uv)`, `worldPointAt(uv)` | functions | Raw depth, and the world position of the pixel. || `worldNormalAt(uv)` | function | The surface normal, found from neighbouring depth samples. It is exact on flat faces and wrong at silhouettes and in the sky. |+| `objectSlotAt(uv)` | function | The position in the pass's `objects` list of the part under `uv`, or -1 for the background, any other part, and a pass with no `objects`. Edges are hard: an object ID cannot be antialiased. A name that is not a part in the frame fails the render and lists part names to use. || `gl_FragCoord` | `vec4` | The pixel position inside this pass. |## Limits and rulesdiff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex 0e059640..5eff4482 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -365,11 +365,13 @@ function draw(record, time = 0, camera = record.scene.camera, frameOptions = {})}record.renderer.world = environment.world;syncMeshBlend(record);- record.renderer.userPasses?.set(record.scene.postPasses ?? []);+ const postPasses = record.scene.postPasses ?? [];+ record.renderer.userPasses?.set(postPasses);applyRecipeWindHeading(record.renderer, environment);const { objectMotion, ...renderOptions } = frameOptions;const frame = { ...camera, environment: litEnvironment(record.scene.parts, environment, Boolean(record.renderer.groundPart)), effects, exposure: effects.exposure, time, lines: environment.lines, ...windFrame(environment), ...renderOptions, effects: effectsWithFStop(renderOptions.effects ?? effects, camera) };- let needsObjectIds = Boolean(objectMotion);+ // Object motion blur, a mask by parts and a user pass with `objects` all read the object ID pass.+ let needsObjectIds = Boolean(objectMotion) || postPasses.some(pass => pass.objects);if (effects.colorBalance || effects.glare || effects.lensDistortion || effects.stylize || effects.composite) {const postSteps = agentSceneModule('postSteps');postSteps.installPostSteps(record.renderer);diff --git a/src/post-mask.js b/src/post-mask.jsindex 3d1dc1cf..93dc4ad3 100644--- a/src/post-mask.js+++ b/src/post-mask.js@@ -19,7 +19,7 @@ function checkedKeys(group, allowed, name) {}}-function resolveObjectNames(objects, name) {+export function resolveObjectNames(objects, name) {if (!Array.isArray(objects) || objects.length === 0 || objects.length > MAXIMUM_MASK_OBJECTS) {throw new TypeError(`${name} must be a list of 1 to ${MAXIMUM_MASK_OBJECTS} part names.`);}@@ -89,7 +89,7 @@ float maskAt(vec2 coordinate) {}`;-function objectIdentifiers(renderer, names) {+export function objectIdentifiers(renderer, names) {if (!renderer.passSupport) throw new Error('A mask by objects needs installRenderPasses(renderer) and the object ID pass drawn first.');const known = renderer.passSupport.objectNames();const identifiers = [];diff --git a/src/user-pass.js b/src/user-pass.jsindex aeb17a17..32d9aea6 100644--- a/src/user-pass.js+++ b/src/user-pass.js@@ -2,6 +2,7 @@ import { compileProgram, fullscreenVertex } from './gl-program.js';import {SNIPPET_LIMITS, requireKnownFields, validateShaderSnippet, validateSnippetUniforms, snippetUniformDeclarations, mapShaderInfoLog,} from './shader-snippet-validator.js';+import { MAXIMUM_MASK_OBJECTS, resolveObjectNames, objectIdentifiers, unbindMask } from './post-mask.js';// User post-process passes: fragment snippets that run on the finished frame, after the stock effects, the grade and the// view transform, as the last step before the canvas. The core renderer carries nothing for them. installUserPasses wraps@@ -9,10 +10,10 @@ import {// A snippet is checked by shader-snippet-validator.js before it is ever compiled./** Names the engine provides to a snippet; a uniform cannot take one. */-export const USER_PASS_INPUTS = Object.freeze(['sceneColor', 'sceneDepth', 'resolution', 'time', 'depthAt', 'worldPointAt', 'worldNormalAt', 'userPass']);+export const USER_PASS_INPUTS = Object.freeze(['sceneColor', 'sceneDepth', 'resolution', 'time', 'depthAt', 'worldPointAt', 'worldNormalAt', 'objectSlotAt', 'sceneObjectIds', 'userPass']);export const USER_PASS_SCALE = Object.freeze({ min: 0.05, max: 1 });const PASS_NAME_PATTERN = /^[a-z][A-Za-z0-9-]{0,31}$/;-const PASS_FIELDS = ['name', 'fragment', 'uniforms', 'scale'];+const PASS_FIELDS = ['name', 'fragment', 'uniforms', 'scale', 'objects'];const ENTRY_FUNCTION = Object.freeze({ pattern: /\bvec4\s+userPass\s*\(\s*vec2\s+[A-Za-z_]\w*\s*\)\s*\{/, example: 'vec4 userPass(vec2 uv) { return texture(sceneColor, uv); }' });/**@@ -38,14 +39,16 @@ export function normalizeUserPasses(input) {}validateShaderSnippet(entry.fragment, { label: `Post pass "${entry.name}" fragment`, requiredFunction: ENTRY_FUNCTION });const uniforms = validateSnippetUniforms(entry.uniforms, { label: `Post pass "${entry.name}"`, reservedNames: USER_PASS_INPUTS });- return { name: entry.name, fragment: entry.fragment, uniforms, scale };+ const normalized = { name: entry.name, fragment: entry.fragment, uniforms, scale };+ if (entry.objects !== undefined) normalized.objects = resolveObjectNames(entry.objects, `postPasses[${index}].objects`);+ return normalized;});}/** What scene_capabilities reports under postPasses; it lives here so the scene entry does not carry the text. */export const USER_PASS_CAPABILITIES = Object.freeze({- fields: 'name, fragment (defines vec4 userPass(vec2 uv)), uniforms, scale 0.05-1 (1)',- inputs: 'sceneColor, sceneDepth (sampler2D), resolution (vec2, this pass), time (s), depthAt(uv), worldPointAt(uv), worldNormalAt(uv, from depth); gl_FragCoord',+ fields: 'name, fragment (defines vec4 userPass(vec2 uv)), uniforms, scale 0.05-1 (1), objects (1-16 part names)',+ inputs: 'sceneColor, sceneDepth (sampler2D), resolution (vec2, this pass), time (s), depthAt(uv), worldPointAt(uv), worldNormalAt(uv, from depth), objectSlotAt(uv) (position of the part under uv in the pass objects list, or -1); gl_FragCoord',uniforms: 'a number is float, [x,y(,z(,w))] is vec2-4, {type:"int",value} is int; name ^[a-z][A-Za-z0-9]{0,31}$',rules: 'no # lines, while, do, uniform/layout declarations, gl_ names but gl_FragCoord, or u67_ names; for loops only as for (int i = 0; i < N; i++) { } with literal bounds, 64 iterations at most, loop work under 4096',limits: Object.freeze({ maxPasses: SNIPPET_LIMITS.maxPasses, maxBytes: SNIPPET_LIMITS.maxBytes, maxUniforms: SNIPPET_LIMITS.maxUniforms, maxLoopIterations: SNIPPET_LIMITS.maxLoopIterations, maxLoopWork: SNIPPET_LIMITS.maxLoopWork }),@@ -57,6 +60,7 @@ export function describeUserPasses(passes) {return passes.map(pass => ({name: pass.name,scale: pass.scale,+ ...(pass.objects ? { objects: pass.objects } : {}),uniforms: Object.entries(pass.uniforms).map(([name, { type }]) => `${name}:${type}`),fragmentBytes: pass.fragment.length,}));@@ -71,12 +75,26 @@ uniform sampler2D sceneDepth;uniform mat4 u67_inverseViewProjection;uniform vec2 resolution;uniform float time;+uniform sampler2D sceneObjectIds;+uniform int u67_objectCount;+uniform int u67_objectIds[${MAXIMUM_MASK_OBJECTS}];out vec4 u67_output;float depthAt(vec2 uv) { return texture(sceneDepth, uv).r; }vec3 worldPointAt(vec2 uv) {vec4 point = u67_inverseViewProjection * vec4(uv * 2.0 - 1.0, depthAt(uv) * 2.0 - 1.0, 1.0);return point.xyz / point.w;}+// Which entry of the pass objects list is under uv: -1 for the background, a part not in the list, or a pass with no objects.+int objectSlotAt(vec2 uv) {+ if (u67_objectCount == 0) return -1;+ vec3 idBytes = floor(texelFetch(sceneObjectIds, ivec2(uv * vec2(textureSize(sceneObjectIds, 0))), 0).rgb * 255.0 + 0.5);+ int identifier = int(idBytes.r + idBytes.g * 256.0 + idBytes.b * 65536.0);+ for (int slot = 0; slot < ${MAXIMUM_MASK_OBJECTS}; slot++) {+ if (slot >= u67_objectCount) break;+ if (u67_objectIds[slot] == identifier) return slot;+ }+ return -1;+}// Normal from the depth buffer: the cross product of the smaller step in x and in y, facing the camera.vec3 worldNormalAt(vec2 uv) {vec2 texel = 1.0 / vec2(textureSize(sceneDepth, 0));@@ -200,7 +218,7 @@ export function installUserPasses(renderer) {throw compilationError(pass, error, built);}if (entry) releaseEntry(context, entry);- const names = ['sceneColor', 'sceneDepth', 'u67_inverseViewProjection', 'resolution', 'time', ...Object.keys(pass.uniforms)];+ const names = ['sceneColor', 'sceneDepth', 'sceneObjectIds', 'u67_objectCount', 'u67_objectIds', 'u67_inverseViewProjection', 'resolution', 'time', ...Object.keys(pass.uniforms)];state.entries[index] = {pass, source: built.source, program, target: null,locations: Object.fromEntries(names.map(name => [name, context.getUniformLocation(program, name)])),@@ -245,7 +263,19 @@ export function installUserPasses(renderer) {context.uniform2f(locations.resolution, width, height);context.uniform1f(locations.time, frame.time ?? 0);for (const [name, uniform] of Object.entries(pass.uniforms)) setUniformValue(context, locations[name], uniform);+ const identifiers = pass.objects ? objectIdentifiers(renderer, pass.objects) : [];+ context.uniform1i(locations.u67_objectCount, identifiers.length);+ if (identifiers.length > 0) {+ const padded = new Int32Array(MAXIMUM_MASK_OBJECTS);+ padded.set(identifiers);+ context.uniform1iv(locations.u67_objectIds, padded);+ context.activeTexture(context.TEXTURE2);+ context.bindTexture(context.TEXTURE_2D, renderer.passSupport.texture);+ context.uniform1i(locations.sceneObjectIds, 2);+ }context.drawArrays(context.TRIANGLES, 0, 3);+ // The next frame draws into the object ID target, so it must not stay bound as a texture.+ if (identifiers.length > 0) unbindMask(context);}function copyToCanvas(texture, width, height) {diff --git a/tests/post-steps.test.js b/tests/post-steps.test.jsindex d7aa7b89..8b4904db 100644--- a/tests/post-steps.test.js+++ b/tests/post-steps.test.js@@ -11,6 +11,7 @@ import { agentSceneModulesNeeded, loadAgentSceneModules } from '../src/agent-sceimport { installCameraEffects } from '../src/camera-effects.js';import { installPostSteps, resolvePostSteps, glareFragment, lensDistortionFragment, colorBalanceFragment, stylizeFragment, compositeFragment, usesObjectMask } from '../src/post-steps.js';import { createAgentScene } from '../src/agent-scene.js';+import { normalizeUserPasses, buildPassFragment, installUserPasses, describeUserPasses } from '../src/user-pass.js';function recordingContext(calls) {return new Proxy({}, {@@ -183,6 +184,40 @@ test('a composite step resolves a background and a key, and says what to pass whassert.throws(() => resolvePostSteps({ ...masked, lensDistortion: { distortion: 0.2 } }), /composite.mask cannot be combined with lensDistortion/);});+const PASS_FRAGMENT = 'vec4 userPass(vec2 uv) { return objectSlotAt(uv) == 0 ? vec4(1.0) : texture(sceneColor, uv); }';++test('a user pass lists the parts it reads and says what to pass when the list is wrong', () => {+ const [pass] = normalizeUserPasses([{ name: 'tint', fragment: PASS_FRAGMENT, objects: ['hero', 'floor'] }]);+ assert.deepEqual(pass.objects, ['hero', 'floor']);+ assert.deepEqual(describeUserPasses([pass])[0].objects, ['hero', 'floor']);+ assert.equal(Object.hasOwn(normalizeUserPasses([{ name: 'plain', fragment: 'vec4 userPass(vec2 uv) { return texture(sceneColor, uv); }' }])[0], 'objects'), false);+ assert.throws(() => normalizeUserPasses([{ name: 'tint', fragment: PASS_FRAGMENT, objects: [] }]), /postPasses\[0\]\.objects must be a list of 1 to 16 part names/);+ assert.throws(() => normalizeUserPasses([{ name: 'tint', fragment: PASS_FRAGMENT, objects: [3] }]), /postPasses\[0\]\.objects\[0\] must be a part name/);+ assert.throws(() => normalizeUserPasses([{ name: 'tint', fragment: PASS_FRAGMENT, objects: Array(17).fill('a') }]), /1 to 16 part names/);+ assert.throws(() => normalizeUserPasses([{ name: 'tint', fragment: PASS_FRAGMENT, uniforms: { objectSlotAt: 1 } }]), /objectSlotAt/);+ assert.match(buildPassFragment(pass).source, /int objectSlotAt\(vec2 uv\)/);+});++test('a user pass binds the object ID pass for its parts and unbinds it after the draw', () => {+ const calls = [];+ const context = recordingContext(calls);+ const renderer = {+ context, canvas: { width: 16, height: 9 }, postStages: [], depthTexture: 'depth', whiteTexture: 'white',+ inverseViewProjection: mat4.create(), resolveFrame() {}, benchmark: false,+ passSupport: { texture: 'ids', objectNames: () => ['floor', 'hero'] },+ };+ installUserPasses(renderer);+ renderer.userPasses.set([{ name: 'tint', fragment: PASS_FRAGMENT, objects: ['hero'] }]);+ renderer.resolveFrame({ time: 0 });+ const uploaded = calls.find(call => call[0] === 'uniform1iv');+ assert.deepEqual(Array.from(uploaded[2]).slice(0, 2), [2, 0], 'the hero part has object ID 2, and the rest of the list is padded');+ assert.ok(calls.some(call => call[0] === 'bindTexture' && call[2] === 'ids'));+ const afterDraw = calls.slice(calls.findLastIndex(call => call[0] === 'drawArrays') + 1);+ assert.ok(afterDraw.some(call => call[0] === 'bindTexture' && call[2] === null), 'the object ID texture is unbound after the draw');+ renderer.userPasses.set([{ name: 'tint', fragment: PASS_FRAGMENT, objects: ['missing'] }]);+ assert.throws(() => renderer.resolveFrame({ time: 0 }), /Mask object "missing" is not a part in the frame/);+});+test('the shaders keep their fixed sample patterns and read no history', () => {for (const fragment of [glareFragment, lensDistortionFragment, colorBalanceFragment, stylizeFragment, compositeFragment]) {assert.match(fragment, /^#version 300 es/);@@ -195,7 +230,7 @@ test('the shaders keep their fixed sample patterns and read no history', () => {const chromePath = process.env.CHROME_PATH ?? 'C:/Program Files/Google/Chrome/Application/chrome.exe';const rendersEnabled = process.env.RENDER_TESTS === '1' && existsSync(chromePath);const STEP_CASES = ['defocus', 'color-balance', 'glare-streaks', 'glare-streaks-6', 'glare-anamorphic', 'glare-fog', 'lens-distortion', 'lens-dispersion',- 'grade-hero', 'grade-depth', 'sharpen', 'kuwahara', 'posterize', 'pixelate', 'pixelate-hero', 'key-chroma', 'key-luma'];+ 'grade-hero', 'grade-depth', 'user-pass-hero', 'sharpen', 'kuwahara', 'posterize', 'pixelate', 'pixelate-hero', 'key-chroma', 'key-luma'];test('each named step changes the picture and the default still keeps its hash', { skip: !rendersEnabled, timeout: 300000 }, () => {const output = execFileSync('node', ['bench/post/post-steps.mjs', '--rounds=1', `--out=${join(tmpdir(), 'post-steps-test')}`], { encoding: 'utf8' });@@ -213,6 +248,9 @@ test('each named step changes the picture and the default still keeps its hash',assert.deepEqual(summary['grade-hero'].regions.floor, summary.default.regions.floor);assert.notDeepEqual(summary['color-balance'].regions.floor, summary.default.regions.floor);assert.deepEqual(summary['pixelate-hero'].regions.floor, summary.default.regions.floor);+ // A user pass that lists the hero part and tints where objectSlotAt says 0 changes the hero and leaves the floor alone.+ assert.notDeepEqual(summary['user-pass-hero'].regions.hero, summary.default.regions.hero);+ assert.deepEqual(summary['user-pass-hero'].regions.floor, summary.default.regions.floor);// Hash of the default still as origin/main drew it before the named steps (bench/post/post-steps.mjs --root=<origin/main checkout>).assert.equal(summary.default.hash, '8741ffc3aaace9a8');});