3D engine that AI agents drive
Refuse a post-step mask with lens distortion, unbind the object ID texture after the mask draw, rename the size note (#707)
Nguyen Xuan An30eccb475123
diff --git a/changes/707-compositing.size.md b/changes/707.size.mdsimilarity index 100%rename from changes/707-compositing.size.mdrename to changes/707.size.mddiff --git a/docs/agent-look.md b/docs/agent-look.mdindex 41c2c8ce..e82d57a7 100644--- a/docs/agent-look.md+++ b/docs/agent-look.md@@ -276,7 +276,7 @@ Ambient occlusion reads the depth texture, so it needs the HDR depth target that| `lensDistortion.distortion` | `0` | `-1` to `1`. Above `0` is barrel, below is pincushion || `lensDistortion.dispersion` | `0` | `0` to `1`. Red and blue split apart toward the edges, up to 3 percent of the distance from the center |-A step with a `mask` applies only where the mask is on; with no `mask` it applies to the whole frame, as before. `mask` takes `objects` (one to 16 part names, as `scene_look` with `pass: "objectId"` lists them), `depth` (`near`, `far` metres along the view direction, and `feather`, the metres over which each end fades; defaults `0`, `100000`, `0`), or both, which multiply, and `invert: true` to flip the result. A grade for the hero part only: `{ "colorBalance": { "gain": [1.3, 1, 0.8], "mask": { "objects": ["hero"] } } }`. A mask by parts reads the object ID pass, so the frame is drawn twice, and its edges are hard because an object ID cannot be antialiased; a mask by depth costs nothing extra. A name that is not a part of the frame fails with the names it knows. `stylize` runs after color balance: pixelate first (the other filters read the blocks), then kuwahara, sharpen, posterize. Every filter is off until named, and a filter set to `0` is off.+A step with a `mask` applies only where the mask is on; with no `mask` it applies to the whole frame, as before. `mask` takes `objects` (one to 16 part names, as `scene_look` with `pass: "objectId"` lists them), `depth` (`near`, `far` metres along the view direction, and `feather`, the metres over which each end fades; defaults `0`, `100000`, `0`), or both, which multiply, and `invert: true` to flip the result. A grade for the hero part only: `{ "colorBalance": { "gain": [1.3, 1, 0.8], "mask": { "objects": ["hero"] } } }`. A mask by parts reads the object ID pass, so the frame is drawn twice, and its edges are hard because an object ID cannot be antialiased; a mask by depth costs nothing extra. A name that is not a part of the frame fails with the names it knows. A mask cannot be combined with an active `lensDistortion` (distortion other than 0, or dispersion above 0): the mask is built in undistorted screen space, so it would not line up with the warped image, and the recipe fails with an error that says so. `stylize` runs after color balance: pixelate first (the other filters read the blocks), then kuwahara, sharpen, posterize. Every filter is off until named, and a filter set to `0` is off.Depth of field and motion blur are deterministic: they read the frame, its depth and the camera, with fixed sample patterns, so a seek to a time gives the same pixels as playing to it.Depth of field applies to every render, including `scene_look`.diff --git a/src/post-mask.js b/src/post-mask.jsindex 4871942c..3d1dc1cf 100644--- a/src/post-mask.js+++ b/src/post-mask.js@@ -125,3 +125,10 @@ export function bindMask(context, uniforms, renderer, frame, mask) {context.uniform1f(uniforms.maskInvert, mask?.invert ? 1 : 0);context.activeTexture(context.TEXTURE1);}++/** Clears the object ID texture from unit 2 after the draw, so the pass framebuffer can be drawn into again. Leaves unit 1 active. */+export function unbindMask(context) {+ context.activeTexture(context.TEXTURE2);+ context.bindTexture(context.TEXTURE_2D, null);+ context.activeTexture(context.TEXTURE1);+}diff --git a/src/post-steps.js b/src/post-steps.jsindex a35aeb7d..0dd96fbd 100644--- a/src/post-steps.js+++ b/src/post-steps.js@@ -1,6 +1,6 @@import { postStepSupport } from './shader-registry.js';import { createStage, sharedFragmentHeader } from './camera-effects.js';-import { resolveMask, usesObjectMask, maskShader, bindMask, MASK_UNIFORM_NAMES } from './post-mask.js';+import { resolveMask, usesObjectMask, maskShader, bindMask, unbindMask, MASK_UNIFORM_NAMES } from './post-mask.js';export { usesObjectMask };@@ -105,9 +105,21 @@ export function resolvePostSteps(effects) {if (effects.glare !== undefined) resolved.glare = resolveGlare(effects.glare);if (effects.lensDistortion !== undefined) resolved.lensDistortion = resolveLensDistortion(effects.lensDistortion);if (effects.stylize !== undefined) resolved.stylize = resolveStylize(effects.stylize);+ refuseMaskWithLensDistortion(resolved);return resolved;}+// A mask reads the object ID pass and the depth, which are in undistorted screen space; lens distortion warps the colour image+// before the masked stages run, so the mask would sit in the wrong place, more so toward the corners and with dispersion.+function refuseMaskWithLensDistortion(resolved) {+ const lens = resolved.lensDistortion;+ const lensIsOn = Boolean(lens) && (lens.distortion !== 0 || lens.dispersion > 0);+ const maskedStep = ['colorBalance', 'stylize'].find(name => resolved[name]?.mask);+ if (lensIsOn && maskedStep) {+ throw new TypeError(`${maskedStep}.mask cannot be combined with lensDistortion: the mask is in undistorted screen space and would not line up with the warped image. Fix: remove the mask, or set lensDistortion.distortion and dispersion to 0.`);+ }+}+// Registers what the look needs: the field names it accepts and how to resolve them (src/shader-registry.js).postStepSupport.fields = ['colorBalance', 'glare', 'lensDistortion', 'stylize'];postStepSupport.resolve = resolvePostSteps;@@ -264,8 +276,21 @@ function isColorBalanceActive(colorBalance) {return Boolean(colorBalance) && ['lift', 'gamma', 'gain'].some(name => colorBalance[name].some((component, index) => component !== NEUTRAL[name][index]));}+// The object ID texture is the colour attachment of the pass framebuffer, which the next frame draws into. Leaving it bound on+// unit 2 would risk a feedback loop, so the unit is cleared after the masked draw.+function withMaskUnbound(stage) {+ return {+ ...stage,+ apply(renderer, frame, effects, sourceTexture) {+ const result = stage.apply(renderer, frame, effects, sourceTexture);+ if (result) unbindMask(renderer.context);+ return result;+ },+ };+}+function createColorBalanceStage() {- return createStage({+ return withMaskUnbound(createStage({name: 'Color balance',fragment: colorBalanceFragment,uniformNames: ['lift', 'inverseGamma', 'gain', ...MASK_UNIFORM_NAMES],@@ -277,7 +302,7 @@ function createColorBalanceStage() {context.uniform3fv(uniforms.gain, colorBalance.gain);bindMask(context, uniforms, renderer, frame, colorBalance.mask);},- });+ }));}function isStylizeActive(stylize) {@@ -285,7 +310,7 @@ function isStylizeActive(stylize) {}function createStylizeStage() {- return createStage({+ return withMaskUnbound(createStage({name: 'Stylize',fragment: stylizeFragment,uniformNames: ['sharpenAmount', 'kuwaharaRadius', 'posterizeLevels', 'pixelateSize', ...MASK_UNIFORM_NAMES],@@ -298,7 +323,7 @@ function createStylizeStage() {context.uniform1f(uniforms.pixelateSize, stylize.pixelate ?? 0);bindMask(context, uniforms, renderer, frame, stylize.mask);},- });+ }));}function createGlareStage() {diff --git a/tests/post-steps.test.js b/tests/post-steps.test.jsindex 5fe3823d..c0fb9910 100644--- a/tests/post-steps.test.js+++ b/tests/post-steps.test.js@@ -146,12 +146,23 @@ test('a stage binds the part numbers of a masked step and refuses an unknown parconst bound = calls.find(call => call[0] === 'uniform1iv');assert.deepEqual([...bound[2]].slice(0, 3), [2, 4, 0]);assert.throws(() => stage.apply(renderer, frame({ objects: ['ghost'] }), effects, 'source'), /"ghost" is not a part in the frame\. Fix: use a part name such as "floor"/);+ const afterDraw = calls.slice(calls.findLastIndex(call => call[0] === 'drawArrays') + 1);+ const unbind = afterDraw.findIndex(call => call[0] === 'bindTexture' && call[2] === null);+ assert.ok(unbind > 0 && afterDraw[unbind - 1][0] === 'activeTexture', 'the object ID texture is unbound from its unit after the draw');renderer.passSupport = null;assert.throws(() => stage.apply(renderer, frame({ objects: ['hero'] }), effects, 'source'), /installRenderPasses/);stage.releaseTargets(context);stage.dispose(context);});+test('a mask cannot be combined with an active lens distortion, which warps the image the mask is not warped with', () => {+ const masked = { colorBalance: { gain: [1.2, 1, 1], mask: { depth: { far: 5 } } } };+ assert.throws(() => resolvePostSteps({ ...masked, lensDistortion: { distortion: 0.2 } }), /colorBalance.mask cannot be combined with lensDistortion/);+ assert.throws(() => resolvePostSteps({ stylize: { pixelate: 4, mask: { depth: { far: 5 } } }, lensDistortion: { dispersion: 0.3 } }), /stylize.mask cannot be combined/);+ assert.ok(resolvePostSteps({ ...masked, lensDistortion: {} }).colorBalance.mask);+ assert.ok(resolvePostSteps({ lensDistortion: { distortion: 0.2 }, colorBalance: { gain: [1.2, 1, 1] } }).lensDistortion);+});+test('the shaders keep their fixed sample patterns and read no history', () => {for (const fragment of [glareFragment, lensDistortionFragment, colorBalanceFragment, stylizeFragment]) {assert.match(fragment, /^#version 300 es/);