3D engine that AI agents drive
Mask post steps by part names or depth, and add stylize filters (part of #707)
Nguyen Xuan And6c3bef6fe05
diff --git a/bench/post/post-steps.js b/bench/post/post-steps.jsindex 4a6d10d5..89137c6e 100644--- a/bench/post/post-steps.js+++ b/bench/post/post-steps.js@@ -2,6 +2,7 @@import { Renderer67 } from '/src/renderer.js';import { box } from '/src/geometry.js';import { installCameraEffects } from '/src/camera-effects.js';+import { installRenderPasses } from '/src/render-passes.js';// The named steps are an optional module: this harness loads it only when the source tree has it, so the same page also// draws the default picture of an older tree.@@ -17,7 +18,7 @@ function sceneParts() {const lamp = { color: [1, 0.9, 0.7], emissiveColor: [1, 0.9, 0.7], metallic: 0, roughness: 1, emission: 14 };const parts = [{ geometry: box(), matrix: placed([0, -0.05, 0], [10, 0.1, 10]), material },- { geometry: box(), matrix: placed([0, 0.6, 0], [0.4, 1.2, 0.4]), material },+ { name: 'hero', geometry: box(), matrix: placed([0, 0.6, 0], [0.4, 1.2, 0.4]), material },{ geometry: box(), matrix: placed([-1.2, 1.6, -0.6], [0.9, 0.06, 0.06]), material: lamp },];for (let index = 0; index < 5; index++) {@@ -37,34 +38,58 @@ const CASES = {'glare-fog': { glare: { fogGlow: 0.6, threshold: 1, size: 0.12 } },'lens-distortion': { lensDistortion: { distortion: 0.8 } },'lens-dispersion': { lensDistortion: { dispersion: 1 } },+ 'grade-hero': { colorBalance: { gain: [1.4, 0.8, 0.5], mask: { objects: ['hero'] } } },+ 'grade-depth': { colorBalance: { gain: [1.4, 0.8, 0.5], mask: { depth: { near: 6, far: 9, feather: 1 } } } },+ sharpen: { stylize: { sharpen: 1.5 } },+ kuwahara: { stylize: { kuwahara: 3 } },+ posterize: { stylize: { posterize: 4 } },+ pixelate: { stylize: { pixelate: 8 } },+ 'pixelate-hero': { stylize: { pixelate: 8, mask: { objects: ['hero'] } } },};const camera = { eye: [3.4, 1.8, 4.6], target: [0, 0.8, -1], fov: 45 };+// Mean color of a 6 by 6 pixel patch at a point of the frame (0 to 1, from the lower left), to check where a step changed the picture.+function regionMean(context, x, y) {+ const size = 6;+ const pixels = new Uint8Array(size * size * 4);+ context.readPixels(Math.round(x * context.drawingBufferWidth) - size / 2, Math.round(y * context.drawingBufferHeight) - size / 2, size, size, context.RGBA, context.UNSIGNED_BYTE, pixels);+ const mean = [0, 0, 0];+ for (let index = 0; index < pixels.length; index += 4) for (let channel = 0; channel < 3; channel++) mean[channel] += pixels[index + channel] / (size * size);+ return mean.map(value => Number(value.toFixed(2)));+}+async function drawCase(name, { warmupFrames = 4, measuredFrames = 20 } = {}) {const canvas = document.querySelector('canvas');- const renderer = installCameraEffects(new Renderer67(canvas, sceneParts(), { ground: false, hdr: true, depth: true, samples: 4, pixelRatio: 1, memoryBudgetMiB: 1024, environment }));+ const renderer = installRenderPasses(installCameraEffects(new Renderer67(canvas, sceneParts(), { ground: false, hdr: true, depth: true, samples: 4, pixelRatio: 1, memoryBudgetMiB: 1024, environment })));if (postSteps) postSteps.installPostSteps(renderer);renderer.resize();const frame = { ...camera, environment, time: 0, exposure: 1, effects: { ambientOcclusion: 0, ...CASES[name] } };+ const usesObjects = postSteps?.usesObjectMask(postSteps.resolvePostSteps(frame.effects)) ?? false;+ const drawFrame = () => {+ // A mask by parts reads the object ID pass, drawn first.+ if (usesObjects) renderer.render({ ...frame, pass: 'objectId' });+ renderer.render(frame);+ };const pixel = new Uint8Array(4);const finishDrawing = () => renderer.context.readPixels(0, 0, 1, 1, renderer.context.RGBA, renderer.context.UNSIGNED_BYTE, pixel);for (let index = 0; index < warmupFrames; index++) {- renderer.render(frame);+ drawFrame();finishDrawing();}const frameMilliseconds = [];for (let index = 0; index < measuredFrames; index++) {const started = performance.now();- renderer.render(frame);+ drawFrame();finishDrawing();frameMilliseconds.push(performance.now() - started);}- renderer.render(frame);+ drawFrame();const glError = renderer.context.getError();+ const regions = { hero: regionMean(renderer.context, 0.44, 0.44), floor: regionMean(renderer.context, 0.5, 0.06) };const image = canvas.toDataURL('image/png').split(',')[1];renderer.dispose();frameMilliseconds.sort((first, second) => first - second);- return { name, glError, image, medianMilliseconds: frameMilliseconds[Math.floor(measuredFrames / 2)] };+ return { name, glError, image, regions, medianMilliseconds: frameMilliseconds[Math.floor(measuredFrames / 2)] };}window.hasPostSteps = Boolean(postSteps);diff --git a/bench/post/post-steps.mjs b/bench/post/post-steps.mjsindex 8fbcc301..0cb105e7 100644--- a/bench/post/post-steps.mjs+++ b/bench/post/post-steps.mjs@@ -27,16 +27,18 @@ try {await page.goto(`${serverUrl}/bench/post/post-steps.html`, { timeout: PAGE_LOAD_TIMEOUT });await page.waitForFunction(() => window.drawCase, null, { timeout: PAGE_LOAD_TIMEOUT });const hasPostSteps = await page.evaluate(() => window.hasPostSteps);- const names = (await page.evaluate(() => window.caseNames)).filter(name => hasPostSteps || !/color-balance|glare|lens/.test(name));+ const names = (await page.evaluate(() => window.caseNames)).filter(name => hasPostSteps || !/color-balance|glare|lens|grade|sharpen|kuwahara|posterize|pixelate/.test(name));await mkdir(outputFolder, { recursive: true });const medians = new Map(names.map(name => [name, []]));const hashes = {};+ const regions = {};for (let round = 0; round < rounds; round++) {for (const name of round % 2 === 0 ? names : [...names].reverse()) {const result = await page.evaluate(caseName => window.drawCase(caseName), name);if (result.glError !== 0) throw new Error(`${name}: GL error ${result.glError}`);medians.get(name).push(result.medianMilliseconds);const png = Buffer.from(result.image, 'base64');+ regions[name] = result.regions;hashes[name] = createHash('sha256').update(png).digest('hex').slice(0, 16);await writeFile(resolve(outputFolder, `${name}.png`), png);}@@ -44,7 +46,7 @@ try {const summary = {};for (const [name, values] of medians) {const sorted = [...values].sort((first, second) => first - second);- summary[name] = { medianMilliseconds: Number(sorted[Math.floor(sorted.length / 2)].toFixed(3)), rangeMilliseconds: [sorted[0], sorted.at(-1)].map(value => Number(value.toFixed(3))), hash: hashes[name] };+ summary[name] = { medianMilliseconds: Number(sorted[Math.floor(sorted.length / 2)].toFixed(3)), rangeMilliseconds: [sorted[0], sorted.at(-1)].map(value => Number(value.toFixed(3))), hash: hashes[name], regions: regions[name] };}console.log(JSON.stringify({ size: [width, height], rounds, summary }, null, 1));} finally {diff --git a/changes/707-compositing.size.md b/changes/707-compositing.size.mdnew file mode 100644index 00000000..06f65292--- /dev/null+++ b/changes/707-compositing.size.md@@ -0,0 +1 @@+- scene:agent-scene: the stylize effects field adds a few bytes of schema; the filter and mask code sits in the lazy post-steps module.diff --git a/changes/707.md b/changes/707.mdnew file mode 100644index 00000000..c053b26a--- /dev/null+++ b/changes/707.md@@ -0,0 +1 @@+- Post steps take a `mask` by part names or a depth range, and `effects.stylize` adds sharpen, kuwahara, posterize and pixelate filters (part of #707)diff --git a/docs/agent-look.md b/docs/agent-look.mdindex 36684252..41c2c8ce 100644--- a/docs/agent-look.md+++ b/docs/agent-look.md@@ -268,16 +268,23 @@ Ambient occlusion reads the depth texture, so it needs the HDR depth target that| `glare.fogGlow` | `0` | `0` to `1`. A soft halo around the same bright pixels || `glare.threshold` | `1` | `0` to `10`. Peak scene brightness (linear, before tone mapping) from which a pixel glares || `glare.size` | `0.15` | `0.01` to `1`. Streak length and halo radius as a fraction of the image height |+| `stylize.sharpen` | off | `0` to `2`. Adds the difference between a pixel and its four neighbours back to it |+| `stylize.kuwahara` | off | Whole number, `0` to `8`. Painterly smoothing: each pixel takes the mean of the least varying quadrant within this many pixels |+| `stylize.posterize` | off | Whole number, `0` to `256`. Rounds each channel to this many steps |+| `stylize.pixelate` | off | Whole number, `0` to `64`. Block size in pixels |+| `colorBalance.mask`, `stylize.mask` | none | Limits the step to part of the frame, see below || `lensDistortion.distortion` | `0` | `-1` to `1`. Above `0` is barrel, below is pincushion || `lensDistortion.dispersion` | `0` | `0` to `1`. Red and blue split apart toward the edges, up to 3 percent of the distance from the center |+A step with a `mask` applies only where the mask is on; with no `mask` it applies to the whole frame, as before. `mask` takes `objects` (one to 16 part names, as `scene_look` with `pass: "objectId"` lists them), `depth` (`near`, `far` metres along the view direction, and `feather`, the metres over which each end fades; defaults `0`, `100000`, `0`), or both, which multiply, and `invert: true` to flip the result. A grade for the hero part only: `{ "colorBalance": { "gain": [1.3, 1, 0.8], "mask": { "objects": ["hero"] } } }`. A mask by parts reads the object ID pass, so the frame is drawn twice, and its edges are hard because an object ID cannot be antialiased; a mask by depth costs nothing extra. A name that is not a part of the frame fails with the names it knows. `stylize` runs after color balance: pixelate first (the other filters read the blocks), then kuwahara, sharpen, posterize. Every filter is off until named, and a filter set to `0` is off.+Depth of field and motion blur are deterministic: they read the frame, its depth and the camera, with fixed sample patterns, so a seek to a time gives the same pixels as playing to it.Depth of field applies to every render, including `scene_look`.Motion blur needs camera `motion` tracks and a render job with `settings.motion`: the job samples the camera at `motion.start + elapsed - shutter` as well, and smears each pixel along the screen movement between the two.With `motionBlur.objects` above zero, a part that moves between `motion.start + elapsed - shutter` and `motion.start + elapsed` also smears along its own screen path while the still background stays sharp (`1` is the full shutter). It reads each pixel's part from an object ID pass, so it costs a second draw of the frame, and it replaces the camera smear on that frame, since it covers the camera's motion too. A part is moved as one rigid body by its node transform: a skinned or morphed part, an imported asset and a translucent part are not smeared per object.Both effects compile their shader and allocate their targets the first time a frame uses them, and a scene that leaves them at zero pays nothing.-The named post steps are `depthOfField` (defocus from depth), `colorBalance`, `glare` and `lensDistortion`. Each is its own group, off until a recipe names it, and a group at its neutral values draws nothing. They run on the resolved HDR frame, before bloom and tone mapping, in this order: depth of field, motion blur, glare, lens distortion, color balance. Like the two above they are deterministic, and `colorBalance`, `glare` and `lensDistortion` load their code and shaders only for a recipe that names one.+The named post steps are `depthOfField` (defocus from depth), `colorBalance`, `glare`, `lensDistortion` and `stylize`. Each is its own group, off until a recipe names it, and a group at its neutral values draws nothing. They run on the resolved HDR frame, before bloom and tone mapping, in this order: depth of field, motion blur, glare, lens distortion, color balance, stylize. Like the two above they are deterministic, and `colorBalance`, `glare`, `lensDistortion` and `stylize` load their code and shaders only for a recipe that names one.## Directional shadowsdiff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex a3315dd9..b974321d 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -286,13 +286,21 @@ function draw(record, time = 0, camera = record.scene.camera, frameOptions = {})applyRecipeWindHeading(record.renderer, environment);const { objectMotion, ...renderOptions } = frameOptions;const frame = { ...camera, environment: litEnvironment(record.scene.parts, environment, Boolean(record.renderer.groundPart)), effects, exposure: effects.exposure, time, lines: environment.lines, ...windFrame(environment), ...renderOptions, effects: effectsWithFStop(renderOptions.effects ?? effects, camera) };- if (effects.colorBalance || effects.glare || effects.lensDistortion) agentSceneModule('postSteps').installPostSteps(record.renderer);+ let needsObjectIds = Boolean(objectMotion);+ if (effects.colorBalance || effects.glare || effects.lensDistortion || effects.stylize) {+ const postSteps = agentSceneModule('postSteps');+ postSteps.installPostSteps(record.renderer);+ needsObjectIds ||= postSteps.usesObjectMask(postSteps.resolvePostSteps(frame.effects));+ }if (objectMotion) {// Object motion blur reads each pixel's part from the object ID pass, drawn first; the lit frame then smears along each part's motion.const { installObjectMotionBlur, packObjectMotion } = agentSceneModule('objectMotionBlur');if (!frame.pass) installObjectMotionBlur(record.renderer);if (!frame.pass) record.renderer.render({ ...frame, pass: 'objectId', previousCamera: undefined });frame.objectMotion = packObjectMotion(record.renderer, objectMotion.current, objectMotion.previous);+ } else if (needsObjectIds && !frame.pass) {+ // A mask by parts reads the same pass.+ record.renderer.render({ ...frame, pass: 'objectId', previousCamera: undefined });}record.renderer.render(frame);const error = record.renderer.context.getError();diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex 06b6b7d5..cb1e1f80 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -313,8 +313,9 @@ const effects = z.strictObject({colorBalance: looseField,glare: looseField,lensDistortion: looseField,+ stylize: looseField,});-const effectsListed = listed('effects', effects, 'Post effects: exposure, saturation, contrast, temperature, vignette, grain, ambientOcclusion, ambientOcclusionRadius, bloom {strength, threshold, radius}, depthOfField {blur, focusDistance}, motionBlur {shutter}, colorBalance, glare, lensDistortion; see scene_capabilities effects.', coreOf(effects.shape, ['exposure']));+const effectsListed = listed('effects', effects, 'Post effects: exposure, saturation, contrast, temperature, vignette, grain, ambientOcclusion, ambientOcclusionRadius, bloom {strength, threshold, radius}, depthOfField {blur, focusDistance}, motionBlur {shutter}, colorBalance, glare, lensDistortion, stylize; see scene_capabilities effects.', coreOf(effects.shape, ['exposure']));const shadow = z.strictObject({target: boundedVector(-1e6, 1e6).optional(),extent: bounded(0.01, 1e5).optional(),diff --git a/skills/build-67js-scene/references/look.md b/skills/build-67js-scene/references/look.mdindex 475f2d1b..eb99cfab 100644--- a/skills/build-67js-scene/references/look.md+++ b/skills/build-67js-scene/references/look.md@@ -91,7 +91,8 @@ Point lights do not scatter in volume fog; the sun does. A sunlit closed room wi`effects` holds `exposure`, `saturation`, `contrast`, `temperature`, `vignette`, `grain`, `ambientOcclusion` (0 = off),`bloom` and `depthOfField` (see `camera.md`). Nested `bloom` fields merge.Named post steps, each off until set: `colorBalance` (`lift`, `gamma`, `gain`, three numbers each), `glare` (`streaks`,-`count`, `angle`, `fogGlow`, `threshold`, `size`) and `lensDistortion` (`distortion`, `dispersion`); ranges in `docs/agent-look.md`.+`count`, `angle`, `fogGlow`, `threshold`, `size`) `lensDistortion` (`distortion`, `dispersion`) and `stylize` (`sharpen`, `kuwahara`, `posterize`, `pixelate`); ranges in `docs/agent-look.md`.+`colorBalance` and `stylize` take a `mask` (`objects` part names, `depth` `{ near, far, feather }`, `invert`) to apply only to a hero part or a depth band.Tool `scene_apply`:diff --git a/src/agent-scene-modules.js b/src/agent-scene-modules.jsindex ce110f90..20e0fff2 100644--- a/src/agent-scene-modules.js+++ b/src/agent-scene-modules.js@@ -90,7 +90,7 @@ const MODULE_MARKERS = {materialSnippet: ['"snippet":{'],// Only a motionBlur object that sets a non-zero `objects` loads it; the common `"shutter":0` look does not.objectMotionBlur: [/"motionBlur":\{[^}]*"objects":(?!0[,}])/],- postSteps: [/"(?:colorBalance|glare|lensDistortion)"/],+ postSteps: [/"(?:colorBalance|glare|lensDistortion|stylize)"/],};/** Whether a recipe, operation list or edit list contains a boolean edit, which needs loadMeshBooleans() first. */diff --git a/src/post-mask.js b/src/post-mask.jsnew file mode 100644index 00000000..4871942c--- /dev/null+++ b/src/post-mask.js@@ -0,0 +1,127 @@+// A mask for the named post steps (post-steps.js): a step applies only where the mask is on. Two sources, which multiply when+// both are named: the parts listed by name, read from the object ID pass, and a range of distance along the view. The mask is+// exact per pixel, so it has hard edges where a part meets another (an object ID cannot be antialiased); a depth range softens+// its ends with `feather`. A step with no mask applies to the whole frame, as before.++export const MAXIMUM_MASK_OBJECTS = 16;+const MAXIMUM_NAME_LENGTH = 128;++function checkedNumber(value, fallback, minimum, maximum, name) {+ const result = value ?? fallback;+ if (!Number.isFinite(result) || result < minimum || result > maximum) throw new RangeError(`${name} must be between ${minimum} and ${maximum}.`);+ return result;+}++function checkedKeys(group, allowed, name) {+ if (!group || typeof group !== 'object' || Array.isArray(group)) throw new TypeError(`${name} must be an object.`);+ for (const key of Object.keys(group)) {+ if (!allowed.includes(key)) throw new TypeError(`Unknown ${name} field "${key}". Fix: use ${allowed.join(', ')}.`);+ }+}++function resolveObjectNames(objects, name) {+ if (!Array.isArray(objects) || objects.length === 0 || objects.length > MAXIMUM_MASK_OBJECTS) {+ throw new TypeError(`${name} must be a list of 1 to ${MAXIMUM_MASK_OBJECTS} part names.`);+ }+ objects.forEach((part, index) => {+ if (typeof part !== 'string' || part.length === 0 || part.length > MAXIMUM_NAME_LENGTH) {+ throw new TypeError(`${name}[${index}] must be a part name of 1 to ${MAXIMUM_NAME_LENGTH} characters. Fix: pass the node id, as the object ID pass lists it.`);+ }+ });+ return [...objects];+}++/** The mask a step names, resolved with defaults: { objects?, depth?: { near, far, feather }, invert }. */+export function resolveMask(input, name) {+ checkedKeys(input, ['objects', 'depth', 'invert'], name);+ if (input.objects === undefined && input.depth === undefined) {+ throw new TypeError(`${name} needs a source. Fix: name objects (a list of part names), depth ({ near, far, feather }), or both.`);+ }+ const mask = { invert: input.invert === true };+ if (input.invert !== undefined && typeof input.invert !== 'boolean') throw new TypeError(`${name}.invert must be true or false.`);+ if (input.objects !== undefined) mask.objects = resolveObjectNames(input.objects, `${name}.objects`);+ if (input.depth !== undefined) {+ checkedKeys(input.depth, ['near', 'far', 'feather'], `${name}.depth`);+ const near = checkedNumber(input.depth.near, 0, 0, 1e5, `${name}.depth.near`);+ const far = checkedNumber(input.depth.far, 1e5, 0, 1e5, `${name}.depth.far`);+ if (far <= near) throw new RangeError(`${name}.depth.far must be greater than near. Fix: distances are metres along the view direction.`);+ mask.depth = { near, far, feather: checkedNumber(input.depth.feather, 0, 0, 1e4, `${name}.depth.feather`) };+ }+ return mask;+}++/** Whether the object ID pass must be drawn before the lit frame: true when any resolved step masks by objects. */+export function usesObjectMask(steps) {+ return Object.values(steps).some(step => Boolean(step?.mask?.objects));+}++export const MASK_UNIFORM_NAMES = ['objectIds', 'maskObjectCount', 'maskObjectIds', 'maskUseDepth', 'maskDepth', 'maskInvert', 'maskEye', 'maskForward'];++/** GLSL for a step that takes a mask: maskAt(coordinate) is 1 where the step applies and 0 where the source shows through. */+export const maskShader = `+uniform sampler2D objectIds;+uniform int maskObjectCount;+uniform int maskObjectIds[${MAXIMUM_MASK_OBJECTS}];+uniform float maskUseDepth;+uniform vec3 maskDepth;+uniform float maskInvert;+uniform vec3 maskEye;+uniform vec3 maskForward;+float maskAt(vec2 coordinate) {+ float mask = 1.0;+ if (maskObjectCount > 0) {+ vec3 idBytes = floor(texelFetch(objectIds, ivec2(coordinate * resolution), 0).rgb * 255.0 + 0.5);+ int identifier = int(idBytes.r + idBytes.g * 256.0 + idBytes.b * 65536.0);+ float hit = 0.0;+ for (int index = 0; index < ${MAXIMUM_MASK_OBJECTS}; index++) {+ if (index >= maskObjectCount) break;+ if (maskObjectIds[index] == identifier) hit = 1.0;+ }+ mask *= hit;+ }+ if (maskUseDepth > 0.5) {+ float distanceAlongView = dot(worldPointAt(coordinate) - maskEye, maskForward);+ float feather = max(maskDepth.z, 0.0001);+ mask *= clamp((distanceAlongView - (maskDepth.x - feather)) / feather, 0.0, 1.0)+ * (1.0 - clamp((distanceAlongView - maskDepth.y) / feather, 0.0, 1.0));+ }+ return mix(mask, 1.0 - mask, maskInvert);+}+`;++function objectIdentifiers(renderer, names) {+ if (!renderer.passSupport) throw new Error('A mask by objects needs installRenderPasses(renderer) and the object ID pass drawn first.');+ const known = renderer.passSupport.objectNames();+ const identifiers = [];+ for (const name of names) {+ const matches = known.flatMap((candidate, index) => (candidate === name ? [index + 1] : []));+ if (matches.length === 0) {+ throw new Error(`Mask object "${name}" is not a part in the frame. Fix: use a part name such as ${known.slice(0, 8).map(part => `"${part}"`).join(', ') || '(the scene has no opaque parts)'}.`);+ }+ identifiers.push(...matches);+ }+ if (identifiers.length > MAXIMUM_MASK_OBJECTS) throw new RangeError(`The mask names ${identifiers.length} parts; the limit is ${MAXIMUM_MASK_OBJECTS}. Fix: name fewer parts.`);+ return identifiers;+}++/** Sets the mask uniforms of a stage. A step with no mask gets a mask of one everywhere. Leaves texture unit 1 active. */+export function bindMask(context, uniforms, renderer, frame, mask) {+ const identifiers = mask?.objects ? objectIdentifiers(renderer, mask.objects) : [];+ context.uniform1i(uniforms.maskObjectCount, identifiers.length);+ if (identifiers.length > 0) {+ const padded = new Int32Array(MAXIMUM_MASK_OBJECTS);+ padded.set(identifiers);+ context.uniform1iv(uniforms.maskObjectIds, padded);+ context.activeTexture(context.TEXTURE2);+ context.bindTexture(context.TEXTURE_2D, renderer.passSupport.texture);+ context.uniform1i(uniforms.objectIds, 2);+ }+ context.uniform1f(uniforms.maskUseDepth, mask?.depth ? 1 : 0);+ if (mask?.depth) {+ context.uniform3f(uniforms.maskDepth, mask.depth.near, mask.depth.far, mask.depth.feather);+ context.uniform3fv(uniforms.maskEye, frame.eye);+ context.uniform3fv(uniforms.maskForward, renderer.viewForward);+ }+ context.uniform1f(uniforms.maskInvert, mask?.invert ? 1 : 0);+ context.activeTexture(context.TEXTURE1);+}diff --git a/src/post-steps.js b/src/post-steps.jsindex 70f5a1b2..a35aeb7d 100644--- a/src/post-steps.js+++ b/src/post-steps.js@@ -1,16 +1,21 @@import { postStepSupport } from './shader-registry.js';import { createStage, sharedFragmentHeader } from './camera-effects.js';+import { resolveMask, usesObjectMask, maskShader, bindMask, MASK_UNIFORM_NAMES } from './post-mask.js';++export { usesObjectMask };// Named post steps for a graded film: color balance (lift, gamma, gain), glare (streaks and fog glow), lens distortion with-// dispersion. They are optional full-frame stages. The core renderer carries none of their code: a recipe that names one+// dispersion, and stylize filters (sharpen, kuwahara, posterize, pixelate). Color balance and stylize take a `mask` (post-mask.js)+// that limits them to named parts or a range of distance. They are optional full-frame stages. The core renderer carries none of their code: a recipe that names one// loads this module (agent-scene-modules.js), and each stage compiles only on the first frame that turns it on. Defocus from// depth is the `depthOfField` group of camera-effects.js, which is the same step under its older name.//-// All three are deterministic: fixed sample patterns and no history, so a seek repeats the same pixels.+// All of them are deterministic: fixed sample patterns and no history, so a seek repeats the same pixels.const MAXIMUM_STREAK_ARMS = 8;const STREAK_STEPS = 24;const FOG_SAMPLES = 32;+const MAXIMUM_KUWAHARA_RADIUS = 8;// Longest lateral color offset at the image corner at full dispersion, as a fraction of the distance from the center.const MAXIMUM_DISPERSION = 0.03;@@ -41,14 +46,34 @@ function triple(value, fallback, minimum, maximum, name) {function resolveColorBalance(input) {const group = groupOf(input, 'colorBalance');- onlyKeys(group, ['lift', 'gamma', 'gain'], 'colorBalance');+ onlyKeys(group, ['lift', 'gamma', 'gain', 'mask'], 'colorBalance');return {lift: triple(group.lift, NEUTRAL.lift, -1, 1, 'colorBalance.lift'),gamma: triple(group.gamma, NEUTRAL.gamma, 0.1, 4, 'colorBalance.gamma'),gain: triple(group.gain, NEUTRAL.gain, 0, 4, 'colorBalance.gain'),+ ...(group.mask === undefined ? {} : { mask: resolveMask(group.mask, 'colorBalance.mask') }),};}+function wholeNumberInRange(value, minimum, maximum, name) {+ const result = numberInRange(value, undefined, minimum, maximum, name);+ if (!Number.isInteger(result)) throw new RangeError(`${name} must be a whole number.`);+ return result;+}++// Each filter is off until named; a filter named with a number turns on.+function resolveStylize(input) {+ const group = groupOf(input, 'stylize');+ onlyKeys(group, ['sharpen', 'kuwahara', 'posterize', 'pixelate', 'mask'], 'stylize');+ const resolved = {};+ if (group.sharpen !== undefined) resolved.sharpen = numberInRange(group.sharpen, undefined, 0, 2, 'stylize.sharpen');+ if (group.kuwahara !== undefined) resolved.kuwahara = wholeNumberInRange(group.kuwahara, 0, MAXIMUM_KUWAHARA_RADIUS, 'stylize.kuwahara');+ if (group.posterize !== undefined) resolved.posterize = wholeNumberInRange(group.posterize, 0, 256, 'stylize.posterize');+ if (group.pixelate !== undefined) resolved.pixelate = wholeNumberInRange(group.pixelate, 0, 64, 'stylize.pixelate');+ if (group.mask !== undefined) resolved.mask = resolveMask(group.mask, 'stylize.mask');+ return resolved;+}+function resolveGlare(input) {const group = groupOf(input, 'glare');onlyKeys(group, ['streaks', 'count', 'angle', 'fogGlow', 'threshold', 'size'], 'glare');@@ -79,14 +104,16 @@ export function resolvePostSteps(effects) {if (effects.colorBalance !== undefined) resolved.colorBalance = resolveColorBalance(effects.colorBalance);if (effects.glare !== undefined) resolved.glare = resolveGlare(effects.glare);if (effects.lensDistortion !== undefined) resolved.lensDistortion = resolveLensDistortion(effects.lensDistortion);+ if (effects.stylize !== undefined) resolved.stylize = resolveStylize(effects.stylize);return resolved;}// Registers what the look needs: the field names it accepts and how to resolve them (src/shader-registry.js).-postStepSupport.fields = ['colorBalance', 'glare', 'lensDistortion'];+postStepSupport.fields = ['colorBalance', 'glare', 'lensDistortion', 'stylize'];postStepSupport.resolve = resolvePostSteps;export const colorBalanceFragment = `${sharedFragmentHeader}+${maskShader}uniform vec3 lift;uniform vec3 inverseGamma;uniform vec3 gain;@@ -94,7 +121,67 @@ void main() {vec4 source = texture(sourceTexture, textureCoordinate);// Lift raises the shadows and fades out toward white, gain scales the highlights, gamma bends the midtones.vec3 graded = source.rgb * gain + lift * (1.0 - clamp(source.rgb, 0.0, 1.0));- outputColor = vec4(pow(max(graded, vec3(0.0)), inverseGamma), source.a);+ graded = pow(max(graded, vec3(0.0)), inverseGamma);+ float mask = maskAt(textureCoordinate);+ outputColor = vec4(mask < 1.0 ? mix(source.rgb, graded, mask) : graded, source.a);+}`;++export const stylizeFragment = `${sharedFragmentHeader}+${maskShader}+uniform float sharpenAmount;+uniform int kuwaharaRadius;+uniform float posterizeLevels;+uniform float pixelateSize;+vec2 snapped(vec2 coordinate) {+ if (pixelateSize < 1.0) return coordinate;+ return (floor(coordinate * resolution / pixelateSize) + 0.5) * pixelateSize / resolution;+}+vec4 sourceAt(vec2 coordinate) {+ return texture(sourceTexture, snapped(coordinate));+}+// Kuwahara: the mean of whichever of the four quadrants around the pixel varies least, so flat areas stay flat and edges stay sharp.+vec3 kuwahara(vec2 coordinate) {+ vec3 bestMean = vec3(0.0);+ float bestVariance = 1e30;+ for (int quadrant = 0; quadrant < 4; quadrant++) {+ vec2 direction = vec2(quadrant == 1 || quadrant == 3 ? -1.0 : 1.0, quadrant >= 2 ? -1.0 : 1.0);+ vec3 sum = vec3(0.0);+ float lumaSum = 0.0;+ float lumaSquareSum = 0.0;+ float count = 0.0;+ for (int row = 0; row <= ${MAXIMUM_KUWAHARA_RADIUS}; row++) {+ if (row > kuwaharaRadius) break;+ for (int column = 0; column <= ${MAXIMUM_KUWAHARA_RADIUS}; column++) {+ if (column > kuwaharaRadius) break;+ vec3 color = sourceAt(coordinate + direction * vec2(float(column), float(row)) / resolution).rgb;+ float luma = dot(color, vec3(0.299, 0.587, 0.114));+ sum += color;+ lumaSum += luma;+ lumaSquareSum += luma * luma;+ count += 1.0;+ }+ }+ float variance = lumaSquareSum / count - (lumaSum / count) * (lumaSum / count);+ if (variance < bestVariance) {+ bestVariance = variance;+ bestMean = sum / count;+ }+ }+ return bestMean;+}+void main() {+ vec4 source = texture(sourceTexture, textureCoordinate);+ vec4 center = sourceAt(textureCoordinate);+ vec3 color = kuwaharaRadius > 0 ? kuwahara(textureCoordinate) : center.rgb;+ if (sharpenAmount > 0.0) {+ vec2 texel = 1.0 / resolution;+ vec3 around = sourceAt(textureCoordinate + vec2(texel.x, 0.0)).rgb + sourceAt(textureCoordinate - vec2(texel.x, 0.0)).rgb+ + sourceAt(textureCoordinate + vec2(0.0, texel.y)).rgb + sourceAt(textureCoordinate - vec2(0.0, texel.y)).rgb;+ color = max(color + sharpenAmount * (center.rgb * 4.0 - around), vec3(0.0));+ }+ if (posterizeLevels > 0.0) color = floor(color * posterizeLevels + 0.5) / posterizeLevels;+ float mask = maskAt(textureCoordinate);+ outputColor = vec4(mask < 1.0 ? mix(source.rgb, color, mask) : color, source.a);}`;export const glareFragment = `${sharedFragmentHeader}@@ -181,13 +268,35 @@ function createColorBalanceStage() {return createStage({name: 'Color balance',fragment: colorBalanceFragment,- uniformNames: ['lift', 'inverseGamma', 'gain'],+ uniformNames: ['lift', 'inverseGamma', 'gain', ...MASK_UNIFORM_NAMES],isActive: (effects, frame) => isColorBalanceActive(stepsOf(frame).colorBalance),setUniforms(context, uniforms, renderer, frame) {const { colorBalance } = stepsOf(frame);context.uniform3fv(uniforms.lift, colorBalance.lift);context.uniform3fv(uniforms.inverseGamma, colorBalance.gamma.map(gamma => 1 / gamma));context.uniform3fv(uniforms.gain, colorBalance.gain);+ bindMask(context, uniforms, renderer, frame, colorBalance.mask);+ },+ });+}++function isStylizeActive(stylize) {+ return Boolean(stylize) && (stylize.sharpen > 0 || stylize.kuwahara > 0 || stylize.posterize > 0 || stylize.pixelate > 0);+}++function createStylizeStage() {+ return createStage({+ name: 'Stylize',+ fragment: stylizeFragment,+ uniformNames: ['sharpenAmount', 'kuwaharaRadius', 'posterizeLevels', 'pixelateSize', ...MASK_UNIFORM_NAMES],+ isActive: (effects, frame) => isStylizeActive(stepsOf(frame).stylize),+ setUniforms(context, uniforms, renderer, frame) {+ const { stylize } = stepsOf(frame);+ context.uniform1f(uniforms.sharpenAmount, stylize.sharpen ?? 0);+ context.uniform1i(uniforms.kuwaharaRadius, stylize.kuwahara ?? 0);+ context.uniform1f(uniforms.posterizeLevels, stylize.posterize ?? 0);+ context.uniform1f(uniforms.pixelateSize, stylize.pixelate ?? 0);+ bindMask(context, uniforms, renderer, frame, stylize.mask);},});}@@ -255,12 +364,13 @@ function createLensDistortionStage() {}/**- * Adds glare, lens distortion and color balance to a renderer, after any depth of field and motion blur it has, in that- * order. Like them it needs `depth: true`. Each stage runs only for a frame whose effects set it.+ * Adds glare, lens distortion, color balance and the stylize filters to a renderer, after any depth of field and motion blur it has, in that+ * order. Like them it needs `depth: true`. Each stage runs only for a frame whose effects set it. A step with a `mask` by+ * objects reads the object ID pass: install the render passes and draw that pass before the lit frame (`usesObjectMask` says when).*/export function installPostSteps(renderer) {if (!renderer.postStages.some(stage => stage.name === 'Glare')) {- renderer.postStages.push(createGlareStage(), createLensDistortionStage(), createColorBalanceStage());+ renderer.postStages.push(createGlareStage(), createLensDistortionStage(), createColorBalanceStage(), createStylizeStage());}return renderer;}diff --git a/tests/post-steps.test.js b/tests/post-steps.test.jsindex b60504b7..5fe3823d 100644--- a/tests/post-steps.test.js+++ b/tests/post-steps.test.js@@ -9,7 +9,7 @@ import { resolveEffects } from '../src/effects.js';import { normalizeAgentLook } from '../src/agent-look.js';import { agentSceneModulesNeeded, loadAgentSceneModules } from '../src/agent-scene-modules.js';import { installCameraEffects } from '../src/camera-effects.js';-import { installPostSteps, resolvePostSteps, glareFragment, lensDistortionFragment, colorBalanceFragment } from '../src/post-steps.js';+import { installPostSteps, resolvePostSteps, glareFragment, lensDistortionFragment, colorBalanceFragment, stylizeFragment, usesObjectMask } from '../src/post-steps.js';import { createAgentScene } from '../src/agent-scene.js';function recordingContext(calls) {@@ -33,12 +33,12 @@ function fakeRenderer(context) {};}-const STAGE_NAMES = ['Glare', 'Lens distortion', 'Color balance'];+const STAGE_NAMES = ['Glare', 'Lens distortion', 'Color balance', 'Stylize'];test('a look that names no step resolves exactly as before', async () => {await loadAgentSceneModules({});const effects = normalizeAgentLook({}).effects;- for (const field of ['colorBalance', 'glare', 'lensDistortion']) assert.equal(Object.hasOwn(effects, field), false);+ for (const field of ['colorBalance', 'glare', 'lensDistortion', 'stylize']) assert.equal(Object.hasOwn(effects, field), false);assert.deepEqual(resolveEffects({}), resolveEffects({ saturation: 1 }));});@@ -86,13 +86,14 @@ test('each stage draws only for a frame that sets its step, and frees its targetconst context = recordingContext(calls);const renderer = installPostSteps(installCameraEffects(fakeRenderer(context)));assert.deepEqual(renderer.postStages.slice(2).map(stage => stage.name), STAGE_NAMES);- assert.equal(installPostSteps(renderer).postStages.length, 5);+ assert.equal(installPostSteps(renderer).postStages.length, 6);const unset = { eye: [0, 1, 5], effects: {} };- const neutral = { eye: [0, 1, 5], effects: { colorBalance: {}, glare: {}, lensDistortion: {} } };+ const neutral = { eye: [0, 1, 5], effects: { colorBalance: {}, glare: {}, lensDistortion: {}, stylize: {} } };const named = [['Glare', { glare: { streaks: 0.6 } }],['Lens distortion', { lensDistortion: { distortion: 0.5, dispersion: 0.5 } }],['Color balance', { colorBalance: { gain: [1.1, 1, 0.9] } }],+ ['Stylize', { stylize: { posterize: 4 } }],];const effects = resolveEffects({});for (const [name, input] of named) {@@ -103,12 +104,56 @@ test('each stage draws only for a frame that sets its step, and frees its targetstage.releaseTargets(context);stage.dispose(context);}- assert.ok(calls.filter(call => call[0] === 'drawArrays').length === 3);- assert.ok(calls.filter(call => call[0] === 'deleteTexture').length === 3);+ assert.ok(calls.filter(call => call[0] === 'drawArrays').length === 4);+ assert.ok(calls.filter(call => call[0] === 'deleteTexture').length === 4);+});++test('a stylize step resolves its filters and a mask, and refuses bad input', () => {+ assert.deepEqual(resolvePostSteps({ stylize: {} }).stylize, {});+ assert.deepEqual(resolvePostSteps({ stylize: { sharpen: 1, kuwahara: 3, posterize: 5, pixelate: 8 } }).stylize, { sharpen: 1, kuwahara: 3, posterize: 5, pixelate: 8 });+ assert.throws(() => resolvePostSteps({ stylize: { kuwahara: 9 } }), /stylize.kuwahara must be between/);+ assert.throws(() => resolvePostSteps({ stylize: { pixelate: 2.5 } }), /whole number/);+ assert.throws(() => resolvePostSteps({ stylize: { sharpen: -1 } }), /stylize.sharpen must be between/);+ assert.throws(() => resolvePostSteps({ stylize: { blur: 1 } }), /Unknown stylize field/);+});++test('a mask names parts or a depth range, and says what to pass when it is wrong', () => {+ const mask = resolvePostSteps({ colorBalance: { mask: { objects: ['hero'], depth: { near: 2, far: 8 }, invert: true } } }).colorBalance.mask;+ assert.deepEqual(mask, { invert: true, objects: ['hero'], depth: { near: 2, far: 8, feather: 0 } });+ assert.deepEqual(resolvePostSteps({ stylize: { mask: { depth: { far: 5 } } } }).stylize.mask, { invert: false, depth: { near: 0, far: 5, feather: 0 } });+ assert.equal(resolvePostSteps({ colorBalance: {} }).colorBalance.mask, undefined);+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: {} } }), /colorBalance.mask needs a source/);+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: [] } } }), /1 to 16 part names/);+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: Array(17).fill('a') } } }), /1 to 16 part names/);+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: [3] } } }), /mask.objects\[0\] must be a part name/);+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { depth: { near: 5, far: 5 } } } }), /far must be greater than near/);+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { depth: { feather: -1 } } } }), /feather must be between/);+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: ['a'], color: 1 } } }), /Unknown colorBalance.mask field/);+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: ['a'], invert: 1 } } }), /invert must be true or false/);+ assert.equal(usesObjectMask(resolvePostSteps({ colorBalance: { mask: { objects: ['hero'] } } })), true);+ assert.equal(usesObjectMask(resolvePostSteps({ colorBalance: { mask: { depth: { far: 5 } } }, stylize: { pixelate: 4 } })), false);+});++test('a stage binds the part numbers of a masked step and refuses an unknown part', () => {+ const calls = [];+ const context = recordingContext(calls);+ const renderer = installPostSteps(installCameraEffects(fakeRenderer(context)));+ renderer.passSupport = { texture: 'ids', objectNames: () => ['floor', 'hero', 'post', 'hero'] };+ const stage = renderer.postStages.find(candidate => candidate.name === 'Color balance');+ const effects = resolveEffects({});+ const frame = mask => ({ eye: [0, 1, 5], effects: { colorBalance: { gain: [1.2, 1, 1], mask } } });+ assert.ok(stage.apply(renderer, frame({ objects: ['hero'] }), effects, 'source'));+ const bound = calls.find(call => call[0] === 'uniform1iv');+ assert.deepEqual([...bound[2]].slice(0, 3), [2, 4, 0]);+ assert.throws(() => stage.apply(renderer, frame({ objects: ['ghost'] }), effects, 'source'), /"ghost" is not a part in the frame\. Fix: use a part name such as "floor"/);+ renderer.passSupport = null;+ assert.throws(() => stage.apply(renderer, frame({ objects: ['hero'] }), effects, 'source'), /installRenderPasses/);+ stage.releaseTargets(context);+ stage.dispose(context);});test('the shaders keep their fixed sample patterns and read no history', () => {- for (const fragment of [glareFragment, lensDistortionFragment, colorBalanceFragment]) {+ for (const fragment of [glareFragment, lensDistortionFragment, colorBalanceFragment, stylizeFragment]) {assert.match(fragment, /^#version 300 es/);assert.doesNotMatch(fragment, /previousViewProjection|time/);}@@ -118,7 +163,8 @@ test('the shaders keep their fixed sample patterns and read no history', () => {// RENDER_TESTS=1 node --test tests/post-steps.test.js (on a shared GPU, start it through the machine GPU lock)const chromePath = process.env.CHROME_PATH ?? 'C:/Program Files/Google/Chrome/Application/chrome.exe';const rendersEnabled = process.env.RENDER_TESTS === '1' && existsSync(chromePath);-const STEP_CASES = ['defocus', 'color-balance', 'glare-streaks', 'glare-streaks-6', 'glare-anamorphic', 'glare-fog', 'lens-distortion', 'lens-dispersion'];+const STEP_CASES = ['defocus', 'color-balance', 'glare-streaks', 'glare-streaks-6', 'glare-anamorphic', 'glare-fog', 'lens-distortion', 'lens-dispersion',+ 'grade-hero', 'grade-depth', 'sharpen', 'kuwahara', 'posterize', 'pixelate', 'pixelate-hero'];test('each named step changes the picture and the default still keeps its hash', { skip: !rendersEnabled, timeout: 300000 }, () => {const output = execFileSync('node', ['bench/post/post-steps.mjs', '--rounds=1', `--out=${join(tmpdir(), 'post-steps-test')}`], { encoding: 'utf8' });@@ -127,6 +173,11 @@ test('each named step changes the picture and the default still keeps its hash',assert.notEqual(summary[name].hash, summary.default.hash, `${name} differs from the default`);assert.ok(summary[name].medianMilliseconds > 0, `${name} has a frame time`);}+ // A grade masked to the hero part changes the hero and leaves the floor as the default drew it; the unmasked grade changes both.+ assert.notDeepEqual(summary['grade-hero'].regions.hero, summary.default.regions.hero);+ assert.deepEqual(summary['grade-hero'].regions.floor, summary.default.regions.floor);+ assert.notDeepEqual(summary['color-balance'].regions.floor, summary.default.regions.floor);+ assert.deepEqual(summary['pixelate-hero'].regions.floor, summary.default.regions.floor);// Hash of the default still as origin/main drew it before the named steps (bench/post/post-steps.mjs --root=<origin/main checkout>).assert.equal(summary.default.hash, '8741ffc3aaace9a8');});