Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Mask post steps by part names or depth, and add stylize filters (part of #707)

Nguyen Xuan And6c3bef6fe05
diff --git a/bench/post/post-steps.js b/bench/post/post-steps.js
index 4a6d10d5..89137c6e 100644
--- a/bench/post/post-steps.js
+++ b/bench/post/post-steps.js
@@ -2,6 +2,7 @@
import { Renderer67 } from '/src/renderer.js';
import { box } from '/src/geometry.js';
import { installCameraEffects } from '/src/camera-effects.js';
+import { installRenderPasses } from '/src/render-passes.js';
// The named steps are an optional module: this harness loads it only when the source tree has it, so the same page also
// draws the default picture of an older tree.
@@ -17,7 +18,7 @@ function sceneParts() {
const lamp = { color: [1, 0.9, 0.7], emissiveColor: [1, 0.9, 0.7], metallic: 0, roughness: 1, emission: 14 };
const parts = [
{ geometry: box(), matrix: placed([0, -0.05, 0], [10, 0.1, 10]), material },
- { geometry: box(), matrix: placed([0, 0.6, 0], [0.4, 1.2, 0.4]), material },
+ { name: 'hero', geometry: box(), matrix: placed([0, 0.6, 0], [0.4, 1.2, 0.4]), material },
{ geometry: box(), matrix: placed([-1.2, 1.6, -0.6], [0.9, 0.06, 0.06]), material: lamp },
];
for (let index = 0; index < 5; index++) {
@@ -37,34 +38,58 @@ const CASES = {
'glare-fog': { glare: { fogGlow: 0.6, threshold: 1, size: 0.12 } },
'lens-distortion': { lensDistortion: { distortion: 0.8 } },
'lens-dispersion': { lensDistortion: { dispersion: 1 } },
+ 'grade-hero': { colorBalance: { gain: [1.4, 0.8, 0.5], mask: { objects: ['hero'] } } },
+ 'grade-depth': { colorBalance: { gain: [1.4, 0.8, 0.5], mask: { depth: { near: 6, far: 9, feather: 1 } } } },
+ sharpen: { stylize: { sharpen: 1.5 } },
+ kuwahara: { stylize: { kuwahara: 3 } },
+ posterize: { stylize: { posterize: 4 } },
+ pixelate: { stylize: { pixelate: 8 } },
+ 'pixelate-hero': { stylize: { pixelate: 8, mask: { objects: ['hero'] } } },
};
const camera = { eye: [3.4, 1.8, 4.6], target: [0, 0.8, -1], fov: 45 };
+// Mean color of a 6 by 6 pixel patch at a point of the frame (0 to 1, from the lower left), to check where a step changed the picture.
+function regionMean(context, x, y) {
+ const size = 6;
+ const pixels = new Uint8Array(size * size * 4);
+ context.readPixels(Math.round(x * context.drawingBufferWidth) - size / 2, Math.round(y * context.drawingBufferHeight) - size / 2, size, size, context.RGBA, context.UNSIGNED_BYTE, pixels);
+ const mean = [0, 0, 0];
+ for (let index = 0; index < pixels.length; index += 4) for (let channel = 0; channel < 3; channel++) mean[channel] += pixels[index + channel] / (size * size);
+ return mean.map(value => Number(value.toFixed(2)));
+}
+
async function drawCase(name, { warmupFrames = 4, measuredFrames = 20 } = {}) {
const canvas = document.querySelector('canvas');
- const renderer = installCameraEffects(new Renderer67(canvas, sceneParts(), { ground: false, hdr: true, depth: true, samples: 4, pixelRatio: 1, memoryBudgetMiB: 1024, environment }));
+ const renderer = installRenderPasses(installCameraEffects(new Renderer67(canvas, sceneParts(), { ground: false, hdr: true, depth: true, samples: 4, pixelRatio: 1, memoryBudgetMiB: 1024, environment })));
if (postSteps) postSteps.installPostSteps(renderer);
renderer.resize();
const frame = { ...camera, environment, time: 0, exposure: 1, effects: { ambientOcclusion: 0, ...CASES[name] } };
+ const usesObjects = postSteps?.usesObjectMask(postSteps.resolvePostSteps(frame.effects)) ?? false;
+ const drawFrame = () => {
+ // A mask by parts reads the object ID pass, drawn first.
+ if (usesObjects) renderer.render({ ...frame, pass: 'objectId' });
+ renderer.render(frame);
+ };
const pixel = new Uint8Array(4);
const finishDrawing = () => renderer.context.readPixels(0, 0, 1, 1, renderer.context.RGBA, renderer.context.UNSIGNED_BYTE, pixel);
for (let index = 0; index < warmupFrames; index++) {
- renderer.render(frame);
+ drawFrame();
finishDrawing();
}
const frameMilliseconds = [];
for (let index = 0; index < measuredFrames; index++) {
const started = performance.now();
- renderer.render(frame);
+ drawFrame();
finishDrawing();
frameMilliseconds.push(performance.now() - started);
}
- renderer.render(frame);
+ drawFrame();
const glError = renderer.context.getError();
+ const regions = { hero: regionMean(renderer.context, 0.44, 0.44), floor: regionMean(renderer.context, 0.5, 0.06) };
const image = canvas.toDataURL('image/png').split(',')[1];
renderer.dispose();
frameMilliseconds.sort((first, second) => first - second);
- return { name, glError, image, medianMilliseconds: frameMilliseconds[Math.floor(measuredFrames / 2)] };
+ return { name, glError, image, regions, medianMilliseconds: frameMilliseconds[Math.floor(measuredFrames / 2)] };
}
window.hasPostSteps = Boolean(postSteps);
diff --git a/bench/post/post-steps.mjs b/bench/post/post-steps.mjs
index 8fbcc301..0cb105e7 100644
--- a/bench/post/post-steps.mjs
+++ b/bench/post/post-steps.mjs
@@ -27,16 +27,18 @@ try {
await page.goto(`${serverUrl}/bench/post/post-steps.html`, { timeout: PAGE_LOAD_TIMEOUT });
await page.waitForFunction(() => window.drawCase, null, { timeout: PAGE_LOAD_TIMEOUT });
const hasPostSteps = await page.evaluate(() => window.hasPostSteps);
- const names = (await page.evaluate(() => window.caseNames)).filter(name => hasPostSteps || !/color-balance|glare|lens/.test(name));
+ const names = (await page.evaluate(() => window.caseNames)).filter(name => hasPostSteps || !/color-balance|glare|lens|grade|sharpen|kuwahara|posterize|pixelate/.test(name));
await mkdir(outputFolder, { recursive: true });
const medians = new Map(names.map(name => [name, []]));
const hashes = {};
+ const regions = {};
for (let round = 0; round < rounds; round++) {
for (const name of round % 2 === 0 ? names : [...names].reverse()) {
const result = await page.evaluate(caseName => window.drawCase(caseName), name);
if (result.glError !== 0) throw new Error(`${name}: GL error ${result.glError}`);
medians.get(name).push(result.medianMilliseconds);
const png = Buffer.from(result.image, 'base64');
+ regions[name] = result.regions;
hashes[name] = createHash('sha256').update(png).digest('hex').slice(0, 16);
await writeFile(resolve(outputFolder, `${name}.png`), png);
}
@@ -44,7 +46,7 @@ try {
const summary = {};
for (const [name, values] of medians) {
const sorted = [...values].sort((first, second) => first - second);
- summary[name] = { medianMilliseconds: Number(sorted[Math.floor(sorted.length / 2)].toFixed(3)), rangeMilliseconds: [sorted[0], sorted.at(-1)].map(value => Number(value.toFixed(3))), hash: hashes[name] };
+ summary[name] = { medianMilliseconds: Number(sorted[Math.floor(sorted.length / 2)].toFixed(3)), rangeMilliseconds: [sorted[0], sorted.at(-1)].map(value => Number(value.toFixed(3))), hash: hashes[name], regions: regions[name] };
}
console.log(JSON.stringify({ size: [width, height], rounds, summary }, null, 1));
} finally {
diff --git a/changes/707-compositing.size.md b/changes/707-compositing.size.md
new file mode 100644
index 00000000..06f65292
--- /dev/null
+++ b/changes/707-compositing.size.md
@@ -0,0 +1 @@
+- scene:agent-scene: the stylize effects field adds a few bytes of schema; the filter and mask code sits in the lazy post-steps module.
diff --git a/changes/707.md b/changes/707.md
new file mode 100644
index 00000000..c053b26a
--- /dev/null
+++ b/changes/707.md
@@ -0,0 +1 @@
+- Post steps take a `mask` by part names or a depth range, and `effects.stylize` adds sharpen, kuwahara, posterize and pixelate filters (part of #707)
diff --git a/docs/agent-look.md b/docs/agent-look.md
index 36684252..41c2c8ce 100644
--- a/docs/agent-look.md
+++ b/docs/agent-look.md
@@ -268,16 +268,23 @@ Ambient occlusion reads the depth texture, so it needs the HDR depth target that
| `glare.fogGlow` | `0` | `0` to `1`. A soft halo around the same bright pixels |
| `glare.threshold` | `1` | `0` to `10`. Peak scene brightness (linear, before tone mapping) from which a pixel glares |
| `glare.size` | `0.15` | `0.01` to `1`. Streak length and halo radius as a fraction of the image height |
+| `stylize.sharpen` | off | `0` to `2`. Adds the difference between a pixel and its four neighbours back to it |
+| `stylize.kuwahara` | off | Whole number, `0` to `8`. Painterly smoothing: each pixel takes the mean of the least varying quadrant within this many pixels |
+| `stylize.posterize` | off | Whole number, `0` to `256`. Rounds each channel to this many steps |
+| `stylize.pixelate` | off | Whole number, `0` to `64`. Block size in pixels |
+| `colorBalance.mask`, `stylize.mask` | none | Limits the step to part of the frame, see below |
| `lensDistortion.distortion` | `0` | `-1` to `1`. Above `0` is barrel, below is pincushion |
| `lensDistortion.dispersion` | `0` | `0` to `1`. Red and blue split apart toward the edges, up to 3 percent of the distance from the center |
+A step with a `mask` applies only where the mask is on; with no `mask` it applies to the whole frame, as before. `mask` takes `objects` (one to 16 part names, as `scene_look` with `pass: "objectId"` lists them), `depth` (`near`, `far` metres along the view direction, and `feather`, the metres over which each end fades; defaults `0`, `100000`, `0`), or both, which multiply, and `invert: true` to flip the result. A grade for the hero part only: `{ "colorBalance": { "gain": [1.3, 1, 0.8], "mask": { "objects": ["hero"] } } }`. A mask by parts reads the object ID pass, so the frame is drawn twice, and its edges are hard because an object ID cannot be antialiased; a mask by depth costs nothing extra. A name that is not a part of the frame fails with the names it knows. `stylize` runs after color balance: pixelate first (the other filters read the blocks), then kuwahara, sharpen, posterize. Every filter is off until named, and a filter set to `0` is off.
+
Depth of field and motion blur are deterministic: they read the frame, its depth and the camera, with fixed sample patterns, so a seek to a time gives the same pixels as playing to it.
Depth of field applies to every render, including `scene_look`.
Motion blur needs camera `motion` tracks and a render job with `settings.motion`: the job samples the camera at `motion.start + elapsed - shutter` as well, and smears each pixel along the screen movement between the two.
With `motionBlur.objects` above zero, a part that moves between `motion.start + elapsed - shutter` and `motion.start + elapsed` also smears along its own screen path while the still background stays sharp (`1` is the full shutter). It reads each pixel's part from an object ID pass, so it costs a second draw of the frame, and it replaces the camera smear on that frame, since it covers the camera's motion too. A part is moved as one rigid body by its node transform: a skinned or morphed part, an imported asset and a translucent part are not smeared per object.
Both effects compile their shader and allocate their targets the first time a frame uses them, and a scene that leaves them at zero pays nothing.
-The named post steps are `depthOfField` (defocus from depth), `colorBalance`, `glare` and `lensDistortion`. Each is its own group, off until a recipe names it, and a group at its neutral values draws nothing. They run on the resolved HDR frame, before bloom and tone mapping, in this order: depth of field, motion blur, glare, lens distortion, color balance. Like the two above they are deterministic, and `colorBalance`, `glare` and `lensDistortion` load their code and shaders only for a recipe that names one.
+The named post steps are `depthOfField` (defocus from depth), `colorBalance`, `glare`, `lensDistortion` and `stylize`. Each is its own group, off until a recipe names it, and a group at its neutral values draws nothing. They run on the resolved HDR frame, before bloom and tone mapping, in this order: depth of field, motion blur, glare, lens distortion, color balance, stylize. Like the two above they are deterministic, and `colorBalance`, `glare`, `lensDistortion` and `stylize` load their code and shaders only for a recipe that names one.
## Directional shadows
diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.js
index a3315dd9..b974321d 100644
--- a/scripts/agent-preview-browser.js
+++ b/scripts/agent-preview-browser.js
@@ -286,13 +286,21 @@ function draw(record, time = 0, camera = record.scene.camera, frameOptions = {})
applyRecipeWindHeading(record.renderer, environment);
const { objectMotion, ...renderOptions } = frameOptions;
const frame = { ...camera, environment: litEnvironment(record.scene.parts, environment, Boolean(record.renderer.groundPart)), effects, exposure: effects.exposure, time, lines: environment.lines, ...windFrame(environment), ...renderOptions, effects: effectsWithFStop(renderOptions.effects ?? effects, camera) };
- if (effects.colorBalance || effects.glare || effects.lensDistortion) agentSceneModule('postSteps').installPostSteps(record.renderer);
+ let needsObjectIds = Boolean(objectMotion);
+ if (effects.colorBalance || effects.glare || effects.lensDistortion || effects.stylize) {
+ const postSteps = agentSceneModule('postSteps');
+ postSteps.installPostSteps(record.renderer);
+ needsObjectIds ||= postSteps.usesObjectMask(postSteps.resolvePostSteps(frame.effects));
+ }
if (objectMotion) {
// Object motion blur reads each pixel's part from the object ID pass, drawn first; the lit frame then smears along each part's motion.
const { installObjectMotionBlur, packObjectMotion } = agentSceneModule('objectMotionBlur');
if (!frame.pass) installObjectMotionBlur(record.renderer);
if (!frame.pass) record.renderer.render({ ...frame, pass: 'objectId', previousCamera: undefined });
frame.objectMotion = packObjectMotion(record.renderer, objectMotion.current, objectMotion.previous);
+ } else if (needsObjectIds && !frame.pass) {
+ // A mask by parts reads the same pass.
+ record.renderer.render({ ...frame, pass: 'objectId', previousCamera: undefined });
}
record.renderer.render(frame);
const error = record.renderer.context.getError();
diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjs
index 06b6b7d5..cb1e1f80 100644
--- a/scripts/mcp-server.mjs
+++ b/scripts/mcp-server.mjs
@@ -313,8 +313,9 @@ const effects = z.strictObject({
colorBalance: looseField,
glare: looseField,
lensDistortion: looseField,
+ stylize: looseField,
});
-const effectsListed = listed('effects', effects, 'Post effects: exposure, saturation, contrast, temperature, vignette, grain, ambientOcclusion, ambientOcclusionRadius, bloom {strength, threshold, radius}, depthOfField {blur, focusDistance}, motionBlur {shutter}, colorBalance, glare, lensDistortion; see scene_capabilities effects.', coreOf(effects.shape, ['exposure']));
+const effectsListed = listed('effects', effects, 'Post effects: exposure, saturation, contrast, temperature, vignette, grain, ambientOcclusion, ambientOcclusionRadius, bloom {strength, threshold, radius}, depthOfField {blur, focusDistance}, motionBlur {shutter}, colorBalance, glare, lensDistortion, stylize; see scene_capabilities effects.', coreOf(effects.shape, ['exposure']));
const shadow = z.strictObject({
target: boundedVector(-1e6, 1e6).optional(),
extent: bounded(0.01, 1e5).optional(),
diff --git a/skills/build-67js-scene/references/look.md b/skills/build-67js-scene/references/look.md
index 475f2d1b..eb99cfab 100644
--- a/skills/build-67js-scene/references/look.md
+++ b/skills/build-67js-scene/references/look.md
@@ -91,7 +91,8 @@ Point lights do not scatter in volume fog; the sun does. A sunlit closed room wi
`effects` holds `exposure`, `saturation`, `contrast`, `temperature`, `vignette`, `grain`, `ambientOcclusion` (0 = off),
`bloom` and `depthOfField` (see `camera.md`). Nested `bloom` fields merge.
Named post steps, each off until set: `colorBalance` (`lift`, `gamma`, `gain`, three numbers each), `glare` (`streaks`,
-`count`, `angle`, `fogGlow`, `threshold`, `size`) and `lensDistortion` (`distortion`, `dispersion`); ranges in `docs/agent-look.md`.
+`count`, `angle`, `fogGlow`, `threshold`, `size`) `lensDistortion` (`distortion`, `dispersion`) and `stylize` (`sharpen`, `kuwahara`, `posterize`, `pixelate`); ranges in `docs/agent-look.md`.
+`colorBalance` and `stylize` take a `mask` (`objects` part names, `depth` `{ near, far, feather }`, `invert`) to apply only to a hero part or a depth band.
Tool `scene_apply`:
diff --git a/src/agent-scene-modules.js b/src/agent-scene-modules.js
index ce110f90..20e0fff2 100644
--- a/src/agent-scene-modules.js
+++ b/src/agent-scene-modules.js
@@ -90,7 +90,7 @@ const MODULE_MARKERS = {
materialSnippet: ['"snippet":{'],
// Only a motionBlur object that sets a non-zero `objects` loads it; the common `"shutter":0` look does not.
objectMotionBlur: [/"motionBlur":\{[^}]*"objects":(?!0[,}])/],
- postSteps: [/"(?:colorBalance|glare|lensDistortion)"/],
+ postSteps: [/"(?:colorBalance|glare|lensDistortion|stylize)"/],
};
/** Whether a recipe, operation list or edit list contains a boolean edit, which needs loadMeshBooleans() first. */
diff --git a/src/post-mask.js b/src/post-mask.js
new file mode 100644
index 00000000..4871942c
--- /dev/null
+++ b/src/post-mask.js
@@ -0,0 +1,127 @@
+// A mask for the named post steps (post-steps.js): a step applies only where the mask is on. Two sources, which multiply when
+// both are named: the parts listed by name, read from the object ID pass, and a range of distance along the view. The mask is
+// exact per pixel, so it has hard edges where a part meets another (an object ID cannot be antialiased); a depth range softens
+// its ends with `feather`. A step with no mask applies to the whole frame, as before.
+
+export const MAXIMUM_MASK_OBJECTS = 16;
+const MAXIMUM_NAME_LENGTH = 128;
+
+function checkedNumber(value, fallback, minimum, maximum, name) {
+ const result = value ?? fallback;
+ if (!Number.isFinite(result) || result < minimum || result > maximum) throw new RangeError(`${name} must be between ${minimum} and ${maximum}.`);
+ return result;
+}
+
+function checkedKeys(group, allowed, name) {
+ if (!group || typeof group !== 'object' || Array.isArray(group)) throw new TypeError(`${name} must be an object.`);
+ for (const key of Object.keys(group)) {
+ if (!allowed.includes(key)) throw new TypeError(`Unknown ${name} field "${key}". Fix: use ${allowed.join(', ')}.`);
+ }
+}
+
+function resolveObjectNames(objects, name) {
+ if (!Array.isArray(objects) || objects.length === 0 || objects.length > MAXIMUM_MASK_OBJECTS) {
+ throw new TypeError(`${name} must be a list of 1 to ${MAXIMUM_MASK_OBJECTS} part names.`);
+ }
+ objects.forEach((part, index) => {
+ if (typeof part !== 'string' || part.length === 0 || part.length > MAXIMUM_NAME_LENGTH) {
+ throw new TypeError(`${name}[${index}] must be a part name of 1 to ${MAXIMUM_NAME_LENGTH} characters. Fix: pass the node id, as the object ID pass lists it.`);
+ }
+ });
+ return [...objects];
+}
+
+/** The mask a step names, resolved with defaults: { objects?, depth?: { near, far, feather }, invert }. */
+export function resolveMask(input, name) {
+ checkedKeys(input, ['objects', 'depth', 'invert'], name);
+ if (input.objects === undefined && input.depth === undefined) {
+ throw new TypeError(`${name} needs a source. Fix: name objects (a list of part names), depth ({ near, far, feather }), or both.`);
+ }
+ const mask = { invert: input.invert === true };
+ if (input.invert !== undefined && typeof input.invert !== 'boolean') throw new TypeError(`${name}.invert must be true or false.`);
+ if (input.objects !== undefined) mask.objects = resolveObjectNames(input.objects, `${name}.objects`);
+ if (input.depth !== undefined) {
+ checkedKeys(input.depth, ['near', 'far', 'feather'], `${name}.depth`);
+ const near = checkedNumber(input.depth.near, 0, 0, 1e5, `${name}.depth.near`);
+ const far = checkedNumber(input.depth.far, 1e5, 0, 1e5, `${name}.depth.far`);
+ if (far <= near) throw new RangeError(`${name}.depth.far must be greater than near. Fix: distances are metres along the view direction.`);
+ mask.depth = { near, far, feather: checkedNumber(input.depth.feather, 0, 0, 1e4, `${name}.depth.feather`) };
+ }
+ return mask;
+}
+
+/** Whether the object ID pass must be drawn before the lit frame: true when any resolved step masks by objects. */
+export function usesObjectMask(steps) {
+ return Object.values(steps).some(step => Boolean(step?.mask?.objects));
+}
+
+export const MASK_UNIFORM_NAMES = ['objectIds', 'maskObjectCount', 'maskObjectIds', 'maskUseDepth', 'maskDepth', 'maskInvert', 'maskEye', 'maskForward'];
+
+/** GLSL for a step that takes a mask: maskAt(coordinate) is 1 where the step applies and 0 where the source shows through. */
+export const maskShader = `
+uniform sampler2D objectIds;
+uniform int maskObjectCount;
+uniform int maskObjectIds[${MAXIMUM_MASK_OBJECTS}];
+uniform float maskUseDepth;
+uniform vec3 maskDepth;
+uniform float maskInvert;
+uniform vec3 maskEye;
+uniform vec3 maskForward;
+float maskAt(vec2 coordinate) {
+ float mask = 1.0;
+ if (maskObjectCount > 0) {
+ vec3 idBytes = floor(texelFetch(objectIds, ivec2(coordinate * resolution), 0).rgb * 255.0 + 0.5);
+ int identifier = int(idBytes.r + idBytes.g * 256.0 + idBytes.b * 65536.0);
+ float hit = 0.0;
+ for (int index = 0; index < ${MAXIMUM_MASK_OBJECTS}; index++) {
+ if (index >= maskObjectCount) break;
+ if (maskObjectIds[index] == identifier) hit = 1.0;
+ }
+ mask *= hit;
+ }
+ if (maskUseDepth > 0.5) {
+ float distanceAlongView = dot(worldPointAt(coordinate) - maskEye, maskForward);
+ float feather = max(maskDepth.z, 0.0001);
+ mask *= clamp((distanceAlongView - (maskDepth.x - feather)) / feather, 0.0, 1.0)
+ * (1.0 - clamp((distanceAlongView - maskDepth.y) / feather, 0.0, 1.0));
+ }
+ return mix(mask, 1.0 - mask, maskInvert);
+}
+`;
+
+function objectIdentifiers(renderer, names) {
+ if (!renderer.passSupport) throw new Error('A mask by objects needs installRenderPasses(renderer) and the object ID pass drawn first.');
+ const known = renderer.passSupport.objectNames();
+ const identifiers = [];
+ for (const name of names) {
+ const matches = known.flatMap((candidate, index) => (candidate === name ? [index + 1] : []));
+ if (matches.length === 0) {
+ throw new Error(`Mask object "${name}" is not a part in the frame. Fix: use a part name such as ${known.slice(0, 8).map(part => `"${part}"`).join(', ') || '(the scene has no opaque parts)'}.`);
+ }
+ identifiers.push(...matches);
+ }
+ if (identifiers.length > MAXIMUM_MASK_OBJECTS) throw new RangeError(`The mask names ${identifiers.length} parts; the limit is ${MAXIMUM_MASK_OBJECTS}. Fix: name fewer parts.`);
+ return identifiers;
+}
+
+/** Sets the mask uniforms of a stage. A step with no mask gets a mask of one everywhere. Leaves texture unit 1 active. */
+export function bindMask(context, uniforms, renderer, frame, mask) {
+ const identifiers = mask?.objects ? objectIdentifiers(renderer, mask.objects) : [];
+ context.uniform1i(uniforms.maskObjectCount, identifiers.length);
+ if (identifiers.length > 0) {
+ const padded = new Int32Array(MAXIMUM_MASK_OBJECTS);
+ padded.set(identifiers);
+ context.uniform1iv(uniforms.maskObjectIds, padded);
+ context.activeTexture(context.TEXTURE2);
+ context.bindTexture(context.TEXTURE_2D, renderer.passSupport.texture);
+ context.uniform1i(uniforms.objectIds, 2);
+ }
+ context.uniform1f(uniforms.maskUseDepth, mask?.depth ? 1 : 0);
+ if (mask?.depth) {
+ context.uniform3f(uniforms.maskDepth, mask.depth.near, mask.depth.far, mask.depth.feather);
+ context.uniform3fv(uniforms.maskEye, frame.eye);
+ context.uniform3fv(uniforms.maskForward, renderer.viewForward);
+ }
+ context.uniform1f(uniforms.maskInvert, mask?.invert ? 1 : 0);
+ context.activeTexture(context.TEXTURE1);
+}
diff --git a/src/post-steps.js b/src/post-steps.js
index 70f5a1b2..a35aeb7d 100644
--- a/src/post-steps.js
+++ b/src/post-steps.js
@@ -1,16 +1,21 @@
import { postStepSupport } from './shader-registry.js';
import { createStage, sharedFragmentHeader } from './camera-effects.js';
+import { resolveMask, usesObjectMask, maskShader, bindMask, MASK_UNIFORM_NAMES } from './post-mask.js';
+
+export { usesObjectMask };
// Named post steps for a graded film: color balance (lift, gamma, gain), glare (streaks and fog glow), lens distortion with
-// dispersion. They are optional full-frame stages. The core renderer carries none of their code: a recipe that names one
+// dispersion, and stylize filters (sharpen, kuwahara, posterize, pixelate). Color balance and stylize take a `mask` (post-mask.js)
+// that limits them to named parts or a range of distance. They are optional full-frame stages. The core renderer carries none of their code: a recipe that names one
// loads this module (agent-scene-modules.js), and each stage compiles only on the first frame that turns it on. Defocus from
// depth is the `depthOfField` group of camera-effects.js, which is the same step under its older name.
//
-// All three are deterministic: fixed sample patterns and no history, so a seek repeats the same pixels.
+// All of them are deterministic: fixed sample patterns and no history, so a seek repeats the same pixels.
const MAXIMUM_STREAK_ARMS = 8;
const STREAK_STEPS = 24;
const FOG_SAMPLES = 32;
+const MAXIMUM_KUWAHARA_RADIUS = 8;
// Longest lateral color offset at the image corner at full dispersion, as a fraction of the distance from the center.
const MAXIMUM_DISPERSION = 0.03;
@@ -41,14 +46,34 @@ function triple(value, fallback, minimum, maximum, name) {
function resolveColorBalance(input) {
const group = groupOf(input, 'colorBalance');
- onlyKeys(group, ['lift', 'gamma', 'gain'], 'colorBalance');
+ onlyKeys(group, ['lift', 'gamma', 'gain', 'mask'], 'colorBalance');
return {
lift: triple(group.lift, NEUTRAL.lift, -1, 1, 'colorBalance.lift'),
gamma: triple(group.gamma, NEUTRAL.gamma, 0.1, 4, 'colorBalance.gamma'),
gain: triple(group.gain, NEUTRAL.gain, 0, 4, 'colorBalance.gain'),
+ ...(group.mask === undefined ? {} : { mask: resolveMask(group.mask, 'colorBalance.mask') }),
};
}
+function wholeNumberInRange(value, minimum, maximum, name) {
+ const result = numberInRange(value, undefined, minimum, maximum, name);
+ if (!Number.isInteger(result)) throw new RangeError(`${name} must be a whole number.`);
+ return result;
+}
+
+// Each filter is off until named; a filter named with a number turns on.
+function resolveStylize(input) {
+ const group = groupOf(input, 'stylize');
+ onlyKeys(group, ['sharpen', 'kuwahara', 'posterize', 'pixelate', 'mask'], 'stylize');
+ const resolved = {};
+ if (group.sharpen !== undefined) resolved.sharpen = numberInRange(group.sharpen, undefined, 0, 2, 'stylize.sharpen');
+ if (group.kuwahara !== undefined) resolved.kuwahara = wholeNumberInRange(group.kuwahara, 0, MAXIMUM_KUWAHARA_RADIUS, 'stylize.kuwahara');
+ if (group.posterize !== undefined) resolved.posterize = wholeNumberInRange(group.posterize, 0, 256, 'stylize.posterize');
+ if (group.pixelate !== undefined) resolved.pixelate = wholeNumberInRange(group.pixelate, 0, 64, 'stylize.pixelate');
+ if (group.mask !== undefined) resolved.mask = resolveMask(group.mask, 'stylize.mask');
+ return resolved;
+}
+
function resolveGlare(input) {
const group = groupOf(input, 'glare');
onlyKeys(group, ['streaks', 'count', 'angle', 'fogGlow', 'threshold', 'size'], 'glare');
@@ -79,14 +104,16 @@ export function resolvePostSteps(effects) {
if (effects.colorBalance !== undefined) resolved.colorBalance = resolveColorBalance(effects.colorBalance);
if (effects.glare !== undefined) resolved.glare = resolveGlare(effects.glare);
if (effects.lensDistortion !== undefined) resolved.lensDistortion = resolveLensDistortion(effects.lensDistortion);
+ if (effects.stylize !== undefined) resolved.stylize = resolveStylize(effects.stylize);
return resolved;
}
// Registers what the look needs: the field names it accepts and how to resolve them (src/shader-registry.js).
-postStepSupport.fields = ['colorBalance', 'glare', 'lensDistortion'];
+postStepSupport.fields = ['colorBalance', 'glare', 'lensDistortion', 'stylize'];
postStepSupport.resolve = resolvePostSteps;
export const colorBalanceFragment = `${sharedFragmentHeader}
+${maskShader}
uniform vec3 lift;
uniform vec3 inverseGamma;
uniform vec3 gain;
@@ -94,7 +121,67 @@ void main() {
vec4 source = texture(sourceTexture, textureCoordinate);
// Lift raises the shadows and fades out toward white, gain scales the highlights, gamma bends the midtones.
vec3 graded = source.rgb * gain + lift * (1.0 - clamp(source.rgb, 0.0, 1.0));
- outputColor = vec4(pow(max(graded, vec3(0.0)), inverseGamma), source.a);
+ graded = pow(max(graded, vec3(0.0)), inverseGamma);
+ float mask = maskAt(textureCoordinate);
+ outputColor = vec4(mask < 1.0 ? mix(source.rgb, graded, mask) : graded, source.a);
+}`;
+
+export const stylizeFragment = `${sharedFragmentHeader}
+${maskShader}
+uniform float sharpenAmount;
+uniform int kuwaharaRadius;
+uniform float posterizeLevels;
+uniform float pixelateSize;
+vec2 snapped(vec2 coordinate) {
+ if (pixelateSize < 1.0) return coordinate;
+ return (floor(coordinate * resolution / pixelateSize) + 0.5) * pixelateSize / resolution;
+}
+vec4 sourceAt(vec2 coordinate) {
+ return texture(sourceTexture, snapped(coordinate));
+}
+// Kuwahara: the mean of whichever of the four quadrants around the pixel varies least, so flat areas stay flat and edges stay sharp.
+vec3 kuwahara(vec2 coordinate) {
+ vec3 bestMean = vec3(0.0);
+ float bestVariance = 1e30;
+ for (int quadrant = 0; quadrant < 4; quadrant++) {
+ vec2 direction = vec2(quadrant == 1 || quadrant == 3 ? -1.0 : 1.0, quadrant >= 2 ? -1.0 : 1.0);
+ vec3 sum = vec3(0.0);
+ float lumaSum = 0.0;
+ float lumaSquareSum = 0.0;
+ float count = 0.0;
+ for (int row = 0; row <= ${MAXIMUM_KUWAHARA_RADIUS}; row++) {
+ if (row > kuwaharaRadius) break;
+ for (int column = 0; column <= ${MAXIMUM_KUWAHARA_RADIUS}; column++) {
+ if (column > kuwaharaRadius) break;
+ vec3 color = sourceAt(coordinate + direction * vec2(float(column), float(row)) / resolution).rgb;
+ float luma = dot(color, vec3(0.299, 0.587, 0.114));
+ sum += color;
+ lumaSum += luma;
+ lumaSquareSum += luma * luma;
+ count += 1.0;
+ }
+ }
+ float variance = lumaSquareSum / count - (lumaSum / count) * (lumaSum / count);
+ if (variance < bestVariance) {
+ bestVariance = variance;
+ bestMean = sum / count;
+ }
+ }
+ return bestMean;
+}
+void main() {
+ vec4 source = texture(sourceTexture, textureCoordinate);
+ vec4 center = sourceAt(textureCoordinate);
+ vec3 color = kuwaharaRadius > 0 ? kuwahara(textureCoordinate) : center.rgb;
+ if (sharpenAmount > 0.0) {
+ vec2 texel = 1.0 / resolution;
+ vec3 around = sourceAt(textureCoordinate + vec2(texel.x, 0.0)).rgb + sourceAt(textureCoordinate - vec2(texel.x, 0.0)).rgb
+ + sourceAt(textureCoordinate + vec2(0.0, texel.y)).rgb + sourceAt(textureCoordinate - vec2(0.0, texel.y)).rgb;
+ color = max(color + sharpenAmount * (center.rgb * 4.0 - around), vec3(0.0));
+ }
+ if (posterizeLevels > 0.0) color = floor(color * posterizeLevels + 0.5) / posterizeLevels;
+ float mask = maskAt(textureCoordinate);
+ outputColor = vec4(mask < 1.0 ? mix(source.rgb, color, mask) : color, source.a);
}`;
export const glareFragment = `${sharedFragmentHeader}
@@ -181,13 +268,35 @@ function createColorBalanceStage() {
return createStage({
name: 'Color balance',
fragment: colorBalanceFragment,
- uniformNames: ['lift', 'inverseGamma', 'gain'],
+ uniformNames: ['lift', 'inverseGamma', 'gain', ...MASK_UNIFORM_NAMES],
isActive: (effects, frame) => isColorBalanceActive(stepsOf(frame).colorBalance),
setUniforms(context, uniforms, renderer, frame) {
const { colorBalance } = stepsOf(frame);
context.uniform3fv(uniforms.lift, colorBalance.lift);
context.uniform3fv(uniforms.inverseGamma, colorBalance.gamma.map(gamma => 1 / gamma));
context.uniform3fv(uniforms.gain, colorBalance.gain);
+ bindMask(context, uniforms, renderer, frame, colorBalance.mask);
+ },
+ });
+}
+
+function isStylizeActive(stylize) {
+ return Boolean(stylize) && (stylize.sharpen > 0 || stylize.kuwahara > 0 || stylize.posterize > 0 || stylize.pixelate > 0);
+}
+
+function createStylizeStage() {
+ return createStage({
+ name: 'Stylize',
+ fragment: stylizeFragment,
+ uniformNames: ['sharpenAmount', 'kuwaharaRadius', 'posterizeLevels', 'pixelateSize', ...MASK_UNIFORM_NAMES],
+ isActive: (effects, frame) => isStylizeActive(stepsOf(frame).stylize),
+ setUniforms(context, uniforms, renderer, frame) {
+ const { stylize } = stepsOf(frame);
+ context.uniform1f(uniforms.sharpenAmount, stylize.sharpen ?? 0);
+ context.uniform1i(uniforms.kuwaharaRadius, stylize.kuwahara ?? 0);
+ context.uniform1f(uniforms.posterizeLevels, stylize.posterize ?? 0);
+ context.uniform1f(uniforms.pixelateSize, stylize.pixelate ?? 0);
+ bindMask(context, uniforms, renderer, frame, stylize.mask);
},
});
}
@@ -255,12 +364,13 @@ function createLensDistortionStage() {
}
/**
- * Adds glare, lens distortion and color balance to a renderer, after any depth of field and motion blur it has, in that
- * order. Like them it needs `depth: true`. Each stage runs only for a frame whose effects set it.
+ * Adds glare, lens distortion, color balance and the stylize filters to a renderer, after any depth of field and motion blur it has, in that
+ * order. Like them it needs `depth: true`. Each stage runs only for a frame whose effects set it. A step with a `mask` by
+ * objects reads the object ID pass: install the render passes and draw that pass before the lit frame (`usesObjectMask` says when).
*/
export function installPostSteps(renderer) {
if (!renderer.postStages.some(stage => stage.name === 'Glare')) {
- renderer.postStages.push(createGlareStage(), createLensDistortionStage(), createColorBalanceStage());
+ renderer.postStages.push(createGlareStage(), createLensDistortionStage(), createColorBalanceStage(), createStylizeStage());
}
return renderer;
}
diff --git a/tests/post-steps.test.js b/tests/post-steps.test.js
index b60504b7..5fe3823d 100644
--- a/tests/post-steps.test.js
+++ b/tests/post-steps.test.js
@@ -9,7 +9,7 @@ import { resolveEffects } from '../src/effects.js';
import { normalizeAgentLook } from '../src/agent-look.js';
import { agentSceneModulesNeeded, loadAgentSceneModules } from '../src/agent-scene-modules.js';
import { installCameraEffects } from '../src/camera-effects.js';
-import { installPostSteps, resolvePostSteps, glareFragment, lensDistortionFragment, colorBalanceFragment } from '../src/post-steps.js';
+import { installPostSteps, resolvePostSteps, glareFragment, lensDistortionFragment, colorBalanceFragment, stylizeFragment, usesObjectMask } from '../src/post-steps.js';
import { createAgentScene } from '../src/agent-scene.js';
function recordingContext(calls) {
@@ -33,12 +33,12 @@ function fakeRenderer(context) {
};
}
-const STAGE_NAMES = ['Glare', 'Lens distortion', 'Color balance'];
+const STAGE_NAMES = ['Glare', 'Lens distortion', 'Color balance', 'Stylize'];
test('a look that names no step resolves exactly as before', async () => {
await loadAgentSceneModules({});
const effects = normalizeAgentLook({}).effects;
- for (const field of ['colorBalance', 'glare', 'lensDistortion']) assert.equal(Object.hasOwn(effects, field), false);
+ for (const field of ['colorBalance', 'glare', 'lensDistortion', 'stylize']) assert.equal(Object.hasOwn(effects, field), false);
assert.deepEqual(resolveEffects({}), resolveEffects({ saturation: 1 }));
});
@@ -86,13 +86,14 @@ test('each stage draws only for a frame that sets its step, and frees its target
const context = recordingContext(calls);
const renderer = installPostSteps(installCameraEffects(fakeRenderer(context)));
assert.deepEqual(renderer.postStages.slice(2).map(stage => stage.name), STAGE_NAMES);
- assert.equal(installPostSteps(renderer).postStages.length, 5);
+ assert.equal(installPostSteps(renderer).postStages.length, 6);
const unset = { eye: [0, 1, 5], effects: {} };
- const neutral = { eye: [0, 1, 5], effects: { colorBalance: {}, glare: {}, lensDistortion: {} } };
+ const neutral = { eye: [0, 1, 5], effects: { colorBalance: {}, glare: {}, lensDistortion: {}, stylize: {} } };
const named = [
['Glare', { glare: { streaks: 0.6 } }],
['Lens distortion', { lensDistortion: { distortion: 0.5, dispersion: 0.5 } }],
['Color balance', { colorBalance: { gain: [1.1, 1, 0.9] } }],
+ ['Stylize', { stylize: { posterize: 4 } }],
];
const effects = resolveEffects({});
for (const [name, input] of named) {
@@ -103,12 +104,56 @@ test('each stage draws only for a frame that sets its step, and frees its target
stage.releaseTargets(context);
stage.dispose(context);
}
- assert.ok(calls.filter(call => call[0] === 'drawArrays').length === 3);
- assert.ok(calls.filter(call => call[0] === 'deleteTexture').length === 3);
+ assert.ok(calls.filter(call => call[0] === 'drawArrays').length === 4);
+ assert.ok(calls.filter(call => call[0] === 'deleteTexture').length === 4);
+});
+
+test('a stylize step resolves its filters and a mask, and refuses bad input', () => {
+ assert.deepEqual(resolvePostSteps({ stylize: {} }).stylize, {});
+ assert.deepEqual(resolvePostSteps({ stylize: { sharpen: 1, kuwahara: 3, posterize: 5, pixelate: 8 } }).stylize, { sharpen: 1, kuwahara: 3, posterize: 5, pixelate: 8 });
+ assert.throws(() => resolvePostSteps({ stylize: { kuwahara: 9 } }), /stylize.kuwahara must be between/);
+ assert.throws(() => resolvePostSteps({ stylize: { pixelate: 2.5 } }), /whole number/);
+ assert.throws(() => resolvePostSteps({ stylize: { sharpen: -1 } }), /stylize.sharpen must be between/);
+ assert.throws(() => resolvePostSteps({ stylize: { blur: 1 } }), /Unknown stylize field/);
+});
+
+test('a mask names parts or a depth range, and says what to pass when it is wrong', () => {
+ const mask = resolvePostSteps({ colorBalance: { mask: { objects: ['hero'], depth: { near: 2, far: 8 }, invert: true } } }).colorBalance.mask;
+ assert.deepEqual(mask, { invert: true, objects: ['hero'], depth: { near: 2, far: 8, feather: 0 } });
+ assert.deepEqual(resolvePostSteps({ stylize: { mask: { depth: { far: 5 } } } }).stylize.mask, { invert: false, depth: { near: 0, far: 5, feather: 0 } });
+ assert.equal(resolvePostSteps({ colorBalance: {} }).colorBalance.mask, undefined);
+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: {} } }), /colorBalance.mask needs a source/);
+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: [] } } }), /1 to 16 part names/);
+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: Array(17).fill('a') } } }), /1 to 16 part names/);
+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: [3] } } }), /mask.objects\[0\] must be a part name/);
+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { depth: { near: 5, far: 5 } } } }), /far must be greater than near/);
+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { depth: { feather: -1 } } } }), /feather must be between/);
+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: ['a'], color: 1 } } }), /Unknown colorBalance.mask field/);
+ assert.throws(() => resolvePostSteps({ colorBalance: { mask: { objects: ['a'], invert: 1 } } }), /invert must be true or false/);
+ assert.equal(usesObjectMask(resolvePostSteps({ colorBalance: { mask: { objects: ['hero'] } } })), true);
+ assert.equal(usesObjectMask(resolvePostSteps({ colorBalance: { mask: { depth: { far: 5 } } }, stylize: { pixelate: 4 } })), false);
+});
+
+test('a stage binds the part numbers of a masked step and refuses an unknown part', () => {
+ const calls = [];
+ const context = recordingContext(calls);
+ const renderer = installPostSteps(installCameraEffects(fakeRenderer(context)));
+ renderer.passSupport = { texture: 'ids', objectNames: () => ['floor', 'hero', 'post', 'hero'] };
+ const stage = renderer.postStages.find(candidate => candidate.name === 'Color balance');
+ const effects = resolveEffects({});
+ const frame = mask => ({ eye: [0, 1, 5], effects: { colorBalance: { gain: [1.2, 1, 1], mask } } });
+ assert.ok(stage.apply(renderer, frame({ objects: ['hero'] }), effects, 'source'));
+ const bound = calls.find(call => call[0] === 'uniform1iv');
+ assert.deepEqual([...bound[2]].slice(0, 3), [2, 4, 0]);
+ assert.throws(() => stage.apply(renderer, frame({ objects: ['ghost'] }), effects, 'source'), /"ghost" is not a part in the frame\. Fix: use a part name such as "floor"/);
+ renderer.passSupport = null;
+ assert.throws(() => stage.apply(renderer, frame({ objects: ['hero'] }), effects, 'source'), /installRenderPasses/);
+ stage.releaseTargets(context);
+ stage.dispose(context);
});
test('the shaders keep their fixed sample patterns and read no history', () => {
- for (const fragment of [glareFragment, lensDistortionFragment, colorBalanceFragment]) {
+ for (const fragment of [glareFragment, lensDistortionFragment, colorBalanceFragment, stylizeFragment]) {
assert.match(fragment, /^#version 300 es/);
assert.doesNotMatch(fragment, /previousViewProjection|time/);
}
@@ -118,7 +163,8 @@ test('the shaders keep their fixed sample patterns and read no history', () => {
// RENDER_TESTS=1 node --test tests/post-steps.test.js (on a shared GPU, start it through the machine GPU lock)
const chromePath = process.env.CHROME_PATH ?? 'C:/Program Files/Google/Chrome/Application/chrome.exe';
const rendersEnabled = process.env.RENDER_TESTS === '1' && existsSync(chromePath);
-const STEP_CASES = ['defocus', 'color-balance', 'glare-streaks', 'glare-streaks-6', 'glare-anamorphic', 'glare-fog', 'lens-distortion', 'lens-dispersion'];
+const STEP_CASES = ['defocus', 'color-balance', 'glare-streaks', 'glare-streaks-6', 'glare-anamorphic', 'glare-fog', 'lens-distortion', 'lens-dispersion',
+ 'grade-hero', 'grade-depth', 'sharpen', 'kuwahara', 'posterize', 'pixelate', 'pixelate-hero'];
test('each named step changes the picture and the default still keeps its hash', { skip: !rendersEnabled, timeout: 300000 }, () => {
const output = execFileSync('node', ['bench/post/post-steps.mjs', '--rounds=1', `--out=${join(tmpdir(), 'post-steps-test')}`], { encoding: 'utf8' });
@@ -127,6 +173,11 @@ test('each named step changes the picture and the default still keeps its hash',
assert.notEqual(summary[name].hash, summary.default.hash, `${name} differs from the default`);
assert.ok(summary[name].medianMilliseconds > 0, `${name} has a frame time`);
}
+ // A grade masked to the hero part changes the hero and leaves the floor as the default drew it; the unmasked grade changes both.
+ assert.notDeepEqual(summary['grade-hero'].regions.hero, summary.default.regions.hero);
+ assert.deepEqual(summary['grade-hero'].regions.floor, summary.default.regions.floor);
+ assert.notDeepEqual(summary['color-balance'].regions.floor, summary.default.regions.floor);
+ assert.deepEqual(summary['pixelate-hero'].regions.floor, summary.default.regions.floor);
// Hash of the default still as origin/main drew it before the named steps (bench/post/post-steps.mjs --root=<origin/main checkout>).
assert.equal(summary.default.hash, '8741ffc3aaace9a8');
});