3D engine that AI agents drive
Per-object motion blur: effects.motionBlur.objects smears moving parts along their own motion (issue #600)
Nguyen Xuan Anaaf4bb6b9546
diff --git a/changes/600-motion-blur.md b/changes/600-motion-blur.mdnew file mode 100644index 00000000..91f8fdc2--- /dev/null+++ b/changes/600-motion-blur.md@@ -0,0 +1 @@+- Per-object motion blur: `effects.motionBlur.objects` (0 to 1, off by default) smears each part that moves during the shutter along its own screen path, while the still background stays sharp; it replaces camera motion blur on that frame and loads on demand (#600)diff --git a/changes/600-motion-blur.size.md b/changes/600-motion-blur.size.mdnew file mode 100644index 00000000..7bf0f67d--- /dev/null+++ b/changes/600-motion-blur.size.md@@ -0,0 +1,4 @@+- scene:agent-scene+lazy: the object blur stage and its packing load on demand and only for a recipe that sets motionBlur.objects; nothing runs for any other scene+- goal:lit-box: 31 bytes gzip in the shared effect resolver (the optional objects field) and the pass texture getter, which the lazy stage needs+- goal:gltf-box: the same 36 bytes gzip of shared effect and pass code, in a scene that never loads the stage+- goal:agent-scene: the same 32 bytes gzip of shared effect and pass code, in a scene that never loads the stagediff --git a/docs/agent-look.md b/docs/agent-look.mdindex 87d756b3..621d6808 100644--- a/docs/agent-look.md+++ b/docs/agent-look.md@@ -257,11 +257,12 @@ Ambient occlusion reads the depth texture, so it needs the HDR depth target that| `depthOfField.blur` | `0` | `0` to `1`. Lens blur from depth, off by default; the widest circle is 5 percent of the image height || `depthOfField.focusDistance` | `5` | `0.01` to `100000` metres from the camera that stay sharp || `motionBlur.shutter` | `0` | `0` to `0.5` seconds of camera motion to smear |+| `motionBlur.objects` | `0` | `0` to `1`. How much of each moving part's own motion to smear over the same shutter; `0` leaves it off |Depth of field and motion blur are deterministic: they read the frame, its depth and the camera, with fixed sample patterns, so a seek to a time gives the same pixels as playing to it.Depth of field applies to every render, including `scene_look`.Motion blur needs camera `motion` tracks and a render job with `settings.motion`: the job samples the camera at `motion.start + elapsed - shutter` as well, and smears each pixel along the screen movement between the two.-It follows camera motion only; moving objects are not blurred.+With `motionBlur.objects` above zero, a part that moves between `motion.start + elapsed - shutter` and `motion.start + elapsed` also smears along its own screen path while the still background stays sharp (`1` is the full shutter). It reads each pixel's part from an object ID pass, so it costs a second draw of the frame, and it replaces the camera smear on that frame, since it covers the camera's motion too. A part is moved as one rigid body by its node transform: a skinned or morphed part, an imported asset and a translucent part are not smeared per object.Both effects compile their shader and allocate their targets the first time a frame uses them, and a scene that leaves them at zero pays nothing.## Directional shadowsdiff --git a/package.json b/package.jsonindex a7a6955c..7204eec7 100644--- a/package.json+++ b/package.json@@ -154,6 +154,7 @@"smoke:agent-material-snippet": "node scripts/smoke-agent-material-snippet.mjs","smoke:agent-texture-sequence": "node scripts/smoke-agent-texture-sequence.mjs","smoke:render-passes": "node scripts/smoke-render-passes.mjs",+ "smoke:object-motion-blur": "node scripts/smoke-object-motion-blur.mjs","smoke:path-tracer": "node scripts/smoke-path-tracer.mjs","smoke:film-timeline": "node scripts/smoke-film-timeline.mjs","smoke:material-layers": "node scripts/smoke-material-layers.mjs",diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex 1fae93e1..0d25ce27 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -257,7 +257,16 @@ function draw(record, time = 0, camera = record.scene.camera, frameOptions = {})syncMeshBlend(record);record.renderer.userPasses?.set(record.scene.postPasses ?? []);applyRecipeWindHeading(record.renderer, environment);- record.renderer.render({ ...camera, environment: litEnvironment(record.scene.parts, environment, Boolean(record.renderer.groundPart)), effects, exposure: effects.exposure, time, lines: environment.lines, ...windFrame(environment), ...frameOptions, effects: effectsWithFStop(frameOptions.effects ?? effects, camera) });+ const { objectMotion, ...renderOptions } = frameOptions;+ const frame = { ...camera, environment: litEnvironment(record.scene.parts, environment, Boolean(record.renderer.groundPart)), effects, exposure: effects.exposure, time, lines: environment.lines, ...windFrame(environment), ...renderOptions, effects: effectsWithFStop(renderOptions.effects ?? effects, camera) };+ if (objectMotion) {+ // Object motion blur reads each pixel's part from the object ID pass, drawn first; the lit frame then smears along each part's motion.+ const { installObjectMotionBlur, packObjectMotion } = agentSceneModule('objectMotionBlur');+ installObjectMotionBlur(record.renderer);+ record.renderer.render({ ...frame, pass: 'objectId', previousCamera: undefined });+ frame.objectMotion = packObjectMotion(record.renderer, objectMotion.current, objectMotion.previous);+ }+ record.renderer.render(frame);const error = record.renderer.context.getError();if (error !== 0) {throw new Error(`WebGL render failed with error ${error}.`);@@ -388,6 +397,19 @@ function previousCameraFor(record, motionTime) {return previousCamera;}+// The parts' world matrices now and at the start of the shutter, sampled from time like the previous camera. Only a scene that+// asks for object blur (and has motion to sample) pays for it; an imported asset animates by skin, which this does not follow.+function objectMotionFor(record, motionTime) {+ const { shutter, objects } = record.scene.effects.motionBlur;+ if (!(shutter > 0 && objects > 0) || motionTime === undefined || record.asset) return undefined;+ const { sampleWorldMatrices } = agentSceneModule('objectMotionBlur');+ const current = sampleWorldMatrices(record.scene.parts);+ record.scene.sampleMotion(Math.max(0, motionTime - shutter));+ const previous = sampleWorldMatrices(record.scene.parts);+ record.scene.sampleMotion(motionTime);+ return { current, previous };+}+function createPreview(scene, { sessionId, memoryBudgetMiB, backend }, asset, placedAssets, sequenceFrames) {if (active) {throw new Error('Dispose the current scene before creating another.');@@ -753,11 +775,12 @@ window.__agentPreview = {const shotCamera = cameraOverrides ? withCameraOverrides(record.scene.camera, cameraOverrides) : undefined;const sampledPreviousCamera = pass ? undefined : previousCameraFor(record, motionTime);const previousCamera = sampledPreviousCamera && cameraOverrides ? withCameraOverrides(sampledPreviousCamera, cameraOverrides) : sampledPreviousCamera;+ const objectMotion = pass ? undefined : objectMotionFor(record, motionTime);const exposure = record.scene.effects.exposure * brightness;const shotEffects = depthOfFieldOverrides? { ...record.scene.effects, depthOfField: { ...record.scene.effects.depthOfField, ...depthOfFieldOverrides } }: undefined;- draw(record, rendererTime, shotCamera, { pass, passDepthRange, previousCamera,+ draw(record, rendererTime, shotCamera, { pass, passDepthRange, previousCamera, ...(objectMotion ? { objectMotion } : {}),...(brightness === 1 ? {} : { exposure }), ...(shotEffects ? { effects: shotEffects } : {}) });const info = renderInfo(record.renderer);return pass === 'objectId' ? { ...info, objectNames: record.renderer.passSupport.objectNames() } : info;diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex aaea8696..e3661d9d 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -278,7 +278,7 @@ const effects = z.strictObject({radius: bounded(0.1, 4).optional(),}).optional(),depthOfField: z.strictObject({ blur: unit.optional(), focusDistance: bounded(0.01, 100000).optional() }).optional().describe('Lens blur: blur 0 to 1, focusDistance metres.'),- motionBlur: z.strictObject({ shutter: bounded(0, 0.5).optional() }).optional().describe('Camera motion smear, shutter in seconds.'),+ motionBlur: z.strictObject({ shutter: bounded(0, 0.5).optional(), objects: bounded(0, 1).optional() }).optional().describe('Motion smear, shutter in seconds. objects 0 to 1 also smears each moving part along its own motion (off at 0).'),});const shadow = z.strictObject({target: boundedVector(-1e6, 1e6).optional(),diff --git a/scripts/smoke-object-motion-blur.mjs b/scripts/smoke-object-motion-blur.mjsnew file mode 100644index 00000000..aef6aada--- /dev/null+++ b/scripts/smoke-object-motion-blur.mjs@@ -0,0 +1,193 @@+// Proves per-object motion blur through the real MCP runtime: a spinning prop smears while the still background stays+// byte for byte sharp, and a scene that leaves `objects` off renders exactly as before. Set OBJECT_MOTION_BLUR_COPY_DIRECTORY+// to keep the before and after stills and a short clip.+import assert from 'node:assert/strict';+import { createHash } from 'node:crypto';+import { mkdtemp, readFile, writeFile } from 'node:fs/promises';+import { tmpdir, constants, setPriority } from 'node:os';+import { join, resolve } from 'node:path';+import { setTimeout as delay } from 'node:timers/promises';+import { inflateSync } from 'node:zlib';+import { Client } from '@modelcontextprotocol/client';+import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';++setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);+const directory = await mkdtemp(join(tmpdir(), '67js-mcp-object-blur-'));+const jobs = join(directory, 'jobs');+const cli = resolve(process.argv[2] ?? 'bin/67js-mcp.mjs');+const backend = process.argv[3] ?? 'cpu';+const transport = new StdioClientTransport({ command: process.execPath,+ args: [cli, '--backend', backend, '--memory', '128', '--jobs', jobs], cwd: directory, stderr: 'pipe' });+let diagnostics = '';+transport.stderr.on('data', bytes => { diagnostics = (diagnostics + bytes).slice(-8192); });+const client = new Client({ name: '67js-object-blur-proof', version: '1' });+const SIZE = 160;+const COPY_DIRECTORY = process.env.OBJECT_MOTION_BLUR_COPY_DIRECTORY;++async function call(name, argumentsValue = {}, timeout = 60000) {+ const result = await client.callTool({ name, arguments: argumentsValue }, { timeout });+ assert.equal(result.isError, undefined, result.content?.find(item => item.type === 'text')?.text);+ return result;+}++async function finished(jobId) {+ const deadline = Date.now() + 120000;+ while (Date.now() < deadline) {+ const record = (await call('scene_render_status', { jobId })).structuredContent;+ if (['completed', 'cancelled', 'failed'].includes(record.state)) return record;+ await delay(100);+ }+ throw new Error('Job did not finish within 120 seconds.');+}++async function runJob(sessionId, settings, expectedRevision) {+ const submitted = (await call('scene_render_submit', { sessionId, expectedRevision, settings })).structuredContent;+ const record = await finished(submitted.jobId);+ assert.equal(record.state, 'completed', record.reason);+ const result = (await call('scene_render_result', { jobId: submitted.jobId })).structuredContent;+ const files = new Map();+ for (const artifact of result.artifacts) {+ const bytes = await readFile(artifact.path);+ assert.equal(createHash('sha256').update(bytes).digest('hex'), artifact.sha256);+ files.set(artifact.path.split(/[\\/]/).pop(), bytes);+ }+ await call('scene_render_release', { jobId: submitted.jobId });+ return { files };+}++function recipe() {+ return {+ format: '67js.scene', schemaVersion: 1,+ geometries: [{ id: 'box', type: 'box' }],+ materials: [+ { id: 'floor', color: [0.55, 0.55, 0.6], roughness: 0.8 },+ { id: 'blade', color: [0.9, 0.15, 0.1], roughness: 0.5 },+ { id: 'post', color: [0.1, 0.3, 0.9], roughness: 0.5 },+ ],+ nodes: [+ { id: 'floor', geometry: 'box', material: 'floor', translation: [0, -0.1, 0], scale: [12, 0.2, 12] },+ { id: 'post', geometry: 'box', material: 'post', translation: [2.2, 0.6, -1], scale: [0.5, 1.2, 0.5] },+ { id: 'propeller', geometry: 'box', material: 'blade', translation: [0, 1.2, 0], scale: [2.4, 0.16, 0.16] },+ ],+ camera: { eye: [0, 1.2, 5], target: [0, 1.2, 0], fov: 40 },+ motion: { duration: 1, loop: true, tracks: [{ kind: 'node', id: 'propeller', property: 'rotation', keyframes: [+ { time: 0, value: [0, 0, 0, 1] }, { time: 0.5, value: [0, 0, Math.SQRT1_2, Math.SQRT1_2] },+ { time: 1, value: [0, 0, 1, 0] },+ ] }] },+ };+}++function paeth(left, up, upLeft) {+ const estimate = left + up - upLeft;+ const toLeft = Math.abs(estimate - left);+ const toUp = Math.abs(estimate - up);+ const toUpLeft = Math.abs(estimate - upLeft);+ if (toLeft <= toUp && toLeft <= toUpLeft) return left;+ return toUp <= toUpLeft ? up : upLeft;+}++// Reads an 8-bit RGB or RGBA PNG (the only kind the render jobs write) into a flat byte array.+function decodePng(bytes) {+ let offset = 8;+ let header = null;+ const chunks = [];+ while (offset < bytes.length) {+ const length = bytes.readUInt32BE(offset);+ const type = bytes.toString('latin1', offset + 4, offset + 8);+ const data = bytes.subarray(offset + 8, offset + 8 + length);+ if (type === 'IHDR') header = { width: data.readUInt32BE(0), height: data.readUInt32BE(4), depth: data[8], colorType: data[9] };+ if (type === 'IDAT') chunks.push(data);+ offset += 12 + length;+ }+ assert.equal(header.depth, 8);+ const channels = header.colorType === 6 ? 4 : 3;+ const stride = header.width * channels;+ const raw = inflateSync(Buffer.concat(chunks));+ const pixels = Buffer.alloc(stride * header.height);+ for (let row = 0; row < header.height; row++) {+ const filter = raw[row * (stride + 1)];+ for (let column = 0; column < stride; column++) {+ const value = raw[row * (stride + 1) + 1 + column];+ const left = column >= channels ? pixels[row * stride + column - channels] : 0;+ const up = row > 0 ? pixels[(row - 1) * stride + column] : 0;+ const upLeft = row > 0 && column >= channels ? pixels[(row - 1) * stride + column - channels] : 0;+ const predictor = [0, left, up, (left + up) >> 1, paeth(left, up, upLeft)][filter];+ pixels[row * stride + column] = (value + predictor) & 255;+ }+ }+ return { ...header, channels, pixels };+}++// The pixels where two renders differ by more than a rounding step, as { x, y } pairs.+function differingPixels(first, second) {+ const before = decodePng(first);+ const after = decodePng(second);+ const found = [];+ for (let y = 0; y < before.height; y++) {+ for (let x = 0; x < before.width; x++) {+ for (let channel = 0; channel < 3; channel++) {+ const index = (y * before.width + x) * before.channels + channel;+ if (Math.abs(before.pixels[index] - after.pixels[index]) > 2) {+ found.push({ x, y });+ break;+ }+ }+ }+ }+ return found;+}++try {+ await client.connect(transport);+ const created = (await call('scene_create', { recipe: recipe() }, 180000)).structuredContent;+ const sessionId = created.sessionId;+ const base = { width: SIZE, height: SIZE, kind: 'image', motion: { start: 0.25 } };+ let revision = 0;++ async function setBlur(motionBlur) {+ const applied = (await call('scene_apply', { sessionId, expectedRevision: revision,+ operations: [{ type: 'set-effects', effects: { motionBlur } }] })).structuredContent;+ assert.equal(applied.committed, true);+ revision = applied.revision;+ }++ async function renderStill() {+ return (await runJob(sessionId, base, revision)).files.get('render.png');+ }++ await setBlur({ shutter: 0 });+ const sharp = await renderStill();+ await setBlur({ shutter: 0.1 });+ const shutterOnly = await renderStill();+ await setBlur({ shutter: 0.1, objects: 0 });+ const objectsOff = await renderStill();+ await setBlur({ shutter: 0.1, objects: 1 });+ const blurred = await renderStill();++ // Off by default: a shutter alone, with or without `objects: 0`, leaves a still camera's frame untouched.+ assert.deepEqual(shutterOnly, sharp, 'A shutter without objects must not change a still camera frame.');+ assert.deepEqual(objectsOff, sharp, 'objects: 0 must render exactly as before.');++ // The spinning prop smears and nothing else moves: every changed pixel sits on the propeller's turn (a 2.4 m blade 5 m+ // from the lens, about 44 px a metre at this size), so the floor, the post and the sky stay sharp.+ const changed = differingPixels(sharp, blurred);+ assert.ok(changed.length > 40, `The spinning propeller must blur; only ${changed.length} pixels changed.`);+ const center = SIZE / 2;+ const reach = 1.25 * 44 + 4;+ for (const { x, y } of changed) {+ assert.ok(Math.hypot(x - center, y - center) <= reach, `Background pixel (${x}, ${y}) changed; only the prop may blur.`);+ }++ if (COPY_DIRECTORY) {+ await writeFile(join(COPY_DIRECTORY, 'before.png'), sharp);+ await writeFile(join(COPY_DIRECTORY, 'after.png'), blurred);+ const clip = await runJob(sessionId, { width: 320, height: 180, kind: 'video', fps: 24, duration: 1, motion: { start: 0 } }, revision);+ for (const [name, bytes] of clip.files) await writeFile(join(COPY_DIRECTORY, `clip-${name}`), bytes);+ }+ console.log(`object motion blur smoke passed on ${backend}: ${changed.length} prop pixels blurred, background identical`);+} catch (error) {+ console.error(diagnostics);+ throw error;+} finally {+ await client.close();+}diff --git a/src/agent-look.js b/src/agent-look.jsindex 85d0b01e..a86f7a00 100644--- a/src/agent-look.js+++ b/src/agent-look.js@@ -25,7 +25,7 @@ const VOLUME_VECTOR_FIELDS = ['minimum', 'maximum', 'color', 'ambient', 'drift']const VOLUME_NUMBER_FIELDS = ['density', 'steps', 'anisotropy', 'noiseAmount', 'noiseScale', 'maxDistance', 'shadowAmbient', 'heightFalloff', 'detailOctaves', 'detailAmount', 'warp', 'contrast', 'profileSoftness', 'keyScattering', 'scatterOctaves', 'multiScatter', 'multiExtinction', 'multiAnisotropy', 'keyTransmittance', 'surfaceTransmittance'];const BLOOM_FIELDS = ['strength', 'threshold', 'radius'];const DEPTH_OF_FIELD_FIELDS = ['blur', 'focusDistance'];-const MOTION_BLUR_FIELDS = ['shutter'];+const MOTION_BLUR_FIELDS = ['shutter', 'objects'];const MAXIMUM_SHADOW_CASCADES = 4;const MAXIMUM_AFFECTED_NODES = 1024;diff --git a/src/agent-scene-modules.js b/src/agent-scene-modules.jsindex d72495ee..47b20520 100644--- a/src/agent-scene-modules.js+++ b/src/agent-scene-modules.js@@ -34,6 +34,7 @@ const MODULE_LOADERS = {coplanar: () => import('./coplanar-overlap.js'),floorSky: () => import('./floor-sky-visibility.js'),userPass: () => import('./user-pass.js'),+ objectMotionBlur: () => import('./object-motion-blur.js'),materialSnippet: () => import('./material-snippet.js'),};@@ -80,6 +81,8 @@ const MODULE_MARKERS = {floorSky: ['"keyColor"', '"keyIntensity"'],userPass: ['"postPasses"'],materialSnippet: ['"snippet":{'],+ // Only a motionBlur object that sets a non-zero `objects` loads it; the common `"shutter":0` look does not.+ objectMotionBlur: [/"motionBlur":\{[^}]*"objects":(?!0[,}])/],};/** Whether a recipe, operation list or edit list contains a boolean edit, which needs loadMeshBooleans() first. */diff --git a/src/camera-effects.js b/src/camera-effects.jsindex d27e86a1..4d78d211 100644--- a/src/camera-effects.js+++ b/src/camera-effects.js@@ -10,11 +10,11 @@ import { compileProgram, fullscreenVertex } from './gl-program.js';// the per-pixel rotation comes from the pixel position, so a seek to a frame repeats the same pixels.const DEPTH_OF_FIELD_SAMPLES = 32;-const MOTION_BLUR_SAMPLES = 16;+export const MOTION_BLUR_SAMPLES = 16;// Largest blur circle as a fraction of image height at full blur; the blur setting scales it from zero.const MAXIMUM_BLUR_HEIGHT_FRACTION = 0.05;// Longest motion streak as a fraction of the frame, so a camera cut cannot smear the whole image.-const MAXIMUM_STREAK = 0.1;+export const MAXIMUM_STREAK = 0.1;export const sharedFragmentHeader = `#version 300 esprecision highp float;@@ -184,7 +184,8 @@ function createMotionBlurStage() {fragment: motionBlurFragment,uniformNames: ['previousViewProjection'],// Without a previous camera there is no known motion, so the frame passes through untouched.- isActive: (effects, frame) => effects.motionBlur.shutter > 0 && Boolean(frame.previousCamera),+ // A frame that carries object motion is blurred by the object stage, which also covers the camera, so the two never stack.+ isActive: (effects, frame) => effects.motionBlur.shutter > 0 && Boolean(frame.previousCamera) && !frame.objectMotion,setUniforms(context, uniforms, renderer, frame) {context.uniformMatrix4fv(uniforms.previousViewProjection, false, previousViewProjectionOf(renderer, frame));},@@ -192,7 +193,7 @@ function createMotionBlurStage() {}// The previous camera only replaces what it names (eye, target, up, lens), so a partial camera works.-function previousViewProjectionOf(renderer, frame) {+export function previousViewProjectionOf(renderer, frame) {const previous = { ...frame, ...frame.previousCamera };const aspect = renderer.canvas.width / renderer.canvas.height;const projection = applyProjection(mat4.create(), previous, aspect, { world: renderer.world });diff --git a/src/effects.js b/src/effects.jsindex 1cb65407..b698c0ba 100644--- a/src/effects.js+++ b/src/effects.js@@ -26,7 +26,9 @@ export function resolveEffects(effects = {}) {blur: numberInRange(depthOfField.blur, 0, 0, 1, 'Depth of field blur'),focusDistance: numberInRange(depthOfField.focusDistance, 5, 0.01, 100000, 'Depth of field focus distance'),},- motionBlur: { shutter: numberInRange(motionBlur.shutter, 0, 0, 0.5, 'Motion blur shutter') },+ // `objects` (0 to 1, how much of each part's own motion to smear) appears only when set, so a scene without it resolves as before.+ motionBlur: { shutter: numberInRange(motionBlur.shutter, 0, 0, 0.5, 'Motion blur shutter'),+ ...(motionBlur.objects > 0 ? { objects: numberInRange(motionBlur.objects, 0, 0, 1, 'Motion blur objects') } : {}) },saturation: numberInRange(effects.saturation, 1, 0, 2, 'Saturation'),contrast: numberInRange(effects.contrast, 1, 0.1, 2, 'Contrast'),temperature: numberInRange(effects.temperature, 0, -1, 1, 'Temperature'),diff --git a/src/object-motion-blur.js b/src/object-motion-blur.jsnew file mode 100644index 00000000..c83bf707--- /dev/null+++ b/src/object-motion-blur.js@@ -0,0 +1,143 @@+import { mat4 } from 'gl-matrix';+import {+ createStage, previousViewProjectionOf, sharedFragmentHeader, MOTION_BLUR_SAMPLES, MAXIMUM_STREAK,+} from './camera-effects.js';++// Per-object motion blur: a fast prop or part smears along its own screen motion while the still background stays sharp.+// It is opt-in (`effects.motionBlur.objects` above zero) and loads on demand, so a scene without it pays nothing.+//+// The caller draws the object ID pass first (frame.objectMotion carries the parts' transforms at the shutter's start and+// end). Each part's delta matrix carries a point from where it is now to where it was one shutter ago; the stage reads the+// part number of each pixel, moves the pixel's world point by that delta and projects it with the earlier camera. The+// difference on screen is the velocity, and it holds the camera's own motion too, so this stage replaces camera motion+// blur on such a frame. Parts are rigid here: a skinned or morphed part blurs only by its node transform.++const MATRICES_PER_ROW = 256;+const TEXTURE_WIDTH = MATRICES_PER_ROW * 4;+const STAGE_NAME = 'Object motion blur';++export const objectMotionBlurFragment = `${sharedFragmentHeader}+uniform sampler2D objectIds;+uniform sampler2D objectDeltas;+uniform mat4 previousViewProjection;+uniform float objectAmount;+uniform int objectCount;+vec2 screenPointOf(vec3 world) {+ vec4 previousClip = previousViewProjection * vec4(world, 1.0);+ if (previousClip.w <= 0.0) return textureCoordinate;+ return previousClip.xy / previousClip.w * 0.5 + 0.5;+}+mat4 deltaOf(int identifier) {+ int row = (identifier - 1) / ${MATRICES_PER_ROW};+ int column = ((identifier - 1) - row * ${MATRICES_PER_ROW}) * 4;+ return mat4(texelFetch(objectDeltas, ivec2(column, row), 0), texelFetch(objectDeltas, ivec2(column + 1, row), 0),+ texelFetch(objectDeltas, ivec2(column + 2, row), 0), texelFetch(objectDeltas, ivec2(column + 3, row), 0));+}+void main() {+ vec3 idBytes = floor(texelFetch(objectIds, ivec2(gl_FragCoord.xy), 0).rgb * 255.0 + 0.5);+ int identifier = int(idBytes.r + idBytes.g * 256.0 + idBytes.b * 65536.0);+ vec3 world = worldPointAt(textureCoordinate);+ vec2 velocity = screenPointOf(world) - textureCoordinate;+ if (identifier > 0 && identifier <= objectCount) {+ vec2 objectVelocity = screenPointOf((deltaOf(identifier) * vec4(world, 1.0)).xyz) - textureCoordinate;+ velocity += (objectVelocity - velocity) * objectAmount;+ }+ float streak = length(velocity);+ if (streak > ${MAXIMUM_STREAK}) velocity *= ${MAXIMUM_STREAK} / streak;+ if (length(velocity * resolution) < 0.5) {+ outputColor = texture(sourceTexture, textureCoordinate);+ return;+ }+ float jitter = pixelRotation() / 6.2831853;+ vec4 sum = vec4(0.0);+ for (int index = 0; index < ${MOTION_BLUR_SAMPLES}; index++) {+ float along = (float(index) + jitter) / ${MOTION_BLUR_SAMPLES}.0;+ sum += texture(sourceTexture, textureCoordinate + velocity * along);+ }+ outputColor = sum / ${MOTION_BLUR_SAMPLES}.0;+}`;++/** The world matrix of every part, by name, at this instant. Matrices are copied, so a later seek does not change them. */+export function sampleWorldMatrices(parts) {+ return new Map(parts.map(part => [part.name, mat4.clone(part.matrix)]));+}++/**+ * Packs one delta matrix per object ID, in the order the object ID pass numbers parts (draw order, grouped by geometry).+ * A part with no matrix in `previous` or one that did not move keeps the identity, and so does a pixel of no object.+ */+export function packObjectMotion(renderer, current, previous) {+ const identifiers = renderer.batches.filter(batch => renderer.drawsInOpaqueLoop(batch)).flatMap(batch => batch.parts.map(part => part.name));+ const rows = Math.max(1, Math.ceil(identifiers.length / MATRICES_PER_ROW));+ const matrices = new Float32Array(TEXTURE_WIDTH * rows * 4);+ const inverse = mat4.create();+ const delta = mat4.create();+ identifiers.forEach((name, index) => {+ const slot = matrices.subarray(index * 16, index * 16 + 16);+ const now = current.get(name);+ const before = previous.get(name);+ // The texture upload is row-major with 256 matrices per row, so the matrix of ID n + 1 starts at float 16 * n.+ if (now && before && mat4.invert(inverse, now)) {+ mat4.multiply(delta, before, inverse);+ slot.set(delta);+ } else {+ slot.set(mat4.create());+ }+ });+ for (let index = identifiers.length; index < MATRICES_PER_ROW * rows; index++) matrices.set(mat4.create(), index * 16);+ return { matrices, rows, count: identifiers.length };+}++function createObjectMotionStage() {+ let deltaTexture = null;+ const stage = createStage({+ name: STAGE_NAME,+ fragment: objectMotionBlurFragment,+ uniformNames: ['objectIds', 'objectDeltas', 'previousViewProjection', 'objectAmount', 'objectCount'],+ isActive: (effects, frame) => effects.motionBlur.shutter > 0 && effects.motionBlur.objects > 0 && Boolean(frame.objectMotion),+ setUniforms(context, uniforms, renderer, frame, effects) {+ const { matrices, rows, count } = frame.objectMotion;+ deltaTexture ??= context.createTexture();+ context.activeTexture(context.TEXTURE3);+ context.bindTexture(context.TEXTURE_2D, deltaTexture);+ context.texImage2D(context.TEXTURE_2D, 0, context.RGBA32F, TEXTURE_WIDTH, rows, 0, context.RGBA, context.FLOAT, matrices);+ context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.NEAREST);+ context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.NEAREST);+ context.uniform1i(uniforms.objectDeltas, 3);+ context.activeTexture(context.TEXTURE2);+ context.bindTexture(context.TEXTURE_2D, renderer.passSupport.texture);+ context.uniform1i(uniforms.objectIds, 2);+ context.uniform1i(uniforms.objectCount, count);+ context.uniform1f(uniforms.objectAmount, effects.motionBlur.objects);+ const viewProjection = frame.previousCamera ? previousViewProjectionOf(renderer, frame) : renderer.viewProjection;+ context.uniformMatrix4fv(uniforms.previousViewProjection, false, viewProjection);+ context.activeTexture(context.TEXTURE1);+ },+ });+ function release(context) {+ if (deltaTexture) context.deleteTexture(deltaTexture);+ deltaTexture = null;+ }+ return {+ ...stage,+ releaseTargets(context) {+ stage.releaseTargets(context);+ release(context);+ },+ dispose(context) {+ stage.dispose(context);+ release(context);+ },+ };+}++/**+ * Adds object motion blur to a renderer that has the camera effects and render passes installed. Draw the object ID pass+ * first and pass `frame.objectMotion` (from packObjectMotion); the stage runs only for such a frame with+ * `effects.motionBlur.shutter` and `effects.motionBlur.objects` above zero.+ */+export function installObjectMotionBlur(renderer) {+ if (!renderer.passSupport) throw new Error('Object motion blur needs installRenderPasses(renderer).');+ if (!renderer.postStages.some(stage => stage.name === STAGE_NAME)) renderer.postStages.push(createObjectMotionStage());+ return renderer;+}diff --git a/src/render-passes.js b/src/render-passes.jsindex 4ec9a8f8..cbbef926 100644--- a/src/render-passes.js+++ b/src/render-passes.js@@ -105,6 +105,8 @@ export function installRenderPasses(renderer) {renderer.passSupport = {prepare,get framebuffer() { return target.framebuffer; },+ // The pass colour of the last pass frame; object motion blur reads the object IDs from it.+ get texture() { return target.texture; },background: pass => PASS_BACKGROUNDS[pass] ?? DEFAULT_BACKGROUND,copyToCanvas,objectNames,diff --git a/tests/object-motion-blur.test.js b/tests/object-motion-blur.test.jsnew file mode 100644index 00000000..e728374f--- /dev/null+++ b/tests/object-motion-blur.test.js@@ -0,0 +1,54 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import { mat4 } from 'gl-matrix';+import { resolveEffects } from '../src/effects.js';+import { normalizeAgentLook } from '../src/agent-look.js';+import { agentSceneModulesNeeded } from '../src/agent-scene-modules.js';+import { installCameraEffects } from '../src/camera-effects.js';+import { installObjectMotionBlur, packObjectMotion, sampleWorldMatrices } from '../src/object-motion-blur.js';++test('object blur is absent from resolved effects until it is set above zero', () => {+ assert.deepEqual(resolveEffects({ motionBlur: { shutter: 0.1 } }).motionBlur, { shutter: 0.1 });+ assert.deepEqual(resolveEffects({ motionBlur: { shutter: 0.1, objects: 0 } }).motionBlur, { shutter: 0.1 });+ assert.deepEqual(resolveEffects({ motionBlur: { shutter: 0.1, objects: 0.5 } }).motionBlur, { shutter: 0.1, objects: 0.5 });+ assert.throws(() => resolveEffects({ motionBlur: { shutter: 0.1, objects: 2 } }), /Motion blur objects/);+ assert.equal(normalizeAgentLook({ effects: { motionBlur: { shutter: 0.1, objects: 1 } } }).effects.motionBlur.objects, 1);+ assert.throws(() => normalizeAgentLook({ effects: { motionBlur: { objects: 'all' } } }), /objects/);+});++test('only a non-zero objects value loads the object blur module', () => {+ const needs = effects => agentSceneModulesNeeded({ effects }).includes('objectMotionBlur');+ assert.equal(needs({ motionBlur: { shutter: 0 } }), false);+ assert.equal(needs({ motionBlur: { shutter: 0.1, objects: 0 } }), false);+ assert.equal(needs({ motionBlur: { shutter: 0.1, objects: 1 } }), true);+ assert.equal(needs({ motionBlur: { objects: 0.25, shutter: 0.1 } }), true);+});++function fakeRenderer(names) {+ return { batches: [{ parts: names.map(name => ({ name })) }], drawsInOpaqueLoop: () => true, postStages: [], passSupport: {} };+}++test('each part gets the transform back to where it was, and a part that did not move gets the identity', () => {+ const parts = [{ name: 'prop', matrix: mat4.fromTranslation(mat4.create(), [1, 0, 0]) }, { name: 'wall', matrix: mat4.create() }];+ const current = sampleWorldMatrices(parts);+ parts[0].matrix[12] = 5;+ assert.equal(current.get('prop')[12], 1, 'Sampled matrices are copies.');+ const previous = new Map([['prop', mat4.fromTranslation(mat4.create(), [-1, 0, 0])], ['wall', mat4.create()]]);+ const packed = packObjectMotion(fakeRenderer(['prop', 'wall']), current, previous);+ assert.equal(packed.count, 2);+ assert.equal(packed.rows, 1);+ assert.deepEqual([...packed.matrices.slice(0, 16)], [...mat4.fromTranslation(mat4.create(), [-2, 0, 0])]);+ assert.deepEqual([...packed.matrices.slice(16, 32)], [...mat4.create()]);+});++test('the object stage replaces camera blur on a frame that carries object motion', () => {+ const renderer = installObjectMotionBlur(installCameraEffects(fakeRenderer([])));+ const [, cameraStage, objectStage] = renderer.postStages;+ assert.equal(objectStage.name, 'Object motion blur');+ const effects = resolveEffects({ motionBlur: { shutter: 0.1, objects: 1 } });+ const probe = stage => stage.apply({ depthTexture: null }, { previousCamera: {}, objectMotion: {} }, effects, null);+ assert.equal(probe(cameraStage), null, 'Camera blur stays out of the way.');+ assert.throws(() => probe(objectStage), /requires depth/);+ assert.equal(objectStage.apply({ depthTexture: null }, { previousCamera: {} }, effects, null), null, 'Without object motion the stage passes through.');+ assert.equal(installObjectMotionBlur(renderer).postStages.length, 3, 'Installing twice adds nothing.');+});