3D engine that AI agents drive
Keep the ground and horizon still while physics plays in render jobs (#1089)
Nguyen Xuan An6d54a285c94f
diff --git a/changes/1089.md b/changes/1089.mdnew file mode 100644index 00000000..d6fce54d--- /dev/null+++ b/changes/1089.md@@ -0,0 +1 @@+- A render job or a look at a time keeps the automatic floor and horizon fixed by the pose at time 0, so physics shards flying or falling no longer resize the floor or move the horizon of a fixed camera (#1089)diff --git a/docs/agent-look.md b/docs/agent-look.mdindex 94df0985..639333f4 100644--- a/docs/agent-look.md+++ b/docs/agent-look.md@@ -176,7 +176,7 @@ Setting `setup` through `set-environment` resets `lightDirection`, `keyColor`, `Remote URLs and HDRI files are not accepted from tool input.This changes the look of every recipe that sets no lighting: before, the default had no environment map, a key color of `[5.2, 4.7, 4.1]` and a fill intensity of `1`.To get the earlier flat look, set `keyColor`, `fillIntensity` and `environmentIntensity: 0` yourself.-The MCP preview adds an automatic floor: a light grey disc under the scene's lowest geometry, at that height, which catches the key light's shadow. It is not a recipe node and is not exported. A part that already forms a floor (spans at least 90 percent of the scene's width and depth, is thinner than a quarter of its width and is the lowest geometry) switches it off, and `environment.ground: false` removes it. Code that builds its own `Renderer67` gets it by passing `createAgentGround(agentGroundPlacement(parts, environment))` as the `ground` option (`src/agent-ground.js`).+The MCP preview adds an automatic floor: a light grey disc under the scene's lowest geometry, at that height, which catches the key light's shadow. It is not a recipe node and is not exported. A part that already forms a floor (spans at least 90 percent of the scene's width and depth, is thinner than a quarter of its width and is the lowest geometry) switches it off, and `environment.ground: false` removes it. While a motion time is sampled (a render job, or `scene_look` at a time) the floor is decided and sized once from the pose at time 0, so flying or falling parts never resize it or move the horizon. Code that builds its own `Renderer67` gets it by passing `createAgentGround(agentGroundPlacement(parts, environment))` as the `ground` option (`src/agent-ground.js`).## Point lightsdiff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex b67cd2ec..0f7a86d2 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -292,10 +292,31 @@ function rebuildRenderer(record) {record.shadowKey = null;}+// While a motion time is sampled (a render job, a look at a time), the automatic floor is fixed by the pose at time 0, so+// shards flying out of a physics bake or a part driving away never resize it, flip it on or off, or move the horizon.+// A scene with no motion time reads its own pose each draw, as before.+const motionGrounds = new WeakMap();++function automaticGroundPlacement(scene) {+ const frozen = motionGrounds.get(scene);+ if (scene.motionTime != null && frozen?.revision === scene.revision) {+ return frozen.placement;+ }+ return agentGroundPlacement(scene.parts, scene.environment);+}++function sampleMotionTime(scene, time) {+ if (motionGrounds.get(scene)?.revision !== scene.revision) {+ scene.sampleMotion(0);+ motionGrounds.set(scene, { revision: scene.revision, placement: agentGroundPlacement(scene.parts, scene.environment) });+ }+ scene.sampleMotion(time);+}+// The automatic floor (src/agent-ground.js) follows the scene's lowest geometry. A renderer gains or loses it only// through a new renderer, so a floor that appears or disappears (a part now forms one, or environment.ground changed) rebuilds.function syncAutomaticGround(record) {- const placement = agentGroundPlacement(record.scene.parts, record.scene.environment);+ const placement = automaticGroundPlacement(record.scene);if (Boolean(placement) !== Boolean(record.renderer.groundPart)) {rebuildRenderer(record);record.renderer.resize();@@ -499,7 +520,7 @@ function cascadesFor(scene) {function initializeRenderer(scene, { memoryBudgetMiB, backend, transparent = false }) {probeSnippets(scene);- const placement = agentGroundPlacement(scene.parts, scene.environment);+ const placement = automaticGroundPlacement(scene);const renderer = installCoplanarDiagnostics(installRenderPasses(installCameraEffects(new Renderer67(canvas, scene.parts, {ground: placement ? createAgentGround(placement) : false,hdr: true,@@ -914,7 +935,7 @@ window.__agentPreview = {record.scene.sampleFrame({ ...(animation === undefined ? {} : { animation }),...(motionTime === undefined ? {} : { motionTime }) });} else if (motionTime !== undefined) {- record.scene.sampleMotion(motionTime);+ sampleMotionTime(record.scene, motionTime);}const shotCamera = cameraOverrides ? withCameraOverrides(record.scene.camera, cameraOverrides) : undefined;const forVelocity = pass === 'velocity';@@ -1167,7 +1188,7 @@ window.__agentPreview = {if (typeof record.scene.sampleMotion !== 'function') {throw new RangeError('Procedural motion time is unavailable for imported assets.');}- record.scene.sampleMotion(motionTime);+ sampleMotionTime(record.scene, motionTime);}draw(record);const render = renderInfo(record.renderer, viewProblems(record, record.scene.camera, record.scene.effects.exposure));diff --git a/tests/physics-film.test.js b/tests/physics-film.test.jsindex 51479829..57d73b8a 100644--- a/tests/physics-film.test.js+++ b/tests/physics-film.test.js@@ -8,6 +8,7 @@ import { mkdtemp, rm, copyFile, readFile } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join, resolve } from 'node:path';import { spawnSync } from 'node:child_process';+import { inflateSync } from 'node:zlib';import { setTimeout as delay } from 'node:timers/promises';import { Client } from '@modelcontextprotocol/client';import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';@@ -138,7 +139,66 @@ test('a 64 piece fractured wall renders as stills with the ball breaking it', asawait call('scene_dispose', { sessionId });});-const hasFfmpeg = spawnSync('ffmpeg', ['-version'], { windowsHide: true }).status === 0;+// Decodes an 8-bit RGB or RGBA PNG to its pixel bytes, undoing the row filters, so the horizon can be read without ffmpeg.+function decodePng(bytes) {+ const width = bytes.readUInt32BE(16);+ const height = bytes.readUInt32BE(20);+ const channels = bytes[25] === 6 ? 4 : 3;+ const compressed = [];+ for (let offset = 8; offset < bytes.length;) {+ const length = bytes.readUInt32BE(offset);+ if (bytes.toString('latin1', offset + 4, offset + 8) === 'IDAT') {+ compressed.push(bytes.subarray(offset + 8, offset + 8 + length));+ }+ offset += 12 + length;+ }+ const raw = inflateSync(Buffer.concat(compressed));+ const stride = width * channels;+ const pixels = Buffer.alloc(stride * height);+ for (let row = 0; row < height; row++) {+ const filter = raw[row * (stride + 1)];+ for (let index = 0; index < stride; index++) {+ const value = raw[row * (stride + 1) + 1 + index];+ const left = index >= channels ? pixels[row * stride + index - channels] : 0;+ const up = row > 0 ? pixels[(row - 1) * stride + index] : 0;+ const upLeft = row > 0 && index >= channels ? pixels[(row - 1) * stride + index - channels] : 0;+ const estimate = left + up - upLeft;+ const distances = [Math.abs(estimate - left), Math.abs(estimate - up), Math.abs(estimate - upLeft)];+ const paeth = distances[0] <= distances[1] && distances[0] <= distances[2] ? left : (distances[1] <= distances[2] ? up : upLeft);+ const predictor = [0, left, up, (left + up) >> 1, paeth][filter];+ pixels[row * stride + index] = (value + predictor) & 255;+ }+ }+ return { width, height, channels, pixels };+}++// The row of the biggest colour jump down the left edge, where only ground and sky show, never the wall or the shards.+function horizonRowAtLeftEdge(image) {+ const rowBytes = image.width * image.channels;+ let bestRow = 0;+ let bestJump = -1;+ for (let row = 1; row < image.height; row++) {+ let jump = 0;+ for (let channel = 0; channel < 3; channel++) {+ jump += Math.abs(image.pixels[row * rowBytes + channel] - image.pixels[(row - 1) * rowBytes + channel]);+ }+ if (jump > bestJump) {+ bestJump = jump;+ bestRow = row;+ }+ }+ return bestRow;+}++test('the horizon stays on the same row in a fixed-camera physics film', async () => {+ const sessionId = await createFracturedWall();+ const frames = await stillBytes(await submitJob(sessionId, 'stills', { motion: { start: 0 } }));+ const rows = frames.map(frame => horizonRowAtLeftEdge(decodePng(frame)));+ assert.ok(Math.max(...rows) - Math.min(...rows) <= 1, `the horizon row at the left edge moved: ${rows.join(', ')}`);+ await call('scene_dispose', { sessionId });+});++const hasFfmpeg =spawnSync('ffmpeg', ['-version'], { windowsHide: true }).status === 0;test('a 64 piece fractured wall encodes as video for the length of the bake', { skip: !hasFfmpeg && 'ffmpeg is not installed' }, async () => {const sessionId = await createFracturedWall();