3D engine that AI agents drive
Drive the ocean time from a geometry motion track (part of #817)
Nguyen Xuan An7f47f4c03218
diff --git a/changes/817-ocean-time.size.md b/changes/817-ocean-time.size.mdnew file mode 100644index 00000000..b9128800--- /dev/null+++ b/changes/817-ocean-time.size.md@@ -0,0 +1,3 @@+- scene:agent-scene: the per-frame ocean geometry sampler and the `geometry` track kind (about 180 bytes gzip) give agents a moving sea in one video job instead of one still per frame; a 72 frame 540p clip rendered in about 3 s.+- scene:agent-scene+lazy: same change, the track kind and the sampler appear in the scene and motion chunks.+- goal:agent-scene: the sampler is 30 lines that reuse generateGeometry and the motion sampler, and a moving sea is an agent-facing capability.diff --git a/changes/817.md b/changes/817.mdnew file mode 100644index 00000000..0bb137cb--- /dev/null+++ b/changes/817.md@@ -0,0 +1 @@+- A motion track `{ "kind": "geometry", "id": "sea", "property": "oceanTime" }` drives the `time` of the ocean nodes in a mesh stack, so one video job or timeline renders moving waves (#817)diff --git a/docs/agent-motion.md b/docs/agent-motion.mdindex a16a0d70..706222e5 100644--- a/docs/agent-motion.md+++ b/docs/agent-motion.md@@ -244,6 +244,18 @@ gives that target; targets without a track keep theirs. Names come from the filethat Blender and face rigs with ARKit names such as `eyeBlinkLeft` and `jawOpen` export); `scene_inspect` returns them as`morphTargetNames`, and an unknown name fails with the first ones. Tracks can address morph targets only on an imported asset.+### Ocean time tracks++A track `{ "kind": "geometry", "id": "sea", "property": "oceanTime", "keyframes": [...] }` drives the `time` (seconds, -1e6 to 1e6)+of every `ocean` node in the `stack` of the mesh geometry `sea`, so one video job or timeline renders moving waves without+a still per frame. Two keyframes `0 -> 0` and `4 -> 4` run the sea in real time; steeper keyframes run it faster. The track+replaces the `time` written in the stack. Each frame rebuilds the geometry from the cached edits and the stack nodes before the+ocean, then runs the ocean and the nodes after it, so the vertices of a frame equal those of a still with that `time` written+into the node, and seeking in any order gives the same bytes. The ocean work limit (50 million wave evaluations) and `maxVertices`+apply to every frame; the per-frame cost grows with vertices times waves, so keep a sea that moves to about 30,000 vertices.+The ID must name a mesh geometry whose stack has an `ocean` node (an `ocean` in `edits` is baked and cannot move). Only+`oceanTime` is a geometry property; other numeric stack fields are not drivable.+### Joint limits and sanity warningsA two-bone solver takes `limits: { "root": { "x": [min, max], "y": [...], "z": [...] }, "middle": {...} }` in degrees. Eachdiff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex 229e4d0e..6df40665 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -591,6 +591,7 @@ const motion = z.strictObject({motionTrack('camera', 'target', vector3, vector3),motionTrack('camera', 'fov', finite.gt(0).lt(180), finite),motionTrack('camera', 'roll', finite, finite),+ motionTrack('geometry', 'oceanTime', finite.min(-1e6).max(1e6), finite),lookTrack,])).max(128),}).describe('One absolute-time procedural clip. Local node transforms propagate through parents. Cubic tangents are derivatives per second. Backend checks references, duplicate tracks, 2048 total keys and every sampled pose.');diff --git a/src/agent-motion.js b/src/agent-motion.jsindex 1b432b7f..8b396188 100644--- a/src/agent-motion.js+++ b/src/agent-motion.js@@ -21,6 +21,8 @@ const PROPERTIES = {camera: { eye: 3, target: 3, fov: 1, roll: 1 },// An imported asset's morph target weight, addressed by the target's name.morph: { weight: 1 },+ // A mesh geometry's modifier stack: oceanTime (seconds) replaces `time` of every ocean node in the stack, evaluated per frame.+ geometry: { oceanTime: 1 },// A light of the scene, addressed as point:N, spot:N or area:N (its index in environment.pointLights and so on). Spot and area lights also take direction.light: { position: 3, direction: 3, color: 3, intensity: 1 },// The key light of the environment: lightDirection, keyColor, keyIntensity.@@ -49,6 +51,7 @@ const LOOK_RANGES = {sunElevation: [-90, 90], sunRotation: [-360, 360], turbidity: [1, 10], groundAlbedo: [0, 1], aerialPerspective: [0, 4], ozone: [0, 10],},effects: { exposure: [0.01, 16], focusDistance: [0.01, 100000] },+ geometry: { oceanTime: [-1e6, 1e6] },};const INTERPOLATIONS = ['LINEAR', 'STEP', 'SMOOTH', 'CUBICSPLINE'];const MAX_IK_SOLVERS = 16;@@ -492,6 +495,7 @@ export function normalizeAgentMotion(input, recipe) {const materials = new Set((recipe.materials ?? []).map(material => material.id));// An imported asset lists its morph targets; a recipe mesh offers the names of its shape keys.const morphTargets = new Set(recipe.morphTargets ?? (recipe.geometries ?? []).flatMap(geometry => (geometry.shapeKeys ?? []).map(key => key.name)));+ const oceanGeometries = new Set((recipe.geometries ?? []).filter(geometry => geometry.type === 'mesh' && (geometry.stack ?? []).some(node => node?.op === 'ocean')).map(geometry => geometry.id));const identities = new Set();let keyframeCount = 0;const tracks = inputTracks.map((track, index) => {@@ -508,9 +512,10 @@ export function normalizeAgentMotion(input, recipe) {if (typeof track.id !== 'string' || !track.id.trim() || track.id.length > 128) {throw invalidField(`motion.tracks[${index}].id`, track.id, 'an ID of 1 to 128 characters, such as "body"', TypeError);}- const references = track.kind === 'node' ? nodes : track.kind === 'morph' ? morphTargets : materials;+ const references = track.kind === 'node' ? nodes : track.kind === 'morph' ? morphTargets : track.kind === 'geometry' ? oceanGeometries : materials;if (track.kind !== 'light' && !references.has(track.id)) {- const hint = track.kind === 'morph' ? `; morph targets of this asset: ${[...morphTargets].slice(0, 24).join(', ') || 'none'}` : '';+ const hint = track.kind === 'morph' ? `; morph targets of this asset: ${[...morphTargets].slice(0, 24).join(', ') || 'none'}`+ : track.kind === 'geometry' ? '; a geometry track needs a mesh geometry whose stack has an ocean node, such as stack [{ "op": "ocean" }]' : '';throw invalidField(`motion.tracks[${index}].id`, track.id, `the ID of a ${track.kind} in this recipe, such as ${JSON.stringify([...references][0] ?? 'body')}${hint}`);}}@@ -808,8 +813,8 @@ export function compileAgentMotion(input, recipe) {})),});const sampler = compileAnimationSampler(clip);- const output = { nodes: new Map(), materials: new Map(), morphs: new Map(), lights: new Map(), sun: null, environment: null, effects: null, camera: null };- const patchesByKind = { node: output.nodes, material: output.materials, morph: output.morphs, light: output.lights };+ const output = { nodes: new Map(), materials: new Map(), morphs: new Map(), geometries: new Map(), lights: new Map(), sun: null, environment: null, effects: null, camera: null };+ const patchesByKind = { node: output.nodes, material: output.materials, morph: output.morphs, geometry: output.geometries, light: output.lights };const bindings = ordinaryTracks.map((track, index) => {let patch;if (LOOK_KINDS.has(track.kind) && track.kind !== 'light') {diff --git a/src/agent-scene.js b/src/agent-scene.jsindex e3be67fe..93d2b7e6 100644--- a/src/agent-scene.js+++ b/src/agent-scene.js@@ -1124,19 +1124,47 @@ function stagePose(stagedState, time) {}const camera = sampled?.camera ? cameraPoseFrom(sampled.camera) : stagedState.recipe.camera;retainedMatrices(stagedState.parts);- const clothGeometries = sampledGeometries(stagedState, time);+ const clothGeometries = sampledGeometries(stagedState, time, sampled);// Staged last: a pose pending on a decoded asset must be committed or discarded, so nothing above may throw after it.const assetClips = stagedState.assetRuntime.animators.length ? agentSceneModule('assets').stageAssetClips(stagedState.assetRuntime, graph, time) : null;const movedLook = stagedState.motionSampler && agentSceneModule('motion').motionMovesLook(sampled) ? sampled : null;return { graph, materialUpdates, camera, time, clothGeometries, assetClips, hiddenNodes, movedLook, morphs: sampled?.morphs ?? null };}+/**+ * Geometries of mesh stacks whose ocean time a geometry track drives, keyed by node ID. Each one is built from the cached edits with+ * every ocean node's `time` set to the sampled value, so it equals the still of a recipe that writes that time into the node. The last+ * result of each geometry is kept, so a repeated time reuses the geometry and the renderer sees no change.+ */+function oceanTimeGeometries(stagedState, sampled) {+ const patches = sampled?.geometries;+ if (!patches?.size) {+ return new Map();+ }+ const frames = stagedState.oceanFrames ??= new Map();+ const geometries = new Map();+ for (const [geometryId, patch] of patches) {+ let frame = frames.get(geometryId);+ if (frame?.time !== patch.oceanTime) {+ const definition = stagedState.recipe.geometries.find(candidate => candidate.id === geometryId);+ const stack = definition.stack.map(node => (node.op === 'ocean' ? { ...node, time: patch.oceanTime } : node));+ const scope = cutterScopeOf(stagedState.recipe.geometries);+ frame = { time: patch.oceanTime, geometry: Object.freeze(generateGeometry({ ...definition, stack }, stagedState.maxGeometryVertices, scope)) };+ frames.set(geometryId, frame);+ }+ for (const node of stagedState.recipe.nodes) {+ if (node.geometry === geometryId) geometries.set(node.id, frame.geometry);+ }+ }+ return geometries;+}+// Cloth and particles replace the geometry of their own nodes at a sampled time; no node belongs to both.-function sampledGeometries(stagedState, time) {+function sampledGeometries(stagedState, time, sampled) {if (time === null) {return null;}- const geometries = new Map();+ const geometries = oceanTimeGeometries(stagedState, sampled);for (const sampler of [stagedState.clothSampler, stagedState.particleSampler]) {sampler?.sample(time)?.forEach((geometry, nodeId) => geometries.set(nodeId, geometry));}@@ -1477,6 +1505,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {stats: geometryStats(geometryRecords.values()),};stagedState.assetRuntime = assetRuntime;+ stagedState.maxGeometryVertices = Math.min(limits.maxVertices, 1_000_000);stagedState.sequences = [...materialHandles.values()].map(handle => handle.holder.sequence).filter(Boolean);stagedState.localBounds = localBounds;stagedState.skinBindings = skinBindings;diff --git a/tests/agent-scene-ocean-time.test.js b/tests/agent-scene-ocean-time.test.jsnew file mode 100644index 00000000..58d6cceb--- /dev/null+++ b/tests/agent-scene-ocean-time.test.js@@ -0,0 +1,70 @@+import './support/load-agent-scene-modules.mjs';+import test from 'node:test';+import assert from 'node:assert/strict';+import { createAgentScene } from '../src/agent-scene.js';++const OCEAN = { op: 'ocean', seed: 3, windSpeed: 4, size: 2, resolution: 4 };++function seaRecipe({ trackTime = true, stack = [{ op: 'subdivide', levels: 3 }, OCEAN] } = {}) {+ return {+ format: '67js.scene', schemaVersion: 1,+ geometries: [{ id: 'sea', type: 'mesh', base: { type: 'box', bevel: 0 }, stack }],+ materials: [{ id: 'water', color: [0.1, 0.3, 0.5] }],+ nodes: [{ id: 'surface', geometry: 'sea', material: 'water' }],+ camera: { eye: [30, 20, 30], target: [0, 0, 0] },+ ...(trackTime ? {+ motion: {+ duration: 4,+ tracks: [{ kind: 'geometry', id: 'sea', property: 'oceanTime', keyframes: [{ time: 0, value: 0 }, { time: 4, value: 8 }] }],+ },+ } : {}),+ };+}++function positionsOf(scene) {+ return Array.from(scene.parts[0].geometry.positions);+}++test('an ocean time track moves the waves, and a frame equals the still written at that time', () => {+ const scene = createAgentScene(seaRecipe());+ scene.sampleMotion(0);+ const atStart = positionsOf(scene);+ scene.sampleMotion(2);+ const atTwoSeconds = positionsOf(scene);+ assert.notDeepEqual(atTwoSeconds, atStart, 'the sea moves between frames');+ const still = createAgentScene(seaRecipe({ trackTime: false, stack: [{ op: 'subdivide', levels: 3 }, { ...OCEAN, time: 4 }] }));+ assert.deepEqual(atTwoSeconds, positionsOf(still), 'time 2 s on a 0 to 8 track is the still at ocean time 4');+});++test('seeking back and forth gives the same vertices, and a repeated time keeps the geometry', () => {+ const scene = createAgentScene(seaRecipe());+ scene.sampleMotion(1.25);+ const first = positionsOf(scene);+ scene.sampleMotion(3.5);+ scene.sampleMotion(1.25);+ assert.deepEqual(positionsOf(scene), first);+ const geometry = scene.parts[0].geometry;+ scene.sampleMotion(1.25);+ assert.equal(scene.parts[0].geometry, geometry);+});++test('an ocean time track names a mesh geometry with an ocean node in its stack', () => {+ const missingGeometry = seaRecipe();+ missingGeometry.motion.tracks[0].id = 'nowhere';+ assert.throws(() => createAgentScene(missingGeometry), /ocean node/);+ assert.throws(() => createAgentScene(seaRecipe({ stack: [{ op: 'subdivide', levels: 2 }] })), /ocean node/);+});++test('ocean time keyframes stay within the bounds of the modifier', () => {+ const recipe = seaRecipe();+ recipe.motion.tracks[0].keyframes[1].value = 2e6;+ assert.throws(() => createAgentScene(recipe), /oceanTime/);+ const wrongSize = seaRecipe();+ wrongSize.motion.tracks[0].keyframes[1].value = [1, 2];+ assert.throws(() => createAgentScene(wrongSize));+});++test('per-frame ocean work stays inside the ocean work limit', () => {+ const scene = createAgentScene(seaRecipe({ stack: [{ op: 'subdivide', levels: 3 }, { ...OCEAN, resolution: 16 }] }));+ assert.doesNotThrow(() => scene.sampleMotion(1));+});