Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Drive the ocean time from a geometry motion track (part of #817)

Nguyen Xuan An7f47f4c03218
diff --git a/changes/817-ocean-time.size.md b/changes/817-ocean-time.size.md
new file mode 100644
index 00000000..b9128800
--- /dev/null
+++ b/changes/817-ocean-time.size.md
@@ -0,0 +1,3 @@
+- scene:agent-scene: the per-frame ocean geometry sampler and the `geometry` track kind (about 180 bytes gzip) give agents a moving sea in one video job instead of one still per frame; a 72 frame 540p clip rendered in about 3 s.
+- scene:agent-scene+lazy: same change, the track kind and the sampler appear in the scene and motion chunks.
+- goal:agent-scene: the sampler is 30 lines that reuse generateGeometry and the motion sampler, and a moving sea is an agent-facing capability.
diff --git a/changes/817.md b/changes/817.md
new file mode 100644
index 00000000..0bb137cb
--- /dev/null
+++ b/changes/817.md
@@ -0,0 +1 @@
+- A motion track `{ "kind": "geometry", "id": "sea", "property": "oceanTime" }` drives the `time` of the ocean nodes in a mesh stack, so one video job or timeline renders moving waves (#817)
diff --git a/docs/agent-motion.md b/docs/agent-motion.md
index a16a0d70..706222e5 100644
--- a/docs/agent-motion.md
+++ b/docs/agent-motion.md
@@ -244,6 +244,18 @@ gives that target; targets without a track keep theirs. Names come from the file
that Blender and face rigs with ARKit names such as `eyeBlinkLeft` and `jawOpen` export); `scene_inspect` returns them as
`morphTargetNames`, and an unknown name fails with the first ones. Tracks can address morph targets only on an imported asset.
+### Ocean time tracks
+
+A track `{ "kind": "geometry", "id": "sea", "property": "oceanTime", "keyframes": [...] }` drives the `time` (seconds, -1e6 to 1e6)
+of every `ocean` node in the `stack` of the mesh geometry `sea`, so one video job or timeline renders moving waves without
+a still per frame. Two keyframes `0 -> 0` and `4 -> 4` run the sea in real time; steeper keyframes run it faster. The track
+replaces the `time` written in the stack. Each frame rebuilds the geometry from the cached edits and the stack nodes before the
+ocean, then runs the ocean and the nodes after it, so the vertices of a frame equal those of a still with that `time` written
+into the node, and seeking in any order gives the same bytes. The ocean work limit (50 million wave evaluations) and `maxVertices`
+apply to every frame; the per-frame cost grows with vertices times waves, so keep a sea that moves to about 30,000 vertices.
+The ID must name a mesh geometry whose stack has an `ocean` node (an `ocean` in `edits` is baked and cannot move). Only
+`oceanTime` is a geometry property; other numeric stack fields are not drivable.
+
### Joint limits and sanity warnings
A two-bone solver takes `limits: { "root": { "x": [min, max], "y": [...], "z": [...] }, "middle": {...} }` in degrees. Each
diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjs
index 229e4d0e..6df40665 100644
--- a/scripts/mcp-server.mjs
+++ b/scripts/mcp-server.mjs
@@ -591,6 +591,7 @@ const motion = z.strictObject({
motionTrack('camera', 'target', vector3, vector3),
motionTrack('camera', 'fov', finite.gt(0).lt(180), finite),
motionTrack('camera', 'roll', finite, finite),
+ motionTrack('geometry', 'oceanTime', finite.min(-1e6).max(1e6), finite),
lookTrack,
])).max(128),
}).describe('One absolute-time procedural clip. Local node transforms propagate through parents. Cubic tangents are derivatives per second. Backend checks references, duplicate tracks, 2048 total keys and every sampled pose.');
diff --git a/src/agent-motion.js b/src/agent-motion.js
index 1b432b7f..8b396188 100644
--- a/src/agent-motion.js
+++ b/src/agent-motion.js
@@ -21,6 +21,8 @@ const PROPERTIES = {
camera: { eye: 3, target: 3, fov: 1, roll: 1 },
// An imported asset's morph target weight, addressed by the target's name.
morph: { weight: 1 },
+ // A mesh geometry's modifier stack: oceanTime (seconds) replaces `time` of every ocean node in the stack, evaluated per frame.
+ geometry: { oceanTime: 1 },
// A light of the scene, addressed as point:N, spot:N or area:N (its index in environment.pointLights and so on). Spot and area lights also take direction.
light: { position: 3, direction: 3, color: 3, intensity: 1 },
// The key light of the environment: lightDirection, keyColor, keyIntensity.
@@ -49,6 +51,7 @@ const LOOK_RANGES = {
sunElevation: [-90, 90], sunRotation: [-360, 360], turbidity: [1, 10], groundAlbedo: [0, 1], aerialPerspective: [0, 4], ozone: [0, 10],
},
effects: { exposure: [0.01, 16], focusDistance: [0.01, 100000] },
+ geometry: { oceanTime: [-1e6, 1e6] },
};
const INTERPOLATIONS = ['LINEAR', 'STEP', 'SMOOTH', 'CUBICSPLINE'];
const MAX_IK_SOLVERS = 16;
@@ -492,6 +495,7 @@ export function normalizeAgentMotion(input, recipe) {
const materials = new Set((recipe.materials ?? []).map(material => material.id));
// An imported asset lists its morph targets; a recipe mesh offers the names of its shape keys.
const morphTargets = new Set(recipe.morphTargets ?? (recipe.geometries ?? []).flatMap(geometry => (geometry.shapeKeys ?? []).map(key => key.name)));
+ const oceanGeometries = new Set((recipe.geometries ?? []).filter(geometry => geometry.type === 'mesh' && (geometry.stack ?? []).some(node => node?.op === 'ocean')).map(geometry => geometry.id));
const identities = new Set();
let keyframeCount = 0;
const tracks = inputTracks.map((track, index) => {
@@ -508,9 +512,10 @@ export function normalizeAgentMotion(input, recipe) {
if (typeof track.id !== 'string' || !track.id.trim() || track.id.length > 128) {
throw invalidField(`motion.tracks[${index}].id`, track.id, 'an ID of 1 to 128 characters, such as "body"', TypeError);
}
- const references = track.kind === 'node' ? nodes : track.kind === 'morph' ? morphTargets : materials;
+ const references = track.kind === 'node' ? nodes : track.kind === 'morph' ? morphTargets : track.kind === 'geometry' ? oceanGeometries : materials;
if (track.kind !== 'light' && !references.has(track.id)) {
- const hint = track.kind === 'morph' ? `; morph targets of this asset: ${[...morphTargets].slice(0, 24).join(', ') || 'none'}` : '';
+ const hint = track.kind === 'morph' ? `; morph targets of this asset: ${[...morphTargets].slice(0, 24).join(', ') || 'none'}`
+ : track.kind === 'geometry' ? '; a geometry track needs a mesh geometry whose stack has an ocean node, such as stack [{ "op": "ocean" }]' : '';
throw invalidField(`motion.tracks[${index}].id`, track.id, `the ID of a ${track.kind} in this recipe, such as ${JSON.stringify([...references][0] ?? 'body')}${hint}`);
}
}
@@ -808,8 +813,8 @@ export function compileAgentMotion(input, recipe) {
})),
});
const sampler = compileAnimationSampler(clip);
- const output = { nodes: new Map(), materials: new Map(), morphs: new Map(), lights: new Map(), sun: null, environment: null, effects: null, camera: null };
- const patchesByKind = { node: output.nodes, material: output.materials, morph: output.morphs, light: output.lights };
+ const output = { nodes: new Map(), materials: new Map(), morphs: new Map(), geometries: new Map(), lights: new Map(), sun: null, environment: null, effects: null, camera: null };
+ const patchesByKind = { node: output.nodes, material: output.materials, morph: output.morphs, geometry: output.geometries, light: output.lights };
const bindings = ordinaryTracks.map((track, index) => {
let patch;
if (LOOK_KINDS.has(track.kind) && track.kind !== 'light') {
diff --git a/src/agent-scene.js b/src/agent-scene.js
index e3be67fe..93d2b7e6 100644
--- a/src/agent-scene.js
+++ b/src/agent-scene.js
@@ -1124,19 +1124,47 @@ function stagePose(stagedState, time) {
}
const camera = sampled?.camera ? cameraPoseFrom(sampled.camera) : stagedState.recipe.camera;
retainedMatrices(stagedState.parts);
- const clothGeometries = sampledGeometries(stagedState, time);
+ const clothGeometries = sampledGeometries(stagedState, time, sampled);
// Staged last: a pose pending on a decoded asset must be committed or discarded, so nothing above may throw after it.
const assetClips = stagedState.assetRuntime.animators.length ? agentSceneModule('assets').stageAssetClips(stagedState.assetRuntime, graph, time) : null;
const movedLook = stagedState.motionSampler && agentSceneModule('motion').motionMovesLook(sampled) ? sampled : null;
return { graph, materialUpdates, camera, time, clothGeometries, assetClips, hiddenNodes, movedLook, morphs: sampled?.morphs ?? null };
}
+/**
+ * Geometries of mesh stacks whose ocean time a geometry track drives, keyed by node ID. Each one is built from the cached edits with
+ * every ocean node's `time` set to the sampled value, so it equals the still of a recipe that writes that time into the node. The last
+ * result of each geometry is kept, so a repeated time reuses the geometry and the renderer sees no change.
+ */
+function oceanTimeGeometries(stagedState, sampled) {
+ const patches = sampled?.geometries;
+ if (!patches?.size) {
+ return new Map();
+ }
+ const frames = stagedState.oceanFrames ??= new Map();
+ const geometries = new Map();
+ for (const [geometryId, patch] of patches) {
+ let frame = frames.get(geometryId);
+ if (frame?.time !== patch.oceanTime) {
+ const definition = stagedState.recipe.geometries.find(candidate => candidate.id === geometryId);
+ const stack = definition.stack.map(node => (node.op === 'ocean' ? { ...node, time: patch.oceanTime } : node));
+ const scope = cutterScopeOf(stagedState.recipe.geometries);
+ frame = { time: patch.oceanTime, geometry: Object.freeze(generateGeometry({ ...definition, stack }, stagedState.maxGeometryVertices, scope)) };
+ frames.set(geometryId, frame);
+ }
+ for (const node of stagedState.recipe.nodes) {
+ if (node.geometry === geometryId) geometries.set(node.id, frame.geometry);
+ }
+ }
+ return geometries;
+}
+
// Cloth and particles replace the geometry of their own nodes at a sampled time; no node belongs to both.
-function sampledGeometries(stagedState, time) {
+function sampledGeometries(stagedState, time, sampled) {
if (time === null) {
return null;
}
- const geometries = new Map();
+ const geometries = oceanTimeGeometries(stagedState, sampled);
for (const sampler of [stagedState.clothSampler, stagedState.particleSampler]) {
sampler?.sample(time)?.forEach((geometry, nodeId) => geometries.set(nodeId, geometry));
}
@@ -1477,6 +1505,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {
stats: geometryStats(geometryRecords.values()),
};
stagedState.assetRuntime = assetRuntime;
+ stagedState.maxGeometryVertices = Math.min(limits.maxVertices, 1_000_000);
stagedState.sequences = [...materialHandles.values()].map(handle => handle.holder.sequence).filter(Boolean);
stagedState.localBounds = localBounds;
stagedState.skinBindings = skinBindings;
diff --git a/tests/agent-scene-ocean-time.test.js b/tests/agent-scene-ocean-time.test.js
new file mode 100644
index 00000000..58d6cceb
--- /dev/null
+++ b/tests/agent-scene-ocean-time.test.js
@@ -0,0 +1,70 @@
+import './support/load-agent-scene-modules.mjs';
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createAgentScene } from '../src/agent-scene.js';
+
+const OCEAN = { op: 'ocean', seed: 3, windSpeed: 4, size: 2, resolution: 4 };
+
+function seaRecipe({ trackTime = true, stack = [{ op: 'subdivide', levels: 3 }, OCEAN] } = {}) {
+ return {
+ format: '67js.scene', schemaVersion: 1,
+ geometries: [{ id: 'sea', type: 'mesh', base: { type: 'box', bevel: 0 }, stack }],
+ materials: [{ id: 'water', color: [0.1, 0.3, 0.5] }],
+ nodes: [{ id: 'surface', geometry: 'sea', material: 'water' }],
+ camera: { eye: [30, 20, 30], target: [0, 0, 0] },
+ ...(trackTime ? {
+ motion: {
+ duration: 4,
+ tracks: [{ kind: 'geometry', id: 'sea', property: 'oceanTime', keyframes: [{ time: 0, value: 0 }, { time: 4, value: 8 }] }],
+ },
+ } : {}),
+ };
+}
+
+function positionsOf(scene) {
+ return Array.from(scene.parts[0].geometry.positions);
+}
+
+test('an ocean time track moves the waves, and a frame equals the still written at that time', () => {
+ const scene = createAgentScene(seaRecipe());
+ scene.sampleMotion(0);
+ const atStart = positionsOf(scene);
+ scene.sampleMotion(2);
+ const atTwoSeconds = positionsOf(scene);
+ assert.notDeepEqual(atTwoSeconds, atStart, 'the sea moves between frames');
+ const still = createAgentScene(seaRecipe({ trackTime: false, stack: [{ op: 'subdivide', levels: 3 }, { ...OCEAN, time: 4 }] }));
+ assert.deepEqual(atTwoSeconds, positionsOf(still), 'time 2 s on a 0 to 8 track is the still at ocean time 4');
+});
+
+test('seeking back and forth gives the same vertices, and a repeated time keeps the geometry', () => {
+ const scene = createAgentScene(seaRecipe());
+ scene.sampleMotion(1.25);
+ const first = positionsOf(scene);
+ scene.sampleMotion(3.5);
+ scene.sampleMotion(1.25);
+ assert.deepEqual(positionsOf(scene), first);
+ const geometry = scene.parts[0].geometry;
+ scene.sampleMotion(1.25);
+ assert.equal(scene.parts[0].geometry, geometry);
+});
+
+test('an ocean time track names a mesh geometry with an ocean node in its stack', () => {
+ const missingGeometry = seaRecipe();
+ missingGeometry.motion.tracks[0].id = 'nowhere';
+ assert.throws(() => createAgentScene(missingGeometry), /ocean node/);
+ assert.throws(() => createAgentScene(seaRecipe({ stack: [{ op: 'subdivide', levels: 2 }] })), /ocean node/);
+});
+
+test('ocean time keyframes stay within the bounds of the modifier', () => {
+ const recipe = seaRecipe();
+ recipe.motion.tracks[0].keyframes[1].value = 2e6;
+ assert.throws(() => createAgentScene(recipe), /oceanTime/);
+ const wrongSize = seaRecipe();
+ wrongSize.motion.tracks[0].keyframes[1].value = [1, 2];
+ assert.throws(() => createAgentScene(wrongSize));
+});
+
+test('per-frame ocean work stays inside the ocean work limit', () => {
+ const scene = createAgentScene(seaRecipe({ stack: [{ op: 'subdivide', levels: 3 }, { ...OCEAN, resolution: 16 }] }));
+ assert.doesNotThrow(() => scene.sampleMotion(1));
+});