Skip to content

3D engine that AI agents drive

Default motion for any recipe that moves, track-only recipes included (#2052)

Nguyen Xuan Anda4bad4a496e
diff --git a/changes/2052.md b/changes/2052.md
index 6635b4a5..a803afa9 100644
--- a/changes/2052.md
+++ b/changes/2052.md
@@ -1,2 +1,2 @@
-- A stills or video job of a recipe whose only moving content is motion tracks, particles, physics or cloth now plays the motion clock from 0 when `settings.motion` is left out, instead of rendering frozen frames.
-- `scene_look` and `scene_review` add a `motion-not-set` hint when the scene has motion tracks, particles, physics or cloth and no `motionTime` was given.
+- A stills or video job of a recipe whose only moving content is motion tracks (with keyframes at different times), particles, physics or cloth now plays the motion clock from 0 when `settings.motion` is left out, instead of rendering frozen frames.
+- `scene_look` and `scene_review` add a `motion-not-set` hint when the scene has such moving content and no `motionTime` was given.
diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.js
index 6744eb12..93ce6253 100644
--- a/scripts/agent-preview-browser.js
+++ b/scripts/agent-preview-browser.js
@@ -52,7 +52,7 @@ import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../src/agent-b
import { agentAssetDecodeOptions, withCatalogLimitAdvice } from '../src/agent-asset-limits.js';
import { describeMarks, validateMarks } from '../src/agent-annotate.js';
import { sceneQueryFor, partMaterialsFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache } from './agent-walkmap.js';
-import { MOTION_NOT_SET_HINT, recipePlaysSimulation } from '../src/motion-hint.js';
+import { MOTION_NOT_SET_HINT, recipeMoves } from '../src/motion-hint.js';
// Open scenes by sessionId, least recently used first. One canvas serves them all, so at most one record owns a renderer;
// a call on another scene hands the canvas over (the scene data stays, the renderer is rebuilt).
@@ -1341,7 +1341,7 @@ window.__agentPreview = {
mimeType: 'image/png',
metadata: { sessionId, revision: record.scene.revision, topologyRevision: record.scene.topologyRevision,
motionTime: record.scene.motionTime ?? null, ...render, ...(labels ? { labels } : {}),
- ...(motionTime === undefined && recipePlaysSimulation(record.scene.exportRecipe()) ? { hints: [MOTION_NOT_SET_HINT] } : {}) },
+ ...(motionTime === undefined && recipeMoves(record.scene.exportRecipe()) ? { hints: [MOTION_NOT_SET_HINT] } : {}) },
};
},
// The findings of a view without drawing it, for a job whose frames the page does not draw (the path tracer). Exposure is
diff --git a/scripts/agent-render-jobs.mjs b/scripts/agent-render-jobs.mjs
index 9a817b21..f9d079e0 100644
--- a/scripts/agent-render-jobs.mjs
+++ b/scripts/agent-render-jobs.mjs
@@ -7,7 +7,7 @@ import { RENDER_PASS_NAMES, RENDER_OUTPUT_FORMATS, MAXIMUM_JOB_FPS, MAXIMUM_JOB_
import { parseQuality } from './agent-render-quality.mjs';
import { estimateFrameBytes } from '../src/budget.js';
import { recipeAssetNeeds } from '../src/agent-scene-assets.js';
-import { recipePlaysSimulation } from '../src/motion-hint.js';
+import { recipeMoves } from '../src/motion-hint.js';
import { normalizeFilmTimeline, filmTimelineDuration, videoFrameCount } from '../src/film-timeline.js';
import { OIDN_DEFAULT_SAMPLES, PATH_TRACER_DENOISERS, PATH_TRACER_JOB_LIMITS } from '../src/path-tracer-capabilities.js';
import { AGENT_ASSET_LIMITS, describeAssetBytes } from '../src/agent-asset-limits.js';
@@ -110,7 +110,7 @@ function playsPhysics(recipe) {
*/
function playsSimulationFromStart(input, recipe, timelineStart) {
return input.kind !== 'image' && timelineStart === undefined && input.animation === undefined
- && recipe?.format === 'crucible.scene' && recipePlaysSimulation(recipe);
+ && recipe?.format === 'crucible.scene' && recipeMoves(recipe);
}
/** The settings of a job as the store keeps them: every default filled in, every limit checked. */
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index ec71139f..0ace14f4 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -61,7 +61,7 @@ import { mergeFrameDiagnostics, pathTracerFindingsLimit } from '../src/frame-che
import { BRANCH_LIMITS, validateBranchName } from '../src/agent-branches.js';
import { createSharedGpuCard } from './agent-gpu-queue.mjs';
import { sessionProblem, SCENE_STORE_LIMITS } from '../src/agent-errors.js';
-import { MOTION_NOT_SET_HINT, recipePlaysSimulation } from '../src/motion-hint.js';
+import { MOTION_NOT_SET_HINT, recipeMoves } from '../src/motion-hint.js';
const SCENE_LIMITS = AGENT_SCENE_DEFAULT_LIMITS;
const MAXIMUM_IMAGE_BYTES = 1024 * 1024;
@@ -1525,7 +1525,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
output = { ...result, metadata: { sessionId: settings.sessionId, sourceRevision: snapshot.sourceRevision,
backend, width: settings.gridWidth, height: settings.gridHeight, shots,
peakNodeBytes, peakBrowserHeap, maximumPendingFrames: 1,
- ...(recipePlaysSimulation(snapshot.recipe) && settings.shots.some(shot => shot.motionTime === null) ? { hints: [MOTION_NOT_SET_HINT] } : {}) } };
+ ...(recipeMoves(snapshot.recipe) && settings.shots.some(shot => shot.motionTime === null) ? { hints: [MOTION_NOT_SET_HINT] } : {}) } };
} catch (error) {
if (deadlineExpired) throw new Error(reviewDeadlineText());
throw error;
diff --git a/src/motion-hint.js b/src/motion-hint.js
index 6bd3fcf7..c2c9c881 100644
--- a/src/motion-hint.js
+++ b/src/motion-hint.js
@@ -5,8 +5,14 @@ export const MOTION_NOT_SET_HINT = {
fix: 'Pass motionTime (seconds from 0) to see the scene at that moment, such as motionTime 1.',
};
-/** Whether a recipe has motion tracks or simulated content that moves only when a motion time is sampled. */
-export function recipePlaysSimulation(recipe) {
- const hasTracks = Array.isArray(recipe?.motion?.tracks) && recipe.motion.tracks.length > 0;
- return Boolean(hasTracks || recipe?.physics || recipe?.cloth || recipe?.particles);
+/** Whether a track has keyframes at two or more different times, so it changes something when the motion clock runs. */
+function trackSpansTime(track) {
+ const times = Array.isArray(track?.keyframes) ? track.keyframes.map(keyframe => keyframe?.time) : [];
+ return new Set(times).size > 1;
+}
+
+/** Whether a recipe has anything that moves on the motion clock: motion tracks with keyframes over time, particles, physics or cloth. */
+export function recipeMoves(recipe) {
+ const hasMovingTrack = Array.isArray(recipe?.motion?.tracks) && recipe.motion.tracks.some(trackSpansTime);
+ return Boolean(hasMovingTrack || recipe?.physics || recipe?.cloth || recipe?.particles);
}
diff --git a/tests/physics-film.test.js b/tests/physics-film.test.js
index 9096e905..15d0be1e 100644
--- a/tests/physics-film.test.js
+++ b/tests/physics-film.test.js
@@ -257,3 +257,22 @@ test('a fixed camera stays locked while a fractured wall flies apart', { skip: !
`the static marker moved on screen from ${JSON.stringify(first)} to ${JSON.stringify(last)}`);
await call('scene_dispose', { sessionId });
});
+
+function trackOnlyScene(motion) {
+ return { ...sceneWithGround([{ id: 'box', geometry: 'slab', material: 'steel', translation: [-1.5, 0.5, 0], scale: [0.8, 1, 0.8] }]), motion };
+}
+
+// A recipe whose only motion is tracks, filmed with no settings.motion, still plays from second 0 (#2052).
+for (const [label, track] of [
+ ['node', { kind: 'node', id: 'box', property: 'translation', keyframes: [{ time: 0, value: [-1.5, 0.5, 0] }, { time: 1, value: [1.5, 0.5, 0] }] }],
+ ['camera', { kind: 'camera', property: 'eye', keyframes: [{ time: 0, value: [4, 2.5, 7] }, { time: 1, value: [-4, 2.5, 7] }] }],
+]) {
+ test(`a video job of a ${label}-track-only recipe moves without a motion setting`, { skip: !hasFfmpeg && 'ffmpeg is not installed' }, async () => {
+ const { sessionId } = await call('scene_create', { recipe: trackOnlyScene({ duration: 1, tracks: [track] }) });
+ const result = await submitJob(sessionId, 'video', { duration: 1 });
+ const frames = decodedFrames(result.artifacts[0].path, FRAME_WIDTH, FRAME_HEIGHT);
+ assert.ok(frames.length > 2);
+ assert.notDeepEqual(frames[0], frames[frames.length - 1], 'the first and last frames differ');
+ await call('scene_dispose', { sessionId });
+ });
+}
diff --git a/tests/render-job-simulation-motion.test.js b/tests/render-job-simulation-motion.test.js
index 415827f8..917a9307 100644
--- a/tests/render-job-simulation-motion.test.js
+++ b/tests/render-job-simulation-motion.test.js
@@ -39,10 +39,10 @@ nodeTest('an image job and a recipe with nothing simulated get no implied motion
});
nodeTest('the motion-not-set hint says what to pass', async () => {
- const { MOTION_NOT_SET_HINT, recipePlaysSimulation } = await import('../src/motion-hint.js');
+ const { MOTION_NOT_SET_HINT, recipeMoves } = await import('../src/motion-hint.js');
assert.match(MOTION_NOT_SET_HINT.fix, /motionTime/);
- assert.equal(recipePlaysSimulation(particleRecipe), true);
- assert.equal(recipePlaysSimulation({ format: 'crucible.scene' }), false);
+ assert.equal(recipeMoves(particleRecipe), true);
+ assert.equal(recipeMoves({ format: 'crucible.scene' }), false);
});
const tracksRecipe = {
@@ -63,8 +63,10 @@ nodeTest('a tracks job longer than the authored motion keeps the held pose inste
});
nodeTest('the motion-not-set hint covers motion tracks, and an empty track list does not count', async () => {
- const { MOTION_NOT_SET_HINT, recipePlaysSimulation } = await import('../src/motion-hint.js');
+ const { MOTION_NOT_SET_HINT, recipeMoves } = await import('../src/motion-hint.js');
assert.match(MOTION_NOT_SET_HINT.message, /motion tracks/);
- assert.equal(recipePlaysSimulation(tracksRecipe), true);
- assert.equal(recipePlaysSimulation({ format: 'crucible.scene', motion: { duration: 2, tracks: [] } }), false);
+ assert.equal(recipeMoves(tracksRecipe), true);
+ assert.equal(recipeMoves({ format: 'crucible.scene', motion: { duration: 2, tracks: [] } }), false);
+ const holdTrack = { kind: 'node', id: 'box', property: 'translation', keyframes: [{ time: 0, value: [0, 0, 0] }] };
+ assert.equal(recipeMoves({ format: 'crucible.scene', motion: { duration: 2, tracks: [holdTrack] } }), false, 'one keyframe never moves');
});
diff --git a/tests/support/preview-harness.mjs b/tests/support/preview-harness.mjs
index 9b6e20b3..78b676f0 100644
--- a/tests/support/preview-harness.mjs
+++ b/tests/support/preview-harness.mjs
@@ -15,7 +15,7 @@ import { assetCopyKey, recipeAssetNeeds } from '../../src/agent-scene-assets.js'
import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../../src/agent-branches.js';
import { sessionProblem, SCENE_STORE_LIMITS } from '../../src/agent-errors.js';
import { imageLimitProblem } from '../../src/image-header.js';
-import { MOTION_NOT_SET_HINT, recipePlaysSimulation } from '../../src/motion-hint.js';
+import { MOTION_NOT_SET_HINT, recipeMoves } from '../../src/motion-hint.js';
import { sampleMotionAtZeroForGround } from '../../scripts/motion-ground-sample.js';
import { TEXTURE_SEQUENCE_LIMITS } from '../../src/texture-sequence-limits.js';
import { AGENT_SCENE_DEFAULT_LIMITS, MEMORY_CEILING_MIB } from '../../src/agent-scene-capabilities.js';
@@ -90,7 +90,7 @@ export function previewHarness({ createImageBitmap = async () => { throw new Err
// The harness loads Manifold from node_modules directly, so the preview's served-file loader is ignored.
setManifoldModuleLoader: () => {},
needsPhysics, loadPhysicsEngine, physicsEngineLoaded, setPhysicsModuleLoader: () => {},
- describeMarks, validateMarks, sceneQueryFor, partMaterialsFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache, parseGltf, hasPatternTextures, rasterOptionsFor, Uint8Array, atob, sessionProblem, SCENE_STORE_LIMITS, AGENT_SCENE_DEFAULT_LIMITS, MEMORY_CEILING_MIB, imageLimitProblem, MOTION_NOT_SET_HINT, recipePlaysSimulation, sampleMotionAtZeroForGround, TEXTURE_SEQUENCE_LIMITS, Blob, createImageBitmap, describeApplication, framingProblems, placementProblems, textureProblems, exposureProblems, lumaShares, assetCopyKey, recipeAssetNeeds, agentAssetDecodeOptions, withCatalogLimitAdvice,
+ describeMarks, validateMarks, sceneQueryFor, partMaterialsFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache, parseGltf, hasPatternTextures, rasterOptionsFor, Uint8Array, atob, sessionProblem, SCENE_STORE_LIMITS, AGENT_SCENE_DEFAULT_LIMITS, MEMORY_CEILING_MIB, imageLimitProblem, MOTION_NOT_SET_HINT, recipeMoves, sampleMotionAtZeroForGround, TEXTURE_SEQUENCE_LIMITS, Blob, createImageBitmap, describeApplication, framingProblems, placementProblems, textureProblems, exposureProblems, lumaShares, assetCopyKey, recipeAssetNeeds, agentAssetDecodeOptions, withCatalogLimitAdvice,
applyProjection, applyCameraFields, mat4, fitAgentShadow, resolveAgentShadow, agentEnvironmentMap, BRANCH_LIMITS,
// Line art needs a real renderer with render passes, so the fault-injected one gets a no-op install.
agentSceneModule: name => (name === 'lineArt' ? { installLineArt: renderer => renderer } : name === 'userPass' ? { ...agentSceneModule(name), installUserPasses: renderer => renderer } : agentSceneModule(name)), agentSceneModuleLoaded, createBranchStore, printableLabel,