3D engine that AI agents drive
Keep the settle capability note out of the core bundle and allow 512 stacking colliders
Nguyen Xuan An67baf83e5dc7
diff --git a/changes/980.size.md b/changes/980.size.mdnew file mode 100644index 00000000..1c51700f--- /dev/null+++ b/changes/980.size.md@@ -0,0 +1,4 @@+- scene:agent-scene+lazy: the settle code (placement, ray tests, the physics definition and the recipe operation) in the lazy chunk the scene loads only for a settle operation (6071 bytes gzip); no other recipe runs it+- goal:agent-scene: the core bundle grows by 39 bytes gzip for the lazy module entry and the settle operation branch; the field list, validation and solver code are all in the lazy chunk+- tool-total: the settle operation is a new scene_apply branch an agent needs to drop rubble and props onto a floor in one call (273 bytes); its fields are served by scene_capabilities topic settle+- tool:scene_apply: the same settle operation branch, a bare settings object whose detail lives in scene_capabilities topic settle (273 bytes)diff --git a/docs/physics.md b/docs/physics.mdindex 8025522e..a6a9996b 100644--- a/docs/physics.md+++ b/docs/physics.md@@ -582,7 +582,7 @@ const { instances, report } = settleInstances({| Setting | Default | Meaning || --- | --- | --- |-| `colliders` | required | Up to 64 `box` ({ translation, rotation, halfExtents }) or `ball` ({ translation, radius }) shapes to land on. List earlier settled props here to stack on them. |+| `colliders` | required | Up to 512 `box` ({ translation, rotation, halfExtents }) or `ball` ({ translation, radius }) shapes to land on. List earlier settled props here to stack on them. || `shape` | `box` | Prop shape, `box` or `ball`. || `area`, `count` or `density` | required | Where to scatter, and how many (256 at most) or how many per square metre. || `instances` | none | Instead of `area`, `count`, `density` and `size`: your own start poses `[{ translation, rotation, size }]` (size is three edge lengths for a box, a diameter for a ball). |diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjsindex 01e7736b..5c54dabe 100644--- a/scripts/agent-runtime.mjs+++ b/scripts/agent-runtime.mjs@@ -8,6 +8,7 @@ import { fileURLToPath } from 'node:url';import packageMetadata from '../package.json' with { type: 'json' };import { VEGETATION_CAPABILITY_NOTE } from '../src/agent-vegetation-notes.js';import { CROWD_CAPABILITY_NOTE } from '../src/agent-crowd-notes.js';+import { SETTLE_CAPABILITY_NOTE } from '../src/settle-notes.js';import { USER_PASS_CAPABILITIES } from '../src/user-pass.js';import { GAME_CAPABILITIES } from '../src/game-setup-notes.js';import { MESH_BLEND_CAPABILITIES } from '../src/mesh-blend.js';@@ -774,7 +775,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browreturn Object.freeze({async capabilities() {return {- scene: { ...AGENT_SCENE_CAPABILITIES, game: GAME_CAPABILITIES, postPasses: USER_PASS_CAPABILITIES, meshBlend: MESH_BLEND_CAPABILITIES, materialSnippet: MATERIAL_SNIPPET_CAPABILITIES, vegetation: VEGETATION_CAPABILITY_NOTE, crowd: CROWD_CAPABILITY_NOTE, textureSequence: TEXTURE_SEQUENCE_CAPABILITIES },+ scene: { ...AGENT_SCENE_CAPABILITIES, game: GAME_CAPABILITIES, postPasses: USER_PASS_CAPABILITIES, meshBlend: MESH_BLEND_CAPABILITIES, materialSnippet: MATERIAL_SNIPPET_CAPABILITIES, vegetation: VEGETATION_CAPABILITY_NOTE, crowd: CROWD_CAPABILITY_NOTE, settle: SETTLE_CAPABILITY_NOTE, textureSequence: TEXTURE_SEQUENCE_CAPABILITIES },sceneLimits: SCENE_LIMITS,preview: { backend, memoryBudgetMiB: memory, maximumWidth: 1024, maximumHeight: 1024, maximumImageBytes: MAXIMUM_IMAGE_BYTES, maximumActiveScenes: SCENE_STORE_LIMITS.maxOpenScenes, maximumOpenSceneBytes: SCENE_STORE_LIMITS.maxOpenSceneBytes, maximumQueuedCalls: MAXIMUM_QUEUED_CALLS, evaluationTimeoutMs: EVALUATION_TIMEOUT_MS },review: { ...REVIEW_LIMITS, backend, memoryBudgetMiB: memory, evaluationTimeoutMs: EVALUATION_TIMEOUT_MS,diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex c293e936..e2a07318 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -734,7 +734,7 @@ const otherOperations = [z.strictObject({ type: z.literal('set-camera'), ...cameraFields }),z.strictObject({ type: z.literal('set-motion'), motion: recipeMotionReference.nullable() }),z.strictObject({ type: z.literal('bake-motion'), nodes: z.array(identifier).min(1).max(32), start: nonnegative.optional(), end: nonnegative.optional(), step: positive.optional(), tolerance: z.record(z.string(), z.number()).optional() }),- z.strictObject({ type: z.literal('settle'), settle: z.record(z.string(), z.unknown()).describe('Drop props onto nodes and keep them as static parts; fields in scene_capabilities topic settle.') }),+ z.strictObject({ type: z.literal('settle'), settle: z.record(z.string(), z.unknown()).describe('See scene_capabilities topic settle.') }),z.strictObject({ type: z.literal('set-armatures'), armatures: armaturesReference.nullable() }),z.strictObject({ type: z.literal('set-physics'), physics: z.record(z.string(), z.unknown()).nullable().describe('Same object as recipe.physics in scene_create.') }),z.strictObject({ type: z.literal('set-cloth'), cloth: z.record(z.string(), z.unknown()).nullable() }),diff --git a/scripts/measure-tool-cost.mjs b/scripts/measure-tool-cost.mjsindex 2f6cc762..6dbe6368 100644--- a/scripts/measure-tool-cost.mjs+++ b/scripts/measure-tool-cost.mjs@@ -5,6 +5,7 @@ import { InMemoryTransport } from '@modelcontextprotocol/server';import { createAgentMcpServer } from './mcp-server.mjs';import { VEGETATION_CAPABILITY_NOTE } from '../src/agent-vegetation-notes.js';import { CROWD_CAPABILITY_NOTE } from '../src/agent-crowd-notes.js';+import { SETTLE_CAPABILITY_NOTE } from '../src/settle-notes.js';import { USER_PASS_CAPABILITIES } from '../src/user-pass.js';import { GAME_CAPABILITIES } from '../src/game-setup-notes.js';import { MESH_BLEND_CAPABILITIES } from '../src/mesh-blend.js';@@ -75,7 +76,7 @@ export async function listTools() {/** What an agent receives from scene_capabilities with no topic: the index on this branch, the whole list on an older one. */async function defaultCapabilitiesResult() {- const scene = { ...AGENT_SCENE_CAPABILITIES, game: GAME_CAPABILITIES, postPasses: USER_PASS_CAPABILITIES, meshBlend: MESH_BLEND_CAPABILITIES, materialSnippet: MATERIAL_SNIPPET_CAPABILITIES, vegetation: VEGETATION_CAPABILITY_NOTE, crowd: CROWD_CAPABILITY_NOTE, textureSequence: TEXTURE_SEQUENCE_CAPABILITIES };+ const scene = { ...AGENT_SCENE_CAPABILITIES, game: GAME_CAPABILITIES, postPasses: USER_PASS_CAPABILITIES, meshBlend: MESH_BLEND_CAPABILITIES, materialSnippet: MATERIAL_SNIPPET_CAPABILITIES, vegetation: VEGETATION_CAPABILITY_NOTE, crowd: CROWD_CAPABILITY_NOTE, settle: SETTLE_CAPABILITY_NOTE, textureSequence: TEXTURE_SEQUENCE_CAPABILITIES };const runtime = { async close() {}, async capabilities() { return { scene, phase: 'idle', availableIn: '0.0.0' }; } };return withClient(runtime, async client => (await client.callTool({ name: 'scene_capabilities', arguments: {} })).structuredContent);}diff --git a/src/agent-scene-capabilities.js b/src/agent-scene-capabilities.jsindex 6a320ef3..684f23c1 100644--- a/src/agent-scene-capabilities.js+++ b/src/agent-scene-capabilities.js@@ -343,7 +343,6 @@ export const AGENT_SCENE_CAPABILITIES = Object.freeze({}),motionModifiers: 'Track extrapolation constant|linear; modifiers [{type: cycles|stepped|noise|generator|limits|envelope, ...}], 8 at most; cycles and stepped remap time first.',motionLimits: Object.freeze({ maxTracks: 128, maxKeyframesPerTrack: 64, maxKeyframes: 2048, maxDuration: 30 }),- settle: 'scene_apply settle { settle: { id, geometry (box or sphere), material (ID or up to 8), colliders [node IDs to land on: boxes and spheres; a group node stands for its children, so a later settle stacks on an earlier one], area { min [x, z], max [x, z] }, count (256 at most) or density (per m2), size [min, max] m (0.1-0.25), aspect 0-1 (0.3, box edge variation), rotation 0-1 (1, tumble; 0 is yaw only), spacing m (0.01), maxSlope deg (60), seed, duration s (3, 6 at most), friction, restitution } } or instances [{ translation, rotation, size }] in place of area/count/density/size: drops the props as rigid bodies (Rapier) onto the colliders at 60 Hz and writes the resting transforms as static child nodes id-0, id-1... of a group node id. Same seed, same transforms. Props that start overlapping or on ground steeper than maxSlope are skipped; ones that fall off the world are left out. Needs the physics peer.',motionBake: 'scene_apply bake-motion { nodes (32 at most), start 0, end (motion or physics duration), step 1/30 (512 samples), tolerance { position m, angle deg, scale } }: samples tracks, IK, constraints, drivers and physics into LINEAR tracks and removes those sources (an IK solver or the physics world only when all its nodes are baked). tolerance drops keys within it; without it a track keeps every sample, 64 keys at most. Procedural scenes only.',});diff --git a/src/settle-notes.js b/src/settle-notes.jsnew file mode 100644index 00000000..8a912898--- /dev/null+++ b/src/settle-notes.js@@ -0,0 +1,2 @@+// What scene_capabilities topic settle says. It lives apart from agent-scene-capabilities.js, which every agent scene bundle carries; the MCP host adds it to the capabilities it returns.+export const SETTLE_CAPABILITY_NOTE = 'scene_apply settle { settle: { id, geometry (box or sphere), material (ID or up to 8), colliders [node IDs to land on: boxes and spheres; a group node stands for its children, so a later settle stacks on an earlier one], area { min [x, z], max [x, z] }, count (256 at most) or density (per m2), size [min, max] m (0.1-0.25), aspect 0-1 (0.3, box edge variation), rotation 0-1 (1, tumble; 0 is yaw only), spacing m (0.01), maxSlope deg (60), seed, duration s (3, 6 at most), friction, restitution } } or instances [{ translation, rotation, size }] in place of area/count/density/size: drops the props as rigid bodies (Rapier) onto the colliders at 60 Hz and writes the resting transforms as static child nodes id-0, id-1... of a group node id. Same seed, same transforms. Props that start overlapping or on ground steeper than maxSlope are skipped; ones that fall off the world are left out. Needs the physics peer.';diff --git a/src/settle.js b/src/settle.jsindex b26bffcd..75123078 100644--- a/src/settle.js+++ b/src/settle.js@@ -8,7 +8,7 @@ import { invalidField } from './field-error.js';import { bakePhysicsScene, loadPhysicsEngine } from './physics.js';import { createRandom } from './vegetation-geometry.js';-export const SETTLE_LIMITS = Object.freeze({ maxProps: 256, maxColliders: 64, maxDuration: 6, maxAttemptsPerProp: 60 });+export const SETTLE_LIMITS = Object.freeze({ maxProps: 256, maxColliders: 512, maxDuration: 6, maxAttemptsPerProp: 60 });const FRAMES_PER_SECOND = 60;const PLACEMENT_ATTEMPTS_BEFORE_RAISING = 20;const COLUMN_GROWTH = 1.15;@@ -287,7 +287,8 @@ export function settleInstances(input) {const report = { requested: settings.instances?.length ?? settings.count, placed: props.length, skipped, fellOff: 0, unsettled: 0, duration: settings.duration };if (!props.length) return { instances: [], report };const frameCount = Math.round(settings.duration * FRAMES_PER_SECOND) + 1;- const scene = bakePhysicsScene(physicsDefinition(settings, props), { frameCount, timestep: 1 / FRAMES_PER_SECOND });+ const definition = physicsDefinition(settings, props);+ const scene = bakePhysicsScene(definition, { frameCount, timestep: 1 / FRAMES_PER_SECOND, maxBodies: definition.bodies.length, maxColliders: definition.colliders.length });const finalPose = scene.sample(scene.duration);const lowestSurface = Math.min(...settings.colliders.map(collider => (collider.shape === 'ball' ? collider.translation[1] - collider.radius : collider.translation[1] - Math.hypot(...collider.halfExtents))));const instances = [];diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.jsindex d1703fa3..9445052d 100644--- a/tests/agent-runtime.test.js+++ b/tests/agent-runtime.test.js@@ -13,6 +13,7 @@ import { randomUUID } from 'node:crypto';import { createContext, Script } from 'node:vm';import { VEGETATION_CAPABILITY_NOTE } from '../src/agent-vegetation-notes.js';import { CROWD_CAPABILITY_NOTE } from '../src/agent-crowd-notes.js';+import { SETTLE_CAPABILITY_NOTE } from '../src/settle-notes.js';import { GAME_CAPABILITIES } from '../src/game-setup-notes.js';import { USER_PASS_CAPABILITIES } from '../src/user-pass.js';import { MESH_BLEND_CAPABILITIES } from '../src/mesh-blend.js';@@ -121,7 +122,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = faconst context = createContext({ createServer, randomUUID, existsSync: () => true,readFile: async () => '', packageMetadata: { version: 'test' },createRequire: () => ({ resolve: () => { throw new Error('no peer in tests'); } }),- VEGETATION_CAPABILITY_NOTE, CROWD_CAPABILITY_NOTE, GAME_CAPABILITIES, USER_PASS_CAPABILITIES, MESH_BLEND_CAPABILITIES, MATERIAL_SNIPPET_CAPABILITIES, AGENT_SCENE_CAPABILITIES, AGENT_SCENE_DEFAULT_LIMITS, RENDER_PASS_NAMES, RENDER_PASS_DEPTH_RANGE, VIDEO_ENCODERS, MAXIMUM_VIDEO_SECONDS, MAXIMUM_JOB_SIDE, MAXIMUM_JOB_PIXELS, MAXIMUM_MEMORY_MIB, BAKE_WORKER_MAXIMUM_SECONDS, PATH_TRACER_CAPABILITIES,+ VEGETATION_CAPABILITY_NOTE, CROWD_CAPABILITY_NOTE, SETTLE_CAPABILITY_NOTE, GAME_CAPABILITIES, USER_PASS_CAPABILITIES, MESH_BLEND_CAPABILITIES, MATERIAL_SNIPPET_CAPABILITIES, AGENT_SCENE_CAPABILITIES, AGENT_SCENE_DEFAULT_LIMITS, RENDER_PASS_NAMES, RENDER_PASS_DEPTH_RANGE, VIDEO_ENCODERS, MAXIMUM_VIDEO_SECONDS, MAXIMUM_JOB_SIDE, MAXIMUM_JOB_PIXELS, MAXIMUM_MEMORY_MIB, BAKE_WORKER_MAXIMUM_SECONDS, PATH_TRACER_CAPABILITIES,recipeAssetNeeds, gameAssetIds, sequenceFrameIds, catalogLookReferences, BUNDLED_ENVIRONMENT_NAMES, BUNDLED_CUBE_LOOK_NAMES, TEXTURE_SEQUENCE_CAPABILITIES, AGENT_ASSET_LIMITS, assertCombinedAssetBytes,audioInputArguments, createAudioLibrary, measureStreamSeconds, filmTimelineDuration, filmTimelineShotStarts, sampleFilmTimeline, normalizeReview, REVIEW_LIMITS, isPreviewHost, BRANCH_LIMITS, validateBranchName, sessionProblem, SCENE_STORE_LIMITS,runAnnotateSession: async () => ({ outcome: 'timeout' }), openInDefaultBrowser: async () => {}, ANNOTATE_LIMITS,diff --git a/tests/agent-settle.test.js b/tests/agent-settle.test.jsindex 420a2a7a..716cf37f 100644--- a/tests/agent-settle.test.js+++ b/tests/agent-settle.test.js@@ -3,6 +3,7 @@ import test from 'node:test';import assert from 'node:assert/strict';import { createAgentScene, loadPhysicsEngine, needsPhysics } from '../src/agent-scene.js';import { AGENT_SCENE_CAPABILITIES } from '../src/agent-scene-capabilities.js';+import { SETTLE_CAPABILITY_NOTE } from '../src/settle-notes.js';await loadPhysicsEngine();@@ -67,5 +68,5 @@ test('settle operation errors name the field and what to pass', () => {test('a settle operation asks the host to load the physics engine and is listed in the capabilities', () => {assert.equal(needsPhysics([{ type: 'settle', settle: rubble }]), true);assert.ok(AGENT_SCENE_CAPABILITIES.operations.includes('settle'));- assert.match(AGENT_SCENE_CAPABILITIES.settle, /scene_apply settle/);+ assert.match(SETTLE_CAPABILITY_NOTE, /scene_apply settle/);});