3D engine that AI agents drive
Let scene_apply place the first instance of a catalog asset (#1063)
Nguyen Xuan Ana14178d0e1b3
diff --git a/changes/1063.md b/changes/1063.mdnew file mode 100644index 00000000..99b4e61c--- /dev/null+++ b/changes/1063.md@@ -0,0 +1 @@+- `scene_apply` can place the first instance of a catalog asset, so an agent builds a scene step by step and adds a model in any step (#1063)diff --git a/changes/1063.size.md b/changes/1063.size.mdnew file mode 100644index 00000000..eafee4d3--- /dev/null+++ b/changes/1063.size.md@@ -0,0 +1,3 @@+- scene:agent-scene: 70 bytes gzip for `provideAssets` and one sentence in the `placedAssets` capability text, which lets scene_apply place the first catalog asset of a scene.+- scene:agent-scene+lazy: the same 86 bytes gzip, the same feature.+- goal:agent-scene: an agent-facing capability (building a scene step by step with downloaded models, #1063) for under 100 bytes gzip.diff --git a/docs/agent-scene.md b/docs/agent-scene.mdindex 5a9b382b..23063afb 100644--- a/docs/agent-scene.md+++ b/docs/agent-scene.md@@ -304,7 +304,7 @@ A node with an `asset` field places a GLB from the `--assets` catalog inside the- `clip` plays that clip, held at its first frame until scene time `start`, then at `speed`, looping unless `loop` is false. Each placement that plays a clip gets a decoded copy of its own (its geometry is not shared), because a pose is state of one decoded asset. A seek gives the same frame as playing to that time. `set-motion` tracks still move the node itself, so a walking figure is a clip plus a motion track on the same node.- Bounds: 512 placements, 16 different assets, 16 placements playing a clip, and `maxNodes` for the generated parts. Placed geometry does not count against `maxVertices`; each asset keeps the decoder limits of the catalog.- `scene_inspect` adds `assets`: per placement the asset id, part count, world bounds, clips (index, name, duration), material names and the playing clip.-- Not yet: a scene created without any placed asset cannot add the first one with `scene_apply`. Render jobs draw placed assets and play their clips on scene time (`settings.motion.start`).+- `scene_apply` places a catalog asset with `add-node` even when the scene had none: the first `add-node` with an `asset` loads it, a second node with the same asset id shares its geometry. An id the catalog does not list fails with an error that names it and points at `scene_asset_catalog`, and the placement and asset bounds below still hold. A session made by `scene_create_asset` holds one imported asset and cannot place others. Render jobs draw placed assets and play their clips on scene time (`settings.motion.start`).### Vegetationdiff --git a/scripts/agent-asset-catalog.mjs b/scripts/agent-asset-catalog.mjsindex 71d2305e..47e1c3b5 100644--- a/scripts/agent-asset-catalog.mjs+++ b/scripts/agent-asset-catalog.mjs@@ -146,7 +146,7 @@ export async function createAgentAssetCatalog(root) {},async read(assetId) {const entry = entries.get(assetId);- if (!entry) throw new Error('Unknown asset ID. Read the configured asset catalog first.');+ if (!entry) throw new Error(`Unknown asset ID ${String(assetId).slice(0, 12)}. Call scene_asset_catalog and pass an assetId it lists.`);const result = await readFile(entry.filename);if (result.assetId !== assetId) throw new Error('Asset bytes changed since catalog creation. Restart the runtime to refresh the catalog.');const clipCount = result.document?.animations?.length ?? 0;diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex c2705229..b67cd2ec 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -1112,7 +1112,11 @@ window.__agentPreview = {async applyWithAssets(input) {const record = requireSession(input.sessionId);if (!record.placedAssets) {- throw new Error('This scene was created without placed assets. Recreate it with an asset node in scene_create, then add more.');+ if (typeof record.scene.provideAssets !== 'function') {+ throw new Error('This session holds an imported asset, which cannot place other assets. Create a scene with scene_create, then add the asset node with scene_apply.');+ }+ record.placedAssets = new Map();+ record.scene.provideAssets(record.placedAssets);}await loadAllAgentSceneModules();await loadPlacedAssets(record.placedAssets, input.assetNeeds ?? input.operations, input.assetData);diff --git a/scripts/smoke-agent-apply-first-asset.mjs b/scripts/smoke-agent-apply-first-asset.mjsnew file mode 100644index 00000000..d2f458fc--- /dev/null+++ b/scripts/smoke-agent-apply-first-asset.mjs@@ -0,0 +1,72 @@+// Through MCP on the CPU backend: a scene created with no placed asset gains its first catalog GLB through one+// scene_apply call, then a second instance of it, and an unknown asset id fails with a clear error and leaves the scene alone.+// Usage: node scripts/smoke-agent-apply-first-asset.mjs [width] [height] (a small still lands in artifacts/apply-first-asset.png)+import assert from 'node:assert/strict';+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';+import { constants, setPriority } from 'node:os';+import { resolve } from 'node:path';+import { Client } from '@modelcontextprotocol/client';+import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';++const [width = 480, height = 320] = process.argv.slice(2).map(Number);+setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);+const assetsDirectory = resolve('artifacts/apply-first-asset-assets');+await rm(assetsDirectory, { recursive: true, force: true });+await mkdir(assetsDirectory, { recursive: true });+await writeFile(resolve(assetsDirectory, 'robot.glb'), await readFile('public/wall-e.glb'));++const transport = new StdioClientTransport({+ command: process.execPath,+ args: [resolve('scripts/mcp-server.mjs'), '--backend', 'cpu', '--memory', '192', '--assets', assetsDirectory],+ stderr: 'pipe', cwd: process.cwd(),+});+let diagnostics = '';+transport.stderr.on('data', chunk => { diagnostics += chunk.toString(); });+const client = new Client({ name: '67js-apply-first-asset-proof', version: '1' }, {+ versionNegotiation: { mode: { pin: '2026-07-28' }, probe: { timeoutMs: 180000 } },+});+async function call(name, argumentsValue = {}, { expectError = false } = {}) {+ const result = await client.callTool({ name, arguments: argumentsValue }, { timeout: 180000 });+ const text = result.content?.find(item => item.type === 'text')?.text;+ if (expectError) {+ assert.equal(result.isError, true, `${name} should have failed`);+ return text;+ }+ assert.equal(result.isError, undefined, text);+ return result;+}++try {+ await client.connect(transport, { timeout: 180000 });+ const robot = (await call('scene_asset_catalog')).structuredContent.assets.find(asset => asset.name === 'robot.glb').assetId;+ const created = (await call('scene_create', { recipe: {+ format: '67js.scene', schemaVersion: 1,+ geometries: [{ id: 'floor', type: 'terrain', size: 12, resolution: 16, amplitude: 0.1 }],+ materials: [{ id: 'sand', color: [0.6, 0.55, 0.4], roughness: 0.9 }],+ nodes: [{ id: 'ground', geometry: 'floor', material: 'sand' }],+ camera: { eye: [0, 3, 8], target: [0, 1, 0], fov: 45 },+ } })).structuredContent;+ const { sessionId } = created;+ assert.equal(created.render.drawBatches > 0, true);++ const unknown = await call('scene_apply', { sessionId, expectedRevision: created.revision,+ operations: [{ type: 'add-node', node: { id: 'ghost', asset: { id: 'f'.repeat(64) } } }] }, { expectError: true });+ assert.match(unknown, /Unknown asset ID ffffffffffff.*scene_asset_catalog/);++ const first = (await call('scene_apply', { sessionId, expectedRevision: created.revision,+ operations: [{ type: 'add-node', node: { id: 'robot-1', translation: [-1.5, 0, 0], asset: { id: robot } } }] })).structuredContent;+ const second = (await call('scene_apply', { sessionId, expectedRevision: first.revision,+ operations: [{ type: 'add-node', node: { id: 'robot-2', translation: [1.5, 0, 0], asset: { id: robot } } }] })).structuredContent;+ const placements = (await call('scene_inspect', { sessionId })).structuredContent.assets;+ assert.deepEqual(placements.map(placement => placement.assetId), [robot, robot]);++ const image = (await call('scene_look', { sessionId, width, height })).content.find(item => item.type === 'image');+ await writeFile('artifacts/apply-first-asset.png', Buffer.from(image.data, 'base64'));+ await call('scene_dispose', { sessionId });+ console.log(JSON.stringify({ unknownAssetRefused: true, revisions: [created.revision, first.revision, second.revision], placements: placements.length }));+} catch (error) {+ console.error(diagnostics);+ throw error;+} finally {+ await client.close();+}diff --git a/src/agent-scene-capabilities.js b/src/agent-scene-capabilities.jsindex abd837e9..553097fb 100644--- a/src/agent-scene-capabilities.js+++ b/src/agent-scene-capabilities.js@@ -201,7 +201,7 @@ export const AGENT_SCENE_CAPABILITIES = Object.freeze({geometryTypes: Object.freeze(['box', 'cylinder', 'sphere', 'cone', 'icoSphere', 'torus', 'lathe', 'extrude', 'tube', 'curve', 'sdf', 'text', 'grid', 'plane', 'circle', 'line', 'terrain', 'mesh', 'fragment']),camera: 'eye, target, up, near, far, then fov (vertical degrees) or focalLength (mm, with sensorWidth mm, 36; not both). projection orthographic with orthoScale (view height in m, default 4). shift [x, y] moves the frame by a fraction of its size. fStop (0.7-64) turns on depth of field focused on target; 1.4 or lower is full blur; effects.depthOfField.blur must stay 0. Orthographic/shifted cameras use the rasterizer.',terrain: 'One mesh, x and z from -size/2 to size/2, hills about y = 0. size 40 m (1-2000), resolution 64 cells (2-256), seed 1, amplitude 1 m (0-200), holes up to 16 of {center:[x,z],radius}, no surface inside; radius a few cells or more.',- placedAssets: 'Node field asset:{id,materials?,clip?,drive?} turns a group node into a catalog GLB (id from asset_catalog). Its transform places it. materials:[{name,color,metallic,roughness,opacity,emission,emissiveColor}] override source materials by name. clip:{index,start=0,speed=1,loop=true} plays that clip from scene time start; set-motion tracks move the node and scene time seeks the clip. drive:{path | legs, cruiseSpeed | speed, wheels, lights} drives a car along a path in the placement frame (docs/vehicle.md). Max 512 placements, 16 assets, 16 animated.',+ placedAssets: 'Node field asset:{id,materials?,clip?,drive?} turns a group node into a catalog GLB (id from asset_catalog). Its transform places it. scene_apply add-node with an asset field places the first or any later instance, so a scene need not start with its assets. materials:[{name,color,metallic,roughness,opacity,emission,emissiveColor}] override source materials by name. clip:{index,start=0,speed=1,loop=true} plays that clip from scene time start; set-motion tracks move the node and scene time seeks the clip. drive:{path | legs, cruiseSpeed | speed, wheels, lights} drives a car along a path in the placement frame (docs/vehicle.md). Max 512 placements, 16 assets, 16 animated.',askUser: 'scene_ask_user opens the current view in their browser; they pin up to maxMarks (16) points, each with a note, and press Send. Ask when a change is hard to say in words ("warmer here", "this bit"), not for facts you can inspect. Each mark returns partId, point, normal, note, distance and material (null on sky), plus view.camera and a PNG with numbered pins. Apply the change the note names to that partId, then to similar parts (same material or name prefix) when the note says so. outcome cancelled or timeout returns no marks: ask in text. Edits made while they mark change revision; the result warns.',navigation: 'scene_inspect path:{ground,from:[x,z],to:[x,z]} returns a route over the ground node around its holes, steep slopes and the nodes in avoid:[ids] (box footprints), as keyframes [{time,value:[x,y,z]}] for a translation track with interpolation LINEAR. radius 0.3 m; maxSlope 40 degrees; cellSize defaults to radius; seconds or speed (1.4 m/s); lift adds y.',meshEdits: Object.freeze({ operations: Object.freeze(Object.keys(MESH_EDIT_OPERATIONS)), maxEdits: MAX_MESH_EDITS }),diff --git a/src/agent-scene.js b/src/agent-scene.jsindex 93b822d3..f29a64cd 100644--- a/src/agent-scene.js+++ b/src/agent-scene.js@@ -1897,6 +1897,10 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {publishPose(state, stagePose(state, state.motionTime));return exported;},+ /** Gives a scene created without placed assets the pool of decoded catalog assets, so a later edit can place the first one. */+ provideAssets(pool) {+ options = { ...options, assets: pool };+ },apply(inputOperations, options = {}) {expected(options);if (!Array.isArray(inputOperations) || inputOperations.length === 0 || inputOperations.length > limits.maxOperations) {diff --git a/tests/agent-scene-assets.test.js b/tests/agent-scene-assets.test.jsindex d58fe258..1a20f175 100644--- a/tests/agent-scene-assets.test.js+++ b/tests/agent-scene-assets.test.js@@ -113,3 +113,25 @@ test('apply can add and remove placed assets', async () => {scene.apply([{ type: 'remove-node', id: 'late-tree' }], { expectedRevision: 2 });assert.equal(scene.parts.filter(part => part.name.startsWith('late-tree')).length, 0);});++test('a scene created without assets places its first catalog asset, then a second instance, through apply', async () => {+ const scene = createAgentScene(recipe([]), {}, {});+ assert.throws(() => scene.apply([{ type: 'add-node', node: { id: 'boat', asset: { id: ASSET_ID } } }], { expectedRevision: scene.revision }), /places catalog assets/);+ assert.equal(scene.exportRecipe().nodes.length, 1, 'the refused edit leaves the scene alone');+ scene.provideAssets(await loadedAssets());+ scene.apply([{ type: 'add-node', node: { id: 'boat', translation: [3, 0, 0], asset: { id: ASSET_ID } } }], { expectedRevision: scene.revision });+ scene.apply([{ type: 'add-node', node: { id: 'boat-2', translation: [-3, 0, 0], asset: { id: ASSET_ID } } }], { expectedRevision: scene.revision });+ const placed = scene.inspect().assets;+ assert.deepEqual(placed.map(placement => placement.assetId), [ASSET_ID, ASSET_ID]);+});++test('an asset that is not in the pool is refused by id, and the 16 distinct asset bound still applies', async () => {+ const scene = createAgentScene(recipe([]), {}, {});+ scene.provideAssets(new Map());+ assert.throws(() => scene.apply([{ type: 'add-node', node: { id: 'boat', asset: { id: ASSET_ID } } }], { expectedRevision: scene.revision }), /Asset abababababab copy 0 is not loaded/);+ const many = Array.from({ length: 17 }, (_, index) => ({ type: 'add-node', node: { id: `asset-${index}`, asset: { id: index.toString(16).padStart(2, '0').repeat(32) } } }));+ const pool = new Map();+ for (const operation of many) pool.set(assetCopyKey(operation.node.asset.id, 0), await strip());+ scene.provideAssets(pool);+ assert.throws(() => scene.apply(many, { expectedRevision: scene.revision }), /at most 512 assets, 16 different ones/);+});