3D engine that AI agents drive
Drop the node inspection from scene_apply, scene_restore and mesh_edit answers (issue #718)
Nguyen Xuan An1ea233657ccc
diff --git a/changes/718.md b/changes/718.mdnew file mode 100644index 00000000..c03f6756--- /dev/null+++ b/changes/718.md@@ -0,0 +1 @@+- `scene_apply`, `scene_restore` and `mesh_edit` answers no longer carry the node inspection: they return the revision, the render status, a result per operation and the diff, and a 20-operation batch answers in under 2 KB instead of about 8 KB. Call `scene_inspect` to read the nodes (#718)diff --git a/changes/718.size.md b/changes/718.size.mdnew file mode 100644index 00000000..73e6b6bb--- /dev/null+++ b/changes/718.size.md@@ -0,0 +1,2 @@+- goal:agent-scene: 9 bytes gzip with no change to any file the bundle imports (this issue only edits the preview script, docs and tests), so the difference looks like build noise between the two measurement checkouts; the issue itself shrinks the agent answer by about 6 KB per edit.+- scene:agent-scene+lazy: the same noise, 15 bytes gzip, no imported file changed.diff --git a/docs/learn/agent-workflows.md b/docs/learn/agent-workflows.mdindex c2f1eadd..da331732 100644--- a/docs/learn/agent-workflows.md+++ b/docs/learn/agent-workflows.md@@ -193,28 +193,6 @@ The image has the size that the agent asked for, here 512 by 320 pixels. It look"sessionId": "1f0ac91e-860f-4332-b6e6-44e7007a81a1","revision": 1,"topologyRevision": 1,- "inspection": {- "revision": 1,- "topologyRevision": 1,- "motion": null,- "physics": null,- "boundsSpace": "authored-base-world",- "counts": {"nodes":6,"parts":5,"geometries":2,"materials":1,"vertices":1578},- "geometryBytes": 70272,- "estimatedGeometryBytes": 88368,- "residentGeometryBytes": 70272,- "residentGeometryVertices": 1578,- "look": {"world":false,"pointLights":0,"spotLights":0,"areaLights":0,"volumeFogDensity":0,"exposure":1},- "nodes": [- {"id":"table","label":"table","parent":null,"geometry":null,"material":null,"vertices":0,"triangles":0},- {"id":"top","label":"top","parent":"table","geometry":"top","material":"wood","vertices":1176,"triangles":2028},- {"id":"leg-a","label":"leg-a","parent":"table","geometry":"leg","material":"wood","vertices":402,"triangles":672},- {"id":"leg-b","label":"leg-b","parent":"table","geometry":"leg","material":"wood","vertices":402,"triangles":672},- {"id":"leg-c","label":"leg-c","parent":"table","geometry":"leg","material":"wood","vertices":402,"triangles":672},- {"id":"leg-d","label":"leg-d","parent":"table","geometry":"leg","material":"wood","vertices":402,"triangles":672}- ],- "truncated": false- },"render": {"graphics": "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device (Subzero) (0x0000C0DE)), SwiftShader driver)","estimatedBytes": 24212100,@@ -231,7 +209,13 @@ The image has the size that the agent asked for, here 512 by 320 pixels. It look"drawBatches": 3},"committed": true,- "renderSucceeded": true+ "renderSucceeded": true,+ "results": [+ { "i": 0, "op": "add-node", "target": "leg-b", "changed": "added node" },+ { "i": 1, "op": "add-node", "target": "leg-c", "changed": "added node" },+ { "i": 2, "op": "add-node", "target": "leg-d", "changed": "added node" }+ ],+ "diff": { "nodes": { "added": ["leg-b", "leg-c", "leg-d"] } }}```diff --git a/docs/learn/optimize-for-agents.md b/docs/learn/optimize-for-agents.mdindex 6c512001..c1180716 100644--- a/docs/learn/optimize-for-agents.md+++ b/docs/learn/optimize-for-agents.md@@ -66,7 +66,7 @@ Rules that follow from this:Every call is a turn. These habits remove turns:- **Batch edits.** One `scene_apply` takes up to 256 operations. Adding four legs in one call took one call and 49 ms in the local test. A call for each leg takes four.-- **Skip the inspect after an edit.** `scene_create`, `scene_apply` and `scene_restore` return the revision, an inspection of the nodes and the render status. A `scene_inspect` call right after them repeats that.+- **Read the answer, not the scene.** `scene_apply` returns the revision, the render status, a result for each operation and a diff of the IDs it changed. Call `scene_inspect` only when you need node bounds or counts.- **Edit, do not rewrite.** A small mesh change as a `mesh_edit` call is 187 bytes. The same change as a new `scene_create` is 1,202 bytes, plus 52 bytes to dispose the old scene. The gap grows with the size of the scene.- **Leave out `sessionId`.** One scene exists at a time. Every tool uses the active scene when the field is missing.- **Send the right revision.** A stale `expectedRevision` costs a failed call and a retry. Pass the `revision` from the last result.diff --git a/docs/mcp.md b/docs/mcp.mdindex b5ef6cfe..bb1bdc4f 100644--- a/docs/mcp.md+++ b/docs/mcp.md@@ -105,7 +105,7 @@ Known gaps, checked by `npm run check:tool-schemas`:10. Call `scene_export_recipe` to retain the canonical recipe.11. Call `scene_dispose` to release the scene before creating another one.-`scene_create`, `scene_apply`, and `scene_restore` return the session ID, current revision, inspection, and render status.+`scene_create` returns the session ID, current revision, inspection, and render status. `scene_apply` and `scene_restore` return the same without the inspection, so an edit answer stays small; call `scene_inspect` to read the nodes.`scene_checkpoint` returns a checkpoint UUID scoped to its session UUID.A restore increments revision and keeps the checkpoint available.A stale revision or invalid command produces an error without publishing partial edits.@@ -126,7 +126,7 @@ Inspect or export the recipe before retrying. Context loss still permits recipe| `scene_asset_catalog` | None | Configured catalog and content-bound GLB, image, `.hdr` (kind `environment`) and `.cube` (kind `look`) IDs; an ID goes in `environment.hdri` or `environment.look`, see [agent-look](agent-look.md#catalog-hdris-and-looks) || `scene_create_asset` | `recipe` using `67js.asset` | Imported scene ID, revision, inspection and render status || `scene_inspect` | `sessionId`, optional `ids`, `limit`, `includeBounds` | Compact node records and revision |-| `scene_apply` | `sessionId`, `expectedRevision`, `operations` | Updated revision, inspection, and render status |+| `scene_apply` | `sessionId`, `expectedRevision`, `operations` | Updated revision, render status, results and diff || `scene_walkmap` | `sessionId` or `assetId` (query a big catalog GLB), `min`, `max`, optional `cellSize`, `clearance`, `keepOut`, `streetY`, `includeCells` | Grid rows of `.` walkable, `x` blocked and `#` no surface, with counts; ground height and headroom per cell with `includeCells`. See [scene-query.md](scene-query.md) || `mesh_inspect` | `sessionId`, optional `geometries` (up to 16 IDs) | Per geometry: polygon counts, face sizes, parts, face tags, bounds and lint problems with fixes || `mesh_edit` | `sessionId`, `expectedRevision`, `geometry`, `edits`, optional `replace` | Updated revision, render status and the mesh report with per-edit results |@@ -134,7 +134,7 @@ Inspect or export the recipe before retrying. Context loss still permits recipe| `scene_look` | `sessionId`, optional `width`, `height`, `motionTime`, `labels` | PNG image and metadata || `scene_review` | `sessionId`, `expectedRevision`, `shots` (each with an optional `branch`), optional `width`, `height`, `labels` | Labeled PNG grid and metadata from an isolated snapshot || `scene_checkpoint` | `sessionId` | Checkpoint ID, session ID, and revision |-| `scene_restore` | `sessionId`, `expectedRevision`, and a `checkpointId` or a `branch` | Updated revision, inspection, and render status |+| `scene_restore` | `sessionId`, `expectedRevision`, and a `checkpointId` or a `branch` | Updated revision and render status || `scene_release_checkpoint` | `sessionId`, `checkpointId` | Release confirmation || `scene_export_recipe` | `sessionId` | Canonical recipe and revision || `scene_export_glb` | `sessionId`, optional `clip`, `motionTime`, `maxTextureSize` | A GLB file under `<jobs>/exports/` with a server-chosen name: path, bytes, SHA-256. See [GLB export](export-glb.md) |diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex b2be53d3..33902dfd 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -314,27 +314,32 @@ function syncMeshBlend(record) {record.renderer.meshBlend.radii = radii;}-function status(record) {+function renderStatus(record) {const render = renderInfo(record.renderer);record.lastRender = render;return {sessionId: record.sessionId,revision: record.scene.revision,topologyRevision: record.scene.topologyRevision,- inspection: record.scene.inspect(),render,};}+function status(record) {+ const { sessionId, revision, topologyRevision, render } = renderStatus(record);+ return { sessionId, revision, topologyRevision, inspection: record.scene.inspect(), render };+}++// The answer to an edit stays short: it holds no inspection, because every answer stays in the agent's context.+// An agent that needs the node list calls scene_inspect.function mutationStatus(record) {try {draw(record);- return { ...status(record), committed: true, renderSucceeded: true };+ return { ...renderStatus(record), committed: true, renderSucceeded: true };} catch (error) {return {sessionId: record.sessionId,revision: record.scene.revision,- inspection: record.scene.inspect(),render: record.lastRender ?? null,committed: true,renderSucceeded: false,diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex 6e657d68..6c51d316 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -880,7 +880,7 @@ export function createAgentMcpServer(runtime, { version = packageMetadata.versioregister('mesh_inspect', 'Check recipe geometries as polygon meshes: vertex, edge and face counts, triangles/quads/n-gons, connected and closed parts, face tags (top, bottom, side, left, right, front, back), bounds, and lint problems (non-manifold, inconsistent winding, inverted, degenerate, open, loose) with a suggested fix each. Also imported asset parts (mesh-0). Read-only and cached per geometry.', z.strictObject({ ...sessionFields, geometry: identifier.optional().describe('One geometry ID to check, as in mesh_edit.'), geometries: z.array(identifier).min(1).max(16).optional().describe('Geometry IDs to check. Omit both to check the first 16 geometries.') }), readOnly, async ({ geometry: geometryId, geometries, ...rest }) => jsonResult(await runtime.inspectMeshes({ ...rest, ...(geometries || geometryId ? { geometries: [...(geometries ?? []), ...(geometryId ? [geometryId] : [])] } : {}) })));register('mesh_edit', 'Append edits to a recipe mesh geometry or an imported asset part such as mesh-0 (or replace its edit log), atomically like scene_apply: extrude, inset, delete, flip, translate with falloff, tag, loop cut, bevel, mirror, array, solidify, UV unwrap, set_position and displace with field expressions (math, noise, attributes), booleans with a primitive cutter, sculpt brush strokes and shrinkwrap onto a primitive. Edits that select no faces or break limits fail without changing the scene. Returns the new revision, render status and the mesh report with per-edit results.', z.strictObject({ ...sessionFields, expectedRevision, geometry: identifier.describe('ID of a geometry with type mesh, or an imported part such as mesh-0.'), edits: meshEdits.min(1), replace: z.boolean().optional().describe('Replace the whole edit log instead of appending.') }), edit, async argumentsValue => jsonResult(await runtime.editMesh(argumentsValue)));register('mesh_stack', 'Set the non-destructive modifier stack of a recipe mesh geometry (ops and fields: scene_capabilities meshModifiers), evaluated after its edits, each prefix cached. apply: n moves the first n nodes into the edit log, as Blender\'s Apply modifier. Atomic like scene_apply. Returns the revision, render status and mesh report.', z.strictObject({ ...sessionFields, expectedRevision, geometry: identifier.describe('ID of a geometry with type mesh.'), stack: meshStack.optional().describe('The whole new stack. Omit to keep the current stack (for example with apply).'), apply: z.number().int().min(0).max(16).optional() }), edit, async argumentsValue => jsonResult(await runtime.setMeshStack(argumentsValue)));- register('scene_apply', 'Atomically edit transforms, look, camera or materials. Procedural scenes support membership and set-motion authoring. Imported assets support set-root, set-asset-node, set-motion and absolute set-animation sampling. Successful batches increment revision once. Returns results (one entry per operation, with a warning when it changed nothing) and a diff of added, removed and changed IDs.', z.strictObject({ ...sessionFields, expectedRevision, operations: z.array(relations ? operation : operationWithoutRelations).min(1).max(256).describe('Node/material/camera/root/animation commands use flat fields. Motion, look patches and add commands contain nested records. Backend validates each scene mode before publication.'), ...branchField }), edit, async argumentsValue => jsonResult(await runtime.apply(argumentsValue)));+ register('scene_apply', 'Atomically edit transforms, look, camera or materials. Procedural scenes support membership and set-motion authoring. Imported assets support set-root, set-asset-node, set-motion and absolute set-animation sampling. Successful batches increment revision once. Returns results (one entry per operation, with a warning when it changed nothing) and a diff of added, removed and changed IDs, not the node list (call scene_inspect for that).', z.strictObject({ ...sessionFields, expectedRevision, operations: z.array(relations ? operation : operationWithoutRelations).min(1).max(256).describe('Node/material/camera/root/animation commands use flat fields. Motion, look patches and add commands contain nested records. Backend validates each scene mode before publication.'), ...branchField }), edit, async argumentsValue => jsonResult(await runtime.apply(argumentsValue)));register('scene_look', 'Render the active scene as a bounded PNG, optionally with part labels. Optional authored motionTime seeks without changing revision; null restores the authored base pose. Omit to retain the last preview pose.', z.strictObject({ ...sessionFields, motionTime: z.number().min(0).max(3600).nullable().optional(), width: z.number().int().min(64).max(1024).default(768).describe('PNG width in pixels.'), height: z.number().int().min(64).max(1024).default(512).describe('PNG height in pixels.'), labels: labelsFlag }), readOnly, async argumentsValue => {const { data, mimeType, metadata } = await runtime.look(argumentsValue);return { content: [{ type: 'image', data, mimeType }, { type: 'text', text: JSON.stringify(metadata) }], structuredContent: metadata };diff --git a/scripts/smoke-agent-post-pass.mjs b/scripts/smoke-agent-post-pass.mjsindex cd9288db..cffabaa9 100644--- a/scripts/smoke-agent-post-pass.mjs+++ b/scripts/smoke-agent-post-pass.mjs@@ -123,7 +123,7 @@ try {// The session is still usable: the fix draws, and the result is quantised and blocky.const fixed = (await call('scene_apply', { sessionId, expectedRevision: brokenRevision, operations: [{ type: 'set-post-passes', postPasses: [ditherPass] }] })).structuredContent;assert.equal(fixed.renderSucceeded, true, fixed.renderError);- assert.deepEqual(fixed.inspection.postPasses, [{ name: 'pixel-dither', scale: SCALE, uniforms: ['levels:float', 'spread:float'], fragmentBytes: ditherFragment.length }]);+ assert.deepEqual((await call('scene_inspect', { sessionId })).structuredContent.postPasses, [{ name: 'pixel-dither', scale: SCALE, uniforms: ['levels:float', 'spread:float'], fragmentBytes: ditherFragment.length }]);const dithered = await measure(await look(sessionId));assert.ok(dithered.colors <= LEVELS ** 3, `the dithered scene should have at most ${LEVELS ** 3} colours, has ${dithered.colors}`);assert.ok(dithered.uniformBlockShare > 0.99, `the dithered scene should be ${BLOCK}x${BLOCK} blocks, share ${dithered.uniformBlockShare}`);diff --git a/scripts/smoke-mcp.mjs b/scripts/smoke-mcp.mjsindex 99f5f6b4..868dab35 100644--- a/scripts/smoke-mcp.mjs+++ b/scripts/smoke-mcp.mjs@@ -66,8 +66,9 @@ try {...fixture.nodes.map(node => ({ type: 'add-node', node })),] })).structuredContent;assert.equal(created.renderSucceeded, true);- assert.equal(created.inspection.counts.parts, 19);- assert.equal(created.inspection.counts.geometries, 7);+ const createdCounts = (await call('scene_inspect', { sessionId })).structuredContent;+ assert.equal(createdCounts.counts.parts, 19);+ assert.equal(createdCounts.counts.geometries, 7);assert.equal(created.revision, 1);assert.match(created.render.graphics, backend === 'gpu' ? /NVIDIA/i : /SwiftShader/i);const meshes = (await call('mesh_inspect', { sessionId })).structuredContent;@@ -90,8 +91,10 @@ try {assert.equal(edited.committed, true);assert.equal(edited.renderSucceeded, true);assert.equal(edited.render.depth, true);- assert.equal(edited.inspection.look.pointLights, 1);- assert.equal(edited.inspection.look.volumeFogDensity, 0.04);+ assert.equal(edited.inspection, undefined);+ const editedLook = (await call('scene_inspect', { sessionId })).structuredContent.look;+ assert.equal(editedLook.pointLights, 1);+ assert.equal(editedLook.volumeFogDensity, 0.04);const after = await look(sessionId, 'after');assert.notDeepEqual(after.bytes, before.bytes);const stale = await client.callTool({ name: 'scene_apply', arguments: {@@ -113,7 +116,7 @@ try {{ type: 'add-node', node: { id: 'ornament', geometry: 'ornament', material: 'ruby', translation: [-1.2, 0.8, 0], scale: [0.28, 0.28, 0.28] } },] })).structuredContent;assert.equal(enlarged.renderSucceeded, true);- assert.equal(enlarged.inspection.counts.parts, 20);+ assert.equal((await call('scene_inspect', { sessionId })).structuredContent.counts.parts, 20);assert.equal(enlarged.render.geometryBuffers, 8);const topology = await look(sessionId, 'topology');assert.notDeepEqual(topology.bytes, before.bytes);@@ -127,7 +130,7 @@ try {...fixture.materials.map(material => ({ type: 'remove-material', id: material.id })),] })).structuredContent;assert.equal(removed.renderSucceeded, true);- assert.equal(removed.inspection.counts.parts, 0);+ assert.equal((await call('scene_inspect', { sessionId })).structuredContent.counts.parts, 0);assert.equal(removed.render.gpuBuffers, 0);await call('scene_restore', { sessionId, checkpointId: checkpoint.checkpointId, expectedRevision: 6 });assert.deepEqual((await look(sessionId, 'recreated')).bytes, before.bytes);@@ -149,7 +152,7 @@ try {const replacement = (await call('scene_create', { recipe: createAgentPreviewFixture() })).structuredContent;assert.notEqual(replacement.sessionId, sessionId);await call('scene_dispose', { sessionId: replacement.sessionId });- report = { backend, era, tools: listing.tools.length, parts: created.inspection.counts.parts, geometryBytes: created.inspection.geometryBytes,+ report = { backend, era, tools: listing.tools.length, parts: createdCounts.counts.parts, geometryBytes: createdCounts.geometryBytes,graphics: created.render.graphics, before: before.metadata, portrait: portrait.metadata, restorePngExact: true, staleRevisionRejected: true,builtFromEmpty: true, topologyRestoreExact: true, disposedGeometryBuffers: true };} finally {diff --git a/tests/mcp-tools.test.js b/tests/mcp-tools.test.jsindex ca4b80c8..81c6b018 100644--- a/tests/mcp-tools.test.js+++ b/tests/mcp-tools.test.js@@ -27,6 +27,8 @@ async function connect(context) {const sessionId = randomUUID();const checkpoints = new Map();const status = () => ({ sessionId, revision: scene.revision, inspection: scene.inspect(), render: { backend: 'fake' } });+ // Edits answer without the inspection, like the preview does.+ const editStatus = () => ({ sessionId, revision: scene.revision, render: { backend: 'fake' } });const runtime = {async capabilities() { return { scene: AGENT_SCENE_CAPABILITIES, phase: 'idle', availableIn: 'test' }; },async create({ recipe: input }) {@@ -59,7 +61,7 @@ async function connect(context) {async apply({ expectedRevision, operations }) {calls++;scene.apply(operations, { expectedRevision });- return status();+ return editStatus();},async look({ width, height, motionTime }) {calls++;@@ -82,7 +84,7 @@ async function connect(context) {},async restore({ checkpointId, expectedRevision }) {scene.restore(checkpoints.get(checkpointId), { expectedRevision });- return status();+ return editStatus();},async releaseCheckpoint({ checkpointId }) {scene.releaseCheckpoint(checkpoints.get(checkpointId));@@ -482,15 +484,18 @@ test('SDK dynamic batches create resources, reject dangling edits, and restore s{ type: 'add-node', node: { id: 'mesh', parent: 'group', geometry: 'cube', material: 'paint' } },] });assert.equal(added.isError, undefined);- assert.equal(added.structuredContent.inspection.counts.parts, 1);+ assert.equal(added.structuredContent.inspection, undefined);+ assert.equal((await call('scene_inspect', { sessionId })).structuredContent.counts.parts, 1);const invalid = await call('scene_apply', { sessionId, expectedRevision: 1, operations: [{ type: 'remove-geometry', id: 'cube' }] });assert.equal(invalid.isError, true);const inspected = (await call('scene_inspect', { sessionId })).structuredContent;assert.equal(inspected.revision, 1);assert.equal(inspected.counts.geometries, 1);const restored = await call('scene_restore', { sessionId, checkpointId: initial.checkpointId, expectedRevision: 1 });- assert.equal(restored.structuredContent.inspection.counts.parts, 0);- assert.equal(restored.structuredContent.inspection.counts.geometries, 0);+ assert.equal(restored.structuredContent.inspection, undefined);+ const afterRestore = (await call('scene_inspect', { sessionId })).structuredContent;+ assert.equal(afterRestore.counts.parts, 0);+ assert.equal(afterRestore.counts.geometries, 0);});test('a tool that takes a sessionId defaults to the active scene, and still rejects a session that is not active', async context => {