3D engine that AI agents drive
Accept the inputs Haiku sent to scene_inspect and scene_capabilities
Nguyen Xuan An183305043c13
diff --git a/changes/761.md b/changes/761.mdnew file mode 100644index 00000000..1ac2b401--- /dev/null+++ b/changes/761.md@@ -0,0 +1 @@+- scene_inspect and scene_capabilities take the inputs Haiku sent in the bench: short mesh edit branch names (meshEdit.inset), dotted fields (environment.spotLights), near topic names (uvs, scene.terrain), source nodes without a mesh of their own, includeBounds with kind hierarchy; an unwalkable path start names the nearest walkable point (#761)diff --git a/changes/761.size.md b/changes/761.size.mdnew file mode 100644index 00000000..bdebc9bc--- /dev/null+++ b/changes/761.size.md@@ -0,0 +1,2 @@+- entry:navmesh: the unwalkable-start error names the nearest walkable point, so an agent fixes its start in one call instead of probing.+- scene:agent-scene+lazy: node IDs in scene_inspect expand to their parts, and scene_capabilities takes branch prefixes, dotted fields and near topic names; these were the failed calls of the bench run.diff --git a/scripts/capabilities-topics.mjs b/scripts/capabilities-topics.mjsindex acc751a5..7b2fb5e2 100644--- a/scripts/capabilities-topics.mjs+++ b/scripts/capabilities-topics.mjs@@ -47,12 +47,31 @@ function unionIndex(topic, branches) {function outlineAnswer(topic, branchName, schema) {const jsonSchema = inputJsonSchema(schema);const branches = unionBranches(jsonSchema);+ const withDefinitions = part => ({ ...part, ...(jsonSchema.$defs ? { $defs: jsonSchema.$defs } : {}) });if (!branchName) return branches ? unionIndex(topic, branches) : outlineSchema(jsonSchema);- if (!branches?.has(branchName)) {- const known = branches ? [...branches.keys()].join(', ') : 'none: this topic has no branches';- throw new Error(`Topic ${topic} has no branch ${branchName}. Branches: ${known}.`);- }- return outlineSchema({ ...branches.get(branchName), ...(jsonSchema.$defs ? { $defs: jsonSchema.$defs } : {}) });+ if (branches?.has(branchName)) return outlineSchema(withDefinitions(branches.get(branchName)));+ // A short name such as inset or extrude stands for every branch that starts with it: inset_region and inset_individual.+ const prefixed = branches ? [...branches].filter(([name]) => name.startsWith(`${branchName}_`)) : [];+ if (prefixed.length === 1) return outlineSchema(withDefinitions(prefixed[0][1]));+ if (prefixed.length > 1) return unionIndex(topic, new Map(prefixed));+ // A dotted field of a plain topic, such as environment.spotLights, is that field's outline.+ const field = jsonSchema.properties?.[branchName];+ if (!branches && field) return outlineSchema(withDefinitions(field));+ const known = branches ? [...branches.keys()] : Object.keys(jsonSchema.properties ?? {});+ const sharedField = branches && [...branches.values()].some(branch => branch.properties?.[branchName]);+ const fieldNote = sharedField ? ` ${branchName} is a field of those branches, not a branch: open one branch to see it.` : '';+ throw new Error(`Topic ${topic} has no ${branches ? 'branch' : 'field'} ${branchName}. ${branches ? 'Branches' : 'Fields'}: ${known.join(', ') || 'none'}.${fieldNote}`);+}++/** A near miss of a topic name: a leading "scene." is dropped, and a name that contains or is contained by exactly one topic (uvs for meshUv) stands for it. */+function closestTopic(name, allNames) {+ if (allNames.includes(name)) return name;+ const stem = name.toLowerCase().replace(/s$/, '');+ const near = stem.length < 2 ? [] : allNames.filter(candidate => {+ const lowered = candidate.toLowerCase();+ return lowered.includes(stem) || (lowered.length > 2 && stem.includes(lowered));+ });+ return near.length === 1 ? near[0] : undefined;}/** The topics a caller can ask for, by name. */@@ -76,14 +95,19 @@ export function answerCapabilities(capabilities, topic, outlineSources) {topics: names,};}- const [name, branchName] = topic.split('.');+ const allNames = [...names.scene, ...names.runtime, ...names.schemas];+ const parts = topic.split('.');+ if (parts[0] === 'scene' && parts.length > 1) parts.shift();+ const [requestedName, branchName] = parts;+ const name = closestTopic(requestedName, allNames) ?? requestedName;const notes = sceneTopics(capabilities.scene).get(name);const runtime = RUNTIME_SECTIONS.includes(name) ? capabilities[name] : undefined;const source = outlineSources.get(name);const found = [notes, runtime, source].some(value => value !== undefined);if (!found) {- throw new Error(`Unknown topic ${topic}. Topics: ${[...names.scene, ...names.runtime, ...names.schemas].join(', ')}.`);+ throw new Error(`Unknown topic ${topic}. Topics: ${allNames.join(', ')}.`);}- if (source) return { topic, ...(notes === undefined ? {} : { notes }), fields: outlineAnswer(name, branchName, source) };- return { topic, value: notes ?? runtime };+ const answeredTopic = [name, branchName].filter(Boolean).join('.');+ if (source) return { topic: answeredTopic, ...(notes === undefined ? {} : { notes }), fields: outlineAnswer(name, branchName, source) };+ return { topic: answeredTopic, value: notes ?? runtime };}diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex 30927342..3439a711 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -817,7 +817,7 @@ async function renderResultContent(result) {// names bundle internals the agent cannot act on, so only the message is returned.export function readableErrorMessage(error) {const message = error instanceof Error ? error.message : String(error);- const readable = message.split(/\s+at (?:\S+ \()?https?:\/\//)[0].replace(/^page\.evaluate: (?:\w*Error: )?/, '').trim();+ const readable = message.split(/\s+at (?:[^\n(]+ \()?https?:\/\//)[0].replace(/^page\.evaluate: (?:\w*Error: )?/, '').trim();if (!isEngineFault(readable)) {return readable;}@@ -892,7 +892,7 @@ export function createAgentMcpServer(runtime, { version = packageMetadata.versioregister('scene_create', 'Create the single active render scene from a JSON recipe. Dispose an existing scene first.', z.strictObject({ recipe: relations ? recipe : recipeWithoutRelations }), { ...edit, destructiveHint: false }, async argumentsValue => jsonResult(await runtime.create(argumentsValue)));register('scene_asset_catalog', 'List content-bound GLB and PNG, JPEG or WebP image IDs from the configured local asset directory. No arbitrary paths or downloads.', z.strictObject({}), readOnly, async () => jsonResult(await runtime.assetCatalog()));register('scene_create_asset', 'Create the active scene from a catalog GLB and editable asset recipe. Preserves supported maps, skins, morphs and clips. Dispose the existing scene first.', z.strictObject({ recipe: assetRecipe }), { ...edit, destructiveHint: false }, async argumentsValue => jsonResult(await runtime.createAsset(argumentsValue)));- register('scene_inspect', 'Read revision, counts and selected nodes. For a recipe with physics, the result lists body final positions, resting times, contacts and sensor enter/leave events. Bounds describe authored geometry rather than shader displacement.', z.strictObject({ ...sessionFields, ids: z.array(identifier).max(4096).optional().describe('Selected node IDs. Omit to inspect scene order.'), kind: z.enum(['meshes', 'hierarchy']).optional().describe('Imported assets only: hierarchy exposes stable source-node IDs and joints.'), offset: z.number().int().min(0).max(4096).optional().describe('Imported assets only: pagination offset.'), limit: z.number().int().min(1).max(4096).optional().describe('Maximum returned node records. Defaults to 64.'), includeBounds: z.boolean().optional().describe('Include world geometry bounds. Defaults to false.'), path: z.record(z.string(), z.unknown()).optional().describe('Walking route over a ground node, around holes: fields in scene_capabilities navigation.') }), readOnly, async argumentsValue => jsonResult(await runtime.inspect(argumentsValue)));+ register('scene_inspect', 'Read revision, counts and selected nodes. For a recipe with physics, the result lists body final positions, resting times, contacts and sensor enter/leave events. Bounds describe authored geometry rather than shader displacement.', z.strictObject({ ...sessionFields, ids: z.array(identifier).max(4096).optional().describe('Selected node IDs; an imported node means its parts and those below. Omit to inspect scene order.'), kind: z.enum(['meshes', 'hierarchy']).optional().describe('Imported assets only: hierarchy exposes stable source-node IDs and joints.'), offset: z.number().int().min(0).max(4096).optional().describe('Imported assets only: pagination offset.'), limit: z.number().int().min(1).max(4096).optional().describe('Maximum returned node records. Defaults to 64.'), includeBounds: z.boolean().optional().describe('Include world geometry bounds. Defaults to false.'), path: z.record(z.string(), z.unknown()).optional().describe('Walking route over a ground node, around holes: fields in scene_capabilities navigation.') }), readOnly, async argumentsValue => jsonResult(await runtime.inspect(argumentsValue)));register('scene_walkmap', 'Walkable map on a grid: rows of . walkable, x blocked, # no surface. keepOut (m) avoids obstacles. assetId: query a big catalog GLB, no render.', z.strictObject({ ...sessionFields, assetId: z.string().regex(/^[a-f0-9]{64}$/).optional(), min: z.tuple([z.number(), z.number()]), max: z.tuple([z.number(), z.number()]), cellSize: z.number().min(0.05).default(1), clearance: z.number().min(0).default(2), keepOut: z.number().min(0).optional(), streetY: z.number().optional(), includeCells: z.boolean().optional() }), readOnly, async argumentsValue => jsonResult(await runtime.walkMap(argumentsValue)));register('mesh_inspect', 'Check recipe geometries as polygon meshes: vertex, edge and face counts, triangles/quads/n-gons, connected and closed parts, face tags (top, bottom, side, left, right, front, back), bounds, and lint problems (non-manifold, inconsistent winding, inverted, degenerate, open, loose) with a suggested fix each. Also imported asset parts (mesh-0). Read-only and cached per geometry.', z.strictObject({ ...sessionFields, geometry: identifier.optional().describe('One geometry ID to check, as in mesh_edit.'), geometries: z.array(identifier).min(1).max(16).optional().describe('Geometry IDs to check. Omit both to check the first 16 geometries.') }), readOnly, async ({ geometry: geometryId, geometries, ...rest }) => jsonResult(await runtime.inspectMeshes({ ...rest, ...(geometries || geometryId ? { geometries: [...(geometries ?? []), ...(geometryId ? [geometryId] : [])] } : {}) })));register('mesh_edit', 'Append edits to a recipe mesh geometry or an imported asset part such as mesh-0 (or replace its edit log), atomically like scene_apply: extrude, inset, delete, flip, translate with falloff, tag, loop cut, bevel, mirror, array, solidify, UV unwrap, set_position and displace with field expressions (math, noise, attributes), booleans with a primitive cutter, sculpt brush strokes and shrinkwrap onto a primitive. Edits that select no faces or break limits fail without changing the scene. Returns the new revision, render status and the mesh report with per-edit results.', z.strictObject({ ...sessionFields, expectedRevision, geometry: identifier.describe('ID of a geometry with type mesh, or an imported part such as mesh-0.'), edits: meshEdits.min(1), replace: z.boolean().optional().describe('Replace the whole edit log instead of appending.') }), edit, async argumentsValue => jsonResult(await runtime.editMesh(argumentsValue)));diff --git a/src/agent-asset.js b/src/agent-asset.jsindex d72d5bb2..7b8e6a3f 100644--- a/src/agent-asset.js+++ b/src/agent-asset.js@@ -395,6 +395,25 @@ export function createAgentAssetScene(asset, inputRecipe, { sources } = {}) {return records.find(value => value.id === partIds[0]);}+ /** The parts scene_inspect lists for one ID: a mesh ID is that part; a source node is every part of it and of the nodes below it. */+ function inspectedRecordsFor(id) {+ const sourceNode = sourceNodes.get(id);+ if (!sourceNode || records.some(value => value.id === id)) {+ return [partRecordFor(id)];+ }+ const partIds = [];+ const pending = [id];+ while (pending.length > 0) {+ const nodeId = pending.pop();+ partIds.push(...(sourceNodes.get(nodeId).meshIds ?? []));+ pending.push(...[...sourceNodes.values()].filter(node => node.parent === nodeId).map(node => node.id));+ }+ if (partIds.length === 0) {+ return [partRecordFor(id)];+ }+ return records.filter(value => partIds.includes(value.id));+ }+function geometryOf(record, edited = editedParts) {return edited.get(record.id)?.geometry ?? record.source.geometry;}@@ -1019,9 +1038,6 @@ export function createAgentAssetScene(asset, inputRecipe, { sources } = {}) {throw invalidField('offset', offset, 'a whole number from 0 to 4096, such as 0');}if (options.kind === 'hierarchy') {- if (options.includeBounds) {- throw invalidField('includeBounds', options.includeBounds, 'false for kind hierarchy, which has no deformed silhouette bounds; drop it, or inspect without kind to get mesh bounds', TypeError);- }if (options.ids !== undefined && (!Array.isArray(options.ids) || options.ids.length > sourceNodes.size)) {throw invalidField('ids', Array.isArray(options.ids) ? `${options.ids.length} IDs` : options.ids, `an array of at most ${sourceNodes.size} source-node IDs, such as []`);}@@ -1035,12 +1051,13 @@ export function createAgentAssetScene(asset, inputRecipe, { sources } = {}) {return { revision, topologyRevision, assetId: canonicalAssetId, kind: 'hierarchy', ...motionIkReport(), ...motionDriveReport(), ...motionLayerReport(), ...diagnosticsReport(),counts: { nodes: sourceNodes.size, joints: [...sourceNodes.values()].filter(node => node.joint).length },nodes, offset, truncated: offset + nodes.length < selected.length,+ ...(options.includeBounds ? { note: 'Hierarchy lists no bounds; inspect without kind, with ids of nodes or meshes, to get them.' } : {}),motion: recipe.motion ? { duration: recipe.motion.duration, loop: recipe.motion.loop, tracks: recipe.motion.tracks.length, time: motionTime } : null };}if (options.ids !== undefined && (!Array.isArray(options.ids) || options.ids.length > records.length)) {throw invalidField('ids', Array.isArray(options.ids) ? `${options.ids.length} IDs` : options.ids, `an array of at most ${records.length} mesh IDs, such as ["mesh-0"]`);}- const selected = options.ids === undefined ? records : options.ids.map(id => partRecordFor(id));+ const selected = options.ids === undefined ? records : options.ids.flatMap(id => inspectedRecordsFor(id));const visible = selected.slice(offset, offset + limit);if (options.includeBounds && visible.reduce((total, record) => total + geometryOf(record).positions.length / 3, 0) > MAX_INSPECTION_VERTICES) {throw invalidField('ids', `${visible.length} meshes`, 'fewer meshes, under two million authored base vertices in all, such as ["mesh-0"]');diff --git a/src/navmesh.js b/src/navmesh.jsindex be335399..83ef9bfe 100644--- a/src/navmesh.js+++ b/src/navmesh.js@@ -200,7 +200,24 @@ export function createNavmesh({ positions, indices, agentRadius, maxSlopeDegreesconst cell = cellOf(worldX, worldZ);if (cell < 0) return `${label} [${worldX}, ${worldZ}] is outside the ground.`;if (heights[cell] === NO_SURFACE) return `${label} [${worldX}, ${worldZ}] has no ground under it (a hole).`;- return `${label} [${worldX}, ${worldZ}] is not walkable: it is on an obstacle or a steep slope, or within the agent radius ${radius} m of a hole or an obstacle.`;+ const nearest = nearestWalkableCenter(worldX, worldZ);+ const suggestion = nearest ? ` The nearest walkable point is [${nearest[0].toFixed(2)}, ${nearest[1].toFixed(2)}].` : '';+ return `${label} [${worldX}, ${worldZ}] is not walkable: it is on an obstacle or a steep slope, or within the agent radius ${radius} m of a hole or an obstacle.${suggestion}`;+ }++ function nearestWalkableCenter(worldX, worldZ) {+ let best = null;+ let bestDistance = Infinity;+ for (let cell = 0; cell < walkable.length; cell++) {+ if (walkable[cell] === 0) continue;+ const [centerX, centerZ] = centerOf(cell);+ const distance = Math.hypot(centerX - worldX, centerZ - worldZ);+ if (distance < bestDistance) {+ bestDistance = distance;+ best = [centerX, centerZ];+ }+ }+ return best;}/** Route from [x, z] to [x, z]: { points: [[x, y, z], ...], length, searchNodes }. Throws when the goal cannot be reached. */diff --git a/tests/agent-asset-mesh.test.js b/tests/agent-asset-mesh.test.jsindex 62a9a7bc..402a8271 100644--- a/tests/agent-asset-mesh.test.js+++ b/tests/agent-asset-mesh.test.js@@ -258,16 +258,17 @@ test('imported parts report as polygon meshes with the shared lint', async () =>asset.dispose();});-test('inspect accepts a source-node ID that holds one part and names the parts of one that holds several', async () => {+test('inspect accepts a source-node ID and lists its parts and those of the nodes below it', async () => {const scene = createScene(await loadRobot());const nodes = scene.inspect({ kind: 'hierarchy', limit: 256 }).nodes;const single = nodes.find(node => node.meshIds.length === 1);const several = nodes.find(node => node.meshIds.length > 1);- const empty = nodes.find(node => node.meshIds.length === 0);- assert.ok(single && several && empty, 'the robot has leaf, group and mesh-less nodes');+ const empty = nodes.find(node => node.meshIds.length === 0 && !nodes.some(other => other.parent === node.id));+ assert.ok(single && several && empty, 'the robot has leaf, group and childless mesh-less nodes');const byNode = scene.inspect({ ids: [single.id] }).nodes;assert.deepEqual(byNode.map(node => node.id), single.meshIds);- assert.throws(() => scene.inspect({ ids: [several.id] }), error => error.message.includes(several.meshIds[0]) && /^geometry: got .*; expected one part ID, such as .*holds \d+ parts/.test(error.message));+ const bySeveral = scene.inspect({ ids: [several.id] }).nodes.map(node => node.id);+ assert.ok(several.meshIds.every(id => bySeveral.includes(id)));assert.throws(() => scene.inspect({ ids: [empty.id] }), /^RangeError: geometry: got .*; expected a source node with a mesh of its own, or a mesh ID such as "mesh-0"/);assert.throws(() => scene.inspect({ ids: ['nothing-here'] }), /^RangeError: geometry: got "nothing-here"; expected an imported mesh ID such as "mesh-0"/);});diff --git a/tests/agent-tool-error-causes.test.js b/tests/agent-tool-error-causes.test.jsnew file mode 100644index 00000000..99bcc9fe--- /dev/null+++ b/tests/agent-tool-error-causes.test.js@@ -0,0 +1,98 @@+import './support/load-agent-scene-modules.mjs';+import test from 'node:test';+import assert from 'node:assert/strict';+import { readFileSync } from 'node:fs';+import { Client } from '@modelcontextprotocol/client';+import { InMemoryTransport } from '@modelcontextprotocol/server';+import { createAgentMcpServer, readableErrorMessage } from '../scripts/mcp-server.mjs';+import { AGENT_SCENE_CAPABILITIES } from '../src/agent-scene.js';+import { createAgentAssetScene } from '../src/agent-asset.js';+import { createNavmesh } from '../src/navmesh.js';+import { createTerrain } from '../src/terrain.js';+import { parseGltf } from '../src/gltf.js';++// Failed scene_capabilities and scene_inspect calls of a Haiku bench run (issue 761): each test sends the input the agent sent.++async function askCapabilities(context, topic) {+ const runtime = {+ async capabilities() { return { scene: AGENT_SCENE_CAPABILITIES, phase: 'idle', availableIn: 'test' }; },+ async close() {},+ activeSessionId() { return null; },+ };+ const server = createAgentMcpServer(runtime);+ const client = new Client({ name: '67js-error-cause-test', version: '1' });+ const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();+ await server.connect(serverTransport);+ await client.connect(clientTransport);+ context.after(async () => {+ await client.close();+ await server.close();+ });+ const result = await client.callTool({ name: 'scene_capabilities', arguments: { topic } });+ return { ...result, text: result.content?.[0]?.text };+}++test('meshEdit.inset and meshEdit.extrude open the one or both branches that start with that name', async context => {+ const inset = await askCapabilities(context, 'meshEdit.inset');+ assert.notEqual(inset.isError, true);+ assert.deepEqual(Object.keys(inset.structuredContent.fields.branches), ['inset_region', 'inset_individual']);+ const extrude = await askCapabilities(context, 'meshEdit.extrude');+ assert.deepEqual(Object.keys(extrude.structuredContent.fields.branches), ['extrude_face_region', 'extrude_individual', 'extrude_edges']);+});++test('meshEdit.select says select is a field of the branches and lists the branches', async context => {+ const answer = await askCapabilities(context, 'meshEdit.select');+ assert.equal(answer.isError, true);+ assert.match(answer.text, /Branches: extrude_face_region/);+ assert.match(answer.text, /select is a field of those branches/);+});++test('environment.spotLights answers with the outline of that environment field', async context => {+ const answer = await askCapabilities(context, 'environment.spotLights');+ assert.notEqual(answer.isError, true);+ assert.match(JSON.stringify(answer.structuredContent.fields), /castShadow/);+ const unknown = await askCapabilities(context, 'environment.nothing');+ assert.equal(unknown.isError, true);+ assert.match(unknown.text, /Fields: .*spotLights/);+});++test('uvs and scene.terrain stand for the topics meshUv and terrain', async context => {+ assert.equal((await askCapabilities(context, 'uvs')).structuredContent.topic, 'meshUv');+ assert.equal((await askCapabilities(context, 'scene.terrain')).structuredContent.topic, 'terrain');+ assert.equal((await askCapabilities(context, 'nope')).isError, true);+});++test('an inspected source node without a mesh of its own lists the parts below it', async () => {+ const bytes = readFileSync(new URL('./fixtures/RobotExpressive.glb', import.meta.url));+ const asset = await parseGltf(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), { maxVertices: 500000 });+ const scene = createAgentAssetScene(asset, { format: '67js.asset', schemaVersion: 1, assetId: 'ab'.repeat(32) });+ const bare = scene.inspect({ kind: 'hierarchy', limit: 256 }).nodes.find(node => node.meshIds.length === 0 && asset.nodes.some(other => other.parent === node.id));+ assert.ok(bare, 'the robot has a group node without parts');+ const inspected = scene.inspect({ ids: [bare.id], includeBounds: true });+ assert.ok(inspected.nodes.length >= 1);+ assert.ok(inspected.nodes.every(node => node.bounds && node.id.startsWith('mesh-')));+});++test('includeBounds with kind hierarchy is accepted and says where bounds are', async () => {+ const bytes = readFileSync(new URL('./fixtures/RobotExpressive.glb', import.meta.url));+ const asset = await parseGltf(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), { maxVertices: 500000 });+ const scene = createAgentAssetScene(asset, { format: '67js.asset', schemaVersion: 1, assetId: 'ab'.repeat(32) });+ const report = scene.inspect({ kind: 'hierarchy', includeBounds: true, limit: 4 });+ assert.equal(report.kind, 'hierarchy');+ assert.match(report.note, /without kind/);+});++test('an unwalkable start names the nearest walkable point and loses the page stack', () => {+ const terrain = createTerrain({ size: 40, chunks: 1, levels: 1, resolution: 64, holes: [], height: () => 0 });+ const { positions, indices } = terrain.parts[0].geometry;+ const navmesh = createNavmesh({ positions, indices, obstacles: [{ min: [-17, -17], max: [-13, -13] }] });+ let thrown;+ try {+ navmesh.findPath([-15, -15], [15, 15]);+ } catch (error) {+ thrown = error;+ }+ assert.match(thrown.message, /Start \[-15, -15\] is not walkable.*nearest walkable point is \[-?\d+\.\d+, -?\d+\.\d+\]/);+ const pageError = new Error(`page.evaluate: RangeError: ${thrown.message}\n at Object.L [as findPath] (http://127.0.0.1:52976/navmesh.js:140:34)`);+ assert.doesNotMatch(readableErrorMessage(pageError), /http|\bat Object/);+});