3D engine that AI agents drive
Make the MCP environment schema match the backend and keep textured scenes renderable (closes #440)
Nguyen Xuan Anb36d5a92f205
diff --git a/changes/440.md b/changes/440.mdnew file mode 100644index 00000000..87ba41de--- /dev/null+++ b/changes/440.md@@ -0,0 +1,2 @@+- Make the MCP `environment` schema and the backend agree (closes #440). The schema now accepts `viewTransform`, `resolveCoplanar`, `objectReflections` and `look`, and `set-environment` accepts `ground`, which the schema already listed and docs/agent-look.md documents (`ground: false` removes the automatic floor). `tests/mcp-environment-schema.test.js` derives the backend list from `src/agent-look.js` and checks the schema and the documented fields against it.+- Rebuild the preview renderer when an edit regroups parts of a scene with any texture map, not only pattern textures. `resolveCoplanar: 'offset'` on a scene with a placed textured GLB used to commit and then fail the next `scene_look` with "replaceParts can add or change only rigid, untextured parts."diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex e856a8d1..3b7a029b 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -17,6 +17,7 @@ import { parseGltf } from '../src/gltf.js';import { assetCopyKey, recipeAssetNeeds } from '../src/agent-scene-assets.js';import { exportGlb } from '../src/export.js';import '../src/texture-update.js';+import { hasPatternTextures } from './agent-preview-textures.js';import { TEXTURE_SEQUENCE_LIMITS } from '../src/texture-sequence-limits.js';import { weightTracksFromGltfAnimation } from '../src/export-skeleton.js';import { clipFromGltfAnimation } from '../src/skeleton-pose.js';@@ -152,10 +153,6 @@ function sceneIsTextured(scene) {return hasPatternTextures(scene.parts) || (scene.sequenceTextures?.().length ?? 0) > 0;}-function hasPatternTextures(parts) {- return parts.some(part => part.material.boxMapScale || part.material.emissiveMap);-}-// replaceParts streams only untextured parts, so a topology change around pattern textures takes a new renderer.function rebuildRenderer(record) {record.renderer.dispose();diff --git a/scripts/agent-preview-textures.js b/scripts/agent-preview-textures.jsnew file mode 100644index 00000000..729d27ef--- /dev/null+++ b/scripts/agent-preview-textures.js@@ -0,0 +1,7 @@+// Any texture map counts: replaceParts adds or changes textured parts only with an addon the preview does not load, so an+// edit that regroups a textured part (a coplanar depth layer on an imported asset) takes a new renderer instead.+const TEXTURE_MAP_FIELDS = ['map', 'normalMap', 'metallicRoughnessMap', 'occlusionMap', 'emissiveMap'];++export function hasPatternTextures(parts) {+ return parts.some(part => part.material.boxMapScale || TEXTURE_MAP_FIELDS.some(field => part.material[field]));+}diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex 5cf22b9f..63442769 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -7,6 +7,7 @@ import { resolve } from 'node:path';import { pathToFileURL } from 'node:url';import packageMetadata from '../package.json' with { type: 'json' };import { MODIFIER_OPERATIONS } from '../src/mesh-operations.js';+import { VIEW_TRANSFORM_NAMES, LOOK_NAMES } from '../src/view-transform-names.js';import { MATERIAL_PATTERN_TYPES, MATERIAL_MAPPINGS, LIGHTING_SETUP_NAMES, RENDER_PASS_NAMES, VIDEO_ENCODERS, MAXIMUM_JOB_SIDE, MAXIMUM_MEMORY_MIB } from '../src/agent-scene-capabilities.js';import { MATERIAL_LAYER_SCALAR_RANGES, MATERIAL_LAYER_COLOR_RANGES } from '../src/material-layers.js';import { SUBSURFACE_RANGES } from '../src/subsurface-limits.js';@@ -223,6 +224,10 @@ const environment = z.strictObject({volumeFog: volumeFog.optional().describe('Directional scattering; needs HDR.'),liquid: liquidOptics.optional().describe('Optics for materials tagged liquid or foam.'),lines: lines.nullable().optional(),+ objectReflections: z.boolean().optional().describe("false turns off reflection probes on smooth metal parts."),+ resolveCoplanar: z.union([z.literal('offset'), z.literal(false)]).optional().describe("'offset' biases the smaller part of each coplanar pair at draw time; false clears."),+ viewTransform: z.enum(VIEW_TRANSFORM_NAMES).optional().describe('Display transform, for example agx to compare with Blender.'),+ look: z.enum(LOOK_NAMES).nullable().optional().describe('Named look preset; null clears.'),wind: z.strictObject({ strength: bounded(0, 4).optional(), direction: bounded(-360, 360).optional().describe('Degrees: 0 toward +x, 90 toward +z.') }).optional().describe('Sway of foliage and grass materials; strength defaults to 0.3.'),});const effects = z.strictObject({diff --git a/src/agent-scene.js b/src/agent-scene.jsindex cc67ea10..5830fe8f 100644--- a/src/agent-scene.js+++ b/src/agent-scene.js@@ -1416,7 +1416,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {knownKeys(operation, ['type', lookField], 'operation');const patch = operation[lookField];// Spot and area lights, cascades and sun links are absent from a recipe until it sets them.- const optionalFields = lookField === 'environment' ? ['spotLights', 'areaLights', 'shadowCascades', 'keyAffects', 'objectReflections', 'lines', 'resolveCoplanar', 'viewTransform', 'look'] : [];+ const optionalFields = lookField === 'environment' ? ['ground', 'spotLights', 'areaLights', 'shadowCascades', 'keyAffects', 'objectReflections', 'lines', 'resolveCoplanar', 'viewTransform', 'look'] : [];knownKeys(patch, [...Object.keys(candidate[lookField]), ...optionalFields], lookField);const nestedField = lookField === 'environment' ? 'volumeFog' : lookField === 'effects' ? 'bloom' : null;const merged = { ...candidate[lookField], ...(lookField === 'environment' ? environmentPatchWithSetup(patch) : patch) };diff --git a/tests/mcp-environment-schema.test.js b/tests/mcp-environment-schema.test.jsnew file mode 100644index 00000000..579d4781--- /dev/null+++ b/tests/mcp-environment-schema.test.js@@ -0,0 +1,95 @@+import './support/load-agent-scene-modules.mjs';+import test from 'node:test';+import assert from 'node:assert/strict';+import { readFile } from 'node:fs/promises';+import { Client } from '@modelcontextprotocol/client';+import { InMemoryTransport } from '@modelcontextprotocol/server';+import { createAgentMcpServer } from '../scripts/mcp-server.mjs';+import { createAgentScene } from '../src/agent-scene.js';+import { hasPatternTextures } from '../scripts/agent-preview-textures.js';++async function sourceText(path) {+ return readFile(new URL(`../${path}`, import.meta.url), 'utf8');+}++async function backendEnvironmentFields() {+ const source = await sourceText('src/agent-look.js');+ const list = source.match(/const ENVIRONMENT_FIELDS = \[([^\]]+)\]/);+ assert.ok(list, 'ENVIRONMENT_FIELDS moved out of src/agent-look.js');+ return [...list[1].matchAll(/'(\w+)'/g)].map(match => match[1]);+}++async function documentedEnvironmentFields() {+ const text = (await sourceText('docs/agent-look.md')).replaceAll('\r\n', '\n');+ const section = text.slice(text.indexOf('\n## Environment\n'));+ const table = section.slice(0, section.indexOf('\n\n', section.indexOf('| --- |')));+ return [...table.matchAll(/^\| `(\w+)` \|/gm)].map(match => match[1]);+}++function findSetEnvironmentSchema(node) {+ if (!node || typeof node !== 'object') return null;+ if (node.properties?.type?.const === 'set-environment' && node.properties.environment) return node.properties.environment;+ for (const value of Object.values(node)) {+ const found = findSetEnvironmentSchema(value);+ if (found) return found;+ }+ return null;+}++async function mcpEnvironmentFields() {+ const server = createAgentMcpServer({ async close() {} });+ const client = new Client({ name: '67js-environment-schema-test', version: '1' });+ const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();+ await server.connect(serverTransport);+ await client.connect(clientTransport);+ const { tools } = await client.listTools();+ await client.close();+ await server.close();+ const schema = findSetEnvironmentSchema(tools.find(tool => tool.name === 'scene_apply').inputSchema);+ assert.ok(schema, 'scene_apply no longer lists a set-environment operation');+ return Object.keys(schema.properties);+}++test('the MCP environment schema lists exactly the fields the backend allows', async () => {+ const backend = (await backendEnvironmentFields()).sort();+ assert.deepEqual((await mcpEnvironmentFields()).sort(), backend);+});++test('every documented environment field is in the MCP schema', async () => {+ const schema = await mcpEnvironmentFields();+ const documented = await documentedEnvironmentFields();+ assert.ok(documented.length > 10);+ assert.deepEqual(documented.filter(field => !schema.includes(field)), []);+ for (const field of ['viewTransform', 'resolveCoplanar', 'ground']) assert.ok(schema.includes(field), field);+});++function coplanarRecipe() {+ return {+ format: '67js.scene', schemaVersion: 1,+ geometries: [{ id: 'box', type: 'box', bevel: 0 }],+ materials: [{ id: 'paint', color: [0.5, 0.5, 0.5] }],+ nodes: [+ { id: 'wall', geometry: 'box', material: 'paint', scale: [10, 4, 0.2] },+ { id: 'sign', geometry: 'box', material: 'paint', scale: [1, 0.5, 0.2] },+ ],+ };+}++test('set-environment accepts ground, viewTransform and resolveCoplanar', () => {+ const scene = createAgentScene(coplanarRecipe());+ scene.apply([{ type: 'set-environment', environment: { ground: false, viewTransform: 'agx', resolveCoplanar: 'offset' } }], { expectedRevision: scene.revision });+ assert.equal(scene.environment.ground, false);+ assert.equal(scene.environment.viewTransform, 'agx');+ assert.equal(scene.environment.resolveCoplanar, 'offset');+});++test('a coplanar depth layer regroups parts, and the preview sees every texture map so it rebuilds the renderer', () => {+ const scene = createAgentScene(coplanarRecipe());+ const before = scene.topologyRevision;+ scene.apply([{ type: 'set-environment', environment: { resolveCoplanar: 'offset' } }], { expectedRevision: scene.revision });+ assert.ok(scene.topologyRevision > before, 'the edit changes topology, which replaceParts refuses for textured parts');+ for (const field of ['map', 'normalMap', 'metallicRoughnessMap', 'occlusionMap', 'emissiveMap']) {+ assert.equal(hasPatternTextures([{ material: { [field]: {} } }]), true, field);+ }+ assert.equal(hasPatternTextures(scene.parts), false);+});diff --git a/tests/support/preview-harness.mjs b/tests/support/preview-harness.mjsindex 9a9d4e30..d97ab398 100644--- a/tests/support/preview-harness.mjs+++ b/tests/support/preview-harness.mjs@@ -7,6 +7,7 @@ import { resolveAgentShadow, agentEnvironmentMap } from '../../src/agent-look.jsimport { agentSceneModule, agentSceneModuleLoaded } from '../../src/agent-scene-modules.js';import { createAgentAssetScene } from '../../src/agent-asset.js';import { parseGltf } from '../../src/gltf.js';+import { hasPatternTextures } from '../../scripts/agent-preview-textures.js';import { assetCopyKey, recipeAssetNeeds } from '../../src/agent-scene-assets.js';import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../../src/agent-branches.js';import { sessionProblem } from '../../src/agent-errors.js';@@ -74,7 +75,7 @@ export function previewHarness() {// The harness loads Manifold from node_modules directly, so the preview's served-file loader is ignored.setManifoldModuleLoader: () => {},needsPhysics, loadPhysicsEngine, physicsEngineLoaded, setPhysicsModuleLoader: () => {},- parseGltf, Uint8Array, atob, sessionProblem, assetCopyKey, recipeAssetNeeds,+ parseGltf, hasPatternTextures, Uint8Array, atob, sessionProblem, assetCopyKey, recipeAssetNeeds,applyProjection, mat4, resolveAgentShadow, agentEnvironmentMap, BRANCH_LIMITS,// Line art needs a real renderer with render passes, so the fault-injected one gets a no-op install.agentSceneModule: name => (name === 'lineArt' ? { installLineArt: renderer => renderer } : name === 'userPass' ? { ...agentSceneModule(name), installUserPasses: renderer => renderer } : agentSceneModule(name)), agentSceneModuleLoaded, createBranchStore, printableLabel,