Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Make the MCP environment schema match the backend and keep textured scenes renderable (closes #440)

Nguyen Xuan Anb36d5a92f205
diff --git a/changes/440.md b/changes/440.md
new file mode 100644
index 00000000..87ba41de
--- /dev/null
+++ b/changes/440.md
@@ -0,0 +1,2 @@
+- Make the MCP `environment` schema and the backend agree (closes #440). The schema now accepts `viewTransform`, `resolveCoplanar`, `objectReflections` and `look`, and `set-environment` accepts `ground`, which the schema already listed and docs/agent-look.md documents (`ground: false` removes the automatic floor). `tests/mcp-environment-schema.test.js` derives the backend list from `src/agent-look.js` and checks the schema and the documented fields against it.
+- Rebuild the preview renderer when an edit regroups parts of a scene with any texture map, not only pattern textures. `resolveCoplanar: 'offset'` on a scene with a placed textured GLB used to commit and then fail the next `scene_look` with "replaceParts can add or change only rigid, untextured parts."
diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.js
index e856a8d1..3b7a029b 100644
--- a/scripts/agent-preview-browser.js
+++ b/scripts/agent-preview-browser.js
@@ -17,6 +17,7 @@ import { parseGltf } from '../src/gltf.js';
import { assetCopyKey, recipeAssetNeeds } from '../src/agent-scene-assets.js';
import { exportGlb } from '../src/export.js';
import '../src/texture-update.js';
+import { hasPatternTextures } from './agent-preview-textures.js';
import { TEXTURE_SEQUENCE_LIMITS } from '../src/texture-sequence-limits.js';
import { weightTracksFromGltfAnimation } from '../src/export-skeleton.js';
import { clipFromGltfAnimation } from '../src/skeleton-pose.js';
@@ -152,10 +153,6 @@ function sceneIsTextured(scene) {
return hasPatternTextures(scene.parts) || (scene.sequenceTextures?.().length ?? 0) > 0;
}
-function hasPatternTextures(parts) {
- return parts.some(part => part.material.boxMapScale || part.material.emissiveMap);
-}
-
// replaceParts streams only untextured parts, so a topology change around pattern textures takes a new renderer.
function rebuildRenderer(record) {
record.renderer.dispose();
diff --git a/scripts/agent-preview-textures.js b/scripts/agent-preview-textures.js
new file mode 100644
index 00000000..729d27ef
--- /dev/null
+++ b/scripts/agent-preview-textures.js
@@ -0,0 +1,7 @@
+// Any texture map counts: replaceParts adds or changes textured parts only with an addon the preview does not load, so an
+// edit that regroups a textured part (a coplanar depth layer on an imported asset) takes a new renderer instead.
+const TEXTURE_MAP_FIELDS = ['map', 'normalMap', 'metallicRoughnessMap', 'occlusionMap', 'emissiveMap'];
+
+export function hasPatternTextures(parts) {
+ return parts.some(part => part.material.boxMapScale || TEXTURE_MAP_FIELDS.some(field => part.material[field]));
+}
diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjs
index 5cf22b9f..63442769 100644
--- a/scripts/mcp-server.mjs
+++ b/scripts/mcp-server.mjs
@@ -7,6 +7,7 @@ import { resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import packageMetadata from '../package.json' with { type: 'json' };
import { MODIFIER_OPERATIONS } from '../src/mesh-operations.js';
+import { VIEW_TRANSFORM_NAMES, LOOK_NAMES } from '../src/view-transform-names.js';
import { MATERIAL_PATTERN_TYPES, MATERIAL_MAPPINGS, LIGHTING_SETUP_NAMES, RENDER_PASS_NAMES, VIDEO_ENCODERS, MAXIMUM_JOB_SIDE, MAXIMUM_MEMORY_MIB } from '../src/agent-scene-capabilities.js';
import { MATERIAL_LAYER_SCALAR_RANGES, MATERIAL_LAYER_COLOR_RANGES } from '../src/material-layers.js';
import { SUBSURFACE_RANGES } from '../src/subsurface-limits.js';
@@ -223,6 +224,10 @@ const environment = z.strictObject({
volumeFog: volumeFog.optional().describe('Directional scattering; needs HDR.'),
liquid: liquidOptics.optional().describe('Optics for materials tagged liquid or foam.'),
lines: lines.nullable().optional(),
+ objectReflections: z.boolean().optional().describe("false turns off reflection probes on smooth metal parts."),
+ resolveCoplanar: z.union([z.literal('offset'), z.literal(false)]).optional().describe("'offset' biases the smaller part of each coplanar pair at draw time; false clears."),
+ viewTransform: z.enum(VIEW_TRANSFORM_NAMES).optional().describe('Display transform, for example agx to compare with Blender.'),
+ look: z.enum(LOOK_NAMES).nullable().optional().describe('Named look preset; null clears.'),
wind: z.strictObject({ strength: bounded(0, 4).optional(), direction: bounded(-360, 360).optional().describe('Degrees: 0 toward +x, 90 toward +z.') }).optional().describe('Sway of foliage and grass materials; strength defaults to 0.3.'),
});
const effects = z.strictObject({
diff --git a/src/agent-scene.js b/src/agent-scene.js
index cc67ea10..5830fe8f 100644
--- a/src/agent-scene.js
+++ b/src/agent-scene.js
@@ -1416,7 +1416,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {
knownKeys(operation, ['type', lookField], 'operation');
const patch = operation[lookField];
// Spot and area lights, cascades and sun links are absent from a recipe until it sets them.
- const optionalFields = lookField === 'environment' ? ['spotLights', 'areaLights', 'shadowCascades', 'keyAffects', 'objectReflections', 'lines', 'resolveCoplanar', 'viewTransform', 'look'] : [];
+ const optionalFields = lookField === 'environment' ? ['ground', 'spotLights', 'areaLights', 'shadowCascades', 'keyAffects', 'objectReflections', 'lines', 'resolveCoplanar', 'viewTransform', 'look'] : [];
knownKeys(patch, [...Object.keys(candidate[lookField]), ...optionalFields], lookField);
const nestedField = lookField === 'environment' ? 'volumeFog' : lookField === 'effects' ? 'bloom' : null;
const merged = { ...candidate[lookField], ...(lookField === 'environment' ? environmentPatchWithSetup(patch) : patch) };
diff --git a/tests/mcp-environment-schema.test.js b/tests/mcp-environment-schema.test.js
new file mode 100644
index 00000000..579d4781
--- /dev/null
+++ b/tests/mcp-environment-schema.test.js
@@ -0,0 +1,95 @@
+import './support/load-agent-scene-modules.mjs';
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { Client } from '@modelcontextprotocol/client';
+import { InMemoryTransport } from '@modelcontextprotocol/server';
+import { createAgentMcpServer } from '../scripts/mcp-server.mjs';
+import { createAgentScene } from '../src/agent-scene.js';
+import { hasPatternTextures } from '../scripts/agent-preview-textures.js';
+
+async function sourceText(path) {
+ return readFile(new URL(`../${path}`, import.meta.url), 'utf8');
+}
+
+async function backendEnvironmentFields() {
+ const source = await sourceText('src/agent-look.js');
+ const list = source.match(/const ENVIRONMENT_FIELDS = \[([^\]]+)\]/);
+ assert.ok(list, 'ENVIRONMENT_FIELDS moved out of src/agent-look.js');
+ return [...list[1].matchAll(/'(\w+)'/g)].map(match => match[1]);
+}
+
+async function documentedEnvironmentFields() {
+ const text = (await sourceText('docs/agent-look.md')).replaceAll('\r\n', '\n');
+ const section = text.slice(text.indexOf('\n## Environment\n'));
+ const table = section.slice(0, section.indexOf('\n\n', section.indexOf('| --- |')));
+ return [...table.matchAll(/^\| `(\w+)` \|/gm)].map(match => match[1]);
+}
+
+function findSetEnvironmentSchema(node) {
+ if (!node || typeof node !== 'object') return null;
+ if (node.properties?.type?.const === 'set-environment' && node.properties.environment) return node.properties.environment;
+ for (const value of Object.values(node)) {
+ const found = findSetEnvironmentSchema(value);
+ if (found) return found;
+ }
+ return null;
+}
+
+async function mcpEnvironmentFields() {
+ const server = createAgentMcpServer({ async close() {} });
+ const client = new Client({ name: '67js-environment-schema-test', version: '1' });
+ const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+ await server.connect(serverTransport);
+ await client.connect(clientTransport);
+ const { tools } = await client.listTools();
+ await client.close();
+ await server.close();
+ const schema = findSetEnvironmentSchema(tools.find(tool => tool.name === 'scene_apply').inputSchema);
+ assert.ok(schema, 'scene_apply no longer lists a set-environment operation');
+ return Object.keys(schema.properties);
+}
+
+test('the MCP environment schema lists exactly the fields the backend allows', async () => {
+ const backend = (await backendEnvironmentFields()).sort();
+ assert.deepEqual((await mcpEnvironmentFields()).sort(), backend);
+});
+
+test('every documented environment field is in the MCP schema', async () => {
+ const schema = await mcpEnvironmentFields();
+ const documented = await documentedEnvironmentFields();
+ assert.ok(documented.length > 10);
+ assert.deepEqual(documented.filter(field => !schema.includes(field)), []);
+ for (const field of ['viewTransform', 'resolveCoplanar', 'ground']) assert.ok(schema.includes(field), field);
+});
+
+function coplanarRecipe() {
+ return {
+ format: '67js.scene', schemaVersion: 1,
+ geometries: [{ id: 'box', type: 'box', bevel: 0 }],
+ materials: [{ id: 'paint', color: [0.5, 0.5, 0.5] }],
+ nodes: [
+ { id: 'wall', geometry: 'box', material: 'paint', scale: [10, 4, 0.2] },
+ { id: 'sign', geometry: 'box', material: 'paint', scale: [1, 0.5, 0.2] },
+ ],
+ };
+}
+
+test('set-environment accepts ground, viewTransform and resolveCoplanar', () => {
+ const scene = createAgentScene(coplanarRecipe());
+ scene.apply([{ type: 'set-environment', environment: { ground: false, viewTransform: 'agx', resolveCoplanar: 'offset' } }], { expectedRevision: scene.revision });
+ assert.equal(scene.environment.ground, false);
+ assert.equal(scene.environment.viewTransform, 'agx');
+ assert.equal(scene.environment.resolveCoplanar, 'offset');
+});
+
+test('a coplanar depth layer regroups parts, and the preview sees every texture map so it rebuilds the renderer', () => {
+ const scene = createAgentScene(coplanarRecipe());
+ const before = scene.topologyRevision;
+ scene.apply([{ type: 'set-environment', environment: { resolveCoplanar: 'offset' } }], { expectedRevision: scene.revision });
+ assert.ok(scene.topologyRevision > before, 'the edit changes topology, which replaceParts refuses for textured parts');
+ for (const field of ['map', 'normalMap', 'metallicRoughnessMap', 'occlusionMap', 'emissiveMap']) {
+ assert.equal(hasPatternTextures([{ material: { [field]: {} } }]), true, field);
+ }
+ assert.equal(hasPatternTextures(scene.parts), false);
+});
diff --git a/tests/support/preview-harness.mjs b/tests/support/preview-harness.mjs
index 9a9d4e30..d97ab398 100644
--- a/tests/support/preview-harness.mjs
+++ b/tests/support/preview-harness.mjs
@@ -7,6 +7,7 @@ import { resolveAgentShadow, agentEnvironmentMap } from '../../src/agent-look.js
import { agentSceneModule, agentSceneModuleLoaded } from '../../src/agent-scene-modules.js';
import { createAgentAssetScene } from '../../src/agent-asset.js';
import { parseGltf } from '../../src/gltf.js';
+import { hasPatternTextures } from '../../scripts/agent-preview-textures.js';
import { assetCopyKey, recipeAssetNeeds } from '../../src/agent-scene-assets.js';
import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../../src/agent-branches.js';
import { sessionProblem } from '../../src/agent-errors.js';
@@ -74,7 +75,7 @@ export function previewHarness() {
// The harness loads Manifold from node_modules directly, so the preview's served-file loader is ignored.
setManifoldModuleLoader: () => {},
needsPhysics, loadPhysicsEngine, physicsEngineLoaded, setPhysicsModuleLoader: () => {},
- parseGltf, Uint8Array, atob, sessionProblem, assetCopyKey, recipeAssetNeeds,
+ parseGltf, hasPatternTextures, Uint8Array, atob, sessionProblem, assetCopyKey, recipeAssetNeeds,
applyProjection, mat4, resolveAgentShadow, agentEnvironmentMap, BRANCH_LIMITS,
// Line art needs a real renderer with render passes, so the fault-injected one gets a no-op install.
agentSceneModule: name => (name === 'lineArt' ? { installLineArt: renderer => renderer } : name === 'userPass' ? { ...agentSceneModule(name), installUserPasses: renderer => renderer } : agentSceneModule(name)), agentSceneModuleLoaded, createBranchStore, printableLabel,