3D engine that AI agents drive
Query large worlds in the preview page instead of a server-side parser (issue #559)
Nguyen Xuan An8a7468374de1
diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex 3a2e690f..904f29b9 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -26,7 +26,7 @@ import { clipFromGltfAnimation } from '../src/skeleton-pose.js';import { sessionProblem } from '../src/agent-errors.js';import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../src/agent-branches.js';import { agentAssetDecodeOptions, withCatalogLimitAdvice } from '../src/agent-asset-limits.js';-import { sceneQueryFor, summarizeWalkMap, walkOptionsFor } from './agent-walkmap.js';+import { sceneQueryFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache } from './agent-walkmap.js';let active = null;let canvas = document.querySelector('canvas');@@ -413,6 +413,7 @@ setPhysicsModuleLoader(() => import(/* @vite-ignore */ new URL('/vendor/rapier.j// The preview bundle is one file, so the on-demand scene modules load together on the first call and later calls stay// synchronous. The boolean engine is WebAssembly and loads asynchronously, once, only for calls that need it.+let queryOnly;const ASSET_DECODE_LIMITS = agentAssetDecodeOptions();// An agent cannot pass loadGltf options, so a limit error names the catalog limits instead of "Fix: pass { ... }".@@ -852,6 +853,10 @@ window.__agentPreview = {if (!allAgentSceneModulesLoaded()) return loadAllAgentSceneModules().then(() => this.inspectMeshes({ sessionId, ...options }));return { sessionId, ...scene.inspectMeshes(options) };},+ async queryOnlyWalkMap({ data, ...input }) {+ queryOnly ??= createQueryOnlyCache(parseGltf);+ return queryOnly.walkMap({ ...input, bytes: data === undefined ? undefined : bytesFromBase64(data) });+ },async walkMap({ sessionId, clearance = 2, groundRange, keepOut, streetY, includeCells = false, ...options }) {const record = requireSession(sessionId);if (!allAgentSceneModulesLoaded()) await loadAllAgentSceneModules();diff --git a/scripts/agent-query-asset.mjs b/scripts/agent-query-asset.mjsindex 05575edd..4df5e031 100644--- a/scripts/agent-query-asset.mjs+++ b/scripts/agent-query-asset.mjs@@ -1,18 +1,5 @@-import { parseGltf } from '../src/gltf.js';-import { createSceneQuery } from '../src/scene-query.js';-import { summarizeWalkMap, walkOptionsFor } from './agent-walkmap.js';--// Query-only load path: raycasts and walk maps over a local catalog GLB that is too big to render through MCP. The file is-// read and parsed in the server process (never sent to the preview page), with its textures dropped before decoding, so-// only positions, indices and node matrices are held. Every number below is a hard ceiling the agent cannot raise.-export const QUERY_ASSET_LIMITS = Object.freeze({- maxVertices: 6_000_000,- maxTriangles: 6_000_000,- maxParts: 16_384,- // Per part BVH budget; the mesh query itself refuses a part over 1,000,000 triangles.- memoryBudgetMiB: 256,-});-+// Server side of the query-only load path: the GLB a catalog file holds, without its images and texture references, so the+// preview page parses geometry only. The page owns the parser and the budget (QUERY_ASSET_LIMITS in agent-walkmap.js).const JSON_CHUNK = 0x4e4f534a;const TEXTURE_KEY = /Texture$/;@@ -51,48 +38,3 @@ export function geometryOnlyGlb(bytes) {header.writeUInt32LE(JSON_CHUNK, 16);return Buffer.concat([header, jsonBytes, rest]);}--/** Parses the geometry of a catalog GLB inside the query-only limits and returns a scene query over it. */-export async function loadQueryOnlyScene(bytes) {- const stripped = geometryOnlyGlb(bytes);- const buffer = stripped.buffer.slice(stripped.byteOffset, stripped.byteOffset + stripped.byteLength);- const asset = await parseGltf(buffer, { maxBytes: bytes.length + 1024, maxVertices: QUERY_ASSET_LIMITS.maxVertices, maxParts: QUERY_ASSET_LIMITS.maxParts });- let triangles = 0;- let vertices = 0;- for (const part of asset.parts) {- triangles += part.geometry.indices.length / 3;- vertices += part.geometry.positions.length / 3;- }- if (triangles > QUERY_ASSET_LIMITS.maxTriangles) {- throw new RangeError(`Query-only asset has ${triangles} triangles, over the query maxTriangles ${QUERY_ASSET_LIMITS.maxTriangles}. Split it or decimate it.`);- }- const parts = asset.parts.map((part, position) => ({ id: part.name ?? `part-${position}`, geometry: part.geometry, matrix: part.matrix, material: part.material?.id }));- const query = createSceneQuery(parts, { memoryBudgetMiB: QUERY_ASSET_LIMITS.memoryBudgetMiB });- return { query, parts: parts.length, triangles, vertices };-}--/** One loaded scene at a time, keyed by the catalog asset ID. A different asset replaces and disposes the first. */-export function createQueryOnlyCache() {- let current = null;- return Object.freeze({- async walkMap({ assetId, bytes, clearance = 2, groundRange, keepOut, streetY, includeCells = false, ...options }) {- const started = performance.now();- if (current?.assetId !== assetId) {- current?.scene.query.dispose();- current = null;- current = { assetId, scene: await loadQueryOnlyScene(await bytes()) };- }- const { query, parts, triangles } = current.scene;- const map = query.walkMap(walkOptionsFor(options, keepOut, streetY));- return {- assetId, queryOnly: true, parts, triangles,- ...summarizeWalkMap(map, { clearance, groundRange, keepOut, includeCells }),- skippedParts: query.skipped.length, milliseconds: Math.round(performance.now() - started),- };- },- dispose() {- current?.scene.query.dispose();- current = null;- },- });-}diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjsindex 177363ab..8deba9e9 100644--- a/scripts/agent-runtime.mjs+++ b/scripts/agent-runtime.mjs@@ -15,7 +15,7 @@ import { AGENT_SCENE_CAPABILITIES, AGENT_SCENE_DEFAULT_LIMITS, RENDER_PASS_NAMESimport { PATH_TRACER_CAPABILITIES } from '../src/path-tracer-capabilities.js';import { chooseDenoiser, finishWithOidn } from './agent-path-trace-denoise.mjs';import { createAgentAssetCatalog } from './agent-asset-catalog.mjs';-import { createQueryOnlyCache } from './agent-query-asset.mjs';+import { geometryOnlyGlb } from './agent-query-asset.mjs';import { AGENT_ASSET_LIMITS, assertCombinedAssetBytes } from '../src/agent-asset-limits.js';import { recipeAssetNeeds } from '../src/agent-scene-assets.js';import { sequenceFrameIds } from '../src/agent-texture-sequence.js';@@ -43,6 +43,7 @@ const MAXIMUM_QUEUED_CALLS = 16;const MAXIMUM_GLB_BYTES = 32 * 1024 * 1024;const MAXIMUM_KEPT_GLBS = 8;const EVALUATION_TIMEOUT_MS = 30000;+const QUERY_ONLY_TIMEOUT_MS = 120000;/** A bake runs on a worker for up to its own limit, so a call that bakes gets that much more than a plain call. */const BAKE_EVALUATION_TIMEOUT_MS = EVALUATION_TIMEOUT_MS + BAKE_WORKER_MAXIMUM_SECONDS * 1000;@@ -106,7 +107,6 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browlet queue = Promise.resolve();let closing;let catalogPromise;- const queryOnly = createQueryOnlyCache();let jobsPromise;const savedGlbPaths = [];function jobs() {@@ -197,7 +197,6 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow}async function shutdownResources() {- queryOnly.dispose();try {await browserInstance?.close();} finally {@@ -643,8 +642,16 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browwalkMap({ assetId, ...input }) {if (assetId === undefined) return enqueue(() => invoke('walkMap', input));if (input.sessionId !== undefined) return Promise.reject(new TypeError('scene_walkmap takes a sessionId or an assetId, not both.'));- // Query-only: parsed in this process from the catalog file and never rendered, so it may pass the render vertex cap.- return enqueue(() => queryOnly.walkMap({ ...input, assetId, bytes: async () => (await catalog()).read(assetId) }));+ // Query-only: the preview page parses the textureless geometry and queries it, with no scene and no render, so it may+ // pass the render vertex cap. The page keeps one scene and asks for the file only when it does not hold it.+ return enqueue(async () => {+ await boot();+ const request = { ...input, assetId };+ const cached = await evaluate('queryOnlyWalkMap', request, page, QUERY_ONLY_TIMEOUT_MS);+ if (!cached.needsData) return cached;+ const stripped = geometryOnlyGlb(await (await catalog()).read(assetId));+ return evaluate('queryOnlyWalkMap', { ...request, data: stripped.toString('base64') }, page, QUERY_ONLY_TIMEOUT_MS);+ });},setMeshStack(input) {return enqueue(async () => {diff --git a/scripts/agent-walkmap.js b/scripts/agent-walkmap.jsindex 87846535..81b8dcb9 100644--- a/scripts/agent-walkmap.js+++ b/scripts/agent-walkmap.js@@ -69,3 +69,52 @@ export function summarizeWalkMap(map, { clearance, groundRange, keepOut, include...(includeCells ? { cells: listed } : {}),};}++// Query-only load path: raycasts and walk maps over a catalog GLB that is too big to render through MCP. The server strips+// its textures and sends the file once; this page parses geometry only and keeps one scene query, with no render targets.+// Every number is a ceiling the agent cannot raise.+export const QUERY_ASSET_LIMITS = Object.freeze({+ maxVertices: 6_000_000,+ maxTriangles: 6_000_000,+ maxParts: 16_384,+ // Per part BVH budget; the mesh query itself refuses a part over 1,000,000 triangles.+ memoryBudgetMiB: 256,+});++/** One loaded scene at a time, keyed by the catalog asset ID. Without `bytes` an unknown asset answers `needsData`. */+export function createQueryOnlyCache(parseGltf) {+ let current = null;+ async function load(bytes) {+ const buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);+ const asset = await parseGltf(buffer, { maxBytes: bytes.length + 1024, maxVertices: QUERY_ASSET_LIMITS.maxVertices, maxParts: QUERY_ASSET_LIMITS.maxParts });+ let triangles = 0;+ for (const part of asset.parts) triangles += part.geometry.indices.length / 3;+ if (triangles > QUERY_ASSET_LIMITS.maxTriangles) {+ throw new RangeError(`Query-only asset has ${triangles} triangles, over the query maxTriangles ${QUERY_ASSET_LIMITS.maxTriangles}. Split it or decimate it.`);+ }+ const parts = asset.parts.map((part, position) => ({ id: part.name ?? `part-${position}`, geometry: part.geometry, matrix: part.matrix, material: part.material?.id }));+ return { query: createSceneQuery(parts, { memoryBudgetMiB: QUERY_ASSET_LIMITS.memoryBudgetMiB }), parts: parts.length, triangles };+ }+ return Object.freeze({+ async walkMap({ assetId, bytes, clearance = 2, groundRange, keepOut, streetY, includeCells = false, ...options }) {+ const started = performance.now();+ if (current?.assetId !== assetId) {+ if (!bytes) return { needsData: true };+ current?.scene.query.dispose();+ current = null;+ current = { assetId, scene: await load(bytes) };+ }+ const { query, parts, triangles } = current.scene;+ const map = query.walkMap(walkOptionsFor(options, keepOut, streetY));+ return {+ assetId, queryOnly: true, parts, triangles,+ ...summarizeWalkMap(map, { clearance, groundRange, keepOut, includeCells }),+ skippedParts: query.skipped.length, milliseconds: Math.round(performance.now() - started),+ };+ },+ dispose() {+ current?.scene.query.dispose();+ current = null;+ },+ });+}diff --git a/tests/agent-query-asset.test.js b/tests/agent-query-asset.test.jsindex edf3d4c3..ee9ea1aa 100644--- a/tests/agent-query-asset.test.js+++ b/tests/agent-query-asset.test.js@@ -1,33 +1,25 @@import test from 'node:test';import assert from 'node:assert/strict';import { carAsset } from './fixtures/car-glb.js';-import { createQueryOnlyCache, geometryOnlyGlb, loadQueryOnlyScene, QUERY_ASSET_LIMITS } from '../scripts/agent-query-asset.mjs';+import { geometryOnlyGlb } from '../scripts/agent-query-asset.mjs';+import { createQueryOnlyCache } from '../scripts/agent-walkmap.js';+import { parseGltf } from '../src/gltf.js';-test('geometryOnlyGlb keeps the geometry readable', async () => {- const stripped = geometryOnlyGlb(Buffer.from(carAsset()));- const scene = await loadQueryOnlyScene(stripped);- assert.ok(scene.parts > 0);- assert.ok(scene.triangles > 0);- assert.ok(scene.triangles <= QUERY_ASSET_LIMITS.maxTriangles);- scene.query.dispose();-});+const region = { min: [-3, -3], max: [3, 3], cellSize: 1, topY: 20, clearance: 1 };-test('the query-only cache walks a catalog asset, reuses its scene and replaces it for another asset', async () => {- const cache = createQueryOnlyCache();- let reads = 0;- const bytes = async () => {- reads++;- return Buffer.from(carAsset());- };- const region = { min: [-3, -3], max: [3, 3], cellSize: 1, topY: 20, clearance: 1 };- const first = await cache.walkMap({ assetId: 'a'.repeat(64), bytes, ...region });- assert.equal(first.queryOnly, true);- assert.equal(first.grid.length, 7);- assert.ok(first.walkableCells > 0);- await cache.walkMap({ assetId: 'a'.repeat(64), bytes, ...region, keepOut: 0.5, streetY: 0 });- assert.equal(reads, 1);- await cache.walkMap({ assetId: 'b'.repeat(64), bytes, ...region });- assert.equal(reads, 2);+test('the query-only cache asks for the file once, walks it and replaces it for another asset', async () => {+ const cache = createQueryOnlyCache(parseGltf);+ const bytes = new Uint8Array(geometryOnlyGlb(Buffer.from(carAsset())));+ const first = { assetId: 'a'.repeat(64), ...region };+ assert.deepEqual(await cache.walkMap(first), { needsData: true });+ const result = await cache.walkMap({ ...first, bytes });+ assert.equal(result.queryOnly, true);+ assert.equal(result.grid.length, 7);+ assert.ok(result.walkableCells > 0);+ assert.ok(result.triangles > 0);+ const guarded = await cache.walkMap({ ...first, keepOut: 0.5, streetY: 0 });+ assert.equal(guarded.queryOnly, true);+ assert.deepEqual(await cache.walkMap({ assetId: 'b'.repeat(64), ...region }), { needsData: true });cache.dispose();});diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.jsindex b8236f07..2c10ca53 100644--- a/tests/agent-runtime.test.js+++ b/tests/agent-runtime.test.js@@ -113,7 +113,6 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = faVEGETATION_CAPABILITY_NOTE, CROWD_CAPABILITY_NOTE, USER_PASS_CAPABILITIES, MESH_BLEND_CAPABILITIES, MATERIAL_SNIPPET_CAPABILITIES, AGENT_SCENE_CAPABILITIES, AGENT_SCENE_DEFAULT_LIMITS, RENDER_PASS_NAMES, RENDER_PASS_DEPTH_RANGE, VIDEO_ENCODERS, MAXIMUM_VIDEO_SECONDS, MAXIMUM_JOB_SIDE, MAXIMUM_JOB_PIXELS, MAXIMUM_MEMORY_MIB, BAKE_WORKER_MAXIMUM_SECONDS, PATH_TRACER_CAPABILITIES,recipeAssetNeeds, sequenceFrameIds, TEXTURE_SEQUENCE_CAPABILITIES, AGENT_ASSET_LIMITS, assertCombinedAssetBytes,audioInputArguments, createAudioLibrary, measureStreamSeconds, filmTimelineDuration, filmTimelineShotStarts, sampleFilmTimeline, normalizeReview, REVIEW_LIMITS, isPreviewHost, BRANCH_LIMITS, validateBranchName,- createQueryOnlyCache: () => ({ walkMap: async () => ({}), dispose() {} }),chromium: { launch: async () => browser }, URL, Buffer, process, performance, clearTimeout,setTimeout: (callback, delay) => { deadlines.push(delay); return setTimeout(callback, delay); },});diff --git a/tests/support/preview-harness.mjs b/tests/support/preview-harness.mjsindex 4629dcb1..bbd9a08c 100644--- a/tests/support/preview-harness.mjs+++ b/tests/support/preview-harness.mjs@@ -13,7 +13,7 @@ import { assetCopyKey, recipeAssetNeeds } from '../../src/agent-scene-assets.js'import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../../src/agent-branches.js';import { sessionProblem } from '../../src/agent-errors.js';import { agentAssetDecodeOptions, withCatalogLimitAdvice } from '../../src/agent-asset-limits.js';-import { sceneQueryFor, summarizeWalkMap, walkOptionsFor } from '../../scripts/agent-walkmap.js';+import { sceneQueryFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache } from '../../scripts/agent-walkmap.js';// Runs the preview page's adapter code in a VM with a fault-injected renderer, the way the MCP preview runs it in a browser.const source = (await readFile(new URL('../../scripts/agent-preview-browser.js', import.meta.url), 'utf8')).replace(/^import .+;\r?$/gm, '');@@ -78,7 +78,7 @@ export function previewHarness() {// The harness loads Manifold from node_modules directly, so the preview's served-file loader is ignored.setManifoldModuleLoader: () => {},needsPhysics, loadPhysicsEngine, physicsEngineLoaded, setPhysicsModuleLoader: () => {},- sceneQueryFor, summarizeWalkMap, walkOptionsFor, parseGltf, hasPatternTextures, Uint8Array, atob, sessionProblem, assetCopyKey, recipeAssetNeeds, agentAssetDecodeOptions, withCatalogLimitAdvice,+ sceneQueryFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache, parseGltf, hasPatternTextures, Uint8Array, atob, sessionProblem, assetCopyKey, recipeAssetNeeds, agentAssetDecodeOptions, withCatalogLimitAdvice,applyProjection, mat4, resolveAgentShadow, agentEnvironmentMap, BRANCH_LIMITS,// Line art needs a real renderer with render passes, so the fault-injected one gets a no-op install.agentSceneModule: name => (name === 'lineArt' ? { installLineArt: renderer => renderer } : name === 'userPass' ? { ...agentSceneModule(name), installUserPasses: renderer => renderer } : agentSceneModule(name)), agentSceneModuleLoaded, createBranchStore, printableLabel,