Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Query large worlds in the preview page instead of a server-side parser (issue #559)

Nguyen Xuan An8a7468374de1
diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.js
index 3a2e690f..904f29b9 100644
--- a/scripts/agent-preview-browser.js
+++ b/scripts/agent-preview-browser.js
@@ -26,7 +26,7 @@ import { clipFromGltfAnimation } from '../src/skeleton-pose.js';
import { sessionProblem } from '../src/agent-errors.js';
import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../src/agent-branches.js';
import { agentAssetDecodeOptions, withCatalogLimitAdvice } from '../src/agent-asset-limits.js';
-import { sceneQueryFor, summarizeWalkMap, walkOptionsFor } from './agent-walkmap.js';
+import { sceneQueryFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache } from './agent-walkmap.js';
let active = null;
let canvas = document.querySelector('canvas');
@@ -413,6 +413,7 @@ setPhysicsModuleLoader(() => import(/* @vite-ignore */ new URL('/vendor/rapier.j
// The preview bundle is one file, so the on-demand scene modules load together on the first call and later calls stay
// synchronous. The boolean engine is WebAssembly and loads asynchronously, once, only for calls that need it.
+let queryOnly;
const ASSET_DECODE_LIMITS = agentAssetDecodeOptions();
// An agent cannot pass loadGltf options, so a limit error names the catalog limits instead of "Fix: pass { ... }".
@@ -852,6 +853,10 @@ window.__agentPreview = {
if (!allAgentSceneModulesLoaded()) return loadAllAgentSceneModules().then(() => this.inspectMeshes({ sessionId, ...options }));
return { sessionId, ...scene.inspectMeshes(options) };
},
+ async queryOnlyWalkMap({ data, ...input }) {
+ queryOnly ??= createQueryOnlyCache(parseGltf);
+ return queryOnly.walkMap({ ...input, bytes: data === undefined ? undefined : bytesFromBase64(data) });
+ },
async walkMap({ sessionId, clearance = 2, groundRange, keepOut, streetY, includeCells = false, ...options }) {
const record = requireSession(sessionId);
if (!allAgentSceneModulesLoaded()) await loadAllAgentSceneModules();
diff --git a/scripts/agent-query-asset.mjs b/scripts/agent-query-asset.mjs
index 05575edd..4df5e031 100644
--- a/scripts/agent-query-asset.mjs
+++ b/scripts/agent-query-asset.mjs
@@ -1,18 +1,5 @@
-import { parseGltf } from '../src/gltf.js';
-import { createSceneQuery } from '../src/scene-query.js';
-import { summarizeWalkMap, walkOptionsFor } from './agent-walkmap.js';
-
-// Query-only load path: raycasts and walk maps over a local catalog GLB that is too big to render through MCP. The file is
-// read and parsed in the server process (never sent to the preview page), with its textures dropped before decoding, so
-// only positions, indices and node matrices are held. Every number below is a hard ceiling the agent cannot raise.
-export const QUERY_ASSET_LIMITS = Object.freeze({
- maxVertices: 6_000_000,
- maxTriangles: 6_000_000,
- maxParts: 16_384,
- // Per part BVH budget; the mesh query itself refuses a part over 1,000,000 triangles.
- memoryBudgetMiB: 256,
-});
-
+// Server side of the query-only load path: the GLB a catalog file holds, without its images and texture references, so the
+// preview page parses geometry only. The page owns the parser and the budget (QUERY_ASSET_LIMITS in agent-walkmap.js).
const JSON_CHUNK = 0x4e4f534a;
const TEXTURE_KEY = /Texture$/;
@@ -51,48 +38,3 @@ export function geometryOnlyGlb(bytes) {
header.writeUInt32LE(JSON_CHUNK, 16);
return Buffer.concat([header, jsonBytes, rest]);
}
-
-/** Parses the geometry of a catalog GLB inside the query-only limits and returns a scene query over it. */
-export async function loadQueryOnlyScene(bytes) {
- const stripped = geometryOnlyGlb(bytes);
- const buffer = stripped.buffer.slice(stripped.byteOffset, stripped.byteOffset + stripped.byteLength);
- const asset = await parseGltf(buffer, { maxBytes: bytes.length + 1024, maxVertices: QUERY_ASSET_LIMITS.maxVertices, maxParts: QUERY_ASSET_LIMITS.maxParts });
- let triangles = 0;
- let vertices = 0;
- for (const part of asset.parts) {
- triangles += part.geometry.indices.length / 3;
- vertices += part.geometry.positions.length / 3;
- }
- if (triangles > QUERY_ASSET_LIMITS.maxTriangles) {
- throw new RangeError(`Query-only asset has ${triangles} triangles, over the query maxTriangles ${QUERY_ASSET_LIMITS.maxTriangles}. Split it or decimate it.`);
- }
- const parts = asset.parts.map((part, position) => ({ id: part.name ?? `part-${position}`, geometry: part.geometry, matrix: part.matrix, material: part.material?.id }));
- const query = createSceneQuery(parts, { memoryBudgetMiB: QUERY_ASSET_LIMITS.memoryBudgetMiB });
- return { query, parts: parts.length, triangles, vertices };
-}
-
-/** One loaded scene at a time, keyed by the catalog asset ID. A different asset replaces and disposes the first. */
-export function createQueryOnlyCache() {
- let current = null;
- return Object.freeze({
- async walkMap({ assetId, bytes, clearance = 2, groundRange, keepOut, streetY, includeCells = false, ...options }) {
- const started = performance.now();
- if (current?.assetId !== assetId) {
- current?.scene.query.dispose();
- current = null;
- current = { assetId, scene: await loadQueryOnlyScene(await bytes()) };
- }
- const { query, parts, triangles } = current.scene;
- const map = query.walkMap(walkOptionsFor(options, keepOut, streetY));
- return {
- assetId, queryOnly: true, parts, triangles,
- ...summarizeWalkMap(map, { clearance, groundRange, keepOut, includeCells }),
- skippedParts: query.skipped.length, milliseconds: Math.round(performance.now() - started),
- };
- },
- dispose() {
- current?.scene.query.dispose();
- current = null;
- },
- });
-}
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index 177363ab..8deba9e9 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -15,7 +15,7 @@ import { AGENT_SCENE_CAPABILITIES, AGENT_SCENE_DEFAULT_LIMITS, RENDER_PASS_NAMES
import { PATH_TRACER_CAPABILITIES } from '../src/path-tracer-capabilities.js';
import { chooseDenoiser, finishWithOidn } from './agent-path-trace-denoise.mjs';
import { createAgentAssetCatalog } from './agent-asset-catalog.mjs';
-import { createQueryOnlyCache } from './agent-query-asset.mjs';
+import { geometryOnlyGlb } from './agent-query-asset.mjs';
import { AGENT_ASSET_LIMITS, assertCombinedAssetBytes } from '../src/agent-asset-limits.js';
import { recipeAssetNeeds } from '../src/agent-scene-assets.js';
import { sequenceFrameIds } from '../src/agent-texture-sequence.js';
@@ -43,6 +43,7 @@ const MAXIMUM_QUEUED_CALLS = 16;
const MAXIMUM_GLB_BYTES = 32 * 1024 * 1024;
const MAXIMUM_KEPT_GLBS = 8;
const EVALUATION_TIMEOUT_MS = 30000;
+const QUERY_ONLY_TIMEOUT_MS = 120000;
/** A bake runs on a worker for up to its own limit, so a call that bakes gets that much more than a plain call. */
const BAKE_EVALUATION_TIMEOUT_MS = EVALUATION_TIMEOUT_MS + BAKE_WORKER_MAXIMUM_SECONDS * 1000;
@@ -106,7 +107,6 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
let queue = Promise.resolve();
let closing;
let catalogPromise;
- const queryOnly = createQueryOnlyCache();
let jobsPromise;
const savedGlbPaths = [];
function jobs() {
@@ -197,7 +197,6 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
}
async function shutdownResources() {
- queryOnly.dispose();
try {
await browserInstance?.close();
} finally {
@@ -643,8 +642,16 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
walkMap({ assetId, ...input }) {
if (assetId === undefined) return enqueue(() => invoke('walkMap', input));
if (input.sessionId !== undefined) return Promise.reject(new TypeError('scene_walkmap takes a sessionId or an assetId, not both.'));
- // Query-only: parsed in this process from the catalog file and never rendered, so it may pass the render vertex cap.
- return enqueue(() => queryOnly.walkMap({ ...input, assetId, bytes: async () => (await catalog()).read(assetId) }));
+ // Query-only: the preview page parses the textureless geometry and queries it, with no scene and no render, so it may
+ // pass the render vertex cap. The page keeps one scene and asks for the file only when it does not hold it.
+ return enqueue(async () => {
+ await boot();
+ const request = { ...input, assetId };
+ const cached = await evaluate('queryOnlyWalkMap', request, page, QUERY_ONLY_TIMEOUT_MS);
+ if (!cached.needsData) return cached;
+ const stripped = geometryOnlyGlb(await (await catalog()).read(assetId));
+ return evaluate('queryOnlyWalkMap', { ...request, data: stripped.toString('base64') }, page, QUERY_ONLY_TIMEOUT_MS);
+ });
},
setMeshStack(input) {
return enqueue(async () => {
diff --git a/scripts/agent-walkmap.js b/scripts/agent-walkmap.js
index 87846535..81b8dcb9 100644
--- a/scripts/agent-walkmap.js
+++ b/scripts/agent-walkmap.js
@@ -69,3 +69,52 @@ export function summarizeWalkMap(map, { clearance, groundRange, keepOut, include
...(includeCells ? { cells: listed } : {}),
};
}
+
+// Query-only load path: raycasts and walk maps over a catalog GLB that is too big to render through MCP. The server strips
+// its textures and sends the file once; this page parses geometry only and keeps one scene query, with no render targets.
+// Every number is a ceiling the agent cannot raise.
+export const QUERY_ASSET_LIMITS = Object.freeze({
+ maxVertices: 6_000_000,
+ maxTriangles: 6_000_000,
+ maxParts: 16_384,
+ // Per part BVH budget; the mesh query itself refuses a part over 1,000,000 triangles.
+ memoryBudgetMiB: 256,
+});
+
+/** One loaded scene at a time, keyed by the catalog asset ID. Without `bytes` an unknown asset answers `needsData`. */
+export function createQueryOnlyCache(parseGltf) {
+ let current = null;
+ async function load(bytes) {
+ const buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+ const asset = await parseGltf(buffer, { maxBytes: bytes.length + 1024, maxVertices: QUERY_ASSET_LIMITS.maxVertices, maxParts: QUERY_ASSET_LIMITS.maxParts });
+ let triangles = 0;
+ for (const part of asset.parts) triangles += part.geometry.indices.length / 3;
+ if (triangles > QUERY_ASSET_LIMITS.maxTriangles) {
+ throw new RangeError(`Query-only asset has ${triangles} triangles, over the query maxTriangles ${QUERY_ASSET_LIMITS.maxTriangles}. Split it or decimate it.`);
+ }
+ const parts = asset.parts.map((part, position) => ({ id: part.name ?? `part-${position}`, geometry: part.geometry, matrix: part.matrix, material: part.material?.id }));
+ return { query: createSceneQuery(parts, { memoryBudgetMiB: QUERY_ASSET_LIMITS.memoryBudgetMiB }), parts: parts.length, triangles };
+ }
+ return Object.freeze({
+ async walkMap({ assetId, bytes, clearance = 2, groundRange, keepOut, streetY, includeCells = false, ...options }) {
+ const started = performance.now();
+ if (current?.assetId !== assetId) {
+ if (!bytes) return { needsData: true };
+ current?.scene.query.dispose();
+ current = null;
+ current = { assetId, scene: await load(bytes) };
+ }
+ const { query, parts, triangles } = current.scene;
+ const map = query.walkMap(walkOptionsFor(options, keepOut, streetY));
+ return {
+ assetId, queryOnly: true, parts, triangles,
+ ...summarizeWalkMap(map, { clearance, groundRange, keepOut, includeCells }),
+ skippedParts: query.skipped.length, milliseconds: Math.round(performance.now() - started),
+ };
+ },
+ dispose() {
+ current?.scene.query.dispose();
+ current = null;
+ },
+ });
+}
diff --git a/tests/agent-query-asset.test.js b/tests/agent-query-asset.test.js
index edf3d4c3..ee9ea1aa 100644
--- a/tests/agent-query-asset.test.js
+++ b/tests/agent-query-asset.test.js
@@ -1,33 +1,25 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { carAsset } from './fixtures/car-glb.js';
-import { createQueryOnlyCache, geometryOnlyGlb, loadQueryOnlyScene, QUERY_ASSET_LIMITS } from '../scripts/agent-query-asset.mjs';
+import { geometryOnlyGlb } from '../scripts/agent-query-asset.mjs';
+import { createQueryOnlyCache } from '../scripts/agent-walkmap.js';
+import { parseGltf } from '../src/gltf.js';
-test('geometryOnlyGlb keeps the geometry readable', async () => {
- const stripped = geometryOnlyGlb(Buffer.from(carAsset()));
- const scene = await loadQueryOnlyScene(stripped);
- assert.ok(scene.parts > 0);
- assert.ok(scene.triangles > 0);
- assert.ok(scene.triangles <= QUERY_ASSET_LIMITS.maxTriangles);
- scene.query.dispose();
-});
+const region = { min: [-3, -3], max: [3, 3], cellSize: 1, topY: 20, clearance: 1 };
-test('the query-only cache walks a catalog asset, reuses its scene and replaces it for another asset', async () => {
- const cache = createQueryOnlyCache();
- let reads = 0;
- const bytes = async () => {
- reads++;
- return Buffer.from(carAsset());
- };
- const region = { min: [-3, -3], max: [3, 3], cellSize: 1, topY: 20, clearance: 1 };
- const first = await cache.walkMap({ assetId: 'a'.repeat(64), bytes, ...region });
- assert.equal(first.queryOnly, true);
- assert.equal(first.grid.length, 7);
- assert.ok(first.walkableCells > 0);
- await cache.walkMap({ assetId: 'a'.repeat(64), bytes, ...region, keepOut: 0.5, streetY: 0 });
- assert.equal(reads, 1);
- await cache.walkMap({ assetId: 'b'.repeat(64), bytes, ...region });
- assert.equal(reads, 2);
+test('the query-only cache asks for the file once, walks it and replaces it for another asset', async () => {
+ const cache = createQueryOnlyCache(parseGltf);
+ const bytes = new Uint8Array(geometryOnlyGlb(Buffer.from(carAsset())));
+ const first = { assetId: 'a'.repeat(64), ...region };
+ assert.deepEqual(await cache.walkMap(first), { needsData: true });
+ const result = await cache.walkMap({ ...first, bytes });
+ assert.equal(result.queryOnly, true);
+ assert.equal(result.grid.length, 7);
+ assert.ok(result.walkableCells > 0);
+ assert.ok(result.triangles > 0);
+ const guarded = await cache.walkMap({ ...first, keepOut: 0.5, streetY: 0 });
+ assert.equal(guarded.queryOnly, true);
+ assert.deepEqual(await cache.walkMap({ assetId: 'b'.repeat(64), ...region }), { needsData: true });
cache.dispose();
});
diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.js
index b8236f07..2c10ca53 100644
--- a/tests/agent-runtime.test.js
+++ b/tests/agent-runtime.test.js
@@ -113,7 +113,6 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fa
VEGETATION_CAPABILITY_NOTE, CROWD_CAPABILITY_NOTE, USER_PASS_CAPABILITIES, MESH_BLEND_CAPABILITIES, MATERIAL_SNIPPET_CAPABILITIES, AGENT_SCENE_CAPABILITIES, AGENT_SCENE_DEFAULT_LIMITS, RENDER_PASS_NAMES, RENDER_PASS_DEPTH_RANGE, VIDEO_ENCODERS, MAXIMUM_VIDEO_SECONDS, MAXIMUM_JOB_SIDE, MAXIMUM_JOB_PIXELS, MAXIMUM_MEMORY_MIB, BAKE_WORKER_MAXIMUM_SECONDS, PATH_TRACER_CAPABILITIES,
recipeAssetNeeds, sequenceFrameIds, TEXTURE_SEQUENCE_CAPABILITIES, AGENT_ASSET_LIMITS, assertCombinedAssetBytes,
audioInputArguments, createAudioLibrary, measureStreamSeconds, filmTimelineDuration, filmTimelineShotStarts, sampleFilmTimeline, normalizeReview, REVIEW_LIMITS, isPreviewHost, BRANCH_LIMITS, validateBranchName,
- createQueryOnlyCache: () => ({ walkMap: async () => ({}), dispose() {} }),
chromium: { launch: async () => browser }, URL, Buffer, process, performance, clearTimeout,
setTimeout: (callback, delay) => { deadlines.push(delay); return setTimeout(callback, delay); },
});
diff --git a/tests/support/preview-harness.mjs b/tests/support/preview-harness.mjs
index 4629dcb1..bbd9a08c 100644
--- a/tests/support/preview-harness.mjs
+++ b/tests/support/preview-harness.mjs
@@ -13,7 +13,7 @@ import { assetCopyKey, recipeAssetNeeds } from '../../src/agent-scene-assets.js'
import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../../src/agent-branches.js';
import { sessionProblem } from '../../src/agent-errors.js';
import { agentAssetDecodeOptions, withCatalogLimitAdvice } from '../../src/agent-asset-limits.js';
-import { sceneQueryFor, summarizeWalkMap, walkOptionsFor } from '../../scripts/agent-walkmap.js';
+import { sceneQueryFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache } from '../../scripts/agent-walkmap.js';
// Runs the preview page's adapter code in a VM with a fault-injected renderer, the way the MCP preview runs it in a browser.
const source = (await readFile(new URL('../../scripts/agent-preview-browser.js', import.meta.url), 'utf8')).replace(/^import .+;\r?$/gm, '');
@@ -78,7 +78,7 @@ export function previewHarness() {
// The harness loads Manifold from node_modules directly, so the preview's served-file loader is ignored.
setManifoldModuleLoader: () => {},
needsPhysics, loadPhysicsEngine, physicsEngineLoaded, setPhysicsModuleLoader: () => {},
- sceneQueryFor, summarizeWalkMap, walkOptionsFor, parseGltf, hasPatternTextures, Uint8Array, atob, sessionProblem, assetCopyKey, recipeAssetNeeds, agentAssetDecodeOptions, withCatalogLimitAdvice,
+ sceneQueryFor, summarizeWalkMap, walkOptionsFor, createQueryOnlyCache, parseGltf, hasPatternTextures, Uint8Array, atob, sessionProblem, assetCopyKey, recipeAssetNeeds, agentAssetDecodeOptions, withCatalogLimitAdvice,
applyProjection, mat4, resolveAgentShadow, agentEnvironmentMap, BRANCH_LIMITS,
// Line art needs a real renderer with render passes, so the fault-injected one gets a no-op install.
agentSceneModule: name => (name === 'lineArt' ? { installLineArt: renderer => renderer } : name === 'userPass' ? { ...agentSceneModule(name), installUserPasses: renderer => renderer } : agentSceneModule(name)), agentSceneModuleLoaded, createBranchStore, printableLabel,