3D engine that AI agents drive
Move the texture sequence hooks and the path query out of the scene entry (part of #339)
Nguyen Xuan An4de6d7456816
diff --git a/changes/339-sequence.size.md b/changes/339-sequence.size.mdindex 4ba82c47..8b3608d2 100644--- a/changes/339-sequence.size.md+++ b/changes/339-sequence.size.md@@ -3,6 +3,4 @@- tool:scene_apply: the same field in set-material and add-material.- tool:scene_create_asset: the same field in the shared material schema.- result:scene_capabilities: textureSequence documents the field, its frame rule, covered maps, image formats and limits for agents.-- scene:agent-scene: the recipe field map.sequence adds validation hooks, a per-material sequence runtime in the material handle and a textureSequences report to the scene entry (about 300 bytes gzip); the frame logic itself is in the lazy sequences module.-- scene:agent-scene+lazy: the new sequences module (frame selection, validation, descriptor runtime), the catalog image kinds and the page decoder for frames.-- goal:agent-scene: needs An to accept, it is about 300 bytes gzip over the goal for an image sequence map that cannot live outside the scene entry. If not accepted, the field can move behind a lazy module hook in a follow-up.+- scene:agent-scene+lazy: the sequence code (frame selection, validation, descriptor runtime) in the lazy assets module, and the path query that moved to the lazy navigation module.diff --git a/src/agent-navigation.js b/src/agent-navigation.jsindex df815a7f..9472554b 100644--- a/src/agent-navigation.js+++ b/src/agent-navigation.js@@ -53,3 +53,25 @@ export function queryAgentPath(query, ground, avoidBounds) {keyframes,};}++/** The path option of scene_inspect: checks the query, resolves the ground and avoided node IDs through `nodeGeometry` and finds the route. */+export function resolvePathQuery(query, nodeGeometry, boundsFor) {+ if (query === null || typeof query !== 'object' || Array.isArray(query)) throw new TypeError('path must be an object.');+ for (const key of Object.keys(query)) {+ if (!NAVIGATION_FIELDS.includes(key)) throw new TypeError(`Unknown path field: ${key}. Allowed fields: ${NAVIGATION_FIELDS.join(', ')}.`);+ }+ if (query.avoid !== undefined && (!Array.isArray(query.avoid) || query.avoid.length > 64)) throw new RangeError('path avoid must list at most 64 node IDs.');+ const { geometry, worldMatrix } = nodeGeometry(query.ground, 'ground');+ const positions = new Float32Array(geometry.positions.length);+ for (let offset = 0; offset < positions.length; offset += 3) {+ for (let axis = 0; axis < 3; axis++) {+ positions[offset + axis] = worldMatrix[axis] * geometry.positions[offset] + worldMatrix[4 + axis] * geometry.positions[offset + 1]+ + worldMatrix[8 + axis] * geometry.positions[offset + 2] + worldMatrix[12 + axis];+ }+ }+ const avoidBounds = (query.avoid ?? []).map(id => {+ const avoided = nodeGeometry(id, 'avoid');+ return boundsFor(avoided.geometry, avoided.worldMatrix);+ });+ return queryAgentPath(query, { positions, indices: geometry.indices }, avoidBounds);+}diff --git a/src/agent-scene-assets.js b/src/agent-scene-assets.jsindex 65fbe163..4284eaca 100644--- a/src/agent-scene-assets.js+++ b/src/agent-scene-assets.js@@ -7,6 +7,9 @@ import { normalizeDrive } from './agent-drive-schema.js';// geometry and the same material objects, so the renderer draws the copies as one instanced batch per material.// A placement with a `clip` gets its own decoded copy (an "instance" above 0), because a pose is state of one decoded asset.+// The `assets` module also serves map sequences, so a recipe with one loads no extra module entry in the scene.+export * from './agent-texture-sequence.js';+export const ASSET_PLACEMENT_LIMITS = Object.freeze({ maxPlacements: 512, maxAssets: 16, maxAnimatedPlacements: 16, maxOverrides: 32 });const ASSET_ID = /^[a-f0-9]{64}$/;diff --git a/src/agent-scene-modules.js b/src/agent-scene-modules.jsindex 23579441..905217d0 100644--- a/src/agent-scene-modules.js+++ b/src/agent-scene-modules.js@@ -26,7 +26,6 @@ const MODULE_LOADERS = {lightLinking: () => import('./light-linking.js'),cascades: () => import('./cascaded-shadows.js'),assets: () => import('./agent-scene-assets.js'),- sequences: () => import('./agent-texture-sequence.js'),vegetation: () => import('./agent-vegetation.js'),lineArt: () => import('./line-art.js'),viewTransform: () => import('./view-transform-shader.js'),@@ -63,8 +62,7 @@ const MODULE_MARKERS = {navigation: [],lightLinking: ['"affects"', '"keyAffects"'],cascades: ['"shadowCascades"'],- assets: ['"asset":{'],- sequences: ['"sequence":{'],+ assets: ['"asset":{', '"sequence":{'],vegetation: ['"vegetation":{', '"wind":{'],lineArt: ['"lines":{', '"toon":'],viewTransform: ['"viewTransform":"agx"', '"viewTransform":"neutral"', '"look":"'],diff --git a/src/agent-scene.js b/src/agent-scene.jsindex b7fab067..5387cb93 100644--- a/src/agent-scene.js+++ b/src/agent-scene.js@@ -24,9 +24,7 @@ const MAX_MESH_REPORTS = 16;const TERRAIN_SKY_VISIBILITY_BYTES = 4;const CAMERA_DEFAULTS = Object.freeze({ eye: [4, 3, 5], target: [0, 0, 0], up: [0, 1, 0], fov: 45, near: 0.1, far: 1000 });const NODE_FIELDS = ['translation', 'rotation', 'scale', 'parent', 'label', 'geometry', 'material', 'skin', 'relations', 'asset', 'vegetation', 'decal', 'depthLayer'];-const MATERIAL_FIELDS = ['color', 'metallic', 'roughness', 'emission', 'emissiveColor', 'opacity', 'alphaMode', 'doubleSided', 'pattern', 'subsurface', ...MATERIAL_LAYER_FIELDS];-// `map` is an image sequence (src/agent-texture-sequence.js). It is a recipe field, and the handle builds its descriptor.-const RECIPE_MATERIAL_FIELDS = [...MATERIAL_FIELDS, 'map'];+const MATERIAL_FIELDS = ['color', 'metallic', 'roughness', 'emission', 'emissiveColor', 'opacity', 'alphaMode', 'doubleSided', 'pattern', 'map', 'subsurface', ...MATERIAL_LAYER_FIELDS];const PATTERN_TEXTURE_FIELDS = ['map', 'normalMap', 'metallicRoughnessMap', 'emissiveMap', 'boxMapScale', 'boxMapAxis'];const CAMERA_FIELDS = Object.keys(CAMERA_DEFAULTS);const LOOK_OPERATION_FIELDS = Object.freeze({@@ -201,9 +199,9 @@ function copyMaterialLayers(value, result) {}function materialFrom(value) {- knownKeys(value, ['id', 'tag', ...RECIPE_MATERIAL_FIELDS], 'material');+ knownKeys(value, ['id', 'tag', ...MATERIAL_FIELDS], 'material');// A pattern brings its own colours, so the color field tints it and leaves it unchanged by default.- const defaultColor = value.pattern === undefined && value.map === undefined ? [0.5, 0.5, 0.5] : [1, 1, 1];+ const defaultColor = value.pattern === undefined ? [0.5, 0.5, 0.5] : [1, 1, 1];const graph = value.pattern?.type === 'graph' ? value.pattern.graph : {};// A graph that makes roughness, metallic or emission passes it through as written, so the factors default to 1.const graphFactor = (output, fallback) => (Object.hasOwn(graph ?? {}, output) ? 1 : fallback);@@ -212,15 +210,11 @@ function materialFrom(value) {if (!AGENT_SCENE_CAPABILITIES.materialTags.includes(result.tag)) {throw new RangeError(`Material tag must be one of ${AGENT_SCENE_CAPABILITIES.materialTags.join(', ')}.`);}+ if (value.map !== undefined) agentSceneModule('assets').applyMapSequence(value, result);if (value.pattern !== undefined) {if (result.tag !== 'surface') throw new RangeError('Material pattern needs the surface tag.');result.pattern = agentSceneModule('patterns').normalizeMaterialPattern(value.pattern);}- if (value.map !== undefined) {- if (result.tag !== 'surface') throw new RangeError('Material map needs the surface tag.');- if (value.pattern !== undefined) throw new TypeError('A material takes a pattern or a map sequence, not both.');- result.map = agentSceneModule('sequences').normalizeMapSequence(value.map);- }result.opacity = defaultValue(value.opacity, 1);finite(result.opacity, 'opacity');if (result.opacity < 0 || result.opacity > 1) {@@ -760,15 +754,16 @@ function geometryStats(records) {function batchSignature(material) {// Textures are fixed per renderer, so a changed pattern rebuilds it like a changed tag does.// Transmission moves a material to the pass that sees the opaque scene, which is fixed per renderer.- return `${material.tag}|${material.alphaMode}|${material.opacity}|${material.transmission > 0}|${material.doubleSided}|${JSON.stringify(material.pattern ?? null)}|${JSON.stringify(material.map ?? null)}`;+ return `${material.tag}|${material.alphaMode}|${material.opacity}|${material.transmission > 0}|${material.doubleSided}|${JSON.stringify(material.pattern ?? material.map ?? null)}`;}-function materialHandle(snapshot, sequence = null) {+function materialHandle(snapshot, sequence) {const holder = { snapshot, sequence };const output = {};for (const field of ['tag', ...MATERIAL_FIELDS]) {Object.defineProperty(output, field, {enumerable: true,+ configurable: true,get() {return holder.snapshot[field];},@@ -916,7 +911,7 @@ function publishPose(stagedState, pose) {if (matrix) Float32Array.prototype.set.call(part.matrix, matrix);}}- for (const handle of stagedState.materialHandles.values()) handle.holder.sequence?.seek(pose.time);+ for (const sequence of stagedState.sequences) sequence.seek(pose.time);stagedState.renderCamera = pose.camera;stagedState.motionTime = pose.time;const nextParts = partsWithCloth(stagedState.restParts, pose.clothGeometries);@@ -925,10 +920,6 @@ function publishPose(stagedState, pose) {return partsChanged;}-function hasSequences(sceneState) {- return [...sceneState.materialHandles.values()].some(handle => handle.holder.sequence);-}-// The recipe an agent wrote: without the geometries, materials and part nodes that placed assets generate.function authoredRecipe(recipe) {const authored = structuredClone(recipe);@@ -998,25 +989,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {}// The route is computed over the authored pose of the ground and of the nodes to avoid, like the bounds in scene_inspect.- function pathQuery(query) {- if (query === null || typeof query !== 'object' || Array.isArray(query)) throw new TypeError('path must be an object.');- const navigation = agentSceneModule('navigation');- knownKeys(query, navigation.NAVIGATION_FIELDS, 'path');- if (query.avoid !== undefined && (!Array.isArray(query.avoid) || query.avoid.length > 64)) throw new RangeError('path avoid must list at most 64 node IDs.');- const { geometry, worldMatrix } = drawnNodeGeometry(query.ground, 'ground');- const positions = new Float32Array(geometry.positions.length);- for (let offset = 0; offset < positions.length; offset += 3) {- for (let axis = 0; axis < 3; axis++) {- positions[offset + axis] = worldMatrix[axis] * geometry.positions[offset] + worldMatrix[4 + axis] * geometry.positions[offset + 1]- + worldMatrix[8 + axis] * geometry.positions[offset + 2] + worldMatrix[12 + axis];- }- }- const avoidBounds = (query.avoid ?? []).map(id => {- const avoided = drawnNodeGeometry(id, 'avoid');- return boundsFor(avoided.geometry, avoided.worldMatrix);- });- return navigation.queryAgentPath(query, { positions, indices: geometry.indices }, avoidBounds);- }+ const pathQuery = query => agentSceneModule('navigation').resolvePathQuery(query, drawnNodeGeometry, boundsFor);function ownedGeometryRecords() {const records = new Set(state?.geometryRecords.values() ?? []);@@ -1116,7 +1089,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {// The renderer keeps tag, blending, opacity and sidedness per batch, so a handle is reused only when all match.const existing = [preferredHandle, activeHandle].find(handle => handle && batchSignature(handle.holder.snapshot) === batchSignature(material));const assetOutput = assetRuntime.materialOutputs.get(material.id);- const handle = assetOutput ? assetMaterialHandle(snapshot, assetOutput, preferredHandle, activeHandle) : existing ?? materialHandle(snapshot, material.map ? agentSceneModule('sequences').createTextureSequence(material.id, material.map, options.sequenceFrames) : null);+ const handle = assetOutput ? assetMaterialHandle(snapshot, assetOutput, preferredHandle, activeHandle) : existing ?? materialHandle(snapshot, material.map && agentSceneModule('assets').createTextureSequence(material.id, material.map, options.sequenceFrames));materialHandles.set(material.id, handle);}const activeParts = new Map(state?.sourceRestParts.map(part => [part.name, part]) ?? []);@@ -1156,6 +1129,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {stats: geometryStats(geometryRecords.values()),};stagedState.assetRuntime = assetRuntime;+ stagedState.sequences = [...materialHandles.values()].map(handle => handle.holder.sequence).filter(Boolean);stagedState.localBounds = localBounds;stagedState.skinBindings = skinBindings;stagedState.skinStages = createSkinStages(skinBindings);@@ -1356,17 +1330,15 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {get motionTime() {return state.motionTime;},- /** The colour map of each sequence material: its descriptor, the frame to show and the pixels of that frame. */sequenceTextures() {- return [...state.materialHandles.values()].filter(handle => handle.holder.sequence)- .map(({ holder }) => ({ descriptor: holder.sequence.descriptor, index: holder.sequence.index, image: holder.sequence.image }));+ return state.sequences;},sampleMotion(time) {if (time !== null && (!Number.isFinite(time) || time < 0 || time > 3600)) {throw new RangeError('Motion time must be null or within zero and 3600 seconds.');}- if (time !== null && !state.motionSampler && !state.physicsSampler && !state.clothSampler && !state.particleSampler && !state.assetRuntime.animators.length && !hasSequences(state)) {- throw new RangeError('This procedural recipe has no motion, physics, cloth or particles, no placed asset clip and no map sequence.');+ if (time !== null && !state.motionSampler && !state.physicsSampler && !state.clothSampler && !state.particleSampler && !state.assetRuntime.animators.length && !state.sequences.length) {+ throw new RangeError('This procedural recipe has no motion, physics, cloth or particles, and no placed asset clip.');}if (publishPose(state, stagePose(state, time))) {topologyRevision++;@@ -1567,7 +1539,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {fields = NODE_FIELDS;target = candidate.nodes.find(node => node.id === operation.id);} else if (operation.type === 'set-material') {- fields = RECIPE_MATERIAL_FIELDS;+ fields = [...MATERIAL_FIELDS, 'map'];target = candidate.materials.find(material => material.id === operation.id);} else if (operation.type === 'set-camera') {fields = CAMERA_FIELDS;@@ -1628,7 +1600,7 @@ export function createAgentScene(input, inputLimits = {}, options = {}) {...(state.particleSampler ? { particles: state.particleSampler.report() } : {}),...(state.assetRuntime.vegetation ? { vegetation: state.assetRuntime.vegetation } : {}),...(state.assetRuntime.placements.length ? { assets: agentSceneModule('assets').describeAssetPlacements(state.assetRuntime, state.graph, geometry => state.localBounds.get(geometry)) } : {}),- ...(hasSequences(state) ? { textureSequences: [...state.materialHandles.values()].filter(handle => handle.holder.sequence).map(handle => handle.holder.sequence.describe()) } : {}),+ ...(state.sequences.length ? { textureSequences: state.sequences.map(sequence => sequence.describe()) } : {}),...(state.bakeReport ? { bake: state.bakeReport } : {}),...(state.reflectionReport ? { reflections: state.reflectionReport } : {}),...(state.recipe.armatures ? { rig: rigReport(state.recipe, state.skinBindings) } : {}),diff --git a/src/agent-texture-sequence.js b/src/agent-texture-sequence.jsindex 641824f6..28f036f5 100644--- a/src/agent-texture-sequence.js+++ b/src/agent-texture-sequence.js@@ -67,6 +67,15 @@ export function sequenceFrameIds(value) {return [...ids];}+/** The material `map` field of a recipe material: validates it against the rest of the material and stores it on `result`. */+export function applyMapSequence(value, result) {+ if (result.tag !== 'surface') throw new RangeError('Material map needs the surface tag.');+ if (value.pattern !== undefined) throw new TypeError('A material takes a pattern or a map sequence, not both.');+ result.map = normalizeMapSequence(value.map);+ // Like a pattern, the frames bring their own colours, so color tints them and defaults to white.+ if (value.color === undefined) result.color = [1, 1, 1];+}+/*** The runtime of one material: a stable colour map descriptor holding the first frame, and the frame picked by `seek`.* `pool` maps catalog image IDs to decoded `{ width, height, data }` RGBA frames.