3D engine that AI agents drive
Send image graphs to the kernel worker as plain data, key bakes by pixel bytes, restore the sequence search depth (#860)
Nguyen Xuan An57fe979f70ce
diff --git a/src/agent-texture-sequence.js b/src/agent-texture-sequence.jsindex 8a550115..ad01c707 100644--- a/src/agent-texture-sequence.js+++ b/src/agent-texture-sequence.js@@ -54,7 +54,7 @@ export function sequenceFrameIndex(sequence, time) {export function sequenceFrameIds(value) {const ids = new Set();const visit = (node, depth) => {- if (!node || typeof node !== 'object' || depth > 24) return;+ if (!node || typeof node !== 'object' || depth > 8) return;if (Array.isArray(node)) {for (const child of node) visit(child, depth + 1);return;diff --git a/src/material-graph.js b/src/material-graph.jsindex 32184531..ca942cac 100644--- a/src/material-graph.js+++ b/src/material-graph.js@@ -671,9 +671,27 @@ function writeColor(data, offset, channels) {data[offset + 3] = 255;}+// The kernel worker takes plain data, so the images a graph uses arrive as { [assetId]: { width, height, data } };+// the host's own pool is a Map. Both work.+function imagePoolMap(images) {+ if (!images) return new Map();+ return images instanceof Map ? images : new Map(Object.entries(images));+}++/** The pixels a graph's image nodes read, as plain data a kernel worker can clone and the cache can hash. */+export function graphImageData(graph, pool) {+ const used = {};+ for (const asset of graphImageAssets(graph)) {+ const id = asset.toLowerCase();+ const frame = pool?.get(id);+ if (frame) used[id] = { width: frame.width, height: frame.height, data: frame.data };+ }+ return used;+}+/** Bake a normalized graph into texture descriptors, one for each output the graph has. Deterministic. */export function bakeMaterialGraph(graph, { resolution, bump, size = 1 }, images) {- const compiled = compileMaterialGraph(graph, { size, images: images ?? new Map() });+ const compiled = compileMaterialGraph(graph, { size, images: imagePoolMap(images) });checkMaterialGraphWork(compiled, resolution);const { color, roughness, metallic, emission, height, opacity, clearcoat, transmission, aov } = compiled.outputs;const pixelCount = resolution * resolution;diff --git a/src/material-patterns.js b/src/material-patterns.jsindex 6532c3f6..e22e0b1f 100644--- a/src/material-patterns.js+++ b/src/material-patterns.js@@ -1,6 +1,6 @@import { MATERIAL_PATTERN_TYPES, MATERIAL_MAPPINGS } from './agent-scene-capabilities.js';-import { bakeMaterialGraph, graphImageAssets, normalizeMaterialGraph } from './material-graph.js';-import { runKernel } from './kernel-offload.js';+import { bakeMaterialGraph, graphImageData, normalizeMaterialGraph } from './material-graph.js';+import { kernelKey, runKernel } from './kernel-offload.js';// Procedural surface patterns for recipe materials: seeded, tileable and baked on the CPU into the same texture// descriptors a loaded material uses. Texture coordinates run 0 to 1 and every noise is periodic over that square, so@@ -273,20 +273,29 @@ const bakedByPatternObject = new WeakMap();/** The baked textures of a pattern, shared by every material with the same definition. */export function materialPatternTextures(pattern, imagePool) {- const images = pattern.graph && graphImageAssets(pattern.graph).length ? imagePool : undefined;+ const images = pattern.graph ? graphImageData(pattern.graph, imagePool) : {};+ const hasImages = Object.keys(images).length > 0;const remembered = bakedByPatternObject.get(pattern);- if (remembered) return remembered;- const key = JSON.stringify(pattern);+ // An image graph is remembered only for the very frames it was baked from, so another pool with the same asset ID+ // but different pixels bakes again.+ if (remembered && (!hasImages || sameFrames(remembered.images, images))) return remembered.baked;+ // The key covers the pixel bytes of the images the graph reads, not only their IDs.+ const key = hasImages ? kernelKey('pattern', [pattern, images]) : JSON.stringify(pattern);let baked = bakedPatterns.get(key);if (baked) {bakedPatterns.delete(key);} else {- baked = runKernel('pattern', images ? [pattern, images] : [pattern], bakeMaterialPattern);+ baked = runKernel('pattern', hasImages ? [pattern, images] : [pattern], bakeMaterialPattern);if (bakedPatterns.size >= MAX_BAKED_PATTERNS) bakedPatterns.delete(bakedPatterns.keys().next().value);}bakedPatterns.set(key, baked);- bakedByPatternObject.set(pattern, baked);+ bakedByPatternObject.set(pattern, { baked, images });return baked;}+function sameFrames(rememberedImages, images) {+ const ids = Object.keys(images);+ return ids.length === Object.keys(rememberedImages).length && ids.every(id => rememberedImages[id]?.data === images[id].data);+}+export { PATTERN_DEFAULTS };diff --git a/tests/material-graph-image.test.js b/tests/material-graph-image.test.jsindex 6bbc99bd..9aab37fe 100644--- a/tests/material-graph-image.test.js+++ b/tests/material-graph-image.test.js@@ -2,8 +2,10 @@ import './support/load-agent-scene-modules.mjs';import assert from 'node:assert/strict';import { createHash } from 'node:crypto';import test from 'node:test';-import { bakeMaterialPattern, normalizeMaterialPattern } from '../src/material-patterns.js';-import { compileMaterialGraph, graphImageAssets } from '../src/material-graph.js';+import { bakeMaterialPattern, materialPatternTextures, normalizeMaterialPattern } from '../src/material-patterns.js';+import { buildWithKernels, kernelKey, kernelStats, resetKernelStats } from '../src/kernel-offload.js';+import { createInProcessWorker } from './support/kernel-offload-fixture.mjs';+import { compileMaterialGraph, graphImageAssets, graphImageData } from '../src/material-graph.js';import { sequenceFrameIds } from '../src/agent-texture-sequence.js';import { createAgentScene } from '../src/agent-scene.js';@@ -79,3 +81,59 @@ test('a scene material with an image graph bakes through the frame pool the hostassert.equal(data.length, 256 * 256 * 4);assert.ok(data[(128 * 256 + 255) * 4] > data[(128 * 256) * 4]);});++function poolOf(pixel) {+ return imagePool(2, 2, () => pixel);+}++function bakedFirstByte(pool, pattern = imagePattern()) {+ return materialPatternTextures(pattern, pool).map.image.data[0];+}++test('an image graph is hashable and served by the kernel worker, never logged as a fallback', async () => {+ resetKernelStats();+ const requestedKernels = [];+ const pool = poolOf([10, 20, 30, 255]);+ const textures = await buildWithKernels(+ () => materialPatternTextures(imagePattern(), pool),+ { createWorker: () => createInProcessWorker(requestedKernels), available: true },+ );+ assert.deepEqual(kernelStats().fallbacks, []);+ assert.ok(requestedKernels.includes('pattern'));+ assert.deepEqual([...textures.map.image.data], [...bakeMaterialPattern(imagePattern(), pool).map.image.data]);+});++test('only the images a graph names reach the kernel, and the key covers their pixels', () => {+ const graph = imagePattern().graph;+ const pool = new Map([...poolOf([1, 2, 3, 255]), [OTHER_ASSET, { width: 1, height: 1, data: new Uint8ClampedArray(4) }]]);+ const used = graphImageData(graph, pool);+ assert.deepEqual(Object.keys(used), [IMAGE_ASSET]);+ const keyA = kernelKey('pattern', [imagePattern(), used]);+ const keyB = kernelKey('pattern', [imagePattern(), graphImageData(graph, poolOf([9, 9, 9, 255]))]);+ assert.ok(keyA);+ assert.notEqual(keyA, keyB);+});++test('the same asset ID with different pixels bakes again, with and without the pattern object shortcut', () => {+ const pattern = imagePattern();+ const first = bakedFirstByte(poolOf([250, 0, 0, 255]), pattern);+ const second = bakedFirstByte(poolOf([0, 0, 0, 255]), pattern);+ assert.ok(first > second, 'the same pattern object re-bakes against a pool with other pixels');+ assert.equal(bakedFirstByte(poolOf([250, 0, 0, 255])), first, 'a new pattern object finds the cached bake of identical pixels');+ assert.equal(bakedFirstByte(poolOf([0, 0, 0, 255])), second);+});++test('an image graph whose image is missing from the pool still says so', () => {+ assert.throws(() => materialPatternTextures(imagePattern(), new Map()), /image aaaaaaaaaaaa was not loaded/);+});++test('sequence frame IDs resolve at the depth of an edit operation, and as before for sequence frames', () => {+ const graph = { color: { image: { asset: IMAGE_ASSET } } };+ const operations = [{ op: 'set_material', material: { id: 'wall', pattern: { type: 'graph', graph } } }];+ assert.deepEqual(sequenceFrameIds(operations), [IMAGE_ASSET]);+ const frames = [IMAGE_ASSET, OTHER_ASSET];+ assert.deepEqual(sequenceFrameIds({ materials: [{ id: 'screen', map: { sequence: { frames, fps: 12, loop: true } } }] }).sort(), frames);+ let deep = { type: 'graph', graph };+ for (let level = 0; level < 12; level++) deep = { child: deep };+ assert.deepEqual(sequenceFrameIds(deep), [], 'a graph nested beyond the search depth is ignored, as sequence frames are');+});