Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Send image graphs to the kernel worker as plain data, key bakes by pixel bytes, restore the sequence search depth (#860)

Nguyen Xuan An57fe979f70ce
diff --git a/src/agent-texture-sequence.js b/src/agent-texture-sequence.js
index 8a550115..ad01c707 100644
--- a/src/agent-texture-sequence.js
+++ b/src/agent-texture-sequence.js
@@ -54,7 +54,7 @@ export function sequenceFrameIndex(sequence, time) {
export function sequenceFrameIds(value) {
const ids = new Set();
const visit = (node, depth) => {
- if (!node || typeof node !== 'object' || depth > 24) return;
+ if (!node || typeof node !== 'object' || depth > 8) return;
if (Array.isArray(node)) {
for (const child of node) visit(child, depth + 1);
return;
diff --git a/src/material-graph.js b/src/material-graph.js
index 32184531..ca942cac 100644
--- a/src/material-graph.js
+++ b/src/material-graph.js
@@ -671,9 +671,27 @@ function writeColor(data, offset, channels) {
data[offset + 3] = 255;
}
+// The kernel worker takes plain data, so the images a graph uses arrive as { [assetId]: { width, height, data } };
+// the host's own pool is a Map. Both work.
+function imagePoolMap(images) {
+ if (!images) return new Map();
+ return images instanceof Map ? images : new Map(Object.entries(images));
+}
+
+/** The pixels a graph's image nodes read, as plain data a kernel worker can clone and the cache can hash. */
+export function graphImageData(graph, pool) {
+ const used = {};
+ for (const asset of graphImageAssets(graph)) {
+ const id = asset.toLowerCase();
+ const frame = pool?.get(id);
+ if (frame) used[id] = { width: frame.width, height: frame.height, data: frame.data };
+ }
+ return used;
+}
+
/** Bake a normalized graph into texture descriptors, one for each output the graph has. Deterministic. */
export function bakeMaterialGraph(graph, { resolution, bump, size = 1 }, images) {
- const compiled = compileMaterialGraph(graph, { size, images: images ?? new Map() });
+ const compiled = compileMaterialGraph(graph, { size, images: imagePoolMap(images) });
checkMaterialGraphWork(compiled, resolution);
const { color, roughness, metallic, emission, height, opacity, clearcoat, transmission, aov } = compiled.outputs;
const pixelCount = resolution * resolution;
diff --git a/src/material-patterns.js b/src/material-patterns.js
index 6532c3f6..e22e0b1f 100644
--- a/src/material-patterns.js
+++ b/src/material-patterns.js
@@ -1,6 +1,6 @@
import { MATERIAL_PATTERN_TYPES, MATERIAL_MAPPINGS } from './agent-scene-capabilities.js';
-import { bakeMaterialGraph, graphImageAssets, normalizeMaterialGraph } from './material-graph.js';
-import { runKernel } from './kernel-offload.js';
+import { bakeMaterialGraph, graphImageData, normalizeMaterialGraph } from './material-graph.js';
+import { kernelKey, runKernel } from './kernel-offload.js';
// Procedural surface patterns for recipe materials: seeded, tileable and baked on the CPU into the same texture
// descriptors a loaded material uses. Texture coordinates run 0 to 1 and every noise is periodic over that square, so
@@ -273,20 +273,29 @@ const bakedByPatternObject = new WeakMap();
/** The baked textures of a pattern, shared by every material with the same definition. */
export function materialPatternTextures(pattern, imagePool) {
- const images = pattern.graph && graphImageAssets(pattern.graph).length ? imagePool : undefined;
+ const images = pattern.graph ? graphImageData(pattern.graph, imagePool) : {};
+ const hasImages = Object.keys(images).length > 0;
const remembered = bakedByPatternObject.get(pattern);
- if (remembered) return remembered;
- const key = JSON.stringify(pattern);
+ // An image graph is remembered only for the very frames it was baked from, so another pool with the same asset ID
+ // but different pixels bakes again.
+ if (remembered && (!hasImages || sameFrames(remembered.images, images))) return remembered.baked;
+ // The key covers the pixel bytes of the images the graph reads, not only their IDs.
+ const key = hasImages ? kernelKey('pattern', [pattern, images]) : JSON.stringify(pattern);
let baked = bakedPatterns.get(key);
if (baked) {
bakedPatterns.delete(key);
} else {
- baked = runKernel('pattern', images ? [pattern, images] : [pattern], bakeMaterialPattern);
+ baked = runKernel('pattern', hasImages ? [pattern, images] : [pattern], bakeMaterialPattern);
if (bakedPatterns.size >= MAX_BAKED_PATTERNS) bakedPatterns.delete(bakedPatterns.keys().next().value);
}
bakedPatterns.set(key, baked);
- bakedByPatternObject.set(pattern, baked);
+ bakedByPatternObject.set(pattern, { baked, images });
return baked;
}
+function sameFrames(rememberedImages, images) {
+ const ids = Object.keys(images);
+ return ids.length === Object.keys(rememberedImages).length && ids.every(id => rememberedImages[id]?.data === images[id].data);
+}
+
export { PATTERN_DEFAULTS };
diff --git a/tests/material-graph-image.test.js b/tests/material-graph-image.test.js
index 6bbc99bd..9aab37fe 100644
--- a/tests/material-graph-image.test.js
+++ b/tests/material-graph-image.test.js
@@ -2,8 +2,10 @@ import './support/load-agent-scene-modules.mjs';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import test from 'node:test';
-import { bakeMaterialPattern, normalizeMaterialPattern } from '../src/material-patterns.js';
-import { compileMaterialGraph, graphImageAssets } from '../src/material-graph.js';
+import { bakeMaterialPattern, materialPatternTextures, normalizeMaterialPattern } from '../src/material-patterns.js';
+import { buildWithKernels, kernelKey, kernelStats, resetKernelStats } from '../src/kernel-offload.js';
+import { createInProcessWorker } from './support/kernel-offload-fixture.mjs';
+import { compileMaterialGraph, graphImageAssets, graphImageData } from '../src/material-graph.js';
import { sequenceFrameIds } from '../src/agent-texture-sequence.js';
import { createAgentScene } from '../src/agent-scene.js';
@@ -79,3 +81,59 @@ test('a scene material with an image graph bakes through the frame pool the host
assert.equal(data.length, 256 * 256 * 4);
assert.ok(data[(128 * 256 + 255) * 4] > data[(128 * 256) * 4]);
});
+
+function poolOf(pixel) {
+ return imagePool(2, 2, () => pixel);
+}
+
+function bakedFirstByte(pool, pattern = imagePattern()) {
+ return materialPatternTextures(pattern, pool).map.image.data[0];
+}
+
+test('an image graph is hashable and served by the kernel worker, never logged as a fallback', async () => {
+ resetKernelStats();
+ const requestedKernels = [];
+ const pool = poolOf([10, 20, 30, 255]);
+ const textures = await buildWithKernels(
+ () => materialPatternTextures(imagePattern(), pool),
+ { createWorker: () => createInProcessWorker(requestedKernels), available: true },
+ );
+ assert.deepEqual(kernelStats().fallbacks, []);
+ assert.ok(requestedKernels.includes('pattern'));
+ assert.deepEqual([...textures.map.image.data], [...bakeMaterialPattern(imagePattern(), pool).map.image.data]);
+});
+
+test('only the images a graph names reach the kernel, and the key covers their pixels', () => {
+ const graph = imagePattern().graph;
+ const pool = new Map([...poolOf([1, 2, 3, 255]), [OTHER_ASSET, { width: 1, height: 1, data: new Uint8ClampedArray(4) }]]);
+ const used = graphImageData(graph, pool);
+ assert.deepEqual(Object.keys(used), [IMAGE_ASSET]);
+ const keyA = kernelKey('pattern', [imagePattern(), used]);
+ const keyB = kernelKey('pattern', [imagePattern(), graphImageData(graph, poolOf([9, 9, 9, 255]))]);
+ assert.ok(keyA);
+ assert.notEqual(keyA, keyB);
+});
+
+test('the same asset ID with different pixels bakes again, with and without the pattern object shortcut', () => {
+ const pattern = imagePattern();
+ const first = bakedFirstByte(poolOf([250, 0, 0, 255]), pattern);
+ const second = bakedFirstByte(poolOf([0, 0, 0, 255]), pattern);
+ assert.ok(first > second, 'the same pattern object re-bakes against a pool with other pixels');
+ assert.equal(bakedFirstByte(poolOf([250, 0, 0, 255])), first, 'a new pattern object finds the cached bake of identical pixels');
+ assert.equal(bakedFirstByte(poolOf([0, 0, 0, 255])), second);
+});
+
+test('an image graph whose image is missing from the pool still says so', () => {
+ assert.throws(() => materialPatternTextures(imagePattern(), new Map()), /image aaaaaaaaaaaa was not loaded/);
+});
+
+test('sequence frame IDs resolve at the depth of an edit operation, and as before for sequence frames', () => {
+ const graph = { color: { image: { asset: IMAGE_ASSET } } };
+ const operations = [{ op: 'set_material', material: { id: 'wall', pattern: { type: 'graph', graph } } }];
+ assert.deepEqual(sequenceFrameIds(operations), [IMAGE_ASSET]);
+ const frames = [IMAGE_ASSET, OTHER_ASSET];
+ assert.deepEqual(sequenceFrameIds({ materials: [{ id: 'screen', map: { sequence: { frames, fps: 12, loop: true } } }] }).sort(), frames);
+ let deep = { type: 'graph', graph };
+ for (let level = 0; level < 12; level++) deep = { child: deep };
+ assert.deepEqual(sequenceFrameIds(deep), [], 'a graph nested beyond the search depth is ignored, as sequence frames are');
+});