Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Load textures that sit next to a GLB when an agent registers it (#1062)

Nguyen Xuan An5df55f5b3262
diff --git a/changes/1062.md b/changes/1062.md
new file mode 100644
index 00000000..dff0e288
--- /dev/null
+++ b/changes/1062.md
@@ -0,0 +1 @@
+- The MCP asset catalog now loads a GLB whose textures sit next to it, such as a Kenney kit's `Textures/colormap.png`, and embeds them, so a CC0 download works as it is (#1062)
diff --git a/docs/mcp.md b/docs/mcp.md
index 1fdd5e7e..00ffcae2 100644
--- a/docs/mcp.md
+++ b/docs/mcp.md
@@ -58,11 +58,11 @@ Using Node directly also avoids shell-specific npm launcher behavior in MCP clie
`--backend` accepts `gpu` or `cpu` and defaults to `gpu`.
`--memory` accepts an integer from 32 to 384 MiB and defaults to 192. A 1920x1080 frame needs about 175 MiB of render targets (measured, 4 samples), so the default fits an empty room at that size and 256 leaves room for textures; raise it for textured scenes.
-`--assets` selects an absolute directory containing self-contained GLB files.
+`--assets` selects an absolute directory containing GLB files.
The asset catalog reads regular files directly inside that directory, with limits of 64 files and 64 MiB per file. One call that places several assets, motion sources and sequence frames sends them to the preview page in one message, so their total is capped at 64 MiB too (`maxCombinedBytes`); a call over it fails with an error that names the limit and the total, and the agent places the rest in later calls. Decoding takes 1,000,000 vertices, 4,096 parts and 1,024 joints at most; `scene_capabilities` reports every number under `assets`. They are lower than the `loadGltf` defaults (256 MiB, 8,000,000 vertices) because the server hands the whole file to the preview page as one base64 string, which crashed the page from 80 MiB in a local Chrome test. A file over a limit fails with an error that names the limit and the value it hit; a larger scene loads through `loadGltf` in code (see [gltf.md](gltf.md)). All limits live in `src/agent-asset-limits.js`.
Asset IDs bind file contents through SHA-256. Changed files require a runtime restart before they receive refreshed IDs.
Catalog imports retain supported textures, skins, morphs and animation clips.
-The catalog takes self-contained GLB only. Poly Haven and similar sites ship a glTF folder (`.gltf`, `.bin` and textures): bundle it first with one command, then put the result in the catalog directory: `npx @gltf-transform/cli copy tree_01_1k.gltf tree_01.glb`. A recipe places a catalog asset with an `asset` node field (see [the scene guide](agent-scene.md#placed-catalog-assets)).
+A GLB that points at image files (a Kenney kit ships `boat.glb` plus `Textures/colormap.png`) loads as downloaded: the catalog reads each relative PNG, JPEG or WebP path from that folder or below it, counts it against the 64 MiB file limit and stores it inside the asset, so the asset ID covers the texture bytes. A remote or `data:` address, a path that leaves the folder (`..`, an absolute path, a link) and a missing or non-image file are refused with an error that names the GLB and the path. Buffers must still be embedded. Poly Haven and similar sites ship a glTF folder (`.gltf`, `.bin` and textures): bundle it first with one command (the `.bin` must be embedded; image files may stay beside the GLB), then put the result in the catalog directory: `npx @gltf-transform/cli copy tree_01_1k.gltf tree_01.glb`. A recipe places a catalog asset with an `asset` node field (see [the scene guide](agent-scene.md#placed-catalog-assets)).
These launch options remain outside tool arguments.
The process runs below normal priority to reduce interference with other work.
diff --git a/scripts/agent-asset-catalog.mjs b/scripts/agent-asset-catalog.mjs
index 71d2305e..09f91f71 100644
--- a/scripts/agent-asset-catalog.mjs
+++ b/scripts/agent-asset-catalog.mjs
@@ -32,29 +32,119 @@ export function imageKind(bytes) {
return null;
}
-function selfContainedGlb(bytes) {
+const JSON_CHUNK = 0x4e4f534a;
+const BINARY_CHUNK = 0x004e4942;
+
+function glbDocument(bytes) {
if (bytes.length < 20 || bytes.readUInt32LE(0) !== 0x46546c67 || bytes.readUInt32LE(4) !== 2 || bytes.readUInt32LE(8) !== bytes.length) {
throw new Error('Asset must be a complete glTF 2.0 binary file.');
}
const jsonLength = bytes.readUInt32LE(12);
- if (bytes.readUInt32LE(16) !== 0x4e4f534a || jsonLength % 4 !== 0 || jsonLength > bytes.length - 20) {
+ if (bytes.readUInt32LE(16) !== JSON_CHUNK || jsonLength % 4 !== 0 || jsonLength > bytes.length - 20) {
throw new Error('GLB must start with a bounded JSON chunk.');
}
const document = JSON.parse(bytes.subarray(20, 20 + jsonLength).toString('utf8'));
if (document.asset?.version !== '2.0') {
throw new Error('Asset must declare glTF version 2.0.');
}
+ return { document, jsonLength };
+}
+
+/** A GLB whose buffers are embedded. Image `uri` fields are allowed here; embedSiblingImages turns them into embedded images. */
+function selfContainedGlb(bytes) {
+ const parsed = glbDocument(bytes);
for (const collection of ['buffers', 'images']) {
- if (document[collection] !== undefined && !Array.isArray(document[collection])) {
+ const resources = parsed.document[collection];
+ if (resources !== undefined && !Array.isArray(resources)) {
throw new Error(`GLB ${collection} must be an array.`);
}
- for (const resource of document[collection] ?? []) {
- if (!resource || typeof resource !== 'object' || Object.hasOwn(resource, 'uri')) {
- throw new Error('Catalog GLBs must embed all buffers and images, without resource URIs.');
+ for (const resource of resources ?? []) {
+ if (!resource || typeof resource !== 'object') {
+ throw new Error(`GLB ${collection} entries must be objects.`);
}
+ if (collection === 'buffers' && Object.hasOwn(resource, 'uri')) {
+ throw new Error('Catalog GLBs must embed all buffers; a .bin file next to the GLB is not loaded. Re-export the model as a single .glb.');
+ }
+ }
+ }
+ return parsed;
+}
+
+/** The path segments of an image beside the GLB, or an error that names the GLB, the URI and what to pass instead. */
+function siblingImageSegments(filename, uri) {
+ const prefix = `${filename}: image uri "${uri}"`;
+ if (typeof uri !== 'string' || !uri || uri.length > 512) {
+ throw new Error(`${prefix} must be a relative file path of at most 512 characters, such as "Textures/colormap.png".`);
+ }
+ if (/^[a-z][a-z0-9+.-]*:/i.test(uri)) {
+ throw new Error(`${prefix} is a remote or embedded address, which the catalog does not fetch. Put the image file next to the GLB and use a relative path, such as "colormap.png", or embed the image in the GLB.`);
+ }
+ let decoded;
+ try {
+ decoded = decodeURIComponent(uri);
+ } catch {
+ throw new Error(`${prefix} is not valid percent-encoding. Use a plain relative path, such as "Textures/colormap.png".`);
+ }
+ const segments = decoded.split(/[\\/]/);
+ if (decoded.startsWith('/') || decoded.startsWith('\\') || /^[a-z]:/i.test(decoded) || segments.includes('..')) {
+ throw new Error(`${prefix} leaves the GLB's folder. Copy the image into the folder (or a folder below it) and use a relative path, such as "colormap.png".`);
+ }
+ return segments.filter(segment => segment && segment !== '.');
+}
+
+function padded(buffer, fill) {
+ const padding = (4 - (buffer.length % 4)) % 4;
+ return padding ? Buffer.concat([buffer, Buffer.alloc(padding, fill)]) : buffer;
+}
+
+/** Rebuilds the GLB with every image `uri` read from the GLB's folder and stored in the binary chunk, so the stored asset is self-contained. */
+async function embedSiblingImages(bytes, parsed, filename, readSibling) {
+ const { document, jsonLength } = parsed;
+ const binaryStart = 20 + jsonLength;
+ const hasBinary = bytes.length >= binaryStart + 8 && bytes.readUInt32LE(binaryStart + 4) === BINARY_CHUNK;
+ if (!hasBinary || !Array.isArray(document.buffers) || document.buffers.length === 0) {
+ throw new Error(`${filename}: a GLB that points at image files needs a binary chunk to hold them. Re-export it with its mesh data embedded.`);
+ }
+ const binaryLength = bytes.readUInt32LE(binaryStart);
+ const chunks = [padded(bytes.subarray(binaryStart + 8, binaryStart + 8 + binaryLength), 0)];
+ let size = chunks[0].length;
+ if (!Array.isArray(document.bufferViews)) document.bufferViews = [];
+ const embeddedByPath = new Map();
+ for (const image of document.images) {
+ if (image.uri === undefined) continue;
+ const segments = siblingImageSegments(filename, image.uri);
+ const key = segments.join('/');
+ if (!embeddedByPath.has(key)) {
+ const { imageBytes, format } = await readSibling(filename, image.uri, segments);
+ const imageChunk = padded(imageBytes, 0);
+ const total = bytes.length + (size - binaryLength) + imageChunk.length;
+ if (total > MAXIMUM_BYTES) throw new RangeError(`${filename}: with its images next to it the asset reaches ${total} bytes. ${assetBytesLimitMessage(total)}`);
+ document.bufferViews.push({ buffer: 0, byteOffset: size, byteLength: imageBytes.length });
+ embeddedByPath.set(key, { view: document.bufferViews.length - 1, mimeType: IMAGE_MIME[format] });
+ chunks.push(imageChunk);
+ size += imageChunk.length;
}
+ const embedded = embeddedByPath.get(key);
+ delete image.uri;
+ image.bufferView = embedded.view;
+ image.mimeType = embedded.mimeType;
}
- return document;
+ document.buffers[0].byteLength = size;
+ const binary = Buffer.concat(chunks);
+ const json = padded(Buffer.from(JSON.stringify(document)), 0x20);
+ const total = 12 + 8 + json.length + 8 + binary.length;
+ const result = Buffer.alloc(total);
+ result.writeUInt32LE(0x46546c67, 0);
+ result.writeUInt32LE(2, 4);
+ result.writeUInt32LE(total, 8);
+ result.writeUInt32LE(json.length, 12);
+ result.writeUInt32LE(JSON_CHUNK, 16);
+ json.copy(result, 20);
+ const binaryOffset = 20 + json.length;
+ result.writeUInt32LE(binary.length, binaryOffset);
+ result.writeUInt32LE(BINARY_CHUNK, binaryOffset + 4);
+ binary.copy(result, binaryOffset + 8);
+ return result;
}
/** Explicit local directory catalog. IDs bind bytes, rather than agent-supplied file paths. */
@@ -68,6 +158,38 @@ export async function createAgentAssetCatalog(root) {
}
const entries = new Map();
+ /** One image file below the catalog folder: a regular file, not a link, inside the folder, a real PNG, JPEG or WebP, within the byte bound. */
+ async function readSibling(glbName, uri, segments) {
+ const label = `${glbName}: image "${uri}"`;
+ const path = join(directory, ...segments);
+ let original;
+ try {
+ original = await lstat(path);
+ } catch {
+ throw new Error(`${label} was not found next to the GLB. Copy that file into the --assets folder at the same relative path.`);
+ }
+ if (!original.isFile() || original.isSymbolicLink()) throw new Error(`${label} must be a regular file, without links.`);
+ const resolved = await realpath(path);
+ const offset = relative(directory, resolved);
+ if (offset.startsWith(`..${sep}`) || offset === '..' || isAbsolute(offset)) throw new Error(`${label} leaves the GLB's folder. Copy the image into the folder and use a relative path.`);
+ if (original.size > MAXIMUM_BYTES) throw new RangeError(`${label}: ${assetBytesLimitMessage(original.size)}`);
+ const handle = await open(resolved, 'r');
+ try {
+ const imageBytes = Buffer.alloc(original.size);
+ let position = 0;
+ while (position < imageBytes.length) {
+ const result = await handle.read(imageBytes, position, imageBytes.length - position, position);
+ if (result.bytesRead === 0) throw new Error(`${label} was truncated during reading.`);
+ position += result.bytesRead;
+ }
+ const format = imageKind(imageBytes);
+ if (!format) throw new Error(`${label} is not a PNG, JPEG or WebP file. Convert it to one of those.`);
+ return { imageBytes, format };
+ } finally {
+ await handle.close();
+ }
+ }
+
async function readFile(filename) {
const path = join(directory, filename);
const original = await lstat(path);
@@ -88,7 +210,7 @@ export async function createAgentAssetCatalog(root) {
if (!current.isFile() || current.dev !== original.dev || current.ino !== original.ino || current.size < 20) {
throw new Error('Asset changed during opening or is too small to be a GLB.');
}
- const bytes = Buffer.alloc(current.size);
+ let bytes = Buffer.alloc(current.size);
let position = 0;
while (position < bytes.length) {
const result = await handle.read(bytes, position, bytes.length - position, position);
@@ -103,7 +225,15 @@ export async function createAgentAssetCatalog(root) {
const imageFormat = lookKind ? null : imageKind(bytes);
if (imageFormat && !IMAGE_EXTENSIONS[imageFormat].test(filename)) throw new Error('Image extension must match its PNG, JPEG or WebP content.');
if (!imageFormat && !lookKind && !/\.glb$/i.test(filename)) throw new Error('Catalog files must be GLB, PNG, JPEG, WebP, HDR or CUBE, with a matching extension.');
- const document = imageFormat || lookKind ? null : selfContainedGlb(bytes);
+ let document = null;
+ if (!imageFormat && !lookKind) {
+ const parsed = selfContainedGlb(bytes);
+ document = parsed.document;
+ if (document.images?.some(image => Object.hasOwn(image, 'uri'))) {
+ bytes = await embedSiblingImages(bytes, parsed, filename, readSibling);
+ document = glbDocument(bytes).document;
+ }
+ }
const assetId = createHash('sha256').update(bytes).digest('hex');
return { bytes, document, assetId, imageFormat, lookKind };
} finally {
diff --git a/tests/agent-asset-catalog.test.js b/tests/agent-asset-catalog.test.js
index e2cf4973..9696781f 100644
--- a/tests/agent-asset-catalog.test.js
+++ b/tests/agent-asset-catalog.test.js
@@ -1,6 +1,7 @@
import test from 'node:test';
import assert from 'node:assert/strict';
-import { mkdtemp, writeFile, rm, symlink, readdir, readFile } from 'node:fs/promises';
+import { mkdtemp, writeFile, rm, symlink, readdir, readFile, cp } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
import { tmpdir } from 'node:os';
import { join, dirname } from 'node:path';
import { mat4 } from 'gl-matrix';
@@ -149,3 +150,78 @@ test('an .hdr that is not Radiance data and a .cube that is not a 3D look are re
await writeFile(join(directory, 'fake.cube'), 'LUT_1D_SIZE 2\n0 0 0\n1 1 1\n'.padEnd(64, ' '));
await assert.rejects(createAgentAssetCatalog(directory), /LUT_3D_SIZE/);
});
+
+const KENNEY_BOAT_FOLDER = new URL('./fixtures/kenney-boat/', import.meta.url);
+
+async function boatFixture(context) {
+ const directory = await mkdtemp(join(tmpdir(), '67js-catalog-'));
+ context.after(() => rm(directory, { recursive: true, force: true }));
+ await cp(KENNEY_BOAT_FOLDER, directory, { recursive: true });
+ return directory;
+}
+
+test('a Kenney GLB loads with the colormap that sits next to it, embedded into the stored asset', async context => {
+ const directory = await boatFixture(context);
+ const catalog = await createAgentAssetCatalog(directory);
+ const listing = catalog.list();
+ assert.equal(listing.assets.length, 1);
+ assert.equal(listing.assets[0].images, 1);
+ const stored = await catalog.read(listing.assets[0].assetId);
+ const jsonLength = stored.readUInt32LE(12);
+ const document = JSON.parse(stored.subarray(20, 20 + jsonLength).toString('utf8'));
+ assert.equal(document.images[0].uri, undefined);
+ assert.equal(document.images[0].mimeType, 'image/png');
+ const view = document.bufferViews[document.images[0].bufferView];
+ const binaryStart = 20 + jsonLength + 8;
+ const pngStart = binaryStart + view.byteOffset;
+ assert.deepEqual(stored.subarray(pngStart, pngStart + 4), Buffer.from([0x89, 0x50, 0x4e, 0x47]));
+ assert.equal(view.byteOffset % 4, 0);
+ assert.equal(document.buffers[0].byteLength, stored.length - binaryStart);
+ assert.equal(listing.assets[0].assetId, createHash('sha256').update(stored).digest('hex'));
+});
+
+test('a GLB texture that is missing, remote, outside the folder or not an image is refused with the file and the fix', async context => {
+ const directory = await boatFixture(context);
+ const boat = await readFile(join(directory, 'boat-fishing-small.glb'));
+ const attempts = [
+ ['Textures/absent.png', /boat-fishing-small\.glb: image "Textures\/absent\.png" was not found next to the GLB/],
+ ['https://example.invalid/colormap.png', /remote or embedded address.*relative path/],
+ ['data:image/png;base64,AAAA', /remote or embedded address/],
+ ['../colormap.png', /leaves the GLB's folder.*relative path/],
+ ['Textures/../../colormap.png', /leaves the GLB's folder/],
+ ['%2e%2e/colormap.png', /leaves the GLB's folder/],
+ ['/etc/colormap.png', /leaves the GLB's folder/],
+ ['C:\\colormap.png', /remote or embedded address/],
+ ];
+ for (const [uri, message] of attempts) {
+ await writeFile(join(directory, 'boat-fishing-small.glb'), withDocument(boat, document => { document.images[0].uri = uri; }));
+ await assert.rejects(createAgentAssetCatalog(directory), message, uri);
+ }
+ await writeFile(join(directory, 'Textures', 'notes.png'), 'not an image');
+ await writeFile(join(directory, 'boat-fishing-small.glb'), withDocument(boat, document => { document.images[0].uri = 'Textures/notes.png'; }));
+ await assert.rejects(createAgentAssetCatalog(directory), /not a PNG, JPEG or WebP/);
+});
+
+test('a texture folder that links out of the catalog folder is refused', async context => {
+ const directory = await boatFixture(context);
+ const outside = await mkdtemp(join(tmpdir(), '67js-outside-'));
+ context.after(() => rm(outside, { recursive: true, force: true }));
+ await cp(join(directory, 'Textures', 'colormap.png'), join(outside, 'colormap.png'));
+ await rm(join(directory, 'Textures'), { recursive: true });
+ try {
+ await symlink(outside, join(directory, 'Textures'), 'junction');
+ } catch (error) {
+ context.skip(`links are not allowed here: ${error.code}`);
+ return;
+ }
+ await assert.rejects(createAgentAssetCatalog(directory), /leaves the GLB's folder|without links/);
+});
+
+test('sibling textures stay inside the asset byte bound', async context => {
+ const directory = await boatFixture(context);
+ const boat = await readFile(join(directory, 'boat-fishing-small.glb'));
+ const huge = Buffer.alloc(AGENT_ASSET_LIMITS.maxBytes - boat.length + 8);
+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(huge);
+ await writeFile(join(directory, 'Textures', 'colormap.png'), huge);
+ await assert.rejects(createAgentAssetCatalog(directory), error => error instanceof RangeError && /boat-fishing-small\.glb: with its images next to it the asset reaches/.test(error.message));
+});
diff --git a/tests/fixtures/kenney-boat/License.txt b/tests/fixtures/kenney-boat/License.txt
new file mode 100644
index 00000000..d78705c5
--- /dev/null
+++ b/tests/fixtures/kenney-boat/License.txt
@@ -0,0 +1,28 @@
+
+
+ Watercraft Kit (2.1)
+
+ Created/distributed by Kenney (www.kenney.nl)
+ Creation date: 19-04-2024 11:37
+
+ ------------------------------
+
+ License: (Creative Commons Zero, CC0)
+ http://creativecommons.org/publicdomain/zero/1.0/
+
+ You can use this content for personal, educational, and commercial purposes.
+
+ Support by crediting 'Kenney' or 'www.kenney.nl' (this is not a requirement)
+
+ ------------------------------
+
+ • Website : www.kenney.nl
+ • Donate : www.kenney.nl/donate
+
+ • Patreon : patreon.com/kenney
+
+ Follow on social media for updates:
+
+ • Twitter: twitter.com/KenneyNL
+ • Instagram: instagram.com/kenney_nl
+ • Mastodon: mastodon.gamedev.place/@kenney
\ No newline at end of file
diff --git a/tests/fixtures/kenney-boat/Textures/colormap.png b/tests/fixtures/kenney-boat/Textures/colormap.png
new file mode 100644
index 00000000..7db813ff
Binary files /dev/null and b/tests/fixtures/kenney-boat/Textures/colormap.png differ
diff --git a/tests/fixtures/kenney-boat/boat-fishing-small.glb b/tests/fixtures/kenney-boat/boat-fishing-small.glb
new file mode 100644
index 00000000..b7eb2124
Binary files /dev/null and b/tests/fixtures/kenney-boat/boat-fishing-small.glb differ