Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Merge origin/main into feat/recipe-assets

Nguyen Xuan An5840859ede04
diff --cc scripts/agent-preview-browser.js
index 7d23eaf8,c9eaf7d6..ec22598e
--- a/scripts/agent-preview-browser.js
+++ b/scripts/agent-preview-browser.js
@@@ -288,36 -307,6 +308,27 @@@ setPhysicsModuleLoader(() => import(/*
// The preview bundle is one file, so the on-demand scene modules load together on the first call and later calls stay
// synchronous. The boolean engine is WebAssembly and loads asynchronously, once, only for calls that need it.
- function decodeBase64(data) {
- if (typeof Uint8Array.fromBase64 === 'function') {
- return Uint8Array.fromBase64(data);
- }
- const binary = atob(data);
- const bytes = new Uint8Array(binary.length);
- for (let index = 0; index < binary.length; index++) bytes[index] = binary.charCodeAt(index);
- return bytes;
- }
+
+const ASSET_DECODE_LIMITS = { maxBytes: 16 * 1024 * 1024, maxVertices: 250000, maxParts: 4096, maxTexturePixels: 4194304, maxJoints: 1024 };
+
+function disposePlacedAssets(pool) {
+ for (const asset of pool?.values() ?? []) asset.dispose();
+ pool?.clear();
+}
+
+// A recipe that places catalog assets arrives with their bytes, keyed by asset id. Copy 0 serves every still placement and
+// each placement that plays a clip takes a copy of its own, since a pose is state of one decoded asset.
+async function loadPlacedAssets(pool, value, assetData) {
+ for (const { id, copies } of recipeAssetNeeds(value)) {
+ for (let copy = 0; copy < copies; copy++) {
+ const key = assetCopyKey(id, copy);
+ if (pool.has(key)) continue;
+ if (!assetData?.[id]) throw new Error(`Asset ${id.slice(0, 12)} was not supplied. Check asset_catalog for its id.`);
- pool.set(key, await parseGltf(decodeBase64(assetData[id]).buffer, ASSET_DECODE_LIMITS));
++ pool.set(key, await parseGltf(bytesFromBase64(assetData[id]).buffer, ASSET_DECODE_LIMITS));
+ }
+ }
+}
+
function afterMeshBooleans(value, run) {
if (!allAgentSceneModulesLoaded()) return loadAllAgentSceneModules().then(() => afterMeshBooleans(value, run));
const loads = [];
@@@ -726,16 -696,13 +737,23 @@@ window.__agentPreview =
return { sessionId, ...scene.inspectMeshes(options) };
},
apply(input) {
+ if (input.sources) {
+ const { scene } = requireSession(input.sessionId);
+ return decodeMotionSources(input.sources).then(decoded => {
+ scene.provideMotionSources?.(decoded);
+ return afterMeshBooleans(input.operations, () => this.applyLoaded(input));
+ });
+ }
+ if (recipeAssetNeeds(input.operations).length) return this.applyWithAssets(input);
+ return afterMeshBooleans(input.operations, () => this.applyLoaded(input));
+ },
+ async applyWithAssets(input) {
+ const record = requireSession(input.sessionId);
+ if (!record.placedAssets) {
+ throw new Error('This scene was created without placed assets. Recreate it with an asset node in scene_create, then add more.');
+ }
+ await loadAllAgentSceneModules();
+ await loadPlacedAssets(record.placedAssets, input.operations, input.assetData);
return afterMeshBooleans(input.operations, () => this.applyLoaded(input));
},
applyLoaded({ sessionId, operations, expectedRevision }) {
diff --cc scripts/agent-runtime.mjs
index 2f9c38f9,c5f96dec..a73065bc
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@@ -130,14 -130,23 +131,31 @@@ export function createAgentRuntime({ ke
return catalogPromise;
}
+ /** Bytes of the catalog assets a recipe or edit list places, base64 by asset id, for the page that decodes them. */
+ async function placedAssetData(value) {
+ const needs = recipeAssetNeeds(value);
+ if (!needs.length) return {};
+ const entries = await Promise.all(needs.map(async ({ id }) => [id, (await (await catalog()).read(id)).toString('base64')]));
+ return { assetData: Object.fromEntries(entries) };
+ }
+
+ // Motion layers can play a clip of another catalog asset (`from.asset`). The page gets those assets' bytes by catalog ID,
+ // never a path, and keeps only their clips and nodes.
+ async function motionSources(motions) {
+ const ids = new Set();
+ for (const motion of motions) {
+ for (const layer of Array.isArray(motion?.layers) ? motion.layers : []) {
+ const id = layer?.from?.asset;
+ if (typeof id === 'string' && /^[a-f0-9]{64}$/i.test(id)) ids.add(id.toLowerCase());
+ }
+ }
+ if (ids.size > MAXIMUM_MOTION_SOURCES) throw new RangeError(`Motion layers can retarget clips of at most ${MAXIMUM_MOTION_SOURCES} other assets at once.`);
+ if (!ids.size) return {};
+ const sources = {};
+ for (const id of ids) sources[id] = (await (await catalog()).read(id)).toString('base64');
+ return { sources };
+ }
+
function enqueue(task) {
if (closing || phase === 'closing' || phase === 'closed' || phase === 'failed') {
return Promise.reject(new Error(`The agent runtime is ${phase}.`));
@@@ -581,7 -590,8 +599,8 @@@
}
const bakeAfter = bakeAfterOperations(input.operations);
const rebakes = activeSceneHasBake || bakeAfter === true;
- const applied = await invoke('apply', { ...input, ...(await placedAssetData(input.operations)) }, rebakes ? BAKE_EVALUATION_TIMEOUT_MS : EVALUATION_TIMEOUT_MS);
+ const motions = (input.operations ?? []).filter(operation => operation.type === 'set-motion').map(operation => operation.motion);
- const applied = await invoke('apply', { ...input, ...(await motionSources(motions)) }, rebakes ? BAKE_EVALUATION_TIMEOUT_MS : EVALUATION_TIMEOUT_MS);
++ const applied = await invoke('apply', { ...input, ...(await motionSources(motions)), ...(await placedAssetData(input.operations)) }, rebakes ? BAKE_EVALUATION_TIMEOUT_MS : EVALUATION_TIMEOUT_MS);
activeSceneHasBake = bakeAfter ?? activeSceneHasBake;
return applied;
});
@@@ -688,8 -698,7 +707,8 @@@
privateSessionId = randomUUID();
await reviewEvaluate(branchRecipe || !assetBytes ? 'create' : 'createAsset', { sessionId: privateSessionId,
recipe: branchRecipe ? branchRecipe.recipe : snapshot.recipe, memoryBudgetMiB: memory, backend,
- ...(!branchRecipe && assetBytes ? { data: assetBytes.toString('base64') } : {}),
- ...(!branchRecipe && assetBytes ? { data: assetBytes.toString('base64'), ...(await motionSources([snapshot.recipe.motion])) } : {}) });
++ ...(!branchRecipe && assetBytes ? { data: assetBytes.toString('base64'), ...(await motionSources([snapshot.recipe.motion])) } : {}),
+ ...(!assetBytes ? await placedAssetData(branchRecipe ? branchRecipe.recipe : snapshot.recipe) : {}) });
privateCreated = true;
for (const index of group.shotIndexes) {
const rendered = await reviewEvaluate('reviewShot', { sessionId: privateSessionId, shot: settings.shots[index], index,
diff --cc src/agent-scene.js
index 7d7997cf,ccea5b29..e255d4d9
--- a/src/agent-scene.js
+++ b/src/agent-scene.js
@@@ -1142,9 -1089,9 +1142,9 @@@ export function createAgentScene(input
stagedState.skinStages = createSkinStages(skinBindings);
stagedState.motionSampler = recipe.motion ? agentSceneModule('motion').compileAgentMotion(recipe.motion, recipe) : null;
stagedState.physicsSampler = stagePhysics(recipe, geometryRecords);
- stagedState.clothSampler = stageCloth(recipe, geometryRecords);
+ stagedState.clothSampler = stageCloth(recipe, geometryRecords, stagedState.motionSampler);
stagedState.particleSampler = stageParticles(recipe, geometryRecords);
- const animated = Boolean(stagedState.motionSampler || stagedState.physicsSampler || stagedState.clothSampler || stagedState.particleSampler);
+ const animated = Boolean(stagedState.motionSampler || stagedState.physicsSampler || stagedState.clothSampler || stagedState.particleSampler || assetRuntime.animators.length);
stagedState.poseGraph = animated ? compile(recipe, geometries, localBounds, limits) : null;
stagedState.initialPose = stagePose(stagedState, animated ? 0 : null);
return stagedState;