3D engine that AI agents drive
Merge origin/main into feat/recipe-assets
Nguyen Xuan An5840859ede04
diff --cc scripts/agent-preview-browser.jsindex 7d23eaf8,c9eaf7d6..ec22598e--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@@ -288,36 -307,6 +308,27 @@@ setPhysicsModuleLoader(() => import(/*// The preview bundle is one file, so the on-demand scene modules load together on the first call and later calls stay// synchronous. The boolean engine is WebAssembly and loads asynchronously, once, only for calls that need it.- function decodeBase64(data) {- if (typeof Uint8Array.fromBase64 === 'function') {- return Uint8Array.fromBase64(data);- }- const binary = atob(data);- const bytes = new Uint8Array(binary.length);- for (let index = 0; index < binary.length; index++) bytes[index] = binary.charCodeAt(index);- return bytes;- }++const ASSET_DECODE_LIMITS = { maxBytes: 16 * 1024 * 1024, maxVertices: 250000, maxParts: 4096, maxTexturePixels: 4194304, maxJoints: 1024 };++function disposePlacedAssets(pool) {+ for (const asset of pool?.values() ?? []) asset.dispose();+ pool?.clear();+}++// A recipe that places catalog assets arrives with their bytes, keyed by asset id. Copy 0 serves every still placement and+// each placement that plays a clip takes a copy of its own, since a pose is state of one decoded asset.+async function loadPlacedAssets(pool, value, assetData) {+ for (const { id, copies } of recipeAssetNeeds(value)) {+ for (let copy = 0; copy < copies; copy++) {+ const key = assetCopyKey(id, copy);+ if (pool.has(key)) continue;+ if (!assetData?.[id]) throw new Error(`Asset ${id.slice(0, 12)} was not supplied. Check asset_catalog for its id.`);- pool.set(key, await parseGltf(decodeBase64(assetData[id]).buffer, ASSET_DECODE_LIMITS));++ pool.set(key, await parseGltf(bytesFromBase64(assetData[id]).buffer, ASSET_DECODE_LIMITS));+ }+ }+}+function afterMeshBooleans(value, run) {if (!allAgentSceneModulesLoaded()) return loadAllAgentSceneModules().then(() => afterMeshBooleans(value, run));const loads = [];@@@ -726,16 -696,13 +737,23 @@@ window.__agentPreview =return { sessionId, ...scene.inspectMeshes(options) };},apply(input) {+ if (input.sources) {+ const { scene } = requireSession(input.sessionId);+ return decodeMotionSources(input.sources).then(decoded => {+ scene.provideMotionSources?.(decoded);+ return afterMeshBooleans(input.operations, () => this.applyLoaded(input));+ });+ }+ if (recipeAssetNeeds(input.operations).length) return this.applyWithAssets(input);+ return afterMeshBooleans(input.operations, () => this.applyLoaded(input));+ },+ async applyWithAssets(input) {+ const record = requireSession(input.sessionId);+ if (!record.placedAssets) {+ throw new Error('This scene was created without placed assets. Recreate it with an asset node in scene_create, then add more.');+ }+ await loadAllAgentSceneModules();+ await loadPlacedAssets(record.placedAssets, input.operations, input.assetData);return afterMeshBooleans(input.operations, () => this.applyLoaded(input));},applyLoaded({ sessionId, operations, expectedRevision }) {diff --cc scripts/agent-runtime.mjsindex 2f9c38f9,c5f96dec..a73065bc--- a/scripts/agent-runtime.mjs+++ b/scripts/agent-runtime.mjs@@@ -130,14 -130,23 +131,31 @@@ export function createAgentRuntime({ kereturn catalogPromise;}+ /** Bytes of the catalog assets a recipe or edit list places, base64 by asset id, for the page that decodes them. */+ async function placedAssetData(value) {+ const needs = recipeAssetNeeds(value);+ if (!needs.length) return {};+ const entries = await Promise.all(needs.map(async ({ id }) => [id, (await (await catalog()).read(id)).toString('base64')]));+ return { assetData: Object.fromEntries(entries) };+ }++ // Motion layers can play a clip of another catalog asset (`from.asset`). The page gets those assets' bytes by catalog ID,+ // never a path, and keeps only their clips and nodes.+ async function motionSources(motions) {+ const ids = new Set();+ for (const motion of motions) {+ for (const layer of Array.isArray(motion?.layers) ? motion.layers : []) {+ const id = layer?.from?.asset;+ if (typeof id === 'string' && /^[a-f0-9]{64}$/i.test(id)) ids.add(id.toLowerCase());+ }+ }+ if (ids.size > MAXIMUM_MOTION_SOURCES) throw new RangeError(`Motion layers can retarget clips of at most ${MAXIMUM_MOTION_SOURCES} other assets at once.`);+ if (!ids.size) return {};+ const sources = {};+ for (const id of ids) sources[id] = (await (await catalog()).read(id)).toString('base64');+ return { sources };+ }+function enqueue(task) {if (closing || phase === 'closing' || phase === 'closed' || phase === 'failed') {return Promise.reject(new Error(`The agent runtime is ${phase}.`));@@@ -581,7 -590,8 +599,8 @@@}const bakeAfter = bakeAfterOperations(input.operations);const rebakes = activeSceneHasBake || bakeAfter === true;- const applied = await invoke('apply', { ...input, ...(await placedAssetData(input.operations)) }, rebakes ? BAKE_EVALUATION_TIMEOUT_MS : EVALUATION_TIMEOUT_MS);+ const motions = (input.operations ?? []).filter(operation => operation.type === 'set-motion').map(operation => operation.motion);- const applied = await invoke('apply', { ...input, ...(await motionSources(motions)) }, rebakes ? BAKE_EVALUATION_TIMEOUT_MS : EVALUATION_TIMEOUT_MS);++ const applied = await invoke('apply', { ...input, ...(await motionSources(motions)), ...(await placedAssetData(input.operations)) }, rebakes ? BAKE_EVALUATION_TIMEOUT_MS : EVALUATION_TIMEOUT_MS);activeSceneHasBake = bakeAfter ?? activeSceneHasBake;return applied;});@@@ -688,8 -698,7 +707,8 @@@privateSessionId = randomUUID();await reviewEvaluate(branchRecipe || !assetBytes ? 'create' : 'createAsset', { sessionId: privateSessionId,recipe: branchRecipe ? branchRecipe.recipe : snapshot.recipe, memoryBudgetMiB: memory, backend,- ...(!branchRecipe && assetBytes ? { data: assetBytes.toString('base64') } : {}),- ...(!branchRecipe && assetBytes ? { data: assetBytes.toString('base64'), ...(await motionSources([snapshot.recipe.motion])) } : {}) });++ ...(!branchRecipe && assetBytes ? { data: assetBytes.toString('base64'), ...(await motionSources([snapshot.recipe.motion])) } : {}),+ ...(!assetBytes ? await placedAssetData(branchRecipe ? branchRecipe.recipe : snapshot.recipe) : {}) });privateCreated = true;for (const index of group.shotIndexes) {const rendered = await reviewEvaluate('reviewShot', { sessionId: privateSessionId, shot: settings.shots[index], index,diff --cc src/agent-scene.jsindex 7d7997cf,ccea5b29..e255d4d9--- a/src/agent-scene.js+++ b/src/agent-scene.js@@@ -1142,9 -1089,9 +1142,9 @@@ export function createAgentScene(inputstagedState.skinStages = createSkinStages(skinBindings);stagedState.motionSampler = recipe.motion ? agentSceneModule('motion').compileAgentMotion(recipe.motion, recipe) : null;stagedState.physicsSampler = stagePhysics(recipe, geometryRecords);- stagedState.clothSampler = stageCloth(recipe, geometryRecords);+ stagedState.clothSampler = stageCloth(recipe, geometryRecords, stagedState.motionSampler);stagedState.particleSampler = stageParticles(recipe, geometryRecords);- const animated = Boolean(stagedState.motionSampler || stagedState.physicsSampler || stagedState.clothSampler || stagedState.particleSampler);+ const animated = Boolean(stagedState.motionSampler || stagedState.physicsSampler || stagedState.clothSampler || stagedState.particleSampler || assetRuntime.animators.length);stagedState.poseGraph = animated ? compile(recipe, geometries, localBounds, limits) : null;stagedState.initialPose = stagePose(stagedState, animated ? 0 : null);return stagedState;