3D engine that AI agents drive
Prove retarget, foot lock and keyed morphs end to end through MCP
Nguyen Xuan Ana55596abeedc
diff --git a/changes/325.md b/changes/325.mdindex 4bcd84dc..5934638c 100644--- a/changes/325.md+++ b/changes/325.md@@ -1 +1,2 @@- Retarget a clip from another catalog asset through MCP with `motion.layers[].from`, plant the feet with `motion.footLock` (two-bone IK on contact phases found from foot height and speed, soles turned to an optional ground normal), and key morph weights by target name with `{ kind: "morph" }` tracks (issue #325). `scene_inspect` hierarchy reports `retargets` (mapped, unmapped and missing bones), `footLock` phases and `morphTargetNames`; measured foot drift in planted phases under 2 cm on the test pair.+- `npm run smoke:agent-retarget` proves the whole path through MCP with two humanoid GLBs in the asset catalog: feet within 2 cm in planted phases (4.6 cm without the lock), the sole within 5 degrees of a slope, and keyed blink and jaw weights that seek back to identical bytes (issue #325).diff --git a/package.json b/package.jsonindex 57fe4020..f423d321 100644--- a/package.json+++ b/package.json@@ -154,6 +154,7 @@"benchmark:update-parts": "node scripts/benchmark-update-parts.mjs","smoke:imported-topology": "node scripts/smoke-imported-topology.mjs","smoke:agent-character": "node scripts/smoke-agent-character.mjs",+ "smoke:agent-retarget": "node scripts/smoke-agent-retarget.mjs","smoke:agent-review": "node scripts/smoke-agent-review.mjs","smoke:agent-branches": "node scripts/smoke-agent-branches.mjs","parity": "node scripts/parity-report.mjs",diff --git a/scripts/smoke-agent-retarget.mjs b/scripts/smoke-agent-retarget.mjsnew file mode 100644index 00000000..089338d6--- /dev/null+++ b/scripts/smoke-agent-retarget.mjs@@ -0,0 +1,160 @@+import assert from 'node:assert/strict';+import { createHash } from 'node:crypto';+import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises';+import { constants, setPriority, tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { mat4 } from 'gl-matrix';+import { Client } from '@modelcontextprotocol/client';+import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';+import { FACE_TARGETS, MIXAMO_LIKE, UNREAL_LIKE, humanoidGlb, humanoidJoints, walkTracks } from '../tests/fixtures/humanoid-glb.js';++// Proof through MCP for retargeting: a walk from one humanoid file plays on a second humanoid with another rest pose and+// proportions, the feet are pinned during contact, the sole follows a slope, and a face blinks and opens its jaw from keyed+// morph weights. Foot positions are read from GLB files the server exports at sampled motion times.++setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);+const directory = await mkdtemp(join(tmpdir(), '67js-mcp-retarget-'));+const jobs = join(directory, 'jobs');+const assets = join(directory, 'assets');+await mkdir(assets);++const sourceJoints = humanoidJoints();+const targetJoints = humanoidJoints({ hipsHeight: 1.1, thigh: 0.62, shin: 0.48, armTiltDegrees: 30 });+const sourceBytes = Buffer.from(humanoidGlb({ names: MIXAMO_LIKE, joints: sourceJoints, clips: [{ name: 'Walk', duration: 1, tracks: walkTracks(sourceJoints) }] }));+const targetBytes = Buffer.from(humanoidGlb({ names: UNREAL_LIKE, joints: targetJoints, face: true }));+const idOf = bytes => createHash('sha256').update(bytes).digest('hex');+const sourceId = idOf(sourceBytes);+const targetId = idOf(targetBytes);+await writeFile(join(assets, 'source.glb'), sourceBytes);+await writeFile(join(assets, 'target.glb'), targetBytes);++const transport = new StdioClientTransport({+ command: process.execPath,+ args: [resolve(process.argv[2] ?? 'bin/67js-mcp.mjs'), '--backend', 'cpu', '--memory', '96', '--jobs', jobs, '--assets', assets],+ cwd: directory, stderr: 'pipe',+});+let diagnostics = '';+transport.stderr.on('data', bytes => { diagnostics = (diagnostics + bytes).slice(-8192); });+const client = new Client({ name: '67js-retarget-proof', version: '1' });++async function call(name, argumentsValue = {}, timeout = 60000) {+ const result = await client.callTool({ name, arguments: argumentsValue }, { timeout });+ assert.equal(result.isError, undefined, result.content?.find(item => item.type === 'text')?.text);+ return result;+}++/** World matrix of a named node in a GLB the server wrote, from the posed node transforms. */+function nodeWorldMatrix(glb, name) {+ const jsonLength = glb.readUInt32LE(12);+ const document = JSON.parse(glb.subarray(20, 20 + jsonLength).toString('utf8'));+ const parents = new Map();+ document.nodes.forEach((node, index) => (node.children ?? []).forEach(child => parents.set(child, index)));+ let index = document.nodes.findIndex(node => node.name === name);+ assert.ok(index >= 0, `the export has no node ${name}`);+ let world = mat4.create();+ while (index !== undefined) {+ const node = document.nodes[index];+ const local = node.matrix+ ? mat4.clone(node.matrix)+ : mat4.fromRotationTranslationScale(mat4.create(), node.rotation ?? [0, 0, 0, 1], node.translation ?? [0, 0, 0], node.scale ?? [1, 1, 1]);+ world = mat4.multiply(mat4.create(), local, world);+ index = parents.get(index);+ }+ return world;+}++async function exportedGlb(sessionId, motionTime) {+ const exported = (await call('scene_export_glb', { sessionId, motionTime })).structuredContent;+ return readFile(exported.path);+}++async function footAt(sessionId, node, motionTime) {+ return Array.from(nodeWorldMatrix(await exportedGlb(sessionId, motionTime), node).subarray(12, 15));+}++async function maximumDrift(sessionId, node, phases) {+ let worst = 0;+ for (const phase of phases) {+ const anchor = await footAt(sessionId, node, phase.start);+ for (let step = 1; step <= 8; step++) {+ const position = await footAt(sessionId, node, phase.start + (phase.end - phase.start) * step / 8);+ worst = Math.max(worst, Math.hypot(position[0] - anchor[0], position[1] - anchor[1], position[2] - anchor[2]));+ }+ }+ return worst;+}++const walkLayer = { from: { asset: sourceId, clip: 0 }, scale: 1.25 };+const footLock = { contactHeight: 0.06, contactSpeed: 1.2 };+const angle = 10 * Math.PI / 180;+const slope = [0, Math.cos(angle), -Math.sin(angle)];++/** Creates the target character with a motion, runs the check on its session, and disposes it: one scene at a time. */+async function withTarget(motion, check) {+ const recipe = { format: '67js.asset', schemaVersion: 1, assetId: targetId, motion: { duration: 1, loop: true, ...motion } };+ const sessionId = (await call('scene_create_asset', { recipe }, 180000)).structuredContent.sessionId;+ try {+ return await check(sessionId);+ } finally {+ await call('scene_dispose', { sessionId });+ }+}++const inspectHierarchy = async sessionId => (await call('scene_inspect', { sessionId, kind: 'hierarchy', limit: 64 })).structuredContent;++try {+ await client.connect(transport);++ const locked = await withTarget({ layers: [walkLayer], footLock }, async sessionId => {+ const hierarchy = await inspectHierarchy(sessionId);+ assert.deepEqual(hierarchy.retargets[0].source.missing, []);+ assert.deepEqual(hierarchy.retargets[0].target.missing, []);+ assert.equal(hierarchy.retargets[0].target.mapped, 15);+ const feet = hierarchy.footLock.feet;+ assert.ok(feet.length === 2 && feet.every(foot => foot.phases.length >= 1), JSON.stringify(feet));+ const drift = [];+ for (const foot of feet) {+ drift.push(await maximumDrift(sessionId, UNREAL_LIKE[foot.bone], foot.phases));+ }+ return { feet, drift };+ });+ locked.drift.forEach((drift, slot) => assert.ok(drift < 0.02, `with the lock ${locked.feet[slot].bone} slides ${drift} m`));++ const slide = await withTarget({ layers: [walkLayer] }, sessionId => maximumDrift(sessionId, UNREAL_LIKE[locked.feet[0].bone], locked.feet[0].phases));+ assert.ok(slide > 0.02, `without the lock the foot should slide, drift ${slide}`);++ const slopeDegrees = await withTarget({ layers: [walkLayer], footLock: { ...footLock, ground: { height: 0, normal: slope } } }, async sessionId => {+ const phase = (await inspectHierarchy(sessionId)).footLock.feet[0].phases[0];+ const sole = nodeWorldMatrix(await exportedGlb(sessionId, (phase.start + phase.end) / 2), UNREAL_LIKE.leftFoot);+ const length = Math.hypot(sole[4], sole[5], sole[6]);+ const cosine = (sole[4] * slope[0] + sole[5] * slope[1] + sole[6] * slope[2]) / length;+ return Math.acos(Math.min(1, cosine)) * 180 / Math.PI;+ });+ assert.ok(slopeDegrees < 5, `the planted sole is ${slopeDegrees} degrees off the ground normal`);++ const faceMotion = { duration: 2, loop: false, tracks: [+ { kind: 'morph', id: 'eyeBlinkLeft', property: 'weight', keyframes: [{ time: 0.5, value: 0 }, { time: 0.6, value: 1 }, { time: 0.7, value: 0 }] },+ { kind: 'morph', id: 'jawOpen', property: 'weight', keyframes: [{ time: 0, value: 0 }, { time: 1, value: 0.5 }] },+ ] };+ await withTarget(faceMotion, async sessionId => {+ assert.deepEqual((await inspectHierarchy(sessionId)).morphTargetNames, FACE_TARGETS);+ const faceLook = async motionTime => (await call('scene_look', { sessionId, motionTime, width: 160, height: 160 })).content.find(item => item.type === 'image').data;+ const faceExport = async motionTime => (await exportedGlb(sessionId, motionTime)).toString('base64');+ const blinkExport = await faceExport(0.6);+ const blinkLook = await faceLook(0.6);+ assert.notEqual(await faceExport(1.5), blinkExport, 'keyed weights change the exported face');+ assert.notEqual(await faceLook(1.5), blinkLook, 'keyed weights change the picture');+ assert.equal(await faceExport(0.6), blinkExport, 'seeking back repeats the exported bytes');+ assert.equal(await faceLook(0.6), blinkLook, 'seeking back repeats the PNG bytes');+ });++ console.log(JSON.stringify({+ unlockedSlideMetres: Number(slide.toFixed(4)),+ lockedDriftMetres: locked.drift.map(value => Number(value.toFixed(4))),+ slopeDegrees: Number(slopeDegrees.toFixed(2)),+ morphSeekExact: true,+ }));+} finally {+ await client.close();+ if (diagnostics.includes('Error')) console.error(diagnostics);+}