Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(rig): support Apple Kinect and MediaPipe bones (#212)

ni3a614eb7d2c660
diff --git a/docs/rig-doctor.md b/docs/rig-doctor.md
index 7d2418ca8..6f8e56ffd 100644
--- a/docs/rig-doctor.md
+++ b/docs/rig-doctor.md
@@ -117,6 +117,9 @@ The convention detector reports which authoring tool your skeleton came from and
| Mixamo | `mixamorig:` joint prefix | `mixamo` |
| VRM / VRoid | `J_Bip_*` humanoid joint names | `vrm-humanoid` |
| MikuMikuDance (PMX/PMD) | Japanese PMX bone names (`センター`, `上半身`, `左腕`) | `custom` |
+| Apple / ARKit joint suffix | Four or more skeleton joints ending in `_joint` | `custom` |
+| Kinect / Azure Kinect | `SpineBase` / `SpineMid` plus full trailing-side limb names | `custom` |
+| MediaPipe Pose | `foot_index` and `heel` landmark joints | `custom` |
| Reallusion Character Creator | `CC_Base_*` joint prefix | `custom` |
| Unreal Engine mannequin | `pelvis`, `thigh_l`, `clavicle_l` | `custom` |
| 3ds Max Biped | `Bip01` / `Bip001` prefix | `custom` |
diff --git a/pages/rig-doctor.html b/pages/rig-doctor.html
index b0ed2acff..9101b1924 100644
--- a/pages/rig-doctor.html
+++ b/pages/rig-doctor.html
@@ -122,7 +122,7 @@
<ul class="rd-hero-facts">
<li><strong>0 bytes</strong> leave your machine</li>
<li><strong id="rd-fact-joints">52</strong> canonical joints scored</li>
- <li><strong id="rd-fact-conventions">12</strong> rig conventions recognised</li>
+ <li><strong id="rd-fact-conventions">15</strong> rig conventions recognised</li>
<li><strong>1</strong> click to download the repair</li>
</ul>
</div>
diff --git a/src/glb-canonicalize.js b/src/glb-canonicalize.js
index 1b00a9e4f..592259569 100644
--- a/src/glb-canonicalize.js
+++ b/src/glb-canonicalize.js
@@ -12,6 +12,9 @@
// • VRM / VRoid: `J_Bip_C_Hips`, `J_Bip_L_UpperArm`, `J_Bip_L_Little1` (alias map)
// • VRM 1.0: `upperChest`, `leftUpperArm`, `leftLowerLeg`, `leftToes` (camelCase)
// • MMD (PMX/PMD): `センター`, `上半身`, `左腕`, `左ひじ`, `左ひざ` (Japanese, 左/右 side prefix)
+// • Apple / ARKit: `hips_joint`, `left_arm_joint`, `left_upLeg_joint` (`_joint` suffix)
+// • Kinect: `SpineBase`, `SpineShoulder`, `ShoulderLeft`, `KneeRight`
+// • MediaPipe Pose: `left_hip`, `left_heel`, `left_foot_index` (landmark joints)
// • Daz / Genesis: `hip`, `abdomen`, `lShldr`, `lForeArm`, `lThigh`, `lShin`, `lCollar`
// • MakeHuman: `upperarm.L`, `shin.L`, `clavicle.L` (shared with Unreal/Blender stems)
// • Simple rigs: `shoulderL`, `elbowL`, `wristL`, `hipL`, `kneeL`, `ankleL`, `chest`
@@ -213,6 +216,9 @@ const EXTRA_ALIASES = (() => {
// Second Life / OpenSim centre chain (`m`-prefixed body bones).
['mPelvis', 'Hips'], ['mTorso', 'Spine'], ['mChest', 'Spine1'],
['mNeck', 'Neck'], ['mHead', 'Head'],
+ // Kinect v2 / Azure Kinect body tracking. SpineShoulder is the upper
+ // torso joint between SpineMid and Neck, not a shoulder-side joint.
+ ['SpineBase', 'Hips'], ['SpineMid', 'Spine'], ['SpineShoulder', 'Spine2'],
// Advanced Skeleton (Maya) tags centre bones with an `_M` side token:
// `Root_M`, `Spine1_M`, `Spine2_M`, `Chest_M`, `Neck_M`, `Head_M`. Its LIMBS
// use `_L`/`_R` and already resolve through the sided table, which is why
@@ -282,6 +288,15 @@ const EXTRA_ALIASES = (() => {
['mElbowLeft', 'LeftForeArm'], ['mWristLeft', 'LeftHand'],
['mHipLeft', 'LeftUpLeg'], ['mKneeLeft', 'LeftLeg'],
['mAnkleLeft', 'LeftFoot'], ['mFootLeft', 'LeftToeBase'], ['mToeLeft', 'LeftToeBase'],
+ // Kinect puts the full side word last. ShoulderLeft is the upper-arm
+ // joint, while the Kinect skeleton has no separate clavicle joint.
+ ['ShoulderLeft', 'LeftArm'], ['ElbowLeft', 'LeftForeArm'],
+ ['WristLeft', 'LeftHand'], ['HipLeft', 'LeftUpLeg'],
+ ['KneeLeft', 'LeftLeg'], ['AnkleLeft', 'LeftFoot'], ['FootLeft', 'LeftToeBase'],
+ // MediaPipe Pose uses its foot-index landmark as the toe endpoint and
+ // heel as the articulating foot. Its other limb names already resolve
+ // through the side-prefix aliases above.
+ ['leftHeel', 'LeftFoot'], ['leftFootIndex', 'LeftToeBase'],
// Side-SUFFIX leg spellings (`UpperLeg.L`, `LowerLeg.R`, `LowerArm.L`). The
// arm twins were added with the Rigify forearm fix, but the LEG twins were
// not, and the side-PREFIX entries above (`leftUpperLeg`, `lUpperLeg`) never
@@ -400,6 +415,13 @@ export function canonicalizeBoneName(name) {
if (typeof name !== 'string' || !name) return null;
const direct = _lookupBone(name);
if (direct) return direct;
+ // Apple Reality Composer, ARKit-to-Maya, and related USD pipelines append
+ // `_joint` to every skeleton joint. Retry without that exact trailing token
+ // only after the full spelling failed, preserving any future literal alias
+ // that legitimately ends in `joint`. `root_joint` stays unmapped because
+ // `root` is a transform above the skeleton, not a body joint.
+ const appleJoint = name.replace(/_joint$/i, '');
+ if (appleJoint !== name) return _lookupBone(appleJoint);
// glTF/FBX node de-dup suffix: exporters (CharacterStudio, Blender's glTF
// writer, FBX2glTF) append `_NN` to keep node names unique, producing
// `mixamorig:Hips_01`, `Spine1_03`, `LeftForeArm_010`. The plain lookup
diff --git a/src/rig-report.js b/src/rig-report.js
index 1714ee8c5..7a962a018 100644
--- a/src/rig-report.js
+++ b/src/rig-report.js
@@ -93,6 +93,21 @@ export const CONVENTIONS = [
test: (ctx) => ctx.joints.some((n) => /^(センター|上半身2?|下半身|[左右](腕|ひじ|ひざ|手首|足首))$/.test(n)),
evidence: 'Japanese PMX bone names (センター / 上半身 / 左腕)',
},
+ {
+ id: 'apple-joint', label: 'Apple / ARKit joint suffix', schema: 'custom',
+ test: (ctx) => ctx.joints.filter((n) => /_joint$/i.test(n)).length >= 4,
+ evidence: 'four or more skeleton joints with the _joint suffix',
+ },
+ {
+ id: 'kinect', label: 'Kinect / Azure Kinect', schema: 'custom',
+ test: (ctx) => ctx.joints.some((n) => /^Spine(Base|Mid|Shoulder)$/i.test(n)) && ctx.joints.some((n) => /^(Shoulder|Elbow|Knee)(Left|Right)$/i.test(n)),
+ evidence: 'SpineBase/SpineMid plus full trailing-side limb names',
+ },
+ {
+ id: 'mediapipe', label: 'MediaPipe Pose', schema: 'custom',
+ test: (ctx) => ctx.joints.some((n) => /^(left|right)_foot_index$/i.test(n)) && ctx.joints.some((n) => /^(left|right)_heel$/i.test(n)),
+ evidence: 'foot_index and heel landmark joints',
+ },
{
id: 'reallusion', label: 'Reallusion Character Creator', schema: 'custom',
test: (ctx) => ctx.joints.some((n) => /^CC_Base_/i.test(n)),
diff --git a/tests/glb-canonicalize.test.js b/tests/glb-canonicalize.test.js
index f50bd57a5..6ca4624ba 100644
--- a/tests/glb-canonicalize.test.js
+++ b/tests/glb-canonicalize.test.js
@@ -485,6 +485,57 @@ describe('canonicalizeBoneName', () => {
expect(canonicalizeBoneName(input)).toBe(expected);
});
+ it.each([
+ ['hips_joint', 'Hips'],
+ ['left_arm_joint', 'LeftArm'],
+ ['right_arm_joint', 'RightArm'],
+ ['left_forearm_joint', 'LeftForeArm'],
+ ['right_hand_joint', 'RightHand'],
+ ['left_upLeg_joint', 'LeftUpLeg'],
+ ['right_leg_joint', 'RightLeg'],
+ ['left_foot_joint', 'LeftFoot'],
+ ])('maps Apple / ARKit _joint skeleton bones: %s → %s', (input, expected) => {
+ expect(canonicalizeBoneName(input)).toBe(expected);
+ });
+
+ it('does not map Apple transform roots as body joints', () => {
+ expect(canonicalizeBoneName('root_joint')).toBeNull();
+ });
+
+ it.each([
+ ['SpineBase', 'Hips'],
+ ['SpineMid', 'Spine'],
+ ['SpineShoulder', 'Spine2'],
+ ['ShoulderLeft', 'LeftArm'],
+ ['ShoulderRight', 'RightArm'],
+ ['ElbowLeft', 'LeftForeArm'],
+ ['WristRight', 'RightHand'],
+ ['HipLeft', 'LeftUpLeg'],
+ ['KneeRight', 'RightLeg'],
+ ['AnkleLeft', 'LeftFoot'],
+ ['FootRight', 'RightToeBase'],
+ ])('maps Kinect body-tracking bones: %s → %s', (input, expected) => {
+ expect(canonicalizeBoneName(input)).toBe(expected);
+ });
+
+ it.each([
+ ['left_heel', 'LeftFoot'],
+ ['right_heel', 'RightFoot'],
+ ['left_foot_index', 'LeftToeBase'],
+ ['right_foot_index', 'RightToeBase'],
+ ])('maps MediaPipe Pose foot landmarks: %s → %s', (input, expected) => {
+ expect(canonicalizeBoneName(input)).toBe(expected);
+ });
+
+ it('never crosses sides for Apple, Kinect, or MediaPipe names', () => {
+ for (const name of ['left_arm_joint', 'ShoulderLeft', 'left_heel', 'left_foot_index']) {
+ expect(canonicalizeBoneName(name)?.startsWith('Right'), name).toBe(false);
+ }
+ for (const name of ['right_arm_joint', 'ShoulderRight', 'right_heel', 'right_foot_index']) {
+ expect(canonicalizeBoneName(name)?.startsWith('Left'), name).toBe(false);
+ }
+ });
+
// Side-SUFFIX leg spellings. The arm twins (`forearm.L`, `upper_arm.L`) were
// covered by the Rigify fix; the LEG twins were not, and no side-PREFIX entry
// reaches `upperlegl`. Found in production by scripts/audit-rig-coverage.mjs:
diff --git a/tests/rig-report.test.js b/tests/rig-report.test.js
index 948ec77bb..cac663326 100644
--- a/tests/rig-report.test.js
+++ b/tests/rig-report.test.js
@@ -91,6 +91,21 @@ describe('detectConvention fingerprints', () => {
expect(detectConvention(ctx(['Hips', 'Spine', 'LeftArm', 'RightArm'])).id).not.toBe('mmd');
});
+ it('identifies Apple joint-suffixed rigs', () => {
+ const c = detectConvention(ctx(['hips_joint', 'spine_joint', 'left_arm_joint', 'right_arm_joint']));
+ expect(c.id).toBe('apple-joint');
+ });
+
+ it('identifies Kinect rigs without confusing SpineShoulder for a limb', () => {
+ const c = detectConvention(ctx(['SpineBase', 'SpineMid', 'SpineShoulder', 'ShoulderLeft', 'KneeRight']));
+ expect(c.id).toBe('kinect');
+ });
+
+ it('identifies MediaPipe Pose from its unique foot landmarks', () => {
+ const c = detectConvention(ctx(['left_hip', 'left_knee', 'left_heel', 'right_foot_index']));
+ expect(c.id).toBe('mediapipe');
+ });
+
it('reports unknown rather than guessing when nothing fingerprints', () => {
expect(detectConvention(ctx(['bone_a', 'bone_b', 'bone_c'])).id).toBe('unknown');
});