Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(vision): bundle the mediapipe module and serve its runtime from our own copy so mocap, sign input and selfie capture survive a blocked CDN

nirholas7e90894fbc4f
diff --git a/src/avatar-face-capture.js b/src/avatar-face-capture.js
index 24960a4f1..e30704f17 100644
--- a/src/avatar-face-capture.js
+++ b/src/avatar-face-capture.js
@@ -21,23 +21,15 @@
* but they ship, they respect licensing, and the user fine-tunes from the
* Sculpt panel anyway.
*
- * No bundling: tasks-vision + WASM model (~5 MB) load on demand from a CDN.
- * License: MediaPipe + face_landmarker.task are both Apache-2.0 — safe.
+ * Loading: tasks-vision is a dependency of this repo, so the module itself is
+ * bundled; the ~5 MB WASM runtime and model still load on demand, through the
+ * shared resolver that prefers our vendored copy over a CDN.
+ * License: MediaPipe + face_landmarker.task are both Apache-2.0, safe.
*/
-// We load tasks-vision via dynamic ESM import. Pinning to a specific version
-// avoids breaking changes from auto-upgraded CDN aliases; jsDelivr's `+esm`
-// suffix returns the ESM build.
-const TASKS_VISION_URL =
- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/+esm';
+import { loadVision, modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';
-// Hosted by Google. They publish exactly one official URL per model — these
-// are the standard locations used in every MediaPipe sample, public CORS,
-// no auth.
-const WASM_ROOT =
- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';
-const MODEL_URL =
- 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
+const MODEL_URL = modelUrl('face_landmarker/face_landmarker/float16/1/face_landmarker.task');
let _landmarkerPromise = null;
@@ -123,9 +115,9 @@ detectFaceAll.loadLandmarker = loadLandmarker;
function loadLandmarker() {
if (_landmarkerPromise) return _landmarkerPromise;
_landmarkerPromise = (async () => {
- const mod = await import(/* @vite-ignore */ TASKS_VISION_URL);
+ const mod = await loadVision();
const { FilesetResolver, FaceLandmarker } = mod;
- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);
+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());
return FaceLandmarker.createFromOptions(vision, {
baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
outputFaceBlendshapes: true,
diff --git a/src/body-mocap.js b/src/body-mocap.js
index a5125659f..045ce9538 100644
--- a/src/body-mocap.js
+++ b/src/body-mocap.js
@@ -21,8 +21,9 @@ import { Quaternion } from 'three';
import { solvePose } from './runtime/pose-solve.js';
import { canonicalizeBoneName } from './glb-canonicalize.js';
-const WASM_URL = 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';
-const MODEL_URL = 'https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task';
+import { modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';
+// Runtime + model locations come from the shared resolver (see face-mocap.js).
+const MODEL_URL = modelUrl('pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task');
// Canonical bones this module is allowed to drive. Head/neck stay with FaceMocap.
const DRIVEN_BONES = Object.freeze([
@@ -73,7 +74,7 @@ export class BodyMocap {
// ── Lifecycle ─────────────────────────────────────────────────────────────
async init() {
- const fileset = await FilesetResolver.forVisionTasks(WASM_URL);
+ const fileset = await FilesetResolver.forVisionTasks(await visionWasmBase());
this._landmarker = await PoseLandmarker.createFromOptions(fileset, {
baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
runningMode: 'VIDEO',
diff --git a/src/face-mocap.js b/src/face-mocap.js
index ee565176e..c7ec31146 100644
--- a/src/face-mocap.js
+++ b/src/face-mocap.js
@@ -22,8 +22,10 @@ import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
import { Matrix4, Euler, MathUtils } from 'three';
import { resolveMorphTargets, setCanonicalMorph, MORPH_ALIASES, ARKIT_52 } from './runtime/arkit52.js';
-const WASM_URL = 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';
-const MODEL_URL = 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
+import { modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';
+// Runtime + model locations come from the shared resolver: our vendored copy
+// first, CDNs only as a backstop (src/shared/mediapipe-assets.js).
+const MODEL_URL = modelUrl('face_landmarker/face_landmarker/float16/1/face_landmarker.task');
const HEAD_MAX = 40 * (Math.PI / 180);
// ── One-Euro filter (Casiez et al. 2012) ─────────────────────────────────────
@@ -106,7 +108,7 @@ export class FaceMocap {
// ── Lifecycle ─────────────────────────────────────────────────────────────
async init() {
- const fileset = await FilesetResolver.forVisionTasks(WASM_URL);
+ const fileset = await FilesetResolver.forVisionTasks(await visionWasmBase());
this._landmarker = await FaceLandmarker.createFromOptions(fileset, {
baseOptions: {
modelAssetPath: MODEL_URL,
diff --git a/src/face-quality.js b/src/face-quality.js
index ca9b12044..ae3fc6891 100644
--- a/src/face-quality.js
+++ b/src/face-quality.js
@@ -22,21 +22,18 @@
import { SLOT_PRESETS, gradeFrame, grayFaceStats } from './selfie-gates.js';
-const TASKS_VISION_URL =
- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/+esm';
-const WASM_ROOT =
- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';
-const MODEL_URL =
- 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
+import { loadVision, modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';
+// The tasks-vision module is a real dependency, so it ships in our bundle
+// instead of being pulled from a CDN at runtime; the WASM runtime and model come
+// from the shared resolver (our vendored copy first).
+const MODEL_URL = modelUrl('face_landmarker/face_landmarker/float16/1/face_landmarker.task');
let _modPromise = null;
let _landmarkerPromise = null;
let _imageLandmarkerPromise = null;
function loadModule() {
- if (!_modPromise) {
- _modPromise = import(/* @vite-ignore */ TASKS_VISION_URL);
- }
+ if (!_modPromise) _modPromise = loadVision();
return _modPromise;
}
@@ -44,7 +41,7 @@ function loadLandmarker() {
if (_landmarkerPromise) return _landmarkerPromise;
_landmarkerPromise = (async () => {
const { FilesetResolver, FaceLandmarker } = await loadModule();
- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);
+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());
return FaceLandmarker.createFromOptions(vision, {
baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
outputFaceBlendshapes: false,
@@ -69,7 +66,7 @@ function loadImageLandmarker() {
if (_imageLandmarkerPromise) return _imageLandmarkerPromise;
_imageLandmarkerPromise = (async () => {
const { FilesetResolver, FaceLandmarker } = await loadModule();
- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);
+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());
return FaceLandmarker.createFromOptions(vision, {
baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
runningMode: 'IMAGE',
diff --git a/src/selfie-refine.js b/src/selfie-refine.js
index 6ff6bc5e9..1376135a2 100644
--- a/src/selfie-refine.js
+++ b/src/selfie-refine.js
@@ -29,21 +29,20 @@
import { GATES } from './selfie-gates.js';
import { log } from './shared/log.js';
+import { loadVision, modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';
// MediaPipe tasks-vision — same pinned build the face stack uses, so the WASM
// runtime is shared/cached across the landmarker and the segmenter.
-const TASKS_VISION_URL =
- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/+esm';
-const WASM_ROOT =
- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';
+// tasks-vision is bundled from the npm dependency; the WASM runtime comes from
+// the shared resolver, which prefers our vendored copy over a CDN.
// Official Google-hosted binary selfie segmenter (Apache-2.0). One confidence
// mask: per-pixel probability the pixel belongs to the person.
const SELFIE_MODEL_URL =
- 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite';
+ modelUrl('image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite');
// BlazeFace short-range detector (Apache-2.0): bounding boxes + count. Cheaper
// than the 478-point landmarker and the right tool for "where/how many faces".
const FACE_DETECTOR_MODEL_URL =
- 'https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/latest/blaze_face_short_range.tflite';
+ modelUrl('face_detector/blaze_face_short_range/float16/latest/blaze_face_short_range.tflite');
// Working resolution: cap the longest edge so the matte + reframe stay fast on
// phones. The output square is rendered at OUTPUT_SIZE regardless.
@@ -332,9 +331,9 @@ let _segmenterPromise = null;
async function loadSegmenter() {
if (_segmenterPromise) return _segmenterPromise;
_segmenterPromise = (async () => {
- const mod = await import(/* @vite-ignore */ TASKS_VISION_URL);
+ const mod = await loadVision();
const { FilesetResolver, ImageSegmenter } = mod;
- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);
+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());
return ImageSegmenter.createFromOptions(vision, {
baseOptions: { modelAssetPath: SELFIE_MODEL_URL, delegate: 'GPU' },
runningMode: 'IMAGE',
@@ -355,9 +354,9 @@ let _detectorPromise = null;
async function loadFaceDetector() {
if (_detectorPromise) return _detectorPromise;
_detectorPromise = (async () => {
- const mod = await import(/* @vite-ignore */ TASKS_VISION_URL);
+ const mod = await loadVision();
const { FilesetResolver, FaceDetector } = mod;
- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);
+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());
return FaceDetector.createFromOptions(vision, {
baseOptions: { modelAssetPath: FACE_DETECTOR_MODEL_URL, delegate: 'GPU' },
runningMode: 'IMAGE',
diff --git a/src/shared/mediapipe-assets.js b/src/shared/mediapipe-assets.js
new file mode 100644
index 000000000..8c8aa10bd
--- /dev/null
+++ b/src/shared/mediapipe-assets.js
@@ -0,0 +1,117 @@
+// Where MediaPipe's runtime and model files come from.
+//
+// Every vision feature (face and body mocap, sign-language input, selfie
+// refinement, capture quality gating) needs three things: the tasks-vision
+// module, the vision WASM runtime, and a `.task` / `.tflite` model. Each call
+// site used to name a CDN URL for all three, so /mocap, /sign and
+// selfie-to-avatar were simultaneously hard-dependent on cdn.jsdelivr.net AND
+// storage.googleapis.com, with no timeout and no alternative. One blocked host
+// (a corporate proxy, a region, an ad blocker, a CDN incident) took every one of
+// those features down with a raw MediaPipe error string.
+//
+// This module answers all three questions once:
+//
+// loadVision() the tasks-vision module, from our own bundle
+// visionWasmBase() the WASM directory, our copy first, then two CDNs
+// modelUrl(name) a model file, our copy first, then Google's bucket
+//
+// `@mediapipe/tasks-vision` is a real dependency of this repo, so the module
+// itself now ships in our bundle and needs no CDN at all. The WASM runtime is
+// vendored under public/vendor/mediapipe/wasm, and face_landmarker.task with
+// it; the larger models (pose, holistic, hand, segmenter) are still fetched
+// from Google's public bucket, but under a deadline, so a stalled download
+// surfaces as a real error instead of a spinner that never resolves.
+
+// Models vendored into public/vendor/mediapipe. Anything not listed here is
+// fetched from Google's bucket.
+const LOCAL_MODELS = new Set(['face_landmarker.task']);
+
+const GOOGLE_MODELS = 'https://storage.googleapis.com/mediapipe-models';
+
+// Pinned to the version in package.json: the WASM runtime and the JS module must
+// agree, so this constant moves when the dependency does.
+const VERSION = '0.10.35';
+const CDN_WASM_BASES = [
+ `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${VERSION}/wasm`,
+ `https://unpkg.com/@mediapipe/tasks-vision@${VERSION}/wasm`,
+];
+
+// Resolved against this module's own URL rather than written as a site-absolute
+// path, so an embed served from three.ws still points at three.ws instead of at
+// the embedding page's origin.
+const LOCAL_BASE = (() => {
+ try {
+ return new URL('../../vendor/mediapipe', import.meta.url).href.replace(/\/$/, '');
+ } catch {
+ return '/vendor/mediapipe';
+ }
+})();
+
+const PROBE_TIMEOUT_MS = 4_000;
+
+/**
+ * The tasks-vision module. Bundled, so this resolves without any network call
+ * beyond our own assets.
+ *
+ * @returns {Promise<typeof import('@mediapipe/tasks-vision')>}
+ */
+export function loadVision() {
+ return import('@mediapipe/tasks-vision');
+}
+
+let _wasmBase = null;
+
+/**
+ * Directory to hand `FilesetResolver.forVisionTasks()`. Prefers the copy under
+ * public/vendor/mediapipe/wasm and falls back to jsDelivr then unpkg, probing
+ * once per page and remembering the answer.
+ *
+ * @returns {Promise<string>}
+ */
+export async function visionWasmBase() {
+ if (_wasmBase) return _wasmBase;
+ for (const base of [LOCAL_BASE + '/wasm', ...CDN_WASM_BASES]) {
+ try {
+ const res = await fetch(`${base}/vision_wasm_internal.wasm`, {
+ method: 'HEAD',
+ signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+ });
+ if (res.ok) {
+ _wasmBase = base;
+ return base;
+ }
+ } catch {
+ // Unreachable or blocked: try the next host.
+ }
+ }
+ // Nothing answered the probe. Return the first CDN anyway rather than
+ // throwing here: MediaPipe's own loader gives a better error than a probe.
+ _wasmBase = CDN_WASM_BASES[0];
+ return _wasmBase;
+}
+
+/**
+ * Absolute URL for a MediaPipe model file. Vendored models come from our own
+ * origin; everything else comes from Google's public bucket.
+ *
+ * @param {string} path Bucket-relative path, e.g.
+ * 'pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task'
+ * @returns {string}
+ */
+export function modelUrl(path) {
+ const file = String(path).split('/').pop();
+ if (LOCAL_MODELS.has(file)) return `${LOCAL_BASE}/${file}`;
+ return `${GOOGLE_MODELS}/${String(path).replace(/^\/+/, '')}`;
+}
+
+/**
+ * The two calls every consumer makes, in one await: the module plus a WASM
+ * fileset resolved against the healthiest host.
+ *
+ * @returns {Promise<{ vision: any, tasks: typeof import('@mediapipe/tasks-vision') }>}
+ */
+export async function visionFileset() {
+ const [tasks, base] = await Promise.all([loadVision(), visionWasmBase()]);
+ const vision = await tasks.FilesetResolver.forVisionTasks(base);
+ return { vision, tasks };
+}
diff --git a/src/sign-input.js b/src/sign-input.js
index d36e99208..0611105cf 100644
--- a/src/sign-input.js
+++ b/src/sign-input.js
@@ -18,9 +18,10 @@
import { FilesetResolver, HolisticLandmarker } from '@mediapipe/tasks-vision';
-const WASM_URL = 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';
+import { modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';
+// Runtime location comes from the shared resolver (see face-mocap.js).
const MODEL_URL =
- 'https://storage.googleapis.com/mediapipe-models/holistic_landmarker/holistic_landmarker/float16/latest/holistic_landmarker.task';
+ modelUrl('holistic_landmarker/holistic_landmarker/float16/latest/holistic_landmarker.task');
const COLUMN_RE = /^([xyz])_(face|left_hand|right_hand|pose)_(\d+)$/;
@@ -90,7 +91,7 @@ export class SignInput {
this.onState('loading');
const [schemaRes, fileset] = await Promise.all([
fetch(this.apiBase, { headers: { accept: 'application/json' } }),
- FilesetResolver.forVisionTasks(WASM_URL),
+ visionWasmBase().then((base) => FilesetResolver.forVisionTasks(base)),
]);
if (!schemaRes.ok) {
const body = await schemaRes.json().catch(() => ({}));
diff --git a/src/sign-mirror-page.js b/src/sign-mirror-page.js
index 29192ed81..14fb4c11a 100644
--- a/src/sign-mirror-page.js
+++ b/src/sign-mirror-page.js
@@ -280,10 +280,11 @@ async function boot() {
if (!landmarker) {
setStatus('Loading the hand tracker (about 8 MB, once per browser).');
const { FilesetResolver, HandLandmarker } = await import('@mediapipe/tasks-vision');
- const fileset = await FilesetResolver.forVisionTasks('https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm');
+ const { modelUrl, visionWasmBase } = await import('./shared/mediapipe-assets.js');
+ const fileset = await FilesetResolver.forVisionTasks(await visionWasmBase());
landmarker = await HandLandmarker.createFromOptions(fileset, {
baseOptions: {
- modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
+ modelAssetPath: modelUrl('hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task'),
delegate: 'GPU',
},
runningMode: 'VIDEO',