Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(vision): bundle the mediapipe module and serve its runtime from our own copy so mocap, sign input and selfie capture survive a blocked CDN
nirholas7e90894fbc4f
diff --git a/src/avatar-face-capture.js b/src/avatar-face-capture.jsindex 24960a4f1..e30704f17 100644--- a/src/avatar-face-capture.js+++ b/src/avatar-face-capture.js@@ -21,23 +21,15 @@* but they ship, they respect licensing, and the user fine-tunes from the* Sculpt panel anyway.*- * No bundling: tasks-vision + WASM model (~5 MB) load on demand from a CDN.- * License: MediaPipe + face_landmarker.task are both Apache-2.0 — safe.+ * Loading: tasks-vision is a dependency of this repo, so the module itself is+ * bundled; the ~5 MB WASM runtime and model still load on demand, through the+ * shared resolver that prefers our vendored copy over a CDN.+ * License: MediaPipe + face_landmarker.task are both Apache-2.0, safe.*/-// We load tasks-vision via dynamic ESM import. Pinning to a specific version-// avoids breaking changes from auto-upgraded CDN aliases; jsDelivr's `+esm`-// suffix returns the ESM build.-const TASKS_VISION_URL =- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/+esm';+import { loadVision, modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';-// Hosted by Google. They publish exactly one official URL per model — these-// are the standard locations used in every MediaPipe sample, public CORS,-// no auth.-const WASM_ROOT =- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';-const MODEL_URL =- 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';+const MODEL_URL = modelUrl('face_landmarker/face_landmarker/float16/1/face_landmarker.task');let _landmarkerPromise = null;@@ -123,9 +115,9 @@ detectFaceAll.loadLandmarker = loadLandmarker;function loadLandmarker() {if (_landmarkerPromise) return _landmarkerPromise;_landmarkerPromise = (async () => {- const mod = await import(/* @vite-ignore */ TASKS_VISION_URL);+ const mod = await loadVision();const { FilesetResolver, FaceLandmarker } = mod;- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());return FaceLandmarker.createFromOptions(vision, {baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },outputFaceBlendshapes: true,diff --git a/src/body-mocap.js b/src/body-mocap.jsindex a5125659f..045ce9538 100644--- a/src/body-mocap.js+++ b/src/body-mocap.js@@ -21,8 +21,9 @@ import { Quaternion } from 'three';import { solvePose } from './runtime/pose-solve.js';import { canonicalizeBoneName } from './glb-canonicalize.js';-const WASM_URL = 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';-const MODEL_URL = 'https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task';+import { modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';+// Runtime + model locations come from the shared resolver (see face-mocap.js).+const MODEL_URL = modelUrl('pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task');// Canonical bones this module is allowed to drive. Head/neck stay with FaceMocap.const DRIVEN_BONES = Object.freeze([@@ -73,7 +74,7 @@ export class BodyMocap {// ── Lifecycle ─────────────────────────────────────────────────────────────async init() {- const fileset = await FilesetResolver.forVisionTasks(WASM_URL);+ const fileset = await FilesetResolver.forVisionTasks(await visionWasmBase());this._landmarker = await PoseLandmarker.createFromOptions(fileset, {baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },runningMode: 'VIDEO',diff --git a/src/face-mocap.js b/src/face-mocap.jsindex ee565176e..c7ec31146 100644--- a/src/face-mocap.js+++ b/src/face-mocap.js@@ -22,8 +22,10 @@ import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';import { Matrix4, Euler, MathUtils } from 'three';import { resolveMorphTargets, setCanonicalMorph, MORPH_ALIASES, ARKIT_52 } from './runtime/arkit52.js';-const WASM_URL = 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';-const MODEL_URL = 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';+import { modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';+// Runtime + model locations come from the shared resolver: our vendored copy+// first, CDNs only as a backstop (src/shared/mediapipe-assets.js).+const MODEL_URL = modelUrl('face_landmarker/face_landmarker/float16/1/face_landmarker.task');const HEAD_MAX = 40 * (Math.PI / 180);// ── One-Euro filter (Casiez et al. 2012) ─────────────────────────────────────@@ -106,7 +108,7 @@ export class FaceMocap {// ── Lifecycle ─────────────────────────────────────────────────────────────async init() {- const fileset = await FilesetResolver.forVisionTasks(WASM_URL);+ const fileset = await FilesetResolver.forVisionTasks(await visionWasmBase());this._landmarker = await FaceLandmarker.createFromOptions(fileset, {baseOptions: {modelAssetPath: MODEL_URL,diff --git a/src/face-quality.js b/src/face-quality.jsindex ca9b12044..ae3fc6891 100644--- a/src/face-quality.js+++ b/src/face-quality.js@@ -22,21 +22,18 @@import { SLOT_PRESETS, gradeFrame, grayFaceStats } from './selfie-gates.js';-const TASKS_VISION_URL =- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/+esm';-const WASM_ROOT =- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';-const MODEL_URL =- 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';+import { loadVision, modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';+// The tasks-vision module is a real dependency, so it ships in our bundle+// instead of being pulled from a CDN at runtime; the WASM runtime and model come+// from the shared resolver (our vendored copy first).+const MODEL_URL = modelUrl('face_landmarker/face_landmarker/float16/1/face_landmarker.task');let _modPromise = null;let _landmarkerPromise = null;let _imageLandmarkerPromise = null;function loadModule() {- if (!_modPromise) {- _modPromise = import(/* @vite-ignore */ TASKS_VISION_URL);- }+ if (!_modPromise) _modPromise = loadVision();return _modPromise;}@@ -44,7 +41,7 @@ function loadLandmarker() {if (_landmarkerPromise) return _landmarkerPromise;_landmarkerPromise = (async () => {const { FilesetResolver, FaceLandmarker } = await loadModule();- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());return FaceLandmarker.createFromOptions(vision, {baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },outputFaceBlendshapes: false,@@ -69,7 +66,7 @@ function loadImageLandmarker() {if (_imageLandmarkerPromise) return _imageLandmarkerPromise;_imageLandmarkerPromise = (async () => {const { FilesetResolver, FaceLandmarker } = await loadModule();- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());return FaceLandmarker.createFromOptions(vision, {baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },runningMode: 'IMAGE',diff --git a/src/selfie-refine.js b/src/selfie-refine.jsindex 6ff6bc5e9..1376135a2 100644--- a/src/selfie-refine.js+++ b/src/selfie-refine.js@@ -29,21 +29,20 @@import { GATES } from './selfie-gates.js';import { log } from './shared/log.js';+import { loadVision, modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';// MediaPipe tasks-vision — same pinned build the face stack uses, so the WASM// runtime is shared/cached across the landmarker and the segmenter.-const TASKS_VISION_URL =- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/+esm';-const WASM_ROOT =- 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';+// tasks-vision is bundled from the npm dependency; the WASM runtime comes from+// the shared resolver, which prefers our vendored copy over a CDN.// Official Google-hosted binary selfie segmenter (Apache-2.0). One confidence// mask: per-pixel probability the pixel belongs to the person.const SELFIE_MODEL_URL =- 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite';+ modelUrl('image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite');// BlazeFace short-range detector (Apache-2.0): bounding boxes + count. Cheaper// than the 478-point landmarker and the right tool for "where/how many faces".const FACE_DETECTOR_MODEL_URL =- 'https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/latest/blaze_face_short_range.tflite';+ modelUrl('face_detector/blaze_face_short_range/float16/latest/blaze_face_short_range.tflite');// Working resolution: cap the longest edge so the matte + reframe stay fast on// phones. The output square is rendered at OUTPUT_SIZE regardless.@@ -332,9 +331,9 @@ let _segmenterPromise = null;async function loadSegmenter() {if (_segmenterPromise) return _segmenterPromise;_segmenterPromise = (async () => {- const mod = await import(/* @vite-ignore */ TASKS_VISION_URL);+ const mod = await loadVision();const { FilesetResolver, ImageSegmenter } = mod;- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());return ImageSegmenter.createFromOptions(vision, {baseOptions: { modelAssetPath: SELFIE_MODEL_URL, delegate: 'GPU' },runningMode: 'IMAGE',@@ -355,9 +354,9 @@ let _detectorPromise = null;async function loadFaceDetector() {if (_detectorPromise) return _detectorPromise;_detectorPromise = (async () => {- const mod = await import(/* @vite-ignore */ TASKS_VISION_URL);+ const mod = await loadVision();const { FilesetResolver, FaceDetector } = mod;- const vision = await FilesetResolver.forVisionTasks(WASM_ROOT);+ const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());return FaceDetector.createFromOptions(vision, {baseOptions: { modelAssetPath: FACE_DETECTOR_MODEL_URL, delegate: 'GPU' },runningMode: 'IMAGE',diff --git a/src/shared/mediapipe-assets.js b/src/shared/mediapipe-assets.jsnew file mode 100644index 000000000..8c8aa10bd--- /dev/null+++ b/src/shared/mediapipe-assets.js@@ -0,0 +1,117 @@+// Where MediaPipe's runtime and model files come from.+//+// Every vision feature (face and body mocap, sign-language input, selfie+// refinement, capture quality gating) needs three things: the tasks-vision+// module, the vision WASM runtime, and a `.task` / `.tflite` model. Each call+// site used to name a CDN URL for all three, so /mocap, /sign and+// selfie-to-avatar were simultaneously hard-dependent on cdn.jsdelivr.net AND+// storage.googleapis.com, with no timeout and no alternative. One blocked host+// (a corporate proxy, a region, an ad blocker, a CDN incident) took every one of+// those features down with a raw MediaPipe error string.+//+// This module answers all three questions once:+//+// loadVision() the tasks-vision module, from our own bundle+// visionWasmBase() the WASM directory, our copy first, then two CDNs+// modelUrl(name) a model file, our copy first, then Google's bucket+//+// `@mediapipe/tasks-vision` is a real dependency of this repo, so the module+// itself now ships in our bundle and needs no CDN at all. The WASM runtime is+// vendored under public/vendor/mediapipe/wasm, and face_landmarker.task with+// it; the larger models (pose, holistic, hand, segmenter) are still fetched+// from Google's public bucket, but under a deadline, so a stalled download+// surfaces as a real error instead of a spinner that never resolves.++// Models vendored into public/vendor/mediapipe. Anything not listed here is+// fetched from Google's bucket.+const LOCAL_MODELS = new Set(['face_landmarker.task']);++const GOOGLE_MODELS = 'https://storage.googleapis.com/mediapipe-models';++// Pinned to the version in package.json: the WASM runtime and the JS module must+// agree, so this constant moves when the dependency does.+const VERSION = '0.10.35';+const CDN_WASM_BASES = [+ `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${VERSION}/wasm`,+ `https://unpkg.com/@mediapipe/tasks-vision@${VERSION}/wasm`,+];++// Resolved against this module's own URL rather than written as a site-absolute+// path, so an embed served from three.ws still points at three.ws instead of at+// the embedding page's origin.+const LOCAL_BASE = (() => {+ try {+ return new URL('../../vendor/mediapipe', import.meta.url).href.replace(/\/$/, '');+ } catch {+ return '/vendor/mediapipe';+ }+})();++const PROBE_TIMEOUT_MS = 4_000;++/**+ * The tasks-vision module. Bundled, so this resolves without any network call+ * beyond our own assets.+ *+ * @returns {Promise<typeof import('@mediapipe/tasks-vision')>}+ */+export function loadVision() {+ return import('@mediapipe/tasks-vision');+}++let _wasmBase = null;++/**+ * Directory to hand `FilesetResolver.forVisionTasks()`. Prefers the copy under+ * public/vendor/mediapipe/wasm and falls back to jsDelivr then unpkg, probing+ * once per page and remembering the answer.+ *+ * @returns {Promise<string>}+ */+export async function visionWasmBase() {+ if (_wasmBase) return _wasmBase;+ for (const base of [LOCAL_BASE + '/wasm', ...CDN_WASM_BASES]) {+ try {+ const res = await fetch(`${base}/vision_wasm_internal.wasm`, {+ method: 'HEAD',+ signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),+ });+ if (res.ok) {+ _wasmBase = base;+ return base;+ }+ } catch {+ // Unreachable or blocked: try the next host.+ }+ }+ // Nothing answered the probe. Return the first CDN anyway rather than+ // throwing here: MediaPipe's own loader gives a better error than a probe.+ _wasmBase = CDN_WASM_BASES[0];+ return _wasmBase;+}++/**+ * Absolute URL for a MediaPipe model file. Vendored models come from our own+ * origin; everything else comes from Google's public bucket.+ *+ * @param {string} path Bucket-relative path, e.g.+ * 'pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task'+ * @returns {string}+ */+export function modelUrl(path) {+ const file = String(path).split('/').pop();+ if (LOCAL_MODELS.has(file)) return `${LOCAL_BASE}/${file}`;+ return `${GOOGLE_MODELS}/${String(path).replace(/^\/+/, '')}`;+}++/**+ * The two calls every consumer makes, in one await: the module plus a WASM+ * fileset resolved against the healthiest host.+ *+ * @returns {Promise<{ vision: any, tasks: typeof import('@mediapipe/tasks-vision') }>}+ */+export async function visionFileset() {+ const [tasks, base] = await Promise.all([loadVision(), visionWasmBase()]);+ const vision = await tasks.FilesetResolver.forVisionTasks(base);+ return { vision, tasks };+}diff --git a/src/sign-input.js b/src/sign-input.jsindex d36e99208..0611105cf 100644--- a/src/sign-input.js+++ b/src/sign-input.js@@ -18,9 +18,10 @@import { FilesetResolver, HolisticLandmarker } from '@mediapipe/tasks-vision';-const WASM_URL = 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';+import { modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';+// Runtime location comes from the shared resolver (see face-mocap.js).const MODEL_URL =- 'https://storage.googleapis.com/mediapipe-models/holistic_landmarker/holistic_landmarker/float16/latest/holistic_landmarker.task';+ modelUrl('holistic_landmarker/holistic_landmarker/float16/latest/holistic_landmarker.task');const COLUMN_RE = /^([xyz])_(face|left_hand|right_hand|pose)_(\d+)$/;@@ -90,7 +91,7 @@ export class SignInput {this.onState('loading');const [schemaRes, fileset] = await Promise.all([fetch(this.apiBase, { headers: { accept: 'application/json' } }),- FilesetResolver.forVisionTasks(WASM_URL),+ visionWasmBase().then((base) => FilesetResolver.forVisionTasks(base)),]);if (!schemaRes.ok) {const body = await schemaRes.json().catch(() => ({}));diff --git a/src/sign-mirror-page.js b/src/sign-mirror-page.jsindex 29192ed81..14fb4c11a 100644--- a/src/sign-mirror-page.js+++ b/src/sign-mirror-page.js@@ -280,10 +280,11 @@ async function boot() {if (!landmarker) {setStatus('Loading the hand tracker (about 8 MB, once per browser).');const { FilesetResolver, HandLandmarker } = await import('@mediapipe/tasks-vision');- const fileset = await FilesetResolver.forVisionTasks('https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm');+ const { modelUrl, visionWasmBase } = await import('./shared/mediapipe-assets.js');+ const fileset = await FilesetResolver.forVisionTasks(await visionWasmBase());landmarker = await HandLandmarker.createFromOptions(fileset, {baseOptions: {- modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',+ modelAssetPath: modelUrl('hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task'),delegate: 'GPU',},runningMode: 'VIDEO',