Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* Real-time face mesh quality engine.
*
* Wraps MediaPipe FaceLandmarker for live video analysis. Provides:
* - 468-point wireframe overlay rendering
* - Head-pose estimation (yaw / pitch / roll)
* - Blur detection via Laplacian variance
* - Lighting analysis (luma mean)
* - Face centering check
* - Per-frame quality verdict with named gates
*
* Usage:
* const session = await createQualitySession(videoEl, canvasEl);
* session.onUpdate = (report) => { ... };
* session.start();
* // later:
* session.stop();
*
* The session lazily loads the ~5MB model on first use and caches it
* for subsequent sessions. CDN-hosted for Vite WASM compatibility.
*/
import { SLOT_PRESETS, gradeFrame, grayFaceStats } from './selfie-gates.js';
import { loadVision, modelUrl, visionWasmBase, withVisionDeadline } from './shared/mediapipe-assets.js';
// The tasks-vision module is a real dependency, so it ships in our bundle
// instead of being pulled from a CDN at runtime; the WASM runtime and model come
// from the shared resolver (our vendored copy first).
const MODEL_URL = modelUrl('face_landmarker/face_landmarker/float16/1/face_landmarker.task');
let _modPromise = null;
let _landmarkerPromise = null;
let _imageLandmarkerPromise = null;
function loadModule() {
if (!_modPromise) _modPromise = loadVision();
return _modPromise;
}
function loadLandmarker() {
if (_landmarkerPromise) return _landmarkerPromise;
_landmarkerPromise = (async () => {
const { FilesetResolver, FaceLandmarker } = await loadModule();
const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());
return FaceLandmarker.createFromOptions(vision, {
baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
outputFaceBlendshapes: false,
outputFacialTransformationMatrixes: false,
runningMode: 'VIDEO',
numFaces: 1,
});
})();
_landmarkerPromise = withVisionDeadline(_landmarkerPromise, 'face landmarker').catch((err) => {
_landmarkerPromise = null;
throw err;
});
return _landmarkerPromise;
}
/**
* IMAGE-mode landmarker for one-shot stills, cached like the VIDEO one.
* Every review shot and every upload used to build and tear down its own,
* which re-reads the ~5MB model and re-initialises the GPU delegate while the
* user waits on "Checking shot..." right after the shutter fires.
*/
function loadImageLandmarker() {
if (_imageLandmarkerPromise) return _imageLandmarkerPromise;
_imageLandmarkerPromise = (async () => {
const { FilesetResolver, FaceLandmarker } = await loadModule();
const vision = await FilesetResolver.forVisionTasks(await visionWasmBase());
return FaceLandmarker.createFromOptions(vision, {
baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
runningMode: 'IMAGE',
numFaces: 1,
outputFaceBlendshapes: false,
});
})();
_imageLandmarkerPromise = withVisionDeadline(_imageLandmarkerPromise, 'image face landmarker').catch((err) => {
_imageLandmarkerPromise = null;
throw err;
});
return _imageLandmarkerPromise;
}
/** Pre-warm the model download. */
export function preload() {
loadLandmarker().catch(() => {});
}
/**
* Slot presets for yaw-angle gating live in selfie-gates.js (the shared,
* unit-tested threshold module); re-exported here so existing consumers keep
* their import path. Yaw sign: looking left (camera's right) → positive.
*/
export { SLOT_PRESETS };
/**
* @typedef {Object} QualityReport
* @property {boolean} faceFound
* @property {boolean} centered
* @property {boolean} yawOk
* @property {boolean} blurOk
* @property {boolean} lumaOk
* @property {boolean} allPass
* @property {number|null} yaw
* @property {number|null} pitch
* @property {number|null} roll
* @property {number} blur
* @property {number} luma
* @property {number} clippedFrac Share of the face crop blown to near-white.
* @property {string|null} reason User-facing retake prompt; null when allPass.
* @property {Array|null} landmarks
*/
/**
* Create a real-time face quality session.
*
* @param {HTMLVideoElement} videoEl - Live camera video element
* @param {HTMLCanvasElement} canvasEl - Canvas overlay for wireframe drawing
* @param {{ slot?: string, onUpdate?: (report: QualityReport) => void }} [opts]
* @returns {Promise<{ start: () => void, stop: () => void, setSlot: (s: string) => void, onUpdate: ((r: QualityReport) => void)|null }>}
*/
export async function createQualitySession(videoEl, canvasEl, opts = {}) {
const landmarker = await loadLandmarker();
const mod = await loadModule();
const { FaceLandmarker, DrawingUtils } = mod;
let _running = false;
let _rafId = 0;
let _lastVideoTime = -1;
let _slot = opts.slot || 'frontal';
let _qCanvas = null;
let _qCtx = null;
const session = {
onUpdate: opts.onUpdate || null,
start() {
if (_running) return;
_running = true;
_lastVideoTime = -1;
tick();
},
stop() {
_running = false;
if (_rafId) {
cancelAnimationFrame(_rafId);
_rafId = 0;
}
},
setSlot(s) {
_slot = s;
},
};
function tick() {
if (!_running) return;
const report = analyze();
if (report && session.onUpdate) session.onUpdate(report);
_rafId = requestAnimationFrame(tick);
}
function analyze() {
const w = videoEl.videoWidth;
const h = videoEl.videoHeight;
if (!w || !h) return null;
if (canvasEl.width !== w || canvasEl.height !== h) {
canvasEl.width = w;
canvasEl.height = h;
}
const ctx = canvasEl.getContext('2d');
ctx.clearRect(0, 0, w, h);
let result = null;
if (videoEl.currentTime !== _lastVideoTime) {
_lastVideoTime = videoEl.currentTime;
try {
result = landmarker.detectForVideo(videoEl, performance.now());
} catch (_) {
return null;
}
}
if (!result) return null;
const lms = result.faceLandmarks?.[0];
if (!lms || lms.length < 468) {
return {
...gradeFrame({ faceFound: false }),
yaw: null, pitch: null, roll: null,
blur: 0, luma: 0, clippedFrac: 0, landmarks: null,
};
}
drawWireframe(ctx, lms, FaceLandmarker, DrawingUtils);
const pose = estimateHeadPose(lms);
const nose = lms[1] || lms[4];
const q = computeFaceQuality(videoEl, lms);
const grade = gradeFrame({
faceFound: true,
slot: _slot,
yaw: pose.yaw,
noseX: nose ? nose.x : null,
noseY: nose ? nose.y : null,
blur: q.blur,
luma: q.luma,
clippedFrac: q.clippedFrac,
});
return {
...grade,
yaw: pose.yaw,
pitch: pose.pitch,
roll: pose.roll,
blur: q.blur,
luma: q.luma,
clippedFrac: q.clippedFrac,
landmarks: lms,
};
}
function drawWireframe(ctx, lms, FL, DU) {
const utils = new DU(ctx);
utils.drawConnectors(lms, FL.FACE_LANDMARKS_TESSELATION, {
color: 'rgba(255, 255, 255, 0.12)', lineWidth: 0.4,
});
utils.drawConnectors(lms, FL.FACE_LANDMARKS_FACE_OVAL, {
color: 'rgba(255, 255, 255, 0.45)', lineWidth: 1.2,
});
utils.drawConnectors(lms, FL.FACE_LANDMARKS_RIGHT_EYE, {
color: 'rgba(255, 255, 255, 0.55)', lineWidth: 1,
});
utils.drawConnectors(lms, FL.FACE_LANDMARKS_LEFT_EYE, {
color: 'rgba(255, 255, 255, 0.55)', lineWidth: 1,
});
utils.drawConnectors(lms, FL.FACE_LANDMARKS_RIGHT_EYEBROW, {
color: 'rgba(255, 255, 255, 0.35)', lineWidth: 0.8,
});
utils.drawConnectors(lms, FL.FACE_LANDMARKS_LEFT_EYEBROW, {
color: 'rgba(255, 255, 255, 0.35)', lineWidth: 0.8,
});
utils.drawConnectors(lms, FL.FACE_LANDMARKS_LIPS, {
color: 'rgba(255, 255, 255, 0.45)', lineWidth: 1,
});
}
function computeFaceQuality(video, lms) {
const SIZE = 64;
if (!_qCanvas) {
_qCanvas = document.createElement('canvas');
_qCanvas.width = _qCanvas.height = SIZE;
_qCtx = _qCanvas.getContext('2d', { willReadFrequently: true });
}
let minX = 1, minY = 1, maxX = 0, maxY = 0;
for (const lm of lms) {
if (lm.x < minX) minX = lm.x;
if (lm.y < minY) minY = lm.y;
if (lm.x > maxX) maxX = lm.x;
if (lm.y > maxY) maxY = lm.y;
}
const pad = 0.05;
const vw = video.videoWidth, vh = video.videoHeight;
const fx = Math.max(0, (minX - pad)) * vw;
const fy = Math.max(0, (minY - pad)) * vh;
const fw = Math.min(vw - fx, (maxX - minX + 2 * pad) * vw);
const fh = Math.min(vh - fy, (maxY - minY + 2 * pad) * vh);
if (fw < 4 || fh < 4) return { luma: 0, blur: 0, clippedFrac: 0 };
_qCtx.drawImage(video, fx, fy, fw, fh, 0, 0, SIZE, SIZE);
const { data: d } = _qCtx.getImageData(0, 0, SIZE, SIZE);
const grays = new Float32Array(SIZE * SIZE);
for (let i = 0, j = 0; i < d.length; i += 4, j++) {
grays[j] = 0.299 * d[i] + 0.587 * d[i + 1] + 0.114 * d[i + 2];
}
const { luma, blurStddev, clippedFrac } = grayFaceStats(grays, SIZE, SIZE);
return { luma, blur: blurStddev, clippedFrac };
}
return session;
}
/**
* Head-pose estimation from 478 MediaPipe landmarks.
* Uses geometric asymmetry — fast, no solvePnP dependency.
*/
export function estimateHeadPose(lms) {
const r = lms[33];
const l = lms[263];
const nose = lms[1];
const dxEye = l.x - r.x;
const dyEye = l.y - r.y;
const roll = (Math.atan2(dyEye, dxEye) * 180) / Math.PI;
const distR = Math.hypot(nose.x - r.x, nose.y - r.y);
const distL = Math.hypot(nose.x - l.x, nose.y - l.y);
const yawRaw = (distR - distL) / (distR + distL);
const yaw = -yawRaw * 90;
const eyeMidY = (l.y + r.y) / 2;
const eyeSpan = Math.hypot(l.x - r.x, l.y - r.y) || 1;
const pitchRaw = (eyeMidY - nose.y) / eyeSpan;
const pitch = pitchRaw * 90;
return { yaw, pitch, roll };
}
/**
* Run a one-shot quality check on a static image (an upload, or the frozen
* still the camera captured after the countdown). Unlike the live session,
* this measures the exact pixels that would be submitted, so countdown motion
* blur and lighting shifts that happened after the live gates passed are
* still caught. Applies the same shared gates as the viewfinder.
*
* @param {HTMLImageElement|ImageBitmap|HTMLCanvasElement} source
* @param {string} [slot='frontal']
* @returns {Promise<QualityReport>}
*/
export async function checkImageQuality(source, slot = 'frontal') {
const imgLandmarker = await loadImageLandmarker();
const result = imgLandmarker.detect(source);
const lms = result.faceLandmarks?.[0];
if (!lms || lms.length < 468) {
return {
...gradeFrame({ faceFound: false }),
yaw: null, pitch: null, roll: null,
blur: 0, luma: 0, clippedFrac: 0, landmarks: null,
};
}
const pose = estimateHeadPose(lms);
const nose = lms[1] || lms[4];
const q = measureStillFace(source, lms);
const grade = gradeFrame({
faceFound: true,
slot,
yaw: pose.yaw,
noseX: nose ? nose.x : null,
noseY: nose ? nose.y : null,
blur: q.blur,
luma: q.luma,
clippedFrac: q.clippedFrac,
});
return {
...grade,
yaw: pose.yaw,
pitch: pose.pitch,
roll: pose.roll,
blur: q.blur,
luma: q.luma,
clippedFrac: q.clippedFrac,
landmarks: lms,
};
}
/**
* Blur/luma measurement for a static source: crop to the landmark bounding
* box (small padding) and sample at the same 64px working size the live
* session uses, so the shared thresholds apply to both paths.
*
* @param {HTMLImageElement|ImageBitmap|HTMLCanvasElement} source
* @param {Array<{x:number,y:number}>} lms normalised landmarks
* @returns {{ luma: number, blur: number }}
*/
function measureStillFace(source, lms) {
const SIZE = 64;
const sw = source.videoWidth || source.naturalWidth || source.width;
const sh = source.videoHeight || source.naturalHeight || source.height;
if (!sw || !sh) return { luma: 0, blur: 0, clippedFrac: 0 };
let minX = 1, minY = 1, maxX = 0, maxY = 0;
for (const lm of lms) {
if (lm.x < minX) minX = lm.x;
if (lm.y < minY) minY = lm.y;
if (lm.x > maxX) maxX = lm.x;
if (lm.y > maxY) maxY = lm.y;
}
const pad = 0.05;
const fx = Math.max(0, minX - pad) * sw;
const fy = Math.max(0, minY - pad) * sh;
const fw = Math.min(sw - fx, (maxX - minX + 2 * pad) * sw);
const fh = Math.min(sh - fy, (maxY - minY + 2 * pad) * sh);
if (fw < 4 || fh < 4) return { luma: 0, blur: 0, clippedFrac: 0 };
const canvas = document.createElement('canvas');
canvas.width = canvas.height = SIZE;
const ctx = canvas.getContext('2d', { willReadFrequently: true });
if (!ctx) return { luma: 0, blur: 0, clippedFrac: 0 };
ctx.drawImage(source, fx, fy, fw, fh, 0, 0, SIZE, SIZE);
const { data: d } = ctx.getImageData(0, 0, SIZE, SIZE);
const grays = new Float32Array(SIZE * SIZE);
for (let i = 0, j = 0; i < d.length; i += 4, j++) {
grays[j] = 0.299 * d[i] + 0.587 * d[i + 1] + 0.114 * d[i + 2];
}
const { luma, blurStddev, clippedFrac } = grayFaceStats(grays, SIZE, SIZE);
return { luma, blur: blurStddev, clippedFrac };
}