A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.
// Webcam face tracking -> puppet params, via MediaPipe Face Landmarker (runs locally in the browser).
import { smooth, type Params } from './params'
import { OneEuro } from './filters'
import { BS, LM, SynthStats, TraceRecorder, TraceSource, synthTrace, type Frame } from './trace'
type Landmarker = import('@mediapipe/tasks-vision').FaceLandmarker
type Result = import('@mediapipe/tasks-vision').FaceLandmarkerResult
// The face model: the MediaPipe runtime (wasm) and the landmarker model, ~17 MB (~8 MB over the wire). Served with a
// year's cache under URLs that carry the package version: bump V with @mediapipe/tasks-vision, or browsers
// keep runtime files that no longer match its JavaScript. Sizes: for the download progress (uncompressed)
export const V = '1.1.1'
const RUNTIME = { simd: ['vision_wasm_internal', 13_519_922], plain: ['vision_wasm_nosimd_internal', 12_627_509] } as const
const MODEL_SIZE = 3_758_596
/** where those files are: livliv.ai serves them itself; the SDK points elsewhere (setFaceModelFiles) before loading */
const FILES = { wasm: '/mediapipe/wasm', model: '/mediapipe/face_landmarker.task' }
export function setFaceModelFiles(f: Partial<typeof FILES>) { Object.assign(FILES, f) }
/** where starting the camera is: downloading the model (progress 0..1; the first time only), preparing it (compiling,
* the GPU), the camera (the permission prompt, the first picture) */
export type StartStep = { step: 'download'; progress: number } | { step: 'prepare' } | { step: 'camera' }
let shared: Promise<Landmarker> | null = null
let progress = 0
const watchers = new Set<(p: number) => void>()
/** the download while it runs, and who still wants it: the trackers waiting for their camera (by their ticket), or anyone
* who asked ahead of time (`ahead`, for good: hovering a button says the model will be wanted) */
let download: AbortController | null = null
const waiting = new Set<object>()
let ahead = false
/** The face model, loaded once per page and kept: every tracker and every camera toggle reuses it. Call it early
* (hovering the camera button, the live dialog) and the camera opens without waiting for it. The files are read with
* progress and land in the HTTP cache, where the live window, opened in another tab, finds them */
export function loadFaceModel(onProgress?: (p: number) => void, ticket?: object): Promise<Landmarker> {
if (onProgress) { watchers.add(onProgress); onProgress(progress) }
const done = () => { if (onProgress) watchers.delete(onProgress) }
if (ticket) waiting.add(ticket); else ahead = true
if (!shared) {
const { signal } = (download = new AbortController())
shared = (async () => {
const vision = await import('@mediapipe/tasks-vision')
const [name, size] = (await vision.FilesetResolver.isSimdSupported()) ? RUNTIME.simd : RUNTIME.plain
const runtime = `${FILES.wasm.replace(/\/$/, '')}/${name}`
const got = [0, 0], total = size + MODEL_SIZE
const read = async (url: string, i: number) => {
const r = await fetch(url, { signal })
if (!r.ok || !r.body) throw new Error(`${url} ${r.status}`)
const reader = r.body.getReader(), parts: Uint8Array[] = []
for (;;) {
const { done, value } = await reader.read()
if (done) break
parts.push(value); got[i] += value.length
progress = Math.min(0.99, (got[0] + got[1]) / total)
watchers.forEach((w) => w(progress))
}
const out = new Uint8Array(got[i])
parts.reduce((at, p) => { out.set(p, at); return at + p.length }, 0)
return out
}
// the runtime is read only to have it cached (MediaPipe loads it by URL again); the model is handed over as it is
const [, model] = await Promise.all([read(`${runtime}.wasm?v=${V}`, 0), read(`${FILES.model}?v=${V}`, 1)])
progress = 1; watchers.forEach((w) => w(1))
download = null
return vision.FaceLandmarker.createFromOptions({ wasmLoaderPath: `${runtime}.js?v=${V}`, wasmBinaryPath: `${runtime}.wasm?v=${V}` }, {
baseOptions: { modelAssetBuffer: model, delegate: 'GPU' },
runningMode: 'VIDEO', numFaces: 1,
outputFaceBlendshapes: true, outputFacialTransformationMatrixes: true,
})
})()
shared.catch(() => { shared = null; progress = 0; download = null }) // a failed download is tried again next time
}
shared.then(done, done)
return shared
}
/** a tracker no longer waits for the model (its camera was refused, or it was stopped): with nobody else waiting and nobody
* having asked ahead, the download stops. It runs beside the camera's prompt to save time, and a refusal should not
* cost 7 MB */
function dropFaceModel(ticket: object) {
waiting.delete(ticket)
if (!waiting.size && !ahead) download?.abort()
}
/** a face unseen this long (s) is lost: the player hands the character back to its idle motion */
export const LOST_AFTER = 0.4
/** a head pose (angles), the head's position in front of the camera (px, py, pz: cm, y up, z larger = closer) and the
* resting blink scores of the user's own left / right eye (narrow eyes, glasses rest well above 0). In the current
* view's terms; stored, x, z and px are in the mirrored view's (they flip with it) */
type Pose = { x: number; y: number; z: number; px: number; py: number; pz: number; bl: number; br: number }
export class FaceTracker {
video = document.createElement('video')
lm?: Landmarker
stream?: MediaStream
last = -1
target: Partial<Params> = {}
mouthGain = 1
exprGain = 1
/** like a mirror (default): turn to your left and the character turns toward the screen's left. Off: it turns the
* way you do, as if it were you seen from the front (left/right, tilt, gaze and which eye blinks swap) */
mirror = true
/** the user's natural pose (calibrated) is 'front': its angles are subtracted, its blink scores are 'eyes open' */
zero: Pose = { x: 0, y: 0, z: 0, px: 0, py: 0, pz: 0, bl: 0, br: 0 }
raw = { x: 0, y: 0, z: 0 }
/** the head's position relative to the calibrated one (cm, smoothed; x in the view's terms): the player moves the
* character with it. Zero until a resting position is known */
pos = { x: 0, y: 0, z: 0 }
/** how much the head's sideways position also leans the body (the live setting `follow`, 0..2) */
follow = 0.3
/** process every n-th camera frame (power saving: 2) */
every = 1
/** detections per second at most. The model runs on the main thread, inside the frame: 14-16 ms each on an M4 with a
* busy machine, so on a phone it can take the frame rate with it. 'auto': a detection takes at most half of the main
* thread's time, by what the last ones took, and never under 5 a second (a face is lost after LOST_AFTER). Unset: every
* new camera frame, as livliv.ai has always done */
maxFps?: number | 'auto'
/** how long a detection takes (ms, smoothed) and when the next may run */
private detectMs = 0
private detectAt = 0
/** set: calibration is remembered on this device for this character (localStorage), and found again next time */
calibId?: string
calibrated = false
/** a calibration saved before positions were part of it: the resting position is taken on the next still second */
private posKnown = false
private frameNo = 0
/** the eye turned away from the camera reads half closed (a wink that isn't): 'turn' links the eyes once the head
* turns past ~12°, 'always' links them all the time, 'off' never */
blinkSync: 'turn' | 'always' | 'off' = 'turn'
ok = false
private stopped = false
/** while start() runs: what it is waiting for */
onStep?: (s: StartStep) => void
/** seconds since a face was last seen (Infinity before the first); faces seen per second; the picture too dark */
lostFor = Infinity
fps = 0
dark = false
/** '' (camera), 'rec' (camera, recorded), 'synth' / 'synth-narrow', or the URL of a recording (see trace.ts) */
trace = ''
rec?: TraceRecorder
private src?: TraceSource
private stats?: SynthStats
private recent: Pose[] = []
private seenAt = -1
private counted = 0
private countAt = 0
private light?: CanvasRenderingContext2D | null
// One Euro filters on the measurements (tuned on the synthetic trace: about 40% less jitter at rest than the old
// per-frame easing alone, and no more lag on turns); the per-frame easing in apply() only bridges the 30 fps camera
private f = { x: new OneEuro(0.5, 0.05), y: new OneEuro(0.5, 0.05), z: new OneEuro(0.5, 0.05),
gx: new OneEuro(1.5, 0.5), gy: new OneEuro(1.5, 0.5), brow: new OneEuro(1, 0.5),
px: new OneEuro(0.6, 0.08), py: new OneEuro(0.6, 0.08), pz: new OneEuro(0.6, 0.08) }
/** the current pose becomes 'front' (the last few frames, so one noisy frame doesn't decide it) */
calibrate() {
const r = this.recent.slice(-10)
if (!r.length) return // no face seen yet: nothing to take as front
this.setZero(r); this.save()
}
/** smoothing, 0 (quick, shakier) .. 1 (steady, slower); 0.5 is what the head filters were tuned to. Both the cutoff at
* rest and how fast it opens with speed scale (the cutoff alone barely changes the jitter: noise opens the filter).
* Synthetic trace, offline: 0 -> jitter 0.33°, lag 36 ms; 0.5 -> 0.19°, 56 ms; 1 -> 0.09°, 96 ms */
setSmoothing(s: number) {
const c = s <= 0.5 ? 0.5 * 6 ** (1 - 2 * s) : 0.5 / (10 / 3) ** (2 * s - 1) // 3 Hz .. 0.5 .. 0.15
for (const f of [this.f.x, this.f.y, this.f.z]) { f.minCutoff = c; f.beta = 0.05 * 4 ** (1 - 2 * s) } // 0.2 .. 0.05 .. 0.0125
}
setMirror(on: boolean) {
if (on === this.mirror) return
this.mirror = on
this.zero = { ...this.zero, x: -this.zero.x, z: -this.zero.z, px: -this.zero.px } // the calibrated front stays the front
this.recent = this.recent.map((p) => ({ ...p, x: -p.x, z: -p.z, px: -p.px }))
this.f.x.reset(); this.f.z.reset(); this.f.gx.reset(); this.f.px.reset()
}
async start() {
if (this.trace && this.trace !== 'rec') {
const frames = this.trace.startsWith('synth') ? synthTrace(this.trace === 'synth-narrow')
: await fetch(this.trace).then((r) => { if (!r.ok) throw new Error(`trace ${r.status}`); return r.json() as Promise<Frame[]> })
this.src = new TraceSource(frames, this.trace.startsWith('synth'))
if (this.src.synth) this.stats = new SynthStats(() => this.mirror)
this.ok = true
return
}
this.restore()
// the camera is asked for at once, so it warms up (and the user answers its prompt) while the model loads
let modelReady = false
const say = () => this.onStep?.(!modelReady ? (progress < 1 ? { step: 'download', progress } : { step: 'prepare' }) : { step: 'camera' })
const model = loadFaceModel(() => say(), this).then((lm) => { modelReady = true; say(); return lm })
const camera = navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 480, facingMode: 'user' } })
say()
let stream: MediaStream
try {
[this.lm, stream] = await Promise.all([model, camera])
waiting.delete(this)
} catch (e) {
dropFaceModel(this)
camera.then((s) => s.getTracks().forEach((t) => t.stop()), () => {}) // the model failed: let the camera go
throw e
}
if (this.stopped) { stream.getTracks().forEach((t) => t.stop()); return } // turned off while starting
this.stream = stream
this.video.srcObject = stream
this.video.muted = true; this.video.playsInline = true
await this.video.play()
if (this.trace === 'rec') this.rec = new TraceRecorder()
this.ok = true
}
/** the camera goes off; the face model stays loaded for the next time (loadFaceModel) */
stop() {
this.stopped = true
dropFaceModel(this)
this.stream?.getTracks().forEach((t) => t.stop())
this.ok = false
}
/** Poll once per frame; returns true while a face is seen. */
poll(): boolean {
const now = performance.now()
let f: Frame | null = null
if (this.src) f = this.src.next(now)
else if (this.lm && this.video.readyState >= 2 && this.video.currentTime !== this.last && now >= this.detectAt) {
this.last = this.video.currentTime
if (++this.frameNo % this.every === 0) {
const r = this.lm.detectForVideo(this.video, now)
if (this.maxFps) { // (the first takes far longer, setting things up: not what later ones will)
const took = Math.min(200, performance.now() - now)
this.detectMs = this.detectMs ? this.detectMs + (took - this.detectMs) * 0.15 : took / 8
this.detectAt = now + (this.maxFps === 'auto' ? Math.min(200, 2 * this.detectMs) : 1000 / this.maxFps)
}
f = toFrame(r, now); this.rec?.push(f)
}
}
if (f) this.read(f, now)
this.lostFor = this.seenAt < 0 ? Infinity : (now - this.seenAt) / 1000
if (now - this.countAt >= 1000) {
this.fps = this.countAt ? Math.round((this.counted * 1000) / (now - this.countAt)) : 0
this.counted = 0; this.countAt = now
this.checkLight()
}
return this.lostFor < LOST_AFTER
}
/** after the frame's params are final: numbers for a synthetic run (trace.ts) */
observe(P: Params) { if (this.stats && this.src) this.stats.observe(this.src.at(performance.now()) / 1000, P) }
private read(f: Frame, now: number) {
if (!f.m && !f.b) return
// back after a lost stretch: the filters start from where the face is now, not ease over from where it left
const dt = this.seenAt < 0 ? 0 : (now - this.seenAt) / 1000
if (!dt || dt > LOST_AFTER) for (const k of Object.values(this.f)) k.reset()
this.seenAt = now; this.counted++
const b = f.b, g = (n: (typeof BS)[number]) => (b ? b[BS.indexOf(n)] ?? 0 : 0)
const m = f.m
if (m) {
// column-major 4x4; extract yaw/pitch/roll. MediaPipe's space is Y up, the face looking down +Z at the camera, so
// pitch > 0 is the chin going down: negate it (+angleY is up). Yaw / roll are negated for the mirror view
const r00 = m[0], r10 = m[1], r20 = m[2], r21 = m[6], r22 = m[10]
const yaw = Math.atan2(-r20, Math.hypot(r21, r22))
const pitch = Math.atan2(r21, r22)
const roll = Math.atan2(r10, r00)
const s = this.mirror ? -1 : 1
this.raw = { x: s * deg(yaw) * 1.3, y: -deg(pitch) * 1.3 - 4, z: s * deg(roll) }
const px = s * m[12], py = m[13], pz = m[14]
this.recent.push({ ...this.raw, px, py, pz, bl: g('eyeBlinkLeft'), br: g('eyeBlinkRight') })
if (this.recent.length > 30) this.recent.shift()
// calibrates itself once, on the first second the head holds still, when this device has none for the character
if ((!this.calibrated || !this.posKnown) && this.recent.length >= 20) {
const r = this.recent.slice(-20)
if (sd(r.map((p) => p.x)) < 1.5 && sd(r.map((p) => p.y)) < 1.5) {
if (this.calibrated) { const z = this.zero; this.setZero(r); this.zero = { ...this.zero, x: z.x, y: z.y, z: z.z, bl: z.bl, br: z.br } }
else this.setZero(r)
this.save()
}
}
if (this.posKnown) {
this.pos = { x: this.f.px.filter(px, dt) - this.zero.px, y: this.f.py.filter(py, dt) - this.zero.py, z: this.f.pz.filter(pz, dt) - this.zero.pz }
}
const x = this.f.x.filter(this.raw.x, dt) - this.zero.x
this.target.angleX = clamp(x, -30, 30)
this.target.angleY = clamp(this.f.y.filter(this.raw.y, dt) - this.zero.y, -30, 30)
// Player roll is Y-down, opposite to MediaPipe's Y-up rotation. Convert after calibration so stored poses keep
// their existing convention; yaw and horizontal position are unchanged.
this.target.angleZ = clamp(this.zero.z - this.f.z.filter(this.raw.z, dt), -30, 30)
this.target.bodyX = clamp(x * 0.2 + this.pos.x * 0.25 * this.follow, -10, 10)
}
if (b) {
// eyes: fully open up to a little above this user's resting score (narrow eyes no longer look half closed);
// fully shut from 0.65: a webcam blink peaks at about 0.5-0.7 for one or two frames, so 0.775 left blinks half
// closed (a 0.62 peak showed 0.43 open); mirrored, the user's left eye drives the image-right eye
const eye = (blink: number, rest: number) => {
const open = clamp(rest + 0.06, 0.15, 0.5), shut = Math.min(0.95, Math.max(0.65, open + 0.4))
return clamp(1 - (blink - open) / (shut - open), 0, 1)
}
const [uL, uR] = [eye(g('eyeBlinkLeft'), this.zero.bl), eye(g('eyeBlinkRight'), this.zero.br)]
let l = this.mirror ? uR : uL, r = this.mirror ? uL : uR
const turned = smooth(9, 15, Math.abs(this.raw.x - this.zero.x))
const k = this.blinkSync === 'always' ? 1 : this.blinkSync === 'turn' ? turned : 0
if (k > 0) { // turned away, the more open eye is the one the camera sees well
const avg = (l + r) / 2, both = avg + (Math.max(l, r) - avg) * (this.blinkSync === 'always' ? turned : 1)
l += (both - l) * k; r += (both - r) * k
}
this.target.eyeLOpen = l; this.target.eyeROpen = r
// mouth: the jaw blendshape, or the lips' own gap when they part without much jaw (quick syllables, "e" / "i")
const L = f.l, d = (i: number, j: number) => L ? Math.hypot(L[2 * i] - L[2 * j], L[2 * i + 1] - L[2 * j + 1]) : 0
const lips = L ? clamp((d(1, 2) / Math.max(1e-3, d(0, 3)) - 0.015) * 7, 0, 1) : 0 // LM: 10, 13, 14, 152
this.target.mouthOpen = clamp(Math.max((g('jawOpen') - 0.04) * 2.2, lips) * this.mouthGain, 0, 1)
// gaze (mirrored): looking to the user's left moves the iris toward screen-left
const lookLeft = (g('eyeLookOutLeft') + g('eyeLookInRight')) / 2
const lookRight = (g('eyeLookInLeft') + g('eyeLookOutRight')) / 2
const up = (g('eyeLookUpLeft') + g('eyeLookUpRight')) / 2, down = (g('eyeLookDownLeft') + g('eyeLookDownRight')) / 2
this.target.eyeBallX = clamp(this.f.gx.filter((lookRight - lookLeft) * 2.2 * (this.mirror ? 1 : -1), dt), -1, 1)
this.target.eyeBallY = clamp(this.f.gy.filter((up - down) * 2.2, dt), -1, 1)
const browUp = g('browInnerUp') * 0.6 + (g('browOuterUpLeft') + g('browOuterUpRight')) * 0.2
const browDown = (g('browDownLeft') + g('browDownRight')) / 2
this.target.browLY = this.target.browRY = clamp(this.f.brow.filter((browUp - browDown * 0.8) * 2.4, dt), -1, 1)
this.target.smile = clamp(((g('mouthSmileLeft') + g('mouthSmileRight')) / 2 - 0.35 / this.exprGain) * 2.5 * this.exprGain, 0, 1)
}
}
/** the median of a few still frames becomes 'front', 'here' and 'eyes open' */
private setZero(r: Pose[]) {
const med = (k: keyof Pose) => median(r.map((p) => p[k]))
this.zero = { x: med('x'), y: med('y'), z: med('z'), px: med('px'), py: med('py'), pz: med('pz'), bl: clamp(med('bl'), 0, 0.5), br: clamp(med('br'), 0, 0.5) }
this.calibrated = this.posKnown = true
this.f.px.reset(); this.f.py.reset(); this.f.pz.reset()
}
/** remembered for the next time on this device (not for traces) */
private save() {
if (!this.calibId || this.src) return
const s = this.mirror ? 1 : -1, z = this.zero
try { localStorage.setItem(`lm_calib:${this.calibId}`, JSON.stringify({ ...z, x: s * z.x, z: s * z.z, px: s * z.px, at: Date.now() })) } catch { /* private mode */ }
}
private restore() {
if (!this.calibId) return
try {
const c = JSON.parse(localStorage.getItem(`lm_calib:${this.calibId}`) || 'null')
if (!c || ![c.x, c.y, c.z, c.bl, c.br].every(Number.isFinite)) return
const s = this.mirror ? 1 : -1
this.posKnown = [c.px, c.py, c.pz].every(Number.isFinite)
this.zero = { x: s * c.x, y: c.y, z: s * c.z, bl: clamp(c.bl, 0, 0.5), br: clamp(c.br, 0, 0.5),
px: this.posKnown ? s * c.px : 0, py: this.posKnown ? c.py : 0, pz: this.posKnown ? c.pz : 0 }
this.calibrated = true
} catch { /* private mode, or something else stored there */ }
}
/** once a second: the mean brightness of a 32x24 copy of the camera picture (dark below ~40 of 255, with hysteresis) */
private checkLight() {
if (this.src || this.video.readyState < 2) return
if (this.light === undefined) {
const c = document.createElement('canvas'); c.width = 32; c.height = 24
this.light = c.getContext('2d', { willReadFrequently: true })
}
if (!this.light) return
this.light.drawImage(this.video, 0, 0, 32, 24)
const d = this.light.getImageData(0, 0, 32, 24).data
let sum = 0
for (let i = 0; i < d.length; i += 4) sum += 0.299 * d[i] + 0.587 * d[i + 1] + 0.114 * d[i + 2]
const luma = sum / (d.length / 4)
this.dark = this.dark ? luma < 55 : luma < 40
}
apply(P: Params, dt: number) {
const a = Math.min(1, dt * 30), fast = Math.min(1, dt * 25)
for (const k of Object.keys(this.target) as (keyof Params)[]) {
const v = this.target[k]
if (v === undefined) continue
// an eye closing follows at once: eased, a blink seen for one or two camera frames was over before it got there
const closing = (k === 'eyeLOpen' || k === 'eyeROpen') && v < P[k]
const rate = closing ? 1 : k.startsWith('eye') || k === 'mouthOpen' ? fast : k === 'smile' ? Math.min(1, dt * 8.4) : a
P[k] += (v - P[k]) * rate
}
}
}
/** one MediaPipe result as a Frame: only what read() uses */
function toFrame(r: Result, t: number): Frame {
if (!r.faceLandmarks?.length) return { t }
const scores = new Map((r.faceBlendshapes?.[0]?.categories || []).map((c) => [c.categoryName, c.score]))
const L = r.faceLandmarks[0], m = r.facialTransformationMatrixes?.[0]?.data
return {
t, m: m ? Array.from(m) : undefined,
b: scores.size ? BS.map((n) => scores.get(n) ?? 0) : undefined,
l: L.length > 152 ? LM.flatMap((i) => [L[i].x, L[i].y]) : undefined,
}
}
const deg = (r: number) => (r * 180) / Math.PI
const clamp = (v: number, a: number, b: number) => Math.min(b, Math.max(a, v))
const sd = (xs: number[]) => { const m = xs.reduce((a, b) => a + b, 0) / xs.length; return Math.sqrt(xs.reduce((a, b) => a + (b - m) ** 2, 0) / xs.length) }
const median = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[s.length >> 1] }