Skip to content

A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.

5.1 KB
Raw Blame
// @livliv-dev/player/tracking: the character follows a face in the webcam (MediaPipe Face Landmarker, in the browser; camera
// images stay on the device, though MediaPipe sends Google usage statistics while it runs). A separate import: it needs
// the @mediapipe/tasks-vision package and downloads ~8 MB of model the first time (the wasm runtime and the face model;
// cached by the browser after that).
import { FaceTracker, LOST_AFTER, V, loadFaceModel, setFaceModelFiles, type StartStep } from './runtime/tracking'
import { defaultParams, type Params } from './runtime/params'
import type { Driver, Host } from './base'

export type { StartStep }

export interface FaceModelFiles {
  /** the folder of MediaPipe's wasm files (vision_wasm_internal.js / .wasm and the nosimd pair). Default: jsDelivr's
   *  copy of @mediapipe/tasks-vision 1.1.1. Host them yourself in production */
  wasmUrl?: string
  /** face_landmarker.task. Default: Google's model bucket */
  modelUrl?: string
}

export interface FaceTrackingOptions extends FaceModelFiles {
  /** move like a mirror image (default true): turn left and the character turns to the screen's left */
  mirror?: boolean
  /** 0 (quick, shakier) .. 1 (steady, slower); default 0.5 */
  smoothing?: number
  /** how much leaning toward / away from the camera moves the character, 0 (default) .. 2 */
  follow?: number
  /** how many times a second to look for the face at most. 'auto' (default): as often as the camera gives a new picture,
   *  until that would take over half of the page's main thread (the model runs there); then less, 5 at least */
  maxFps?: number | 'auto'
  /** keep the calibration (the user's resting pose) on this device under this name (localStorage) */
  remember?: string
  /** what starting waits for: the download (progress 0..1, first time only), preparing the model, the camera */
  onProgress?: (step: StartStep) => void
  /** play a recorded face trace (a URL) or 'synth' (a built-in 24 s loop: turns, a nod, a lost face, a blink, talking)
   *  instead of the camera: tests without anyone in front of one */
  trace?: string
}

export interface FaceTracking {
  /** a face is seen now */
  readonly face: boolean
  /** faces seen per second */
  readonly fps: number
  /** the picture is too dark to track well */
  readonly dark: boolean
  mirror: boolean
  /** the user's current pose becomes "facing front" (it also calibrates itself on the first still second) */
  calibrate(): void
  /** camera off, the character back to its idle motion (the face model stays loaded for the next start) */
  stop(): void
}

const DEFAULT_FILES = {
  wasm: `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${V}/wasm`,
  model: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
}
const files = (o: FaceModelFiles) => setFaceModelFiles({ wasm: o.wasmUrl ?? DEFAULT_FILES.wasm, model: o.modelUrl ?? DEFAULT_FILES.model })

/** Downloads the face model ahead of time (on hover over a camera button, say), so starting is quick. Once per page */
export async function preloadFaceModel(options: FaceModelFiles & { onProgress?: (p: number) => void } = {}): Promise<void> {
  files(options)
  await loadFaceModel(options.onProgress)
}

/** Asks for the camera and drives the character with the face it sees. While no face is seen, the character eases back
 *  to its idle motion */
export async function startFaceTracking(player: Host, options: FaceTrackingOptions = {}): Promise<FaceTracking> {
  files(options)
  const t = new FaceTracker()
  t.mirror = options.mirror ?? true
  t.setSmoothing(options.smoothing ?? 0.5)
  t.maxFps = options.maxFps ?? 'auto'
  t.calibId = options.remember
  t.onStep = options.onProgress
  t.trace = options.trace ?? ''
  await t.start()
  const pose = defaultParams()
  const follow = Math.max(0, Math.min(2, options.follow ?? 0))
  let w = 0  // how much the tracked pose shows (eases in 0.3 s after a face appears, out 0.5 s after it is lost)
  const driver: Driver = {
    order: -10,
    update(f) {
      if (!t.ok) return
      t.follow = follow
      const seen = t.poll()
      t.apply(pose, f.dt)
      w = Math.max(0, Math.min(1, w + (seen ? f.dt / 0.3 : -f.dt / 0.5)))
      f.tracked = true
      const k = w * w * (3 - 2 * w)
      if (!k) return
      const P = f.params
      for (const key of Object.keys(t.target) as (keyof Params)[]) P[key] += (pose[key] - P[key]) * k
      // leaning moves the picture: 10 cm sideways 6% of it, 10 cm closer 12% bigger (at follow 1)
      const d = t.pos, c = (v: number, a: number, b: number) => Math.max(a, Math.min(b, v))
      f.move.x += c(d.x * 0.006 * follow * k, -0.15, 0.15)
      f.move.y += c(d.y * 0.006 * follow * k, -0.15, 0.15)
      f.move.s *= 1 + c(d.z * 0.012 * follow * k, -0.15, 0.2)
    },
    dispose() { t.stop() },
  }
  const remove = player.use(driver)
  return {
    get face() { return t.lostFor < LOST_AFTER },
    get fps() { return t.fps },
    get dark() { return t.dark },
    get mirror() { return t.mirror },
    set mirror(on: boolean) { t.setMirror(on) },
    calibrate: () => t.calibrate(),
    stop: remove,
  }
}