Skip to content

A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.

18.6 KB
Raw Blame
// What Player and BodyPlayer have in common, in one place: the frame loop, the pose each frame, expressions, actions,
// drivers, the pointer and the order loads run in. A subclass says what it draws and how it loads.
//
// Each frame the pose is put together from scratch: your `params`, then the idle motion (head, eyes, brows, body, breath;
// follow.ts: livliv.ai's, following the pointer with the eyes and the face), then the drivers in order, then actions and
// expressions, kept in the ranges of PARAM_RANGES. Nothing the player adds is written back to `params`, so a value you set
// stays until you change it, and turning a driver off leaves no trace.
import { ActionPlayer, type ActionId as AnyAction } from './runtime/actions'
import { uiLang } from './runtime/format'
import { PARAM_RANGES, defaultParams, type Params } from './runtime/params'
import { LivLivError } from './errors'
import { Idle, type Seen } from './follow'
import { releaseCanvas } from './release'

export { LivLivError, type LivLivErrorCode } from './errors'

/** a short move: see act() */
export type ActionId = Exclude<AnyAction, 'away'>

export interface PlayerOptions {
  /** move by itself: breathe, blink, sway, look around (default true). Off: only your params and drivers move it */
  idle?: boolean
  /** the eyes and head follow the pointer anywhere on the page ('page', the default), only over the canvas
   *  ('canvas'), or not at all (false) */
  followPointer?: 'page' | 'canvas' | false
  /** drawing-buffer pixels per CSS pixel at most (default 2; 1 saves power on high-DPI screens) */
  maxPixelRatio?: number
  /** start the frame loop at once (default true) */
  autoplay?: boolean
  /** the language expression names are picked in (`expressions[].label`; a BCP 47 tag, default the browser's) */
  lang?: string
}

/** where the character sits in the canvas: x, y move it by shares of the canvas (y up), zoom scales it about the
 *  center; head: frame the head instead of the whole drawing (avatars, chat bubbles; a BodyPlayer does not use it) */
export interface View { x: number; y: number; zoom: number; head: boolean }

/** one frame being put together, as a driver sees it */
export interface Frame {
  /** the pose so far: write to it (for this frame only) */
  readonly params: Params
  /** seconds since the last frame, and since the model loaded */
  readonly dt: number
  readonly time: number
  /** a face tracker drives the head this frame (lip sync then leaves the head alone) */
  tracked: boolean
  /** add to params, kept in their ranges */
  add(offsets: Partial<Params>): void
  /** move the picture: x, y shares of the canvas (y up), s a scale */
  readonly move: { x: number; y: number; s: number }
}

/** something that moves the character each frame: face tracking and lip sync are drivers, and your own code can be */
export interface Driver {
  /** lower runs first: face tracking -10, yours 0 (default), lip sync 10 */
  order?: number
  update(frame: Frame, player: Host): void
  /** called when it is removed, or the player disposed */
  dispose?(): void
}

/** what a driver, the lip sync and the face tracking need of a player: a Player, or a BodyPlayer (@livliv-dev/player/body) */
export interface Host {
  /** adds a driver; returns a function that removes it */
  use(driver: Driver): () => void
  /** blinks now (with the idle motion on) */
  blink(): void
}

/** how far a load has come: the files coming in (0..1, in step order, 1 once they are all here; where the size is not
 *  known it stays at 0 until then), then the character going in */
export type LoadProgress = { step: 'download'; progress: number } | { step: 'build' }

export interface LoadOptions {
  /** calls the load off: it rejects with an AbortError (or the signal's own reason) and the character before stays. An
   *  already aborted signal rejects at once and leaves a load that is running alone */
  signal?: AbortSignal
  onProgress?: (progress: LoadProgress) => void
  /** the URL of a picture to show until the first frame (a character's cover.webp), as the canvas's CSS background: it
   *  fits the canvas's box, adds nothing to the page, and the canvas's own background comes back after. Only while the
   *  canvas shows nothing yet: over a character, that one stays until the next is ready */
  poster?: string
}

/** what a running load gives its steps
 *  @internal */
export interface LoadJob {
  signal: AbortSignal
  /** the files have come this far (0..1) */
  progress(share: number): void
  /** they are all here, and the character goes in */
  build(): void
}

/** @internal */
export const clamp = (v: number, a: number, b: number) => Math.min(b, Math.max(a, v))

/** what a load rejects with when it is called off. A DOMException like a fetch's, so `e.name === 'AbortError'` tells
 *  @internal */
export const aborted = (why: string) => new DOMException(`LivLiv: ${why}`, 'AbortError')

/** @internal */
export const contextGone = () => new LivLivError('context-lost', 'LivLiv: this canvas has lost its WebGL context: use a new canvas and a new player')

/** a response's bytes, saying how far they have come
 *  @internal */
export async function readBody(r: Response, job: LoadJob, share = (p: number) => p): Promise<Blob> {
  const size = r.headers.get('content-encoding') ? 0 : Number(r.headers.get('content-length')) || 0
  if (!r.body) return r.blob()
  const reader = r.body.getReader(), parts: Uint8Array<ArrayBuffer>[] = []
  let got = 0
  for (;;) {
    const { done, value } = await reader.read()
    if (done) break
    parts.push(value)
    got += value.byteLength
    if (size) job.progress(share(Math.min(1, got / size)))
  }
  return new Blob(parts)
}

const DEFAULTS = defaultParams()
const KEYS = Object.keys(PARAM_RANGES) as (keyof Params)[]

export abstract class PlayerBase<Info extends { expressions: { id: string }[] }> implements Host {
  readonly canvas: HTMLCanvasElement
  /** your values for the pose and face (ranges in PARAM_RANGES). The player never changes them */
  readonly params: Params = defaultParams()
  /** see PlayerOptions.idle */
  idle: boolean
  /** see PlayerOptions.followPointer */
  followPointer: 'page' | 'canvas' | false
  readonly view: View = { x: 0, y: 0, zoom: 1, head: false }
  /** see PlayerOptions.lang */
  readonly lang: string
  /** Called when the canvas's WebGL context is lost (a phone reclaiming the page's GPU memory, a driver reset, too many
   *  contexts on the page). The character is gone from this canvas and does not come back by itself: put a new canvas in
   *  its place and make a new player. On a page with many players, a new context can push out another: release those
   *  that have left (dispose({ release: true })) */
  onContextLost?: () => void

  /** @internal */
  protected info: Info | null = null
  /** @internal */
  protected disposed = false
  private idler = new Idle()
  /** last frame's: a face tracker drove it, the mouth moved (the idle starts no move while the character talks) */
  private was = { tracked: false, mouth: 0 }
  /** @internal */
  protected actions = new ActionPlayer()
  private drivers: Driver[] = []
  private expr: Record<string, number> = {}
  private smile = 0
  private raf = 0
  private last = 0
  private visible = true
  private io?: IntersectionObserver
  private queue: Promise<unknown> = Promise.resolve()
  private lost = false
  /** the load that a new one replaces */
  private running: AbortController | null = null
  /** the canvas's own background while a poster is showing */
  private posterOver: Record<string, string> | null = null
  /** @internal */
  protected frame: Frame & { move: { x: number; y: number; s: number } }

  /** pose: the Params the subclass draws from, written to each frame */
  constructor(canvas: HTMLCanvasElement, pose: Params, options: PlayerOptions) {
    this.canvas = canvas
    this.idle = options.idle ?? true
    this.followPointer = options.followPointer ?? 'page'
    this.lang = options.lang || uiLang()
    this.frame = {
      params: pose, dt: 0, time: 0, tracked: false, move: { x: 0, y: 0, s: 1 },
      add(o) {
        for (const [k, v] of Object.entries(o) as [keyof Params, number | undefined][]) {
          if (v) pose[k] = clamp(pose[k] + v, PARAM_RANGES[k][0], PARAM_RANGES[k][1])
        }
      },
    }
    addEventListener('pointermove', this.onMove)
    document.addEventListener('pointerleave', this.onLeave)
    canvas.addEventListener('webglcontextlost', this.onLost)
    if (typeof IntersectionObserver !== 'undefined') {
      this.io = new IntersectionObserver((e) => { this.visible = e[e.length - 1].isIntersecting })
      this.io.observe(canvas)
    }
    if (options.autoplay ?? true) this.play()
  }

  /** something is loaded to draw
   *  @internal */
  protected abstract get drawable(): boolean
  /** seconds this character has played
   *  @internal */
  protected abstract get clock(): number
  /** each expression's weight as shown, eased in draw()
   *  @internal */
  protected abstract get shownExpr(): Record<string, number>
  /** where the head is drawn, for where the eyes look
   *  @internal */
  protected abstract seen(): Seen
  /** draws the pose put together in frame
   *  @internal */
  protected abstract show(dt: number): void
  /** frees what it drew with
   *  @internal */
  protected abstract free(): void

  /** the context this draws with, where it has one and has started it
   *  @internal */
  protected abstract get gl(): WebGLRenderingContext | WebGL2RenderingContext | null

  /** the canvas's WebGL context is gone: nothing draws, and load() rejects with code 'context-lost' */
  get contextLost(): boolean { return this.lost || !!this.gl?.isContextLost() }

  /** the loaded character (null before the first load) */
  get model(): Info | null { return this.info }
  get expressions() { return this.info?.expressions ?? [] as Info['expressions'] }
  get playing() { return this.raf !== 0 }

  /** runs a load: the one running is called off, and this one's steps are told how to be. A load called off rejects like
   *  any other, but nobody has to catch it (player.load(a); player.load(b) reports nothing); a real error is still
   *  reported when nobody catches it
   *  @internal */
  protected start<T>({ signal, onProgress, poster }: LoadOptions, load: (job: LoadJob) => Promise<T>): Promise<T> {
    let give!: (v: T) => void, refuse!: (e: unknown) => void
    const out = new Promise<T>((a, b) => { give = a; refuse = b })
    const fail = (e: unknown) => {
      if ((e as Error | null)?.name === 'AbortError') out.catch(() => {})
      refuse(e)
    }
    if (signal?.aborted) { fail(signal.reason); return out }
    if (this.disposed) { fail(aborted('the player was disposed')); return out }
    if (this.contextLost) { fail(contextGone()); return out }
    this.running?.abort(aborted('a newer load took its place'))
    const ctl = this.running = new AbortController()
    const calledOff = () => ctl.abort(signal!.reason)
    signal?.addEventListener('abort', calledOff, { once: true })
    this.setPoster(poster)
    const tell = (p: LoadProgress) => { try { onProgress?.(p) } catch (e) { console.error(e) } }
    let shown = -1, built = false
    const job: LoadJob = {
      signal: ctl.signal,
      progress(share) { if (share > shown && !built) { shown = share; tell({ step: 'download', progress: share }) } },
      build() { if (!built) { if (shown >= 0 && shown < 1) tell({ step: 'download', progress: 1 }); built = true; tell({ step: 'build' }) } },
    }
    const end = () => {
      signal?.removeEventListener('abort', calledOff)
      if (this.running === ctl) this.running = null
    }
    load(job).then((v) => { end(); give(v) }, (e) => {
      const mine = this.running === ctl
      end()
      if (mine) this.setPoster()  // the page can say why; a newer load has already set its own
      fail(ctl.signal.aborted ? ctl.signal.reason : e)
    })
    return out
  }

  /** the poster over an empty canvas, or none */
  private setPoster(url?: string) {
    const s = this.canvas.style
    if (this.posterOver) { for (const [k, v] of Object.entries(this.posterOver)) s.setProperty(k, v); this.posterOver = null }
    if (!url || this.info) return
    const set = { 'background-image': `url(${JSON.stringify(String(url))})`, 'background-position': 'center', 'background-size': 'contain', 'background-repeat': 'no-repeat' }
    this.posterOver = Object.fromEntries(Object.keys(set).map((k) => [k, s.getPropertyValue(k)]))
    for (const [k, v] of Object.entries(set)) s.setProperty(k, v)
  }

  /** runs `load` after the loads before it
   *  @internal */
  protected inTurn<T>(load: () => Promise<T>): Promise<T> {
    const run = this.queue.then(load)
    this.queue = run.catch(() => {})
    return run
  }

  /** a new character starts on a neutral face
   *  @internal */
  protected neutral() { this.expr = {}; this.smile = 0 }

  /** Shows an expression (an id from `expressions`), easing over from the one before; null goes back to neutral.
   *  weight: how strongly, 0..1. False when the model has no such expression */
  setExpression(id: string | null, weight = 1): boolean {
    if (id !== null && !this.expressions.some((e) => e.id === id)) return false
    const w = clamp(weight, 0, 1)
    this.smile = id === 'smile' ? w : 0
    this.expr = id === null || id === 'smile' ? {} : { [id]: w }
    return true
  }

  /** Plays a short move over whatever drives the character: nod, shake, tilt, hop, shy */
  act(action: ActionId) {
    if ((action as string) === 'away') throw new TypeError("LivLiv: act('away') is setAway(true) now")
    this.actions.play(action)
  }

  /** Dozes (eyes closed, head down, a slow breath, expressions hidden), easing in, until setAway(false) */
  setAway(on: boolean) { this.actions.setAway(on) }

  /** Looks toward (x, y), each -1..1 from the canvas center (y up), instead of following the pointer; null goes back to
   *  following it. Needs the idle motion on */
  lookAt(x: number | null, y = 0) {
    this.idler.lookAt(this.seen(), x === null ? null : clamp(x, -1, 1), clamp(y, -1, 1), this.canvas.clientWidth, this.canvas.clientHeight)
  }

  /** Blinks now (with the idle motion on) */
  blink() { this.idler.blink(this.clock) }

  /** Adds a driver; returns a function that removes it. On a disposed player it is disposed at once: what started
   *  while the player went (a camera, a microphone) does not stay on */
  use(driver: Driver): () => void {
    if (this.disposed) { driver.dispose?.(); return () => {} }
    this.drivers.push(driver)
    this.drivers.sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
    return () => {
      const i = this.drivers.indexOf(driver)
      if (i >= 0) { this.drivers.splice(i, 1); driver.dispose?.() }
    }
  }

  play() {
    if (this.raf || this.disposed) return
    this.last = performance.now()
    const loop = (now: number) => {
      this.raf = requestAnimationFrame(loop)
      const dt = Math.min(0.05, (now - this.last) / 1000)
      this.last = now
      if (this.visible) this.draw(dt)
    }
    this.raf = requestAnimationFrame(loop)
  }

  pause() { cancelAnimationFrame(this.raf); this.raf = 0 }

  /** Draws now at the canvas's current size. A playing player follows the size by itself; call this after resizing a
   *  paused one */
  resize() { this.draw(0) }

  /** The current frame as an image (PNG data URL), as the pose is now, the mark included */
  snapshot(): string { this.draw(0); return this.canvas.toDataURL('image/png') }

  /** Stops, removes the drivers and frees the GPU memory. A load still running rejects. The canvas keeps its WebGL context
   *  and can take a new player afterwards. release: also give the context up, when the canvas is leaving the page: a
   *  browser allows about 16 at once and drops the oldest past that, which may be a character still showing. A canvas
   *  released this way takes no new player, unless it is made at once, in the same task (a framework's unmount and mount).
   *  Safari does not get the room back: reuse canvases there */
  dispose({ release = false }: { release?: boolean } = {}) {
    const gl = this.gl
    this.stop()
    if (release) releaseCanvas(this.canvas, gl)
  }

  private stop() {
    if (this.disposed) return
    this.pause()
    this.disposed = true
    this.running?.abort(aborted('the player was disposed'))
    this.setPoster()
    removeEventListener('pointermove', this.onMove)
    document.removeEventListener('pointerleave', this.onLeave)
    this.canvas.removeEventListener('webglcontextlost', this.onLost)
    this.io?.disconnect()
    for (const d of this.drivers.splice(0)) d.dispose?.()
    this.free()
    this.info = null
  }

  private draw(dt: number) {
    // (the context itself, not only the event, which comes a little later: three.js throws starting on a lost context)
    if (!this.drawable || this.disposed || this.contextLost) return
    const f = this.frame, P = f.params, t = this.clock, E = this.shownExpr, A = this.actions
    // the pose from scratch: your values, the idle motion, the drivers
    this.idler.shown(P)
    Object.assign(P, this.params)
    if (this.idle) {
      this.idler.apply(P, t, dt, { busy: A.busy || this.was.mouth > 0.05, tracked: this.was.tracked,
        faceFree: !this.smile && !Object.values(E).some((v) => v > 0.02) })
    }
    Object.assign(f, { dt, time: t, tracked: false })
    Object.assign(f.move, { x: 0, y: 0, s: 1 })
    for (const d of this.drivers) d.update(f, this)
    this.was = { tracked: f.tracked, mouth: P.mouthOpen }
    f.add(A.update(dt))
    if (this.smile) P.smile = Math.max(P.smile, this.smile)
    if (A.awayW) for (const [k, v] of Object.entries(A.awayPose(t)) as [keyof Params, number][]) P[k] += (v - P[k]) * A.awayW
    // in range whoever set it; a value that is no number (one NaN stays in the springs for good) counts as not set
    for (const k of KEYS) {
      const v = +P[k]
      P[k] = v === v ? clamp(v, PARAM_RANGES[k][0], PARAM_RANGES[k][1]) : DEFAULTS[k]
    }
    // expressions ease in and out (one no longer shown eases back to 0); dozing hides them, each is a face with open eyes
    for (const k of Object.keys(this.expr)) if (!(k in E)) E[k] = 0
    for (const k of Object.keys(E)) E[k] += ((this.expr[k] ?? 0) * (1 - A.awayW) - E[k]) * Math.min(1, dt * 8)
    this.show(dt)
    if (this.posterOver) this.setPoster()
  }

  private onMove = (e: PointerEvent) => {
    if (!this.followPointer) return
    const r = this.canvas.getBoundingClientRect()
    // 'canvas': off it the pointer has left the stage, and the character looks back at the viewer
    const inside = !(e.clientX < r.left || e.clientX > r.right || e.clientY < r.top || e.clientY > r.bottom)
    this.idler.pointer(this.seen(), e.clientX - r.left, e.clientY - r.top, r.width, r.height, this.followPointer === 'page' || inside)
  }
  private onLeave = () => this.idler.leave()
  private onLost = () => {
    this.lost = true
    this.running?.abort(contextGone())
    this.onContextLost?.()
  }
}