A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.
// @livliv-dev/player/body: characters that aren't LivLiv models, on the surface of Player. A Body draws one (a PNGTuber's
// pictures: @livliv-dev/player/png; a VRM: @livliv-dev/player/vrm; or your own); the BodyPlayer gives it what a LivLiv model gets
// from Player: the idle motion, expressions, actions and drivers, so lip sync (@livliv-dev/player/voice) and face tracking
// (@livliv-dev/player/tracking) work for it the same way. Each frame the pose is put together as Player does it (your params,
// the idle motion, the drivers, actions, expressions); the Body turns it into its own picture.
import { pickLabel, type Labels } from './runtime/format'
import { defaultParams, type Params } from './runtime/params'
import { LivLivError, PlayerBase, aborted, contextGone, readBody, type LoadJob, type LoadOptions, type PlayerOptions } from './base'
import type { Seen } from './follow'
import { claimCanvas } from './release'
export { LivLivError }
export type { ActionId, Driver, Frame, Host, LivLivErrorCode, LoadOptions, LoadProgress, PlayerOptions, View } from './base'
/** an expression a Body can show: `labels` its names by language (BCP 47 tag -> name), `label` the one to fall back on */
export interface BodyExpression { id: string; label: string; labels?: Labels }
/** one frame's pose, as a Body draws it: the params, the picture's move (x, y shares of the canvas, y up; s a scale)
* and each expression's weight (0..1, eased) */
export interface Pose { params: Params; move: { x: number; y: number; s: number }; expr: Record<string, number> }
export interface Body {
/** what kind of character it draws ('png', 'vrm', ...): BodyInfo.format */
readonly format: string
/** optional: called before load with the canvas it will draw on; throw if it cannot draw there (the canvas already has
* another kind of context), and load() rejects with that */
attach?(canvas: HTMLCanvasElement): void
/** loads from its files (by file name); its name and the expressions it can show. If it rejects, the character before
* must stay as it is */
load(files: Map<string, Blob>): Promise<{ name: string; expressions: BodyExpression[]; meta?: BodyMeta }>
/** optional: the WebGL context it draws with, once it has one: what lets the player say when it is lost and give it up
* (dispose({ release: true })). A Body without WebGL leaves this out */
context?(): WebGLRenderingContext | WebGL2RenderingContext | null
/** draws the pose into the canvas, w × h device pixels, dt seconds after the last frame */
render(canvas: HTMLCanvasElement, pose: Pose, w: number, h: number, dt: number): void
/** where the head is, in shares of the canvas (from the top left) and its radii */
head(): { x: number; y: number; rx: number; ry: number }
dispose(): void
}
/** who made a body character and under what terms, as its file says */
export interface BodyMeta { author?: { name: string } | null; license?: { id?: string; url?: string } | null }
/** a loaded body character */
export interface BodyInfo {
name: string
/** the Body's format: 'png', 'vrm', ... */
format: string
/** what setExpression() takes (smile is always there: a param, not an expression) */
expressions: { id: string; label: string; labels: Labels }[]
/** what the file says about itself, where it does: a VRM's author and terms (VrmTerms). null otherwise */
meta: BodyMeta | null
}
/** its files: a URL (one file, a .vrm say), a File / Blob, a FileList / File[], or a map or object of file name -> Blob
* or URL (`{ closed: 'me-closed.png', open: 'me-open.png' }`). Files are known by their name, without folders */
export type BodySource = string | URL | Blob | FileList | Iterable<File> | Map<string, Blob | string> | Record<string, Blob | string>
export type BodyPlayerOptions = PlayerOptions
/** the most a body's files may hold in all (a VRM is usually 5-50 MB) */
export const BODY_LIMIT = 256 * 2 ** 20
const SMILE: Labels = { en: 'Smile', zh: '笑', 'zh-Hant': '笑', ja: '笑顔' }
/** Plays a character a Body draws, with the API of Player where it applies (no mark, no `puppet`, no `view.head`) */
export class BodyPlayer extends PlayerBase<BodyInfo> {
readonly body: Body
/** see PlayerOptions.maxPixelRatio */
maxPixelRatio: number
private pose: Pose
private time = 0
constructor(canvas: HTMLCanvasElement, body: Body, options: BodyPlayerOptions = {}) {
claimCanvas(canvas)
const pose: Pose = { params: defaultParams(), move: { x: 0, y: 0, s: 1 }, expr: {} }
super(canvas, pose.params, options)
this.pose = pose
this.body = body
this.maxPixelRatio = options.maxPixelRatio ?? 2
}
/** the Body's own word for it: asking the canvas would start a context on one that may be meant for 2D
* @internal */
protected get gl() { return this.body.context?.() ?? null }
/** @internal */
protected get drawable() { return !!this.info }
/** @internal */
protected get clock() { return this.time }
/** @internal */
protected get shownExpr() { return this.pose.expr }
/** the body's head in the terms the idle's pointAt reads a puppet in (its canvas shares as clip space)
* @internal */
protected seen(): Seen {
const hd = this.info ? this.body.head() : { x: 0.5, y: 0.3, rx: 0.15, ry: 0.12 }
return { model: { anchors: { head: { cx: 2 * hd.x - 1, cy: 2 * hd.y - 1, rx: 2 * hd.rx, ry: 2 * hd.ry } } } as Seen['model'], lastView: [1, 1, 0, 0, 2, 2] }
}
/** Loads a character from its files (see BodySource) in place of the one before (a failed load keeps it). A new load
* replaces one still downloading or waiting, which rejects with an AbortError (see LoadOptions); one whose character is
* already going in finishes first, since a Body swaps it inside, and the newer one then follows */
load(source: BodySource, options: LoadOptions = {}): Promise<BodyInfo> {
return this.start(options, (job) => this.inTurn(async () => {
const { signal } = job
if (signal.aborted) throw signal.reason
this.body.attach?.(this.canvas)
const files = await filesOf(source, job)
if (signal.aborted) throw signal.reason
job.build()
const got = await this.body.load(files)
if (this.disposed) { this.body.dispose(); throw aborted('the player was disposed') } // disposed while it loaded: nothing of it may stay
if (this.contextLost) throw contextGone()
const ex = [{ id: 'smile', label: '', labels: SMILE }, ...got.expressions.filter((e) => e.id !== 'smile')
.map((e) => ({ id: e.id, label: e.label, labels: e.labels ?? { en: e.label } }))]
this.info = { name: got.name, format: this.body.format, meta: got.meta ?? null,
expressions: ex.map((e) => ({ id: e.id, label: pickLabel(e.labels, this.lang, e.label || e.id), labels: e.labels })) }
this.neutral(); this.pose.expr = {}
if (!this.playing) this.resize()
return this.info
}))
}
/** @internal */
protected show(dt: number) {
const f = this.frame, V = this.view
this.pose.move = { x: f.move.x + V.x, y: f.move.y + this.actions.y + V.y, s: f.move.s * V.zoom }
const dpr = Math.min(this.maxPixelRatio, window.devicePixelRatio || 1)
const w = Math.round(this.canvas.clientWidth * dpr), h = Math.round(this.canvas.clientHeight * dpr)
if (w >= 2 && h >= 2) this.body.render(this.canvas, this.pose, w, h, dt)
this.time += dt
}
/** @internal */
protected free() { this.body.dispose() }
}
/** a body's files by name, URLs fetched */
async function filesOf(source: BodySource, job: LoadJob): Promise<Map<string, Blob>> {
const out = new Map<string, Blob>()
let total = 0
// files given by URL come one after another: each is its share of the download
const given = source instanceof Map ? [...source.values()] : typeof source === 'object' && !(source instanceof URL) && !(Symbol.iterator in source) && !(source instanceof Blob) ? Object.values(source) : []
const urls = typeof source === 'string' || source instanceof URL ? 1 : given.filter((v) => typeof v === 'string').length
let came = 0
const add = async (key: string, v: Blob | string) => {
const name = key.replace(/\\/g, '/').split('/').filter(Boolean).pop() || 'file'
let b: Blob
if (typeof v === 'string') {
const url = new URL(v, document.baseURI).href
job.progress(came / urls)
const unreachable = (e: unknown) => {
if (job.signal.aborted) throw e
throw new LivLivError('unreachable', `LivLiv: could not load ${url}: unreachable, or on another origin that sends no CORS header`, { cause: e })
}
const r = await fetch(url, { signal: job.signal }).catch(unreachable)
if (!r.ok) throw new LivLivError('http', `LivLiv: could not load ${url} (HTTP ${r.status})`, { status: r.status })
b = await readBody(r, job, (p) => (came + p) / urls).catch(unreachable)
came++
job.progress(came / urls)
} else b = v
if ((total += b.size) > BODY_LIMIT) throw new LivLivError('too-large', `LivLiv: the files are larger than ${BODY_LIMIT} bytes`)
out.set(name, b)
}
if (typeof source === 'string' || source instanceof URL) {
const u = new URL(String(source), document.baseURI)
await add(decodeURIComponent(u.pathname), u.href)
} else if (source instanceof Blob) await add((source as File).name || 'file', source)
else if (source instanceof Map) for (const [k, v] of source) await add(k, v)
else if (Symbol.iterator in source) for (const f of source as Iterable<File>) await add(f.name, f)
else for (const [k, v] of Object.entries(source)) await add(k, v)
return out
}