Skip to content

A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.

19.8 KB
Raw Blame
// @livliv-dev/player/vrm: a VRM (the open 3D avatar format: VRoid and others) on a BodyPlayer (@livliv-dev/player/body), drawn with
// three.js and @pixiv/three-vrm, which the page installs (optional peer dependencies) and which load only here. The
// shared params become its pose: the head and neck turn and tilt (angleX / Y / Z), the spine sways (bodyX / Z), the
// eyes blink (eyeLOpen / ROpen) and look (eyeBallX / Y), the mouth takes the visemes (mouthOpen with its vowel: aa open,
// ee wide, ou round), the face its expressions (happy with smile; angry, sad, relaxed, surprised); its hair and clothes
// swing by the model's own spring bones. Framed from the head down to the hips, arms down. VRM 1.0 and 0.x, whoever
// exported them; a file is looked through before three.js parses it (checkGlb).
import type * as T from 'three'
import type { VRM } from '@pixiv/three-vrm'
import type { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'
import type { Body, BodyExpression, BodyMeta, Pose } from './body'
import { LivLivError, type LivLivErrorCode } from './errors'

const FEELINGS = ['happy', 'angry', 'sad', 'relaxed', 'surprised'] as const
const LABELS: Record<(typeof FEELINGS)[number], Record<string, string>> = {
  happy: { en: 'Happy', zh: '开心', 'zh-Hant': '開心', ja: '喜び' },
  angry: { en: 'Angry', zh: '生气', 'zh-Hant': '生氣', ja: '怒り' },
  sad: { en: 'Sad', zh: '难过', 'zh-Hant': '難過', ja: '悲しい' },
  relaxed: { en: 'Relaxed', zh: '放松', 'zh-Hant': '放鬆', ja: 'リラックス' },
  surprised: { en: 'Surprised', zh: '惊讶', 'zh-Hant': '驚訝', ja: '驚き' },
}
const D = Math.PI / 180

/** what a VRM may hold before it is decoded: a real avatar stays far below these (VRoid: about 27 M pixels) */
export const VRM_LIMITS = { json: 32 * 2 ** 20, textureSide: 8192, texels: 512 * 2 ** 20, vertices: 16 * 2 ** 20 }

export interface VrmBodyOptions {
  /** given the GLTFLoader before it reads a file: for compressed files, register your decoders here (setDRACOLoader,
   *  setMeshoptDecoder, setKTX2Loader). The player itself ships none */
  configureLoader?: (loader: GLTFLoader) => void
}

/** the terms a VRM's author set (its meta, 1.0 and 0.x said alike, the file's own words in `raw`). A page playing other
 *  people's avatars should show them and keep to them */
export interface VrmTerms extends BodyMeta {
  /** the spec the file follows: '1' or '0' */
  spec: string
  /** who may play the avatar: only its author, people the author licensed, or everyone */
  use: 'author' | 'licensed' | 'everyone' | null
  /** the avatar in commercial work. 1.0 has grades, 0.x only allow and disallow */
  commercial: 'disallow' | 'personal non-profit' | 'personal profit' | 'allow' | 'corporation' | null
  violence: boolean | null
  sexual: boolean | null
  /** passing the file on, and changing it (1.0; null in 0.x, whose terms are in its licence) */
  redistribution: boolean | null
  modification: string | null
  raw: Record<string, unknown>
}

export class VrmBody implements Body {
  readonly format = 'vrm'
  private options: VrmBodyOptions
  /** VRM 0.x stands facing the other way, turned round by three-vrm: rotations about X and Z count the other way round */
  private flip = 1
  /** the file has only one blink, for both eyes */
  private blinkBoth = false
  /** the expressions the file adds to the preset ones: the id it goes by here -> its name in the file */
  private custom = new Map<string, string>()
  private THREE!: typeof T
  private vrm: VRM | null = null
  private scene!: T.Scene
  private camera!: T.PerspectiveCamera
  private renderer: T.WebGLRenderer | null = null
  private gl: WebGL2RenderingContext | null = null
  private bones: Partial<Record<'head' | 'neck' | 'spine' | 'chest', T.Object3D>> = {}
  /** the framing: the world height shown (m), and its center */
  private frame = { height: 1, y: 1 }
  private has = new Set<string>()
  private canvas: HTMLCanvasElement | null = null
  private headAt = { x: 0.5, y: 0.25 }
  private width = 0.6

  constructor(options: VrmBodyOptions = {}) { this.options = options }

  attach(canvas: HTMLCanvasElement) {
    // the attributes render() gives three.js, which then finds this same context
    const gl = canvas.getContext('webgl2', { alpha: true, antialias: true, premultipliedAlpha: true, preserveDrawingBuffer: true })
    if (!gl) throw new LivLivError('no-webgl', 'LivLiv: a VRM needs WebGL2: this browser has none, or this canvas already draws in 2D (a PNGTuber)')
    this.gl = gl
    if (gl.isContextLost()) throw new LivLivError('context-lost', 'LivLiv: this canvas has lost its WebGL context: use a new canvas')
  }

  context() { return this.gl }

  async load(files: Map<string, Blob>) {
    const entry = [...files].find(([n]) => /\.(vrm|glb)$/i.test(n))
    if (!entry) throw new LivLivError('missing-file', 'LivLiv: no .vrm file among the files')
    await checkGlb(entry[1], entry[0], !this.options.configureLoader)
    const [THREE, { GLTFLoader }, { VRMLoaderPlugin, VRMUtils }] = await Promise.all([
      import('three'), import('three/examples/jsm/loaders/GLTFLoader.js'), import('@pixiv/three-vrm')]).catch((e) => {
      throw new LivLivError('unreachable', `LivLiv: could not load three or @pixiv/three-vrm, which a VRM needs: are they installed, and can this page reach them? (${(e as Error).message})`)
    })
    this.THREE = THREE
    const loader = new GLTFLoader()
    loader.register((parser) => new VRMLoaderPlugin(parser))
    this.options.configureLoader?.(loader)
    const url = URL.createObjectURL(entry[1])
    let gltf
    try {
      gltf = await loader.loadAsync(url)
    } catch (e) {
      throw new LivLivError('not-a-model', `LivLiv: ${entry[0]} is not a VRM this player can read (${(e as Error).message})`, { cause: e })
    } finally {
      URL.revokeObjectURL(url)
    }
    const vrm = gltf.userData.vrm as VRM | undefined
    if (!vrm) {
      VRMUtils.deepDispose(gltf.scene)
      throw new LivLivError('not-a-model', `LivLiv: ${entry[0]} is a glTF without VRM data (exported as plain glTF? VRM exporters write a .vrm)`)
    }
    let meshes = 0
    vrm.scene.traverse((o) => { if ((o as T.Mesh).isMesh) meshes++ })
    if (!meshes) {
      VRMUtils.deepDispose(gltf.scene)
      throw new LivLivError('invalid', `LivLiv: ${entry[0]} has a skeleton but no mesh, so there is nothing to draw (export the armature together with its meshes)`)
    }
    // the new character, apart from the one shown (a failed load leaves that on the canvas): it takes over at the end
    VRMUtils.removeUnnecessaryVertices(gltf.scene)
    VRMUtils.combineSkeletons(gltf.scene)
    VRMUtils.rotateVRM0(vrm)  // VRM 0.x faces the other way
    vrm.scene.traverse((o) => { o.frustumCulled = false })
    const h = vrm.humanoid
    const flip = vrm.meta?.metaVersion === '0' ? -1 : 1
    const bones: typeof this.bones = {}
    for (const b of ['head', 'neck', 'spine', 'chest'] as const) bones[b] = h.getNormalizedBoneNode(b) ?? undefined
    // arms down from the T pose, a little forward
    const lu = h.getNormalizedBoneNode('leftUpperArm'), ru = h.getNormalizedBoneNode('rightUpperArm')
    if (lu) lu.rotation.set(0, 0, -1.2 * flip)
    if (ru) ru.rotation.set(0, 0, 1.2 * flip)
    const ll = h.getNormalizedBoneNode('leftLowerArm'), rl = h.getNormalizedBoneNode('rightLowerArm')
    if (ll) ll.rotation.set(0, -0.25, 0)
    if (rl) rl.rotation.set(0, 0.25, 0)
    if (vrm.lookAt) vrm.lookAt.autoUpdate = false
    vrm.update(0)

    const scene = new THREE.Scene()
    scene.add(vrm.scene)
    const light = new THREE.DirectionalLight(0xffffff, Math.PI)
    light.position.set(1, 1, 1).normalize()
    scene.add(light, new THREE.AmbientLight(0xffffff, 0.6))
    // the head's top to the hips: what a LivLiv character's canvas shows of a person
    // in the model's own units, whatever they are (an exporter's unit can make a character 170 long), and its own shape
    // (a head as big as the rest of it)
    const at = (b: 'head' | 'hips' | 'leftUpperArm' | 'rightUpperArm') => { const v = new THREE.Vector3(); h.getNormalizedBoneNode(b)?.getWorldPosition(v); return v }
    const head = at('head'), hips = at('hips'), torso = Math.max(1e-3, head.y - hips.y)
    const box = new THREE.Box3().setFromObject(vrm.scene)
    const crown = Number.isFinite(box.max.y) ? box.max.y : head.y
    const top = Math.max(head.y + torso / 3, Math.min(crown + torso / 20, head.y + torso)), bottom = hips.y
    const shoulders = at('leftUpperArm').distanceTo(at('rightUpperArm'))
    const has = new Set((vrm.expressionManager?.expressions ?? []).map((e) => e.expressionName))
    const meta = vrm.meta as { name?: string; title?: string } | undefined
    const expressions: BodyExpression[] = FEELINGS.filter((f) => f !== 'happy' && has.has(f)).map((f) => ({ id: f, label: LABELS[f].en, labels: LABELS[f] }))
    if (has.has('happy')) expressions.unshift({ id: 'happy', label: LABELS.happy.en, labels: LABELS.happy })
    // what the file adds of its own, under the name it gives; a feeling 0.x has no preset for (VRoid's Surprised) by our id
    const custom = new Map<string, string>()
    for (const n of Object.keys(vrm.expressionManager?.customExpressionMap ?? {})) {
      const lower = n.toLowerCase(), id = (FEELINGS as readonly string[]).includes(lower) ? lower : n
      custom.set(id, n)
      if (!expressions.some((x) => x.id === id)) expressions.push(id === lower && id in LABELS ? { id, label: LABELS[id as keyof typeof LABELS].en, labels: LABELS[id as keyof typeof LABELS] } : { id, label: n })
    }
    // an exporter's placeholder is no name ("undefined": VRM Add-on for Blender, until it is filled in)
    const given = (meta?.name || meta?.title || '').trim()
    const name = (given && given !== 'undefined' ? given : entry[0].replace(/\.(vrm|glb)$/i, '')).slice(0, 200)

    const old = this.vrm
    this.vrm = vrm
    this.scene = scene
    this.bones = bones
    this.has = has
    this.custom = custom
    this.blinkBoth = has.has('blink') && !(has.has('blinkLeft') && has.has('blinkRight'))
    this.flip = flip
    this.frame = { height: Math.max(1e-3, top - bottom), y: (top + bottom) / 2 }
    this.width = Math.max(shoulders * 1.5, this.frame.height / 2)
    this.camera = new THREE.PerspectiveCamera(20, 2 / 3, this.frame.height / 100, this.frame.height * 100)
    this.place(2 / 3)  // render() gives it the canvas's aspect
    if (old) VRMUtils.deepDispose(old.scene)
    return { name, expressions, meta: vrmTerms(vrm.meta as unknown as Record<string, unknown>) }
  }

  /** the camera where the framing fits the canvas's aspect */
  private place(aspect: number) {
    const c = this.camera, half = this.frame.height / 2
    // tall enough for the height, wide enough for the shoulders and arms on narrow canvases
    const dist = Math.max(half, this.width / 2 / aspect) / Math.tan((c.fov * D) / 2)
    c.aspect = aspect
    c.position.set(0, this.frame.y, dist)
    c.lookAt(0, this.frame.y, 0)
    c.updateProjectionMatrix()
  }

  render(canvas: HTMLCanvasElement, { params: P, move, expr }: Pose, w: number, h: number, dt: number) {
    const THREE = this.THREE, vrm = this.vrm
    if (!vrm) return
    if (!this.renderer || this.canvas !== canvas) {
      this.renderer?.dispose()
      this.renderer = new THREE.WebGLRenderer({ canvas, alpha: true, antialias: true, premultipliedAlpha: true, preserveDrawingBuffer: true })
      this.renderer.setClearColor(0x000000, 0)
      this.canvas = canvas
    }
    if (canvas.width !== w || canvas.height !== h) this.renderer.setSize(w, h, false)
    if (this.camera.aspect !== w / h) this.place(w / h)  // a new character's camera, on a canvas already this size too
    const b = this.bones, s = this.flip
    // the head turns most, the neck some (yaw: to the screen's right; pitch: up; roll: its top to the screen's right, as a
    // Player's does). The neck is optional in VRM: without one the head takes it all
    const hk = b.neck ? 0.65 : 1
    b.neck?.rotation.set(-s * P.angleY * D * 0.35, P.angleX * D * 0.35, -s * P.angleZ * D * 0.35)
    b.head?.rotation.set(-s * P.angleY * D * hk, P.angleX * D * hk, -s * P.angleZ * D * hk)
    b.spine?.rotation.set(0, P.bodyX * D * 0.8, -s * P.bodyZ * D * 0.8)
    b.chest?.rotation.set(-s * (P.breath - 0.5) * 0.02, 0, 0)
    vrm.scene.position.set(move.x * this.frame.height * 0.5, move.y * this.frame.height, 0)
    vrm.scene.scale.setScalar(move.s)
    const em = vrm.expressionManager
    if (em) {
      const set = (n: string, v: number) => { if (this.has.has(n)) em.setValue(n, Math.max(0, Math.min(1, v))) }
      if (this.blinkBoth) set('blink', 1 - Math.max(P.eyeLOpen, P.eyeROpen))
      else { set('blinkLeft', 1 - P.eyeLOpen); set('blinkRight', 1 - P.eyeROpen) }
      const o = P.mouthOpen, v = P.vowel
      // wide and round: the file's ee and ou, or else its ih and oh
      set('aa', o * (1 - Math.abs(v) * 0.6)); set(this.has.has('ee') ? 'ee' : 'ih', o * Math.max(0, v) * 0.7); set(this.has.has('ou') ? 'ou' : 'oh', o * Math.max(0, -v) * 0.7)
      // the smile param is a small one (the idle motion's, the face tracker's): a little over half the model's happy, which
      // often closes the eyes. Over half: an expression marked isBinary is off below it
      set('happy', Math.max(P.smile * 0.6, expr.happy ?? 0))
      for (const f of FEELINGS) if (f !== 'happy' && !this.custom.has(f)) set(f, expr[f] ?? 0)
      for (const [id, n] of this.custom) set(n, expr[id] ?? 0)
    }
    // three-vrm's pitch is positive looking down
    if (vrm.lookAt) { vrm.lookAt.yaw = P.eyeBallX * 25; vrm.lookAt.pitch = -P.eyeBallY * 15 }
    vrm.update(dt)
    this.renderer.render(this.scene, this.camera)
    // where the head is now, for the pointer
    const p = new THREE.Vector3()
    b.head?.getWorldPosition(p)
    p.y += 0.06
    p.project(this.camera)
    this.headAt = { x: (p.x + 1) / 2, y: (1 - p.y) / 2 }
  }

  head() { return { ...this.headAt, rx: 0.14, ry: 0.1 } }

  dispose() {
    this.renderer?.dispose(); this.renderer = null; this.canvas = null
    const vrm = this.vrm
    this.vrm = null
    if (vrm) void import('@pixiv/three-vrm').then(({ VRMUtils }) => VRMUtils.deepDispose(vrm.scene))
  }
}

/** the VRM's meta in one shape, whichever spec wrote it */
function vrmTerms(m: Record<string, unknown> | undefined): VrmTerms | undefined {
  if (!m) return undefined
  const text = (v: unknown) => (typeof v === 'string' && v.trim() && v !== 'undefined' ? v.trim() : undefined)
  const yes = (v: unknown) => (typeof v === 'boolean' ? v : v === 'Allow' ? true : v === 'Disallow' ? false : null)
  if (m.metaVersion === '0') {
    const url = text(m.otherLicenseUrl), id = text(m.licenseName)
    return {
      spec: '0', author: text(m.author) ? { name: text(m.author)! } : null, license: id || url ? { id, url } : null,
      use: m.allowedUserName === 'Everyone' ? 'everyone' : m.allowedUserName === 'ExplicitlyLicensedPerson' ? 'licensed' : m.allowedUserName === 'OnlyAuthor' ? 'author' : null,
      commercial: m.commercialUssageName === 'Allow' ? 'allow' : m.commercialUssageName === 'Disallow' ? 'disallow' : null,
      violence: yes(m.violentUssageName), sexual: yes(m.sexualUssageName), redistribution: null, modification: null, raw: m,
    }
  }
  const authors = (Array.isArray(m.authors) ? m.authors : []).map(text).filter(Boolean)
  const url = text(m.licenseUrl)
  return {
    spec: '1', author: authors.length ? { name: authors.join(', ') } : null, license: url ? { url } : null,
    use: m.avatarPermission === 'everyone' ? 'everyone' : m.avatarPermission === 'onlySeparatelyLicensedPerson' ? 'licensed' : m.avatarPermission === 'onlyAuthor' ? 'author' : null,
    commercial: m.commercialUsage === 'corporation' ? 'corporation' : m.commercialUsage === 'personalProfit' ? 'personal profit' : m.commercialUsage === 'personalNonProfit' ? 'personal non-profit' : null,
    violence: yes(m.allowExcessivelyViolentUsage), sexual: yes(m.allowExcessivelySexualUsage), redistribution: yes(m.allowRedistribution),
    modification: text(m.modification) ?? null, raw: m,
  }
}

const COMPRESSED: Record<string, string> = { KHR_draco_mesh_compression: 'Draco', EXT_meshopt_compression: 'Meshopt', KHR_texture_basisu: 'KTX2' }

/** a .vrm looked through before three.js sees it, as any file from anyone: a GLB, no outside files (it would make the page
 *  fetch them), its compression one the page can decode, and images and meshes within VRM_LIMITS, read from the file's
 *  headers before anything is decoded */
async function checkGlb(file: Blob, name: string, noDecoders: boolean) {
  const bad = (why: string, code: LivLivErrorCode = 'invalid'): never => { throw new LivLivError(code, `LivLiv: ${name} ${why}`) }
  const head = new DataView(await file.slice(0, 20).arrayBuffer())
  if (head.byteLength < 20 || head.getUint32(0, true) !== 0x46546c67) bad('is not a VRM: it should begin as a binary glTF does', 'not-a-model')
  if (head.getUint32(4, true) !== 2) bad(`is a glTF of version ${head.getUint32(4, true)}, and VRM uses 2`, 'not-a-model')
  const jsonLen = head.getUint32(12, true)
  if (head.getUint32(16, true) !== 0x4e4f534a) bad('is damaged (no JSON where a binary glTF has it)')
  if (jsonLen > VRM_LIMITS.json) bad(`is not one this player reads: its JSON has ${jsonLen} bytes, more than ${VRM_LIMITS.json}`, 'too-large')
  if (20 + jsonLen > file.size) bad('is damaged or cut short')
  let gltf: { buffers?: { uri?: string }[]; images?: { uri?: string; bufferView?: number }[]; bufferViews?: { byteOffset?: number; byteLength?: number }[]
    accessors?: { count?: number }[]; meshes?: { primitives?: { attributes?: { POSITION?: number } }[] }[]; extensionsUsed?: string[] }
  try { gltf = JSON.parse(await file.slice(20, 20 + jsonLen).text()) } catch { return bad('is damaged (its JSON does not parse)') }
  for (const o of [...(gltf.buffers ?? []), ...(gltf.images ?? [])]) {
    if (o?.uri && !o.uri.startsWith('data:')) bad(`names a file outside itself (${String(o.uri).slice(0, 80)}), and a .vrm has to hold everything it needs`)
  }
  if (noDecoders) {
    for (const [ext, what] of Object.entries(COMPRESSED)) {
      if (gltf.extensionsUsed?.includes(ext)) bad(`is compressed with ${what}, and this page has no decoder for that. Export it uncompressed, or pass VrmBody a configureLoader`, 'missing-feature')
    }
  }
  const positions = new Set<number>()
  for (const m of gltf.meshes ?? []) for (const p of m?.primitives ?? []) if (typeof p?.attributes?.POSITION === 'number') positions.add(p.attributes.POSITION)
  let vertices = 0
  for (const i of positions) vertices += gltf.accessors?.[i]?.count ?? 0
  if (vertices > VRM_LIMITS.vertices) bad(`has ${vertices} vertices, more than ${VRM_LIMITS.vertices}`, 'too-large')
  // where the binary chunk starts, and each picture's size from its first bytes
  const binary = 20 + jsonLen + 8
  let texels = 0
  for (const im of gltf.images ?? []) {
    const bv = typeof im?.bufferView === 'number' ? gltf.bufferViews?.[im.bufferView] : undefined
    if (!bv) continue
    const size = await pictureSize(file.slice(binary + (bv.byteOffset ?? 0), binary + (bv.byteOffset ?? 0) + (bv.byteLength ?? 0)))
    if (!size) continue
    if (size[0] > VRM_LIMITS.textureSide || size[1] > VRM_LIMITS.textureSide || (texels += size[0] * size[1]) > VRM_LIMITS.texels) {
      bad(`has pictures too large for this player (${size[0]} x ${size[1]}; ${VRM_LIMITS.textureSide} px a side and ${VRM_LIMITS.texels} pixels in all at most)`, 'too-large')
    }
  }
}

/** width and height from the start of a PNG or a JPEG (other kinds: unknown, left to the browser) */
async function pictureSize(b: Blob): Promise<[number, number] | null> {
  const d = new DataView(await b.slice(0, 64 * 1024).arrayBuffer())
  if (d.byteLength >= 24 && d.getUint32(0) === 0x89504e47) return [d.getUint32(16), d.getUint32(20)]
  if (d.byteLength < 4 || d.getUint16(0) !== 0xffd8) return null
  for (let i = 2; i + 9 < d.byteLength;) {
    if (d.getUint8(i) !== 0xff) { i++; continue }
    const m = d.getUint8(i + 1)
    if (m >= 0xc0 && m <= 0xcf && m !== 0xc4 && m !== 0xc8 && m !== 0xcc) return [d.getUint16(i + 7), d.getUint16(i + 5)]
    i += 2 + d.getUint16(i + 2)
  }
  return null
}