Skip to content

A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.

19.9 KB
Raw Blame
<!doctype html>
<html lang="en">
<meta charset="utf-8">
<title>Bodies smoke test</title>
<script type="importmap">{ "imports": {
  "three": "/node_modules/three/build/three.module.js",
  "three/examples/jsm/": "/node_modules/three/examples/jsm/",
  "@pixiv/three-vrm": "/node_modules/@pixiv/three-vrm/lib/three-vrm.module.js",
  "@mediapipe/tasks-vision": "/node_modules/@mediapipe/tasks-vision/vision_bundle.mjs"
} }</script>
<canvas id="png" style="width: 200px; height: 240px"></canvas>
<canvas id="vrm" style="width: 200px; height: 300px"></canvas>
<script type="module">
  // Run by smoke.mjs: a PNGTuber shows the picture for each mouth / eye state, and follows the face tracker (its
  // synthetic trace) and a speech clip; a VRM (tiny-vrm.mjs, served by smoke.mjs) loads, draws, takes its expressions
  // and follows the face tracker.
  import { Player } from '../dist/index.js'
  import { BodyPlayer } from '../dist/body.js'
  import { PngBody } from '../dist/png.js'
  import { VrmBody } from '../dist/vrm.js'
  import { createSpeech } from '../dist/voice.js'
  import { preloadFaceModel, startFaceTracking } from '../dist/tracking.js'

  const g = document.createElement('canvas').getContext('2d', { willReadFrequently: true })
  const read = (canvas) => {
    g.canvas.width = canvas.width; g.canvas.height = canvas.height
    g.clearRect(0, 0, canvas.width, canvas.height); g.drawImage(canvas, 0, 0)
    return g.getImageData(0, 0, canvas.width, canvas.height)
  }
  const pixel = (canvas, x, y) => { const d = read(canvas); const i = 4 * (Math.round(y) * d.width + Math.round(x)); return [...d.data.slice(i, i + 3)] }
  const covered = (canvas) => { const d = read(canvas).data; let n = 0; for (let i = 3; i < d.length; i += 4) if (d[i] > 0) n++; return n / (d.length / 4) }
  const solid = (color) => { const c = new OffscreenCanvas(100, 120), x = c.getContext('2d'); x.fillStyle = color; x.fillRect(0, 0, 100, 120); return c.convertToBlob() }
  const wait = (ms) => new Promise((r) => setTimeout(r, ms))
  /** until the condition holds, or some seconds: what a browser does next may take a while on a busy machine */
  const until = async (cond, ms = 5000) => { for (const t0 = performance.now(); !cond() && performance.now() - t0 < ms;) await wait(10) }
  /** what the drivers did, seen by a driver that runs after them */
  const watch = (player) => {
    const seen = { tracked: false, mouth: 0, angle: 0 }
    player.use({ order: 100, update(f) { seen.tracked ||= f.tracked; seen.mouth = Math.max(seen.mouth, f.params.mouthOpen); seen.angle = Math.max(seen.angle, Math.abs(f.params.angleX)) } })
    return seen
  }
  /** a voice-like clip: 220 Hz with syllables, 16-bit mono WAV */
  function wav(secs, sr = 24000) {
    const n = Math.round(secs * sr), b = new DataView(new ArrayBuffer(44 + 2 * n))
    const s = (o, t) => [...t].forEach((c, i) => b.setUint8(o + i, c.charCodeAt(0)))
    s(0, 'RIFF'); b.setUint32(4, 36 + 2 * n, true); s(8, 'WAVEfmt '); b.setUint32(16, 16, true); b.setUint16(20, 1, true); b.setUint16(22, 1, true)
    b.setUint32(24, sr, true); b.setUint32(28, 2 * sr, true); b.setUint16(32, 2, true); b.setUint16(34, 16, true); s(36, 'data'); b.setUint32(40, 2 * n, true)
    for (let i = 0; i < n; i++) { const t = i / sr; b.setInt16(44 + 2 * i, 12000 * Math.sin(2 * Math.PI * 220 * t) * (Math.sin(Math.PI * 4 * t) > 0 ? 1 : 0.05), true) }
    return new Blob([b.buffer], { type: 'audio/wav' })
  }

  /** what Player and BodyPlayer promise alike: one test, run on both (see PlayerBase) */
  window.runContract = async (kind, id) => {
    const c = document.createElement('canvas')
    c.style.cssText = 'width: 200px; height: 240px'
    document.body.append(c)
    const options = { autoplay: false, followPointer: false, idle: false, maxPixelRatio: 2 }
    const p = kind === 'player' ? new Player(c, options) : new BodyPlayer(c, new PngBody(), options)
    const good = kind === 'player' ? `../characters/${id}.livliv` : new Map([['closed.png', await solid('#f00')], ['open.png', await solid('#0f0')]])
    const bad = kind === 'player' ? new Map([['model.json', new Blob(['{'])]]) : new Map([['closed.png', await solid('#f00')]])
    const out = { before: p.model, playing: p.playing }
    const info = await p.load(good)
    out.name = info.name === p.model?.name
    out.expressions = p.expressions === info.expressions
    // the pixel ratio can be read and changed
    out.ratio = [p.maxPixelRatio]
    p.maxPixelRatio = 0.5; p.resize(); out.ratio.push(p.maxPixelRatio, c.width)
    p.maxPixelRatio = 2; p.resize(); out.ratio.push(c.width)
    // a snapshot is the pose as it is now, on a paused player too
    const a = p.snapshot()
    p.params.mouthOpen = 1
    const b = p.snapshot()
    out.snapshot = a !== b && b.startsWith('data:image/png')
    p.params.mouthOpen = 0
    // what a driver gets; the time counts the frames before this one
    const times = []
    let frame = null
    const remove = p.use({ update(f) { times.push(f.time); frame = f } })
    p.draw(0.01); p.draw(0.02); p.draw(0.03)
    out.times = [times[1] - times[0], times[2] - times[1]]
    out.frame = !!frame.params && typeof frame.add === 'function' && frame.dt === 0.03 && 'x' in frame.move
    remove()
    p.draw(0.01)
    out.removed = times.length === 3
    // the pose stays in range, and what is no number is the default, whoever set it
    const pose = () => p.frame.params
    const P = p.params
    P.angleX = 1e9; P.eyeLOpen = -5; p.draw(0.01); out.clamped = [pose().angleX, pose().eyeLOpen]
    P.angleX = NaN; P.eyeLOpen = NaN; P.bodyX = 'x'; p.draw(0.01); out.nan = [pose().angleX, pose().eyeLOpen, pose().bodyX]
    const undo = p.use({ order: 100, update(f) { f.params.angleY = NaN; f.params.mouthOpen = 7 } })
    p.draw(0.01); out.driver = [pose().angleY, pose().mouthOpen]
    undo()
    Object.assign(P, { angleX: 0, eyeLOpen: 1, bodyX: 0 })
    p.draw(0.01)
    out.alive = kind === 'player' ? !p.puppet.layers.some((L) => Number.isNaN(L.out[0])) : true
    // away is a switch, on both
    p.setAway(true)
    for (let i = 0; i < 90; i++) p.draw(1 / 30)
    out.away = pose().eyeLOpen
    p.setAway(false)
    for (let i = 0; i < 90; i++) p.draw(1 / 30)
    out.back = pose().eyeLOpen
    out.oldAway = (() => { try { p.act('away'); return 'took it' } catch (e) { return e.message } })()
    // the rest of the surface
    out.unknown = p.setExpression('no-such-expression')
    out.neutral = p.setExpression(null)
    for (const a2 of ['nod', 'shake', 'tilt', 'hop', 'shy']) p.act(a2)
    p.lookAt(0.5, 0.5); p.lookAt(null); p.blink(); p.draw(0.01)
    p.play(); out.plays = p.playing; p.pause(); out.paused = p.playing
    // Player alone: from the canvas to the model, and a flick
    if (kind === 'player') {
      const [mx, my] = p.canvasToModel(c.clientWidth / 2, c.clientHeight / 2), [mw, mh] = p.model.size
      out.toModel = mx > 0 && my > 0 && mx < mw && my < mh
      const R = p.puppet.chainRuns[0]?.R
      out.poke = [p.poke(mx, my), R ? p.poke(R[R.length - 2], R[R.length - 1]) : -1]
    }
    // a load that fails keeps the character; a disposed player says so and takes nothing more
    const name = p.model.name
    out.failed = await p.load(bad).then(() => 'loaded', () => 'refused')
    out.kept = p.model?.name === name
    p.dispose(); p.dispose()
    out.after = p.model
    out.disposedLoad = await p.load(good).then(() => 'loaded', (e) => e.message)
    let gone = 0
    p.use({ update() {}, dispose() { gone++ } })
    out.gone = gone
    c.remove()
    return out
  }

  /** BodyPlayer: several loads at once. Those still downloading or waiting stop; one already going in (inside the Body,
   *  which cannot be stopped) finishes, and the newer one follows */
  window.runBodyLoads = async () => {
    const out = {}
    const slow = (color) => ({ closed: `/__slow-png/${color}.png`, open: `/__slow-png/${color}.png` })
    {
      const c = document.createElement('canvas'), p = new BodyPlayer(c, new PngBody(), { autoplay: false, followPointer: false, idle: false })
      c.style.cssText = 'width: 40px; height: 40px'
      document.body.append(c)
      const say = (e) => e.name
      const a = p.load(slow('ff0000')).then(() => 'loaded', say)
      await wait(60)
      const b = p.load(slow('00ff00')).then(() => 'loaded', say)
      await wait(60)
      const steps = []
      const last = await p.load(slow('0000ff'), { onProgress: (s) => steps.push(s) }).then(() => 'loaded', say)
      out.row = { a: await a, b: await b, last, pixel: pixel(c, 20, 20), steps }
      p.dispose(); c.remove()
    }
    {
      // a Body holding its load: the page decides when it ends
      const held = [], calls = []
      const body = {
        format: 'test',
        load(files) { calls.push([...files.keys()][0]); return new Promise((res) => held.push(() => res({ name: [...files.keys()][0], expressions: [] }))) },
        render() {}, head: () => ({ x: 0.5, y: 0.3, rx: 0.1, ry: 0.1 }), dispose() {},
      }
      const c = document.createElement('canvas'), p = new BodyPlayer(c, body, { autoplay: false, followPointer: false })
      const file = (n) => new Map([[n, new Blob(['x'])]])
      const a = p.load(file('a')).then((m) => m.name, (e) => e.name)
      await wait(20)
      const b = p.load(file('b')).then((m) => m.name, (e) => e.name)
      const d = p.load(file('d')).then((m) => m.name, (e) => e.name)
      await wait(20)
      held.shift()()
      out.held = { a: await a, b: await b }
      await wait(20)
      held.shift()()
      out.held.d = await d
      out.held.calls = calls
      out.held.shown = p.model?.name
      p.dispose()
    }
    {
      // a Body that takes its 2D context only at its first frame: asking about the context must not start a WebGL one
      const c = document.createElement('canvas')
      c.style.cssText = 'width: 40px; height: 40px'
      document.body.append(c)
      let got = null
      const body = {
        format: 'late', async load() { return { name: 'late', expressions: [] } },
        render(cv) { got = cv.getContext('2d') }, head: () => ({ x: 0.5, y: 0.3, rx: 0.1, ry: 0.1 }), dispose() {},
      }
      const p = new BodyPlayer(c, body, { autoplay: false, followPointer: false })
      out.lateLost = p.contextLost
      const going = p.load(new Map([['a', new Blob(['x'])]]))
      out.lateLost2 = p.contextLost
      await going
      out.late2d = !!got
      p.dispose({ release: true })
      await wait(20)
      out.lateAgain = (() => { try { new BodyPlayer(c, body, { autoplay: false }).dispose(); return 'made' } catch (e) { return e.code } })()
      c.remove()
    }
    return out
  }

  /** the context on a BodyPlayer: a VRM is told and released like a Player; a PNGTuber has none to lose */
  window.runBodyContext = async () => {
    const out = {}
    const code = (e) => e.code ?? e.name
    {
      const c = document.createElement('canvas'), p = new BodyPlayer(c, new VrmBody(), { followPointer: false })
      c.style.cssText = 'width: 60px; height: 80px'
      document.body.append(c)
      let told = 0
      p.onContextLost = () => told++
      await p.load('/__tiny.vrm')
      const gl = c.getContext('webgl2')
      gl.getExtension('WEBGL_lose_context').loseContext()
      await until(() => told)
      out.vrm = { told, lost: p.contextLost, load: await p.load('/__tiny.vrm').then(() => 'loaded', code) }
      p.dispose(); c.remove()
    }
    {
      const c = document.createElement('canvas'), p = new BodyPlayer(c, new VrmBody(), { followPointer: false })
      c.style.cssText = 'width: 60px; height: 80px'
      document.body.append(c)
      await p.load('/__tiny.vrm')
      const gl = c.getContext('webgl2')
      p.dispose({ release: true })
      await until(() => gl.isContextLost())
      out.vrmReleased = { lost: gl.isContextLost(), reuse: (() => { try { new BodyPlayer(c, new VrmBody()); return 'made' } catch (e) { return code(e) } })() }
      c.remove()
    }
    {
      const c = document.createElement('canvas')
      c.style.cssText = 'width: 60px; height: 80px'
      document.body.append(c)
      const files = new Map([['closed.png', await solid('#f00')], ['open.png', await solid('#0f0')]])
      const p = new BodyPlayer(c, new PngBody(), { autoplay: false, followPointer: false, idle: false })
      await p.load(files)
      out.png = { lost: p.contextLost }
      p.dispose({ release: true })
      await wait(30)
      const q = new BodyPlayer(c, new PngBody(), { autoplay: false, followPointer: false, idle: false })
      out.png.again = await q.load(files).then((m) => m.name, code)
      q.dispose(); c.remove()
    }
    return out
  }

  /** the camera refused while the face model comes in (the slow route of smoke.mjs); ahead: it was asked for before */
  window.runRefused = async (ahead = false) => {
    const files = { wasmUrl: '/node_modules/@mediapipe/tasks-vision/wasm', modelUrl: '/__model.task' }
    if (ahead) preloadFaceModel(files).catch(() => {})
    const real = navigator.mediaDevices.getUserMedia
    navigator.mediaDevices.getUserMedia = async () => { await wait(150); throw new DOMException('refused', 'NotAllowedError') }
    const p = new BodyPlayer(document.createElement('canvas'), new PngBody(), { autoplay: false })
    const name = await startFaceTracking(p, files).then(() => 'started', (e) => e.name)
    navigator.mediaDevices.getUserMedia = real
    p.dispose()
    return name
  }

  /** what a VRM player says of a file it cannot play, keeping the character before */
  window.runVrmRefusals = async (urls) => {
    const c = document.createElement('canvas')
    const p = new BodyPlayer(c, new VrmBody(), { autoplay: false, idle: false, followPointer: false })
    await p.load('/__tiny.vrm')
    const out = {}
    for (const [k, u] of Object.entries(urls)) out[k] = await p.load(u).then(() => 'loaded', (e) => e.message)
    out.plainFile = await p.load(new File([new Blob(['not a glb at all, a text file'])], 'x.vrm')).then(() => 'loaded', (e) => e.message)
    out.kept = p.model?.name
    p.dispose()
    return out
  }

  /** how much of the canvas a VRM covers */
  window.runVrmShare = async (url) => {
    const c = document.createElement('canvas')
    c.style.cssText = 'width: 300px; height: 400px'
    document.body.append(c)
    const p = new BodyPlayer(c, new VrmBody(), { autoplay: false, idle: false, followPointer: false })
    await p.load(url)
    p.resize()
    const share = covered(c)
    p.dispose(); c.remove()
    return share
  }

  /** which way a VRM stands and moves, 1.0 and 0.x alike: arms down, head turning right and up, top of the head to the right
   *  on a positive roll, eyes up and to the character's left (the screen's right) */
  window.runVrmPose = async (url) => {
    const THREE = await import('three')
    const c = document.createElement('canvas')
    c.style.cssText = 'width: 300px; height: 400px'
    document.body.append(c)
    const p = new BodyPlayer(c, new VrmBody(), { autoplay: false, idle: false, followPointer: false })
    await p.load(url)
    const vrm = p.body.vrm, H = vrm.humanoid, em = vrm.expressionManager
    const set = (o) => { Object.assign(p.params, { angleX: 0, angleY: 0, angleZ: 0, eyeBallX: 0, eyeBallY: 0, bodyZ: 0 }, o); p.draw(0); p.draw(0) }
    const world = (b) => { const v = new THREE.Vector3(); H.getRawBoneNode(b).getWorldPosition(v); return v }
    const q = (b) => { const r = new THREE.Quaternion(); H.getRawBoneNode(b).getWorldQuaternion(r); return r }
    set({})
    const rest = q('head'), turned = (v) => new THREE.Vector3(...v).applyQuaternion(q('head').multiply(rest.clone().invert()))
    const out = { spec: vrm.meta.metaVersion, hand: world('leftHand').y - world('leftUpperArm').y, terms: p.model.meta }
    set({ angleX: 20 }); out.yaw = turned([0, 0, 1]).x
    set({ angleY: 20 }); out.pitch = turned([0, 0, 1]).y
    set({ angleZ: 20 }); out.roll = turned([0, 1, 0]).x
    set({ eyeBallY: 1 }); out.up = em.getValue('lookUp'); out.down = em.getValue('lookDown')
    set({ eyeBallX: 1 }); out.right = em.getValue('lookLeft')  // the character's left, seen from the front
    p.dispose(); c.remove()
    return out
  }

  window.runBodies = async () => {
    const out = {}
    try {
      const canvas = document.getElementById('png')
      const p = new BodyPlayer(canvas, new PngBody(), { autoplay: false, idle: false })
      const m = await p.load({ closed: await solid('#ff0000'), open: await solid('#00ff00'), blink: await solid('#0000ff'), 'blink-open': await solid('#ffff00') })
      const at = () => { p.resize(); return pixel(canvas, canvas.width / 2, canvas.height * 0.8) }
      const shown = { closed: at() }
      p.params.mouthOpen = 1; shown.open = at()
      p.params.eyeLOpen = p.params.eyeROpen = 0; shown.blinkOpen = at()
      p.params.mouthOpen = 0; shown.blink = at()
      p.params.eyeLOpen = p.params.eyeROpen = 1
      const seen = watch(p)
      p.idle = true; p.play()
      const tracking = await startFaceTracking(p, { trace: 'synth' })
      const speech = createSpeech(p)
      speech.unlock(); speech.say(wav(1.2))
      await wait(1600)
      tracking.stop(); speech.dispose()
      out.png = { name: m.name, format: m.format, expressions: m.expressions.map((e) => e.id), shown, ...seen }
      p.dispose()
      // a failed load leaves the pictures shown, and the name; on a canvas with WebGL it says so
      const canvas2 = document.createElement('canvas')
      canvas2.style.cssText = 'width: 100px; height: 120px'
      document.body.append(canvas2)
      const q = new BodyPlayer(canvas2, new PngBody(), { autoplay: false, idle: false })
      await q.load({ closed: await solid('#ff0000'), open: await solid('#00ff00'), 'name.txt': new Blob(['First']) })
      const refused = await q.load({ closed: await solid('#0000ff') }).then(() => 'loaded', (e) => e.message)
      q.resize()
      out.pngFailed = { refused, kept: q.model?.name, pixel: pixel(canvas2, canvas2.width / 2, canvas2.height * 0.8) }
      q.dispose(); canvas2.remove()
      const gl = document.createElement('canvas')
      gl.getContext('webgl2')
      out.pngOnGl = await new BodyPlayer(gl, new PngBody(), { autoplay: false }).load({ closed: await solid('#ff0000'), open: await solid('#00ff00') }).then(() => 'loaded', (e) => e.message)
    } catch (e) { out.png = { error: String(e.message || e) } }
    try {
      const canvas = document.getElementById('vrm')
      const p = new BodyPlayer(canvas, new VrmBody(), { autoplay: false, lang: 'en' })
      const m = await p.load('/__tiny.vrm')
      p.resize()
      const share = covered(canvas)
      const happy = p.setExpression('happy'), smile = p.setExpression('smile'), unknown = p.setExpression('nope')
      p.params.mouthOpen = 1; p.resize()
      const seen = watch(p)
      p.play()
      const tracking = await startFaceTracking(p, { trace: 'synth' })
      await wait(800)
      tracking.stop()
      // a second load, on a canvas that is not 2:3 and already has its size, and a failed one
      const wide = document.createElement('canvas')
      wide.style.cssText = 'width: 400px; height: 200px'
      document.body.append(wide)
      wide.width = 400; wide.height = 200
      const w = new BodyPlayer(wide, new VrmBody(), { autoplay: false, idle: false })
      await w.load('/__tiny.vrm'); w.resize()
      const aspect1 = w.body.camera.aspect
      await w.load('/__tiny.vrm'); w.resize()
      const aspect2 = w.body.camera.aspect
      const bad = new File([new Blob(['not a glb'])], 'x.vrm')
      const refused = await w.load(bad).then(() => 'loaded', (e) => e.message)
      w.resize()
      out.vrmAgain = { aspect1, aspect2, refused, kept: w.model?.name, share: covered(wide) }
      w.dispose(); wide.remove()
      const flat = document.createElement('canvas')
      flat.getContext('2d')
      out.vrmOn2d = await new BodyPlayer(flat, new VrmBody(), { autoplay: false }).load('/__tiny.vrm').then(() => 'loaded', (e) => e.message)
      const head = p.body.head()
      out.vrm = { name: m.name, format: m.format, expressions: m.expressions.map((e) => e.label), share, happy, smile, unknown, tracked: seen.tracked,
        head: [head.x, head.y].every(Number.isFinite) }
      p.dispose()
    } catch (e) { out.vrm = { error: String(e.message || e) } }
    return out
  }
</script>
</html>