A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.
<!doctype html>
<html lang="en">
<meta charset="utf-8">
<title>Bodies smoke test</title>
<script type="importmap">{ "imports": {
"three": "/node_modules/three/build/three.module.js",
"three/examples/jsm/": "/node_modules/three/examples/jsm/",
"@pixiv/three-vrm": "/node_modules/@pixiv/three-vrm/lib/three-vrm.module.js",
"@mediapipe/tasks-vision": "/node_modules/@mediapipe/tasks-vision/vision_bundle.mjs"
} }</script>
<canvas id="png" style="width: 200px; height: 240px"></canvas>
<canvas id="vrm" style="width: 200px; height: 300px"></canvas>
<script type="module">
// Run by smoke.mjs: a PNGTuber shows the picture for each mouth / eye state, and follows the face tracker (its
// synthetic trace) and a speech clip; a VRM (tiny-vrm.mjs, served by smoke.mjs) loads, draws, takes its expressions
// and follows the face tracker.
import { Player } from '../dist/index.js'
import { BodyPlayer } from '../dist/body.js'
import { PngBody } from '../dist/png.js'
import { VrmBody } from '../dist/vrm.js'
import { createSpeech } from '../dist/voice.js'
import { preloadFaceModel, startFaceTracking } from '../dist/tracking.js'
const g = document.createElement('canvas').getContext('2d', { willReadFrequently: true })
const read = (canvas) => {
g.canvas.width = canvas.width; g.canvas.height = canvas.height
g.clearRect(0, 0, canvas.width, canvas.height); g.drawImage(canvas, 0, 0)
return g.getImageData(0, 0, canvas.width, canvas.height)
}
const pixel = (canvas, x, y) => { const d = read(canvas); const i = 4 * (Math.round(y) * d.width + Math.round(x)); return [...d.data.slice(i, i + 3)] }
const covered = (canvas) => { const d = read(canvas).data; let n = 0; for (let i = 3; i < d.length; i += 4) if (d[i] > 0) n++; return n / (d.length / 4) }
const solid = (color) => { const c = new OffscreenCanvas(100, 120), x = c.getContext('2d'); x.fillStyle = color; x.fillRect(0, 0, 100, 120); return c.convertToBlob() }
const wait = (ms) => new Promise((r) => setTimeout(r, ms))
/** until the condition holds, or some seconds: what a browser does next may take a while on a busy machine */
const until = async (cond, ms = 5000) => { for (const t0 = performance.now(); !cond() && performance.now() - t0 < ms;) await wait(10) }
/** what the drivers did, seen by a driver that runs after them */
const watch = (player) => {
const seen = { tracked: false, mouth: 0, angle: 0 }
player.use({ order: 100, update(f) { seen.tracked ||= f.tracked; seen.mouth = Math.max(seen.mouth, f.params.mouthOpen); seen.angle = Math.max(seen.angle, Math.abs(f.params.angleX)) } })
return seen
}
/** a voice-like clip: 220 Hz with syllables, 16-bit mono WAV */
function wav(secs, sr = 24000) {
const n = Math.round(secs * sr), b = new DataView(new ArrayBuffer(44 + 2 * n))
const s = (o, t) => [...t].forEach((c, i) => b.setUint8(o + i, c.charCodeAt(0)))
s(0, 'RIFF'); b.setUint32(4, 36 + 2 * n, true); s(8, 'WAVEfmt '); b.setUint32(16, 16, true); b.setUint16(20, 1, true); b.setUint16(22, 1, true)
b.setUint32(24, sr, true); b.setUint32(28, 2 * sr, true); b.setUint16(32, 2, true); b.setUint16(34, 16, true); s(36, 'data'); b.setUint32(40, 2 * n, true)
for (let i = 0; i < n; i++) { const t = i / sr; b.setInt16(44 + 2 * i, 12000 * Math.sin(2 * Math.PI * 220 * t) * (Math.sin(Math.PI * 4 * t) > 0 ? 1 : 0.05), true) }
return new Blob([b.buffer], { type: 'audio/wav' })
}
/** what Player and BodyPlayer promise alike: one test, run on both (see PlayerBase) */
window.runContract = async (kind, id) => {
const c = document.createElement('canvas')
c.style.cssText = 'width: 200px; height: 240px'
document.body.append(c)
const options = { autoplay: false, followPointer: false, idle: false, maxPixelRatio: 2 }
const p = kind === 'player' ? new Player(c, options) : new BodyPlayer(c, new PngBody(), options)
const good = kind === 'player' ? `../characters/${id}.livliv` : new Map([['closed.png', await solid('#f00')], ['open.png', await solid('#0f0')]])
const bad = kind === 'player' ? new Map([['model.json', new Blob(['{'])]]) : new Map([['closed.png', await solid('#f00')]])
const out = { before: p.model, playing: p.playing }
const info = await p.load(good)
out.name = info.name === p.model?.name
out.expressions = p.expressions === info.expressions
// the pixel ratio can be read and changed
out.ratio = [p.maxPixelRatio]
p.maxPixelRatio = 0.5; p.resize(); out.ratio.push(p.maxPixelRatio, c.width)
p.maxPixelRatio = 2; p.resize(); out.ratio.push(c.width)
// a snapshot is the pose as it is now, on a paused player too
const a = p.snapshot()
p.params.mouthOpen = 1
const b = p.snapshot()
out.snapshot = a !== b && b.startsWith('data:image/png')
p.params.mouthOpen = 0
// what a driver gets; the time counts the frames before this one
const times = []
let frame = null
const remove = p.use({ update(f) { times.push(f.time); frame = f } })
p.draw(0.01); p.draw(0.02); p.draw(0.03)
out.times = [times[1] - times[0], times[2] - times[1]]
out.frame = !!frame.params && typeof frame.add === 'function' && frame.dt === 0.03 && 'x' in frame.move
remove()
p.draw(0.01)
out.removed = times.length === 3
// the pose stays in range, and what is no number is the default, whoever set it
const pose = () => p.frame.params
const P = p.params
P.angleX = 1e9; P.eyeLOpen = -5; p.draw(0.01); out.clamped = [pose().angleX, pose().eyeLOpen]
P.angleX = NaN; P.eyeLOpen = NaN; P.bodyX = 'x'; p.draw(0.01); out.nan = [pose().angleX, pose().eyeLOpen, pose().bodyX]
const undo = p.use({ order: 100, update(f) { f.params.angleY = NaN; f.params.mouthOpen = 7 } })
p.draw(0.01); out.driver = [pose().angleY, pose().mouthOpen]
undo()
Object.assign(P, { angleX: 0, eyeLOpen: 1, bodyX: 0 })
p.draw(0.01)
out.alive = kind === 'player' ? !p.puppet.layers.some((L) => Number.isNaN(L.out[0])) : true
// away is a switch, on both
p.setAway(true)
for (let i = 0; i < 90; i++) p.draw(1 / 30)
out.away = pose().eyeLOpen
p.setAway(false)
for (let i = 0; i < 90; i++) p.draw(1 / 30)
out.back = pose().eyeLOpen
out.oldAway = (() => { try { p.act('away'); return 'took it' } catch (e) { return e.message } })()
// the rest of the surface
out.unknown = p.setExpression('no-such-expression')
out.neutral = p.setExpression(null)
for (const a2 of ['nod', 'shake', 'tilt', 'hop', 'shy']) p.act(a2)
p.lookAt(0.5, 0.5); p.lookAt(null); p.blink(); p.draw(0.01)
p.play(); out.plays = p.playing; p.pause(); out.paused = p.playing
// Player alone: from the canvas to the model, and a flick
if (kind === 'player') {
const [mx, my] = p.canvasToModel(c.clientWidth / 2, c.clientHeight / 2), [mw, mh] = p.model.size
out.toModel = mx > 0 && my > 0 && mx < mw && my < mh
const R = p.puppet.chainRuns[0]?.R
out.poke = [p.poke(mx, my), R ? p.poke(R[R.length - 2], R[R.length - 1]) : -1]
}
// a load that fails keeps the character; a disposed player says so and takes nothing more
const name = p.model.name
out.failed = await p.load(bad).then(() => 'loaded', () => 'refused')
out.kept = p.model?.name === name
p.dispose(); p.dispose()
out.after = p.model
out.disposedLoad = await p.load(good).then(() => 'loaded', (e) => e.message)
let gone = 0
p.use({ update() {}, dispose() { gone++ } })
out.gone = gone
c.remove()
return out
}
/** BodyPlayer: several loads at once. Those still downloading or waiting stop; one already going in (inside the Body,
* which cannot be stopped) finishes, and the newer one follows */
window.runBodyLoads = async () => {
const out = {}
const slow = (color) => ({ closed: `/__slow-png/${color}.png`, open: `/__slow-png/${color}.png` })
{
const c = document.createElement('canvas'), p = new BodyPlayer(c, new PngBody(), { autoplay: false, followPointer: false, idle: false })
c.style.cssText = 'width: 40px; height: 40px'
document.body.append(c)
const say = (e) => e.name
const a = p.load(slow('ff0000')).then(() => 'loaded', say)
await wait(60)
const b = p.load(slow('00ff00')).then(() => 'loaded', say)
await wait(60)
const steps = []
const last = await p.load(slow('0000ff'), { onProgress: (s) => steps.push(s) }).then(() => 'loaded', say)
out.row = { a: await a, b: await b, last, pixel: pixel(c, 20, 20), steps }
p.dispose(); c.remove()
}
{
// a Body holding its load: the page decides when it ends
const held = [], calls = []
const body = {
format: 'test',
load(files) { calls.push([...files.keys()][0]); return new Promise((res) => held.push(() => res({ name: [...files.keys()][0], expressions: [] }))) },
render() {}, head: () => ({ x: 0.5, y: 0.3, rx: 0.1, ry: 0.1 }), dispose() {},
}
const c = document.createElement('canvas'), p = new BodyPlayer(c, body, { autoplay: false, followPointer: false })
const file = (n) => new Map([[n, new Blob(['x'])]])
const a = p.load(file('a')).then((m) => m.name, (e) => e.name)
await wait(20)
const b = p.load(file('b')).then((m) => m.name, (e) => e.name)
const d = p.load(file('d')).then((m) => m.name, (e) => e.name)
await wait(20)
held.shift()()
out.held = { a: await a, b: await b }
await wait(20)
held.shift()()
out.held.d = await d
out.held.calls = calls
out.held.shown = p.model?.name
p.dispose()
}
{
// a Body that takes its 2D context only at its first frame: asking about the context must not start a WebGL one
const c = document.createElement('canvas')
c.style.cssText = 'width: 40px; height: 40px'
document.body.append(c)
let got = null
const body = {
format: 'late', async load() { return { name: 'late', expressions: [] } },
render(cv) { got = cv.getContext('2d') }, head: () => ({ x: 0.5, y: 0.3, rx: 0.1, ry: 0.1 }), dispose() {},
}
const p = new BodyPlayer(c, body, { autoplay: false, followPointer: false })
out.lateLost = p.contextLost
const going = p.load(new Map([['a', new Blob(['x'])]]))
out.lateLost2 = p.contextLost
await going
out.late2d = !!got
p.dispose({ release: true })
await wait(20)
out.lateAgain = (() => { try { new BodyPlayer(c, body, { autoplay: false }).dispose(); return 'made' } catch (e) { return e.code } })()
c.remove()
}
return out
}
/** the context on a BodyPlayer: a VRM is told and released like a Player; a PNGTuber has none to lose */
window.runBodyContext = async () => {
const out = {}
const code = (e) => e.code ?? e.name
{
const c = document.createElement('canvas'), p = new BodyPlayer(c, new VrmBody(), { followPointer: false })
c.style.cssText = 'width: 60px; height: 80px'
document.body.append(c)
let told = 0
p.onContextLost = () => told++
await p.load('/__tiny.vrm')
const gl = c.getContext('webgl2')
gl.getExtension('WEBGL_lose_context').loseContext()
await until(() => told)
out.vrm = { told, lost: p.contextLost, load: await p.load('/__tiny.vrm').then(() => 'loaded', code) }
p.dispose(); c.remove()
}
{
const c = document.createElement('canvas'), p = new BodyPlayer(c, new VrmBody(), { followPointer: false })
c.style.cssText = 'width: 60px; height: 80px'
document.body.append(c)
await p.load('/__tiny.vrm')
const gl = c.getContext('webgl2')
p.dispose({ release: true })
await until(() => gl.isContextLost())
out.vrmReleased = { lost: gl.isContextLost(), reuse: (() => { try { new BodyPlayer(c, new VrmBody()); return 'made' } catch (e) { return code(e) } })() }
c.remove()
}
{
const c = document.createElement('canvas')
c.style.cssText = 'width: 60px; height: 80px'
document.body.append(c)
const files = new Map([['closed.png', await solid('#f00')], ['open.png', await solid('#0f0')]])
const p = new BodyPlayer(c, new PngBody(), { autoplay: false, followPointer: false, idle: false })
await p.load(files)
out.png = { lost: p.contextLost }
p.dispose({ release: true })
await wait(30)
const q = new BodyPlayer(c, new PngBody(), { autoplay: false, followPointer: false, idle: false })
out.png.again = await q.load(files).then((m) => m.name, code)
q.dispose(); c.remove()
}
return out
}
/** the camera refused while the face model comes in (the slow route of smoke.mjs); ahead: it was asked for before */
window.runRefused = async (ahead = false) => {
const files = { wasmUrl: '/node_modules/@mediapipe/tasks-vision/wasm', modelUrl: '/__model.task' }
if (ahead) preloadFaceModel(files).catch(() => {})
const real = navigator.mediaDevices.getUserMedia
navigator.mediaDevices.getUserMedia = async () => { await wait(150); throw new DOMException('refused', 'NotAllowedError') }
const p = new BodyPlayer(document.createElement('canvas'), new PngBody(), { autoplay: false })
const name = await startFaceTracking(p, files).then(() => 'started', (e) => e.name)
navigator.mediaDevices.getUserMedia = real
p.dispose()
return name
}
/** what a VRM player says of a file it cannot play, keeping the character before */
window.runVrmRefusals = async (urls) => {
const c = document.createElement('canvas')
const p = new BodyPlayer(c, new VrmBody(), { autoplay: false, idle: false, followPointer: false })
await p.load('/__tiny.vrm')
const out = {}
for (const [k, u] of Object.entries(urls)) out[k] = await p.load(u).then(() => 'loaded', (e) => e.message)
out.plainFile = await p.load(new File([new Blob(['not a glb at all, a text file'])], 'x.vrm')).then(() => 'loaded', (e) => e.message)
out.kept = p.model?.name
p.dispose()
return out
}
/** how much of the canvas a VRM covers */
window.runVrmShare = async (url) => {
const c = document.createElement('canvas')
c.style.cssText = 'width: 300px; height: 400px'
document.body.append(c)
const p = new BodyPlayer(c, new VrmBody(), { autoplay: false, idle: false, followPointer: false })
await p.load(url)
p.resize()
const share = covered(c)
p.dispose(); c.remove()
return share
}
/** which way a VRM stands and moves, 1.0 and 0.x alike: arms down, head turning right and up, top of the head to the right
* on a positive roll, eyes up and to the character's left (the screen's right) */
window.runVrmPose = async (url) => {
const THREE = await import('three')
const c = document.createElement('canvas')
c.style.cssText = 'width: 300px; height: 400px'
document.body.append(c)
const p = new BodyPlayer(c, new VrmBody(), { autoplay: false, idle: false, followPointer: false })
await p.load(url)
const vrm = p.body.vrm, H = vrm.humanoid, em = vrm.expressionManager
const set = (o) => { Object.assign(p.params, { angleX: 0, angleY: 0, angleZ: 0, eyeBallX: 0, eyeBallY: 0, bodyZ: 0 }, o); p.draw(0); p.draw(0) }
const world = (b) => { const v = new THREE.Vector3(); H.getRawBoneNode(b).getWorldPosition(v); return v }
const q = (b) => { const r = new THREE.Quaternion(); H.getRawBoneNode(b).getWorldQuaternion(r); return r }
set({})
const rest = q('head'), turned = (v) => new THREE.Vector3(...v).applyQuaternion(q('head').multiply(rest.clone().invert()))
const out = { spec: vrm.meta.metaVersion, hand: world('leftHand').y - world('leftUpperArm').y, terms: p.model.meta }
set({ angleX: 20 }); out.yaw = turned([0, 0, 1]).x
set({ angleY: 20 }); out.pitch = turned([0, 0, 1]).y
set({ angleZ: 20 }); out.roll = turned([0, 1, 0]).x
set({ eyeBallY: 1 }); out.up = em.getValue('lookUp'); out.down = em.getValue('lookDown')
set({ eyeBallX: 1 }); out.right = em.getValue('lookLeft') // the character's left, seen from the front
p.dispose(); c.remove()
return out
}
window.runBodies = async () => {
const out = {}
try {
const canvas = document.getElementById('png')
const p = new BodyPlayer(canvas, new PngBody(), { autoplay: false, idle: false })
const m = await p.load({ closed: await solid('#ff0000'), open: await solid('#00ff00'), blink: await solid('#0000ff'), 'blink-open': await solid('#ffff00') })
const at = () => { p.resize(); return pixel(canvas, canvas.width / 2, canvas.height * 0.8) }
const shown = { closed: at() }
p.params.mouthOpen = 1; shown.open = at()
p.params.eyeLOpen = p.params.eyeROpen = 0; shown.blinkOpen = at()
p.params.mouthOpen = 0; shown.blink = at()
p.params.eyeLOpen = p.params.eyeROpen = 1
const seen = watch(p)
p.idle = true; p.play()
const tracking = await startFaceTracking(p, { trace: 'synth' })
const speech = createSpeech(p)
speech.unlock(); speech.say(wav(1.2))
await wait(1600)
tracking.stop(); speech.dispose()
out.png = { name: m.name, format: m.format, expressions: m.expressions.map((e) => e.id), shown, ...seen }
p.dispose()
// a failed load leaves the pictures shown, and the name; on a canvas with WebGL it says so
const canvas2 = document.createElement('canvas')
canvas2.style.cssText = 'width: 100px; height: 120px'
document.body.append(canvas2)
const q = new BodyPlayer(canvas2, new PngBody(), { autoplay: false, idle: false })
await q.load({ closed: await solid('#ff0000'), open: await solid('#00ff00'), 'name.txt': new Blob(['First']) })
const refused = await q.load({ closed: await solid('#0000ff') }).then(() => 'loaded', (e) => e.message)
q.resize()
out.pngFailed = { refused, kept: q.model?.name, pixel: pixel(canvas2, canvas2.width / 2, canvas2.height * 0.8) }
q.dispose(); canvas2.remove()
const gl = document.createElement('canvas')
gl.getContext('webgl2')
out.pngOnGl = await new BodyPlayer(gl, new PngBody(), { autoplay: false }).load({ closed: await solid('#ff0000'), open: await solid('#00ff00') }).then(() => 'loaded', (e) => e.message)
} catch (e) { out.png = { error: String(e.message || e) } }
try {
const canvas = document.getElementById('vrm')
const p = new BodyPlayer(canvas, new VrmBody(), { autoplay: false, lang: 'en' })
const m = await p.load('/__tiny.vrm')
p.resize()
const share = covered(canvas)
const happy = p.setExpression('happy'), smile = p.setExpression('smile'), unknown = p.setExpression('nope')
p.params.mouthOpen = 1; p.resize()
const seen = watch(p)
p.play()
const tracking = await startFaceTracking(p, { trace: 'synth' })
await wait(800)
tracking.stop()
// a second load, on a canvas that is not 2:3 and already has its size, and a failed one
const wide = document.createElement('canvas')
wide.style.cssText = 'width: 400px; height: 200px'
document.body.append(wide)
wide.width = 400; wide.height = 200
const w = new BodyPlayer(wide, new VrmBody(), { autoplay: false, idle: false })
await w.load('/__tiny.vrm'); w.resize()
const aspect1 = w.body.camera.aspect
await w.load('/__tiny.vrm'); w.resize()
const aspect2 = w.body.camera.aspect
const bad = new File([new Blob(['not a glb'])], 'x.vrm')
const refused = await w.load(bad).then(() => 'loaded', (e) => e.message)
w.resize()
out.vrmAgain = { aspect1, aspect2, refused, kept: w.model?.name, share: covered(wide) }
w.dispose(); wide.remove()
const flat = document.createElement('canvas')
flat.getContext('2d')
out.vrmOn2d = await new BodyPlayer(flat, new VrmBody(), { autoplay: false }).load('/__tiny.vrm').then(() => 'loaded', (e) => e.message)
const head = p.body.head()
out.vrm = { name: m.name, format: m.format, expressions: m.expressions.map((e) => e.label), share, happy, smile, unknown, tracked: seen.tracked,
head: [head.x, head.y].every(Number.isFinite) }
p.dispose()
} catch (e) { out.vrm = { error: String(e.message || e) } }
return out
}
</script>
</html>