A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.
// The motion that needs no GPU: actions, the idle motion, talking, the lip-sync envelope, the filters, the face trace
// the tracking tests play, and the mark's timing and place.
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { ActionPlayer } from '../src/runtime/actions'
import { IdleDriver, PARAM_RANGES, defaultParams, type Params } from '../src/runtime/puppet'
import { TalkMotion } from '../src/runtime/talk'
import { envelope } from '../src/runtime/speech'
import { VowelCue } from '../src/runtime/vowel'
import { OneEuro } from '../src/runtime/filters'
import { BS, synthTrace } from '../src/runtime/trace'
import { cornerSpot, eyeOpen } from '../src/runtime/brand'
import { Idle, type Seen } from '../src/follow'
const outOfRange = (P: Params) => {
for (const [k, [lo, hi]] of Object.entries(PARAM_RANGES)) {
const v = P[k as keyof Params]
if (!(v >= lo - 1e-9 && v <= hi + 1e-9)) return `${k} = ${v}`
}
return null
}
const max = (xs: ArrayLike<number>) => Math.max(...Array.from(xs))
describe('actions', () => {
for (const id of ['nod', 'shake', 'tilt', 'hop', 'shy'] as const) {
it(`${id} moves, then ends with nothing left`, () => {
const a = new ActionPlayer()
a.play(id)
let most = 0
for (let i = 0; i < 180; i++) {
const o = a.update(1 / 60)
most = Math.max(most, ...Object.values(o).map((v) => Math.abs(v ?? 0)), Math.abs(a.y))
}
assert.ok(most > 0)
assert.equal(a.busy, false)
assert.ok(Object.values(a.update(1 / 60)).every((v) => !v))
})
}
it('away eases in, and out when pressed again', () => {
const a = new ActionPlayer()
a.play('away')
for (let i = 0; i < 180; i++) a.update(1 / 60)
assert.ok(a.awayW > 0.95)
a.play('away')
for (let i = 0; i < 180; i++) a.update(1 / 60)
assert.equal(a.awayW, 0)
})
})
describe('idle motion', () => {
it('blinks, turns toward the pointer and stays in range', () => {
const idle = new IdleDriver(), P = defaultParams()
idle.look = { x: 1, y: 0 }
let closed = false
for (let t = 0; t < 30; t += 1 / 60) {
idle.apply(P, t, 1 / 60)
closed ||= P.eyeLOpen < 0.1
assert.equal(outOfRange(P), null)
}
assert.ok(closed, 'blinked')
assert.ok(P.angleX > 10, `turned: ${P.angleX}`)
})
})
describe('the players\' idle (attend: follows the pointer with the eyes and the face)', () => {
// a face in the upper middle of a 400 x 500 canvas, as pointAt reads a drawn puppet
const seen = { model: { anchors: { head: { cx: 0, cy: -0.4, rx: 0.3, ry: 0.3 } } }, lastView: [1, 1, 0, 0, 2, 2] } as unknown as Seen
const run = (idle: Idle, secs: number, each: (t: number) => void = () => {}) => {
const P = defaultParams(), seenP: Params[] = []
for (let i = 0; i < secs * 60; i++) {
each(i / 60)
idle.apply(P, i / 60, 1 / 60, { busy: false, tracked: false, faceFree: true })
idle.shown(P)
assert.equal(outOfRange(P), null)
seenP.push({ ...P })
}
return seenP
}
for (const side of [1, -1]) {
it(`a pointer moving on the ${side > 0 ? 'right' : 'left'}: the eyes lead there, the head turns at most ~13°`, () => {
const idle = new Idle()
const frames = run(idle, 3, (t) => idle.pointer(seen, 200 + side * (150 + 30 * Math.sin(t * 6)), 140, 400, 500, true))
assert.ok(Math.max(...frames.map((P) => side * P.eyeBallX)) > 0.3, 'eyes toward it')
assert.ok(Math.max(...frames.map((P) => Math.abs(P.angleX))) <= 14, 'head within ~13°')
})
}
it('a pointer at rest lets go after a while: the character looks back', () => {
const idle = new Idle()
const frames = run(idle, 5, () => idle.pointer(seen, 380, 140, 400, 500, true))
assert.ok(Math.abs(frames[frames.length - 1].eyeBallX) < 0.3)
})
it('lookAt holds the attention, whatever the pointer does; null lets go', () => {
const idle = new Idle()
idle.lookAt(seen, -0.9, 0, 400, 500)
const frames = run(idle, 4, () => idle.pointer(seen, 390, 250, 400, 500, true))
const P = frames[frames.length - 1]
assert.ok(P.eyeBallX < -0.5 && P.angleX < -5, `eyes ${P.eyeBallX}, head ${P.angleX}`)
idle.lookAt(seen, null, 0, 400, 500)
const after = run(idle, 4)
assert.ok(Math.abs(after[after.length - 1].eyeBallX) < 0.5)
})
})
describe('talking', () => {
it('stays still in silence', () => {
const t = new TalkMotion(), P = defaultParams()
for (let i = 0; i < 120; i++) {
const o = t.update(0, 1 / 60, P)
assert.ok(Math.abs(o.angleX ?? 0) + Math.abs(o.angleY ?? 0) + Math.abs(o.browLY ?? 0) < 1e-6)
}
})
it('nods on syllables', () => {
const t = new TalkMotion(), P = defaultParams()
for (let i = 0; i < 240; i++) t.update(i % 15 < 7 ? 0.8 : 0.05, 1 / 60, P) // four syllables a second
assert.ok(t.stats.onsets > 10, `onsets: ${t.stats.onsets}`)
assert.ok(t.stats.nods > 0)
})
})
describe('lip-sync envelope', () => {
const sr = 24000
const tone = (secs: number, on: (t: number) => boolean) =>
Float32Array.from({ length: secs * sr }, (_, i) => (on(i / sr) ? 0.3 * Math.sin(2 * Math.PI * 400 * (i / sr)) : 0))
it('is closed for silence', () => {
assert.equal(max(envelope(new Float32Array(sr), sr)), 0)
})
it('opens while the voice sounds and closes after it', () => {
const e = envelope(tone(2, (t) => t > 0.5 && t < 1), sr) // 100 frames a second
assert.ok(max(e.slice(55, 95)) > 0.5)
assert.ok(max(e.slice(0, 40)) < 0.05)
assert.ok(max(e.slice(130)) < 0.05)
})
it('the vowel cue stays 0 in silence', () => {
assert.equal(new VowelCue().push(new Float32Array(512), 46.875, 0, 0.02), 0)
})
})
describe('filters and traces', () => {
it('One Euro leaves a constant alone and follows a step', () => {
const f = new OneEuro(1, 0.1)
for (let i = 0; i < 30; i++) assert.ok(Math.abs(f.filter(5, 1 / 30) - 5) < 1e-9)
let v = 5
for (let i = 0; i < 60; i++) v = f.filter(10, 1 / 30)
assert.ok(v > 9.5, `${v}`)
})
it('the synthetic face trace: 24 s at 30 fps, the same each time, no face at 10-12 s', () => {
const a = synthTrace()
assert.equal(a.length, 720)
assert.deepEqual(a, synthTrace())
const lost = a.filter((f) => !f.m && !f.b).map((f) => f.t)
assert.equal(lost.length, 60)
assert.ok(Math.min(...lost) >= 10_000 && Math.max(...lost) < 12_000)
assert.equal(a.find((f) => f.b)!.b!.length, BS.length)
})
})
describe('the mark', () => {
it('blinks once every 5.5 s', () => {
assert.equal(eyeOpen(0), 1)
assert.ok(Math.min(...Array.from({ length: 550 }, (_, i) => eyeOpen(i * 10))) < 0.2)
})
it('sits in the bottom-right corner', () => {
const c = cornerSpot(1920, 1080)
assert.ok(c.right > 1600 && c.right < 1920 && c.y > 900 && c.y < 1080 && c.logo > 0)
})
})